dram_ctrl.cc (10209:ac71c857e1e1) | dram_ctrl.cc (10210:793e5ff26e0b) |
---|---|
1/* 2 * Copyright (c) 2010-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 61 unchanged lines hidden (view full) --- 70 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 71 readBufferSize(p->read_buffer_size), 72 writeBufferSize(p->write_buffer_size), 73 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 74 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 75 minWritesPerSwitch(p->min_writes_per_switch), 76 writesThisTime(0), readsThisTime(0), 77 tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), | 1/* 2 * Copyright (c) 2010-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 61 unchanged lines hidden (view full) --- 70 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 71 readBufferSize(p->read_buffer_size), 72 writeBufferSize(p->write_buffer_size), 73 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 74 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 75 minWritesPerSwitch(p->min_writes_per_switch), 76 writesThisTime(0), readsThisTime(0), 77 tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), |
78 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), | 78 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR), |
79 tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 80 tXAW(p->tXAW), activationLimit(p->activation_limit), 81 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 82 pageMgmt(p->page_policy), 83 maxAccessesPerRow(p->max_accesses_per_row), 84 frontendLatency(p->static_frontend_latency), 85 backendLatency(p->static_backend_latency), 86 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), --- 467 unchanged lines hidden (view full) --- 554 "tRCD %d ticks\n" \ 555 "tCL %d ticks\n" \ 556 "tRP %d ticks\n" \ 557 "tBURST %d ticks\n" \ 558 "tRFC %d ticks\n" \ 559 "tREFI %d ticks\n" \ 560 "tWTR %d ticks\n" \ 561 "tRTW %d ticks\n" \ | 79 tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 80 tXAW(p->tXAW), activationLimit(p->activation_limit), 81 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 82 pageMgmt(p->page_policy), 83 maxAccessesPerRow(p->max_accesses_per_row), 84 frontendLatency(p->static_frontend_latency), 85 backendLatency(p->static_backend_latency), 86 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), --- 467 unchanged lines hidden (view full) --- 554 "tRCD %d ticks\n" \ 555 "tCL %d ticks\n" \ 556 "tRP %d ticks\n" \ 557 "tBURST %d ticks\n" \ 558 "tRFC %d ticks\n" \ 559 "tREFI %d ticks\n" \ 560 "tWTR %d ticks\n" \ 561 "tRTW %d ticks\n" \ |
562 "tWR %d ticks\n" \ |
|
562 "tXAW (%d) %d ticks\n", 563 name(), tRCD, tCL, tRP, tBURST, tRFC, tREFI, tWTR, | 563 "tXAW (%d) %d ticks\n", 564 name(), tRCD, tCL, tRP, tBURST, tRFC, tREFI, tWTR, |
564 tRTW, activationLimit, tXAW); | 565 tRTW, tWR, activationLimit, tXAW); |
565} 566 567void 568DRAMCtrl::printQs() const { 569 DPRINTF(DRAM, "===READ QUEUE===\n\n"); 570 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 571 DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 572 } --- 229 unchanged lines hidden (view full) --- 802 // *after* that does it take to finish the request, depending 803 // on bank status and page open policy. Note that this method 804 // considers only the time taken for the actual read or write 805 // to complete, NOT any additional time thereafter for tRAS or 806 // tRP. 807 Tick accLat = 0; 808 Tick bankLat = 0; 809 rowHitFlag = false; | 566} 567 568void 569DRAMCtrl::printQs() const { 570 DPRINTF(DRAM, "===READ QUEUE===\n\n"); 571 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 572 DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 573 } --- 229 unchanged lines hidden (view full) --- 803 // *after* that does it take to finish the request, depending 804 // on bank status and page open policy. Note that this method 805 // considers only the time taken for the actual read or write 806 // to complete, NOT any additional time thereafter for tRAS or 807 // tRP. 808 Tick accLat = 0; 809 Tick bankLat = 0; 810 rowHitFlag = false; |
810 Tick potentialActTick; | |
811 812 const Bank& bank = dram_pkt->bankRef; 813 814 if (bank.openRow == dram_pkt->row) { 815 // When we have a row-buffer hit, 816 // we don't care about tRAS having expired or not, 817 // but do care about bank being free for access 818 rowHitFlag = true; --- 8 unchanged lines hidden (view full) --- 827 accLat += bank.freeAt - inTime; 828 bankLat += 0; 829 } else { 830 // CAS latency only 831 accLat += tCL; 832 bankLat += tCL; 833 } 834 } else { | 811 812 const Bank& bank = dram_pkt->bankRef; 813 814 if (bank.openRow == dram_pkt->row) { 815 // When we have a row-buffer hit, 816 // we don't care about tRAS having expired or not, 817 // but do care about bank being free for access 818 rowHitFlag = true; --- 8 unchanged lines hidden (view full) --- 827 accLat += bank.freeAt - inTime; 828 bankLat += 0; 829 } else { 830 // CAS latency only 831 accLat += tCL; 832 bankLat += tCL; 833 } 834 } else { |
835 // Row-buffer miss, need to close existing row 836 // once tRAS has expired, then open the new one, 837 // then add cas latency. 838 Tick freeTime = std::max(bank.tRASDoneAt, bank.freeAt); | 835 // Row-buffer miss, need to potentially close an existing row, 836 // then open the new one, then add CAS latency 837 Tick free_at = bank.freeAt; 838 Tick precharge_delay = 0; |
839 | 839 |
840 if (freeTime > inTime) 841 accLat += freeTime - inTime; | 840 // Check if we first need to precharge 841 if (bank.openRow != Bank::NO_ROW) { 842 free_at = std::max(bank.preAllowedAt, free_at); 843 precharge_delay = tRP; 844 } |
842 | 845 |
843 // If the there is no open row, then there is no precharge 844 // delay, otherwise go with tRP 845 Tick precharge_delay = bank.openRow == Bank::NO_ROW ? 0 : tRP; | 846 // If the earliest time to issue the command is in the future, 847 // add it to the access latency 848 if (free_at > inTime) 849 accLat += free_at - inTime; |
846 | 850 |
847 //The bank is free, and you may be able to activate 848 potentialActTick = inTime + accLat + precharge_delay; 849 if (potentialActTick < bank.actAllowedAt) 850 accLat += bank.actAllowedAt - potentialActTick; | 851 // We also need to account for the earliest activation time, 852 // and potentially add that as well to the access latency 853 Tick act_at = inTime + accLat + precharge_delay; 854 if (act_at < bank.actAllowedAt) 855 accLat += bank.actAllowedAt - act_at; |
851 852 accLat += precharge_delay + tRCD + tCL; 853 bankLat += precharge_delay + tRCD + tCL; 854 } 855 856 DPRINTF(DRAM, "Returning < %lld, %lld > from estimateLatency()\n", 857 bankLat, accLat); 858 859 return make_pair(bankLat, accLat); 860} 861 862void | 856 857 accLat += precharge_delay + tRCD + tCL; 858 bankLat += precharge_delay + tRCD + tCL; 859 } 860 861 DPRINTF(DRAM, "Returning < %lld, %lld > from estimateLatency()\n", 862 bankLat, accLat); 863 864 return make_pair(bankLat, accLat); 865} 866 867void |
863DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank, 864 uint16_t row) | 868DRAMCtrl::activateBank(Tick act_tick, uint8_t rank, uint8_t bank, 869 uint16_t row, Bank& bank_ref) |
865{ 866 assert(0 <= rank && rank < ranksPerChannel); 867 assert(actTicks[rank].size() == activationLimit); 868 869 DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 870 871 // update the open row | 870{ 871 assert(0 <= rank && rank < ranksPerChannel); 872 assert(actTicks[rank].size() == activationLimit); 873 874 DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 875 876 // update the open row |
872 assert(banks[rank][bank].openRow == Bank::NO_ROW); 873 banks[rank][bank].openRow = row; | 877 assert(bank_ref.openRow == Bank::NO_ROW); 878 bank_ref.openRow = row; |
874 875 // start counting anew, this covers both the case when we 876 // auto-precharged, and when this access is forced to 877 // precharge | 879 880 // start counting anew, this covers both the case when we 881 // auto-precharged, and when this access is forced to 882 // precharge |
878 banks[rank][bank].bytesAccessed = 0; 879 banks[rank][bank].rowAccesses = 0; | 883 bank_ref.bytesAccessed = 0; 884 bank_ref.rowAccesses = 0; |
880 881 ++numBanksActive; 882 assert(numBanksActive <= banksPerRank * ranksPerChannel); 883 884 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n", 885 act_tick, numBanksActive); 886 887 // start by enforcing tRRD 888 for(int i = 0; i < banksPerRank; i++) { | 885 886 ++numBanksActive; 887 assert(numBanksActive <= banksPerRank * ranksPerChannel); 888 889 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n", 890 act_tick, numBanksActive); 891 892 // start by enforcing tRRD 893 for(int i = 0; i < banksPerRank; i++) { |
889 // next activate must not happen before tRRD 890 banks[rank][i].actAllowedAt = act_tick + tRRD; | 894 // next activate to any bank in this rank must not happen 895 // before tRRD 896 banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 897 banks[rank][i].actAllowedAt); |
891 } 892 | 898 } 899 |
893 // tRC should be added to activation tick of the bank currently accessed, 894 // where tRC = tRAS + tRP, this is just for a check as actAllowedAt for same 895 // bank is already captured by bank.freeAt and bank.tRASDoneAt 896 banks[rank][bank].actAllowedAt = act_tick + tRAS + tRP; 897 | |
898 // next, we deal with tXAW, if the activation limit is disabled 899 // then we are done 900 if (actTicks[rank].empty()) 901 return; 902 903 // sanity check 904 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { | 900 // next, we deal with tXAW, if the activation limit is disabled 901 // then we are done 902 if (actTicks[rank].empty()) 903 return; 904 905 // sanity check 906 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { |
905 // @todo For now, stick with a warning 906 warn("Got %d activates in window %d (%d - %d) which is smaller " 907 "than %d\n", activationLimit, act_tick - actTicks[rank].back(), 908 act_tick, actTicks[rank].back(), tXAW); | 907 panic("Got %d activates in window %d (%llu - %llu) which is smaller " 908 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 909 act_tick, actTicks[rank].back(), tXAW); |
909 } 910 911 // shift the times used for the book keeping, the last element 912 // (highest index) is the oldest one and hence the lowest value 913 actTicks[rank].pop_back(); 914 915 // record an new activation (in the future) 916 actTicks[rank].push_front(act_tick); 917 918 // cannot activate more than X times in time window tXAW, push the 919 // next one (the X + 1'st activate) to be tXAW away from the 920 // oldest in our window of X 921 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 922 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " | 910 } 911 912 // shift the times used for the book keeping, the last element 913 // (highest index) is the oldest one and hence the lowest value 914 actTicks[rank].pop_back(); 915 916 // record an new activation (in the future) 917 actTicks[rank].push_front(act_tick); 918 919 // cannot activate more than X times in time window tXAW, push the 920 // next one (the X + 1'st activate) to be tXAW away from the 921 // oldest in our window of X 922 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 923 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " |
923 "than %d\n", activationLimit, actTicks[rank].back() + tXAW); | 924 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); |
924 for(int j = 0; j < banksPerRank; j++) 925 // next activate must not happen before end of window | 925 for(int j = 0; j < banksPerRank; j++) 926 // next activate must not happen before end of window |
926 banks[rank][j].actAllowedAt = actTicks[rank].back() + tXAW; | 927 banks[rank][j].actAllowedAt = 928 std::max(actTicks[rank].back() + tXAW, 929 banks[rank][j].actAllowedAt); |
927 } 928 929 // at the point when this activate takes place, make sure we 930 // transition to the active power state 931 if (!activateEvent.scheduled()) 932 schedule(activateEvent, act_tick); 933 else if (activateEvent.when() > act_tick) 934 // move it sooner in time --- 66 unchanged lines hidden (view full) --- 1001 Tick bankLat = lat.first; 1002 Tick accessLat = lat.second; 1003 Tick actTick; 1004 1005 // This request was woken up at this time based on a prior call 1006 // to estimateLatency(). However, between then and now, both the 1007 // accessLatency and/or busBusyUntil may have changed. We need 1008 // to correct for that. | 930 } 931 932 // at the point when this activate takes place, make sure we 933 // transition to the active power state 934 if (!activateEvent.scheduled()) 935 schedule(activateEvent, act_tick); 936 else if (activateEvent.when() > act_tick) 937 // move it sooner in time --- 66 unchanged lines hidden (view full) --- 1004 Tick bankLat = lat.first; 1005 Tick accessLat = lat.second; 1006 Tick actTick; 1007 1008 // This request was woken up at this time based on a prior call 1009 // to estimateLatency(). However, between then and now, both the 1010 // accessLatency and/or busBusyUntil may have changed. We need 1011 // to correct for that. |
1009 | |
1010 Tick addDelay = (curTick() + accessLat < busBusyUntil) ? 1011 busBusyUntil - (curTick() + accessLat) : 0; 1012 | 1012 Tick addDelay = (curTick() + accessLat < busBusyUntil) ? 1013 busBusyUntil - (curTick() + accessLat) : 0; 1014 |
1015 // Update request parameters 1016 dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST; 1017 1018 DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \ 1019 "readytime is %lld busbusyuntil is %lld. " \ 1020 "Scheduling at readyTime\n", dram_pkt->addr, 1021 curTick(), accessLat, dram_pkt->readyTime, busBusyUntil); 1022 1023 // Make sure requests are not overlapping on the databus 1024 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 1025 |
|
1013 Bank& bank = dram_pkt->bankRef; 1014 1015 // Update bank state 1016 if (rowHitFlag) { 1017 bank.freeAt = curTick() + addDelay + accessLat; 1018 } else { 1019 // If there is a page open, precharge it. 1020 if (bank.openRow != Bank::NO_ROW) { 1021 prechargeBank(bank, std::max(std::max(bank.freeAt, | 1026 Bank& bank = dram_pkt->bankRef; 1027 1028 // Update bank state 1029 if (rowHitFlag) { 1030 bank.freeAt = curTick() + addDelay + accessLat; 1031 } else { 1032 // If there is a page open, precharge it. 1033 if (bank.openRow != Bank::NO_ROW) { 1034 prechargeBank(bank, std::max(std::max(bank.freeAt, |
1022 bank.tRASDoneAt), | 1035 bank.preAllowedAt), |
1023 curTick()) + tRP); 1024 } 1025 1026 // Any precharge is already part of the latency 1027 // estimation, so update the bank free time 1028 bank.freeAt = curTick() + addDelay + accessLat; 1029 1030 // any waiting for banks account for in freeAt 1031 actTick = bank.freeAt - tCL - tRCD; 1032 | 1036 curTick()) + tRP); 1037 } 1038 1039 // Any precharge is already part of the latency 1040 // estimation, so update the bank free time 1041 bank.freeAt = curTick() + addDelay + accessLat; 1042 1043 // any waiting for banks account for in freeAt 1044 actTick = bank.freeAt - tCL - tRCD; 1045 |
1033 // If you activated a new row do to this access, the next access 1034 // will have to respect tRAS for this bank 1035 bank.tRASDoneAt = actTick + tRAS; | 1046 // The next access has to respect tRAS for this bank 1047 bank.preAllowedAt = actTick + tRAS; |
1036 | 1048 |
1037 recordActivate(actTick, dram_pkt->rank, dram_pkt->bank, 1038 dram_pkt->row); | 1049 // Record the activation and deal with all the global timing 1050 // constraints caused be a new activation (tRRD and tXAW) 1051 activateBank(actTick, dram_pkt->rank, dram_pkt->bank, 1052 dram_pkt->row, bank); 1053 |
1039 } 1040 | 1054 } 1055 |
1056 // If this is a write, we also need to respect the write 1057 // recovery time before a precharge 1058 if (!dram_pkt->isRead) { 1059 bank.preAllowedAt = std::max(bank.preAllowedAt, 1060 dram_pkt->readyTime + tWR); 1061 } 1062 1063 // We also have to respect tRP, and any constraints on when we may 1064 // precharge the bank, in the case of reads this is really only 1065 // going to cause any change if we did not have a row hit and are 1066 // now forced to respect tRAS 1067 bank.actAllowedAt = std::max(bank.actAllowedAt, 1068 bank.preAllowedAt + tRP); 1069 |
|
1041 // increment the bytes accessed and the accesses per row 1042 bank.bytesAccessed += burstSize; 1043 ++bank.rowAccesses; 1044 1045 // if we reached the max, then issue with an auto-precharge 1046 bool auto_precharge = pageMgmt == Enums::close || 1047 bank.rowAccesses == maxAccessesPerRow; 1048 --- 40 unchanged lines hidden (view full) --- 1089 // 2) close_adaptive policy and we have not got any more hits 1090 auto_precharge = !got_more_hits && 1091 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1092 } 1093 1094 // if this access should use auto-precharge, then we are 1095 // closing the row 1096 if (auto_precharge) { | 1070 // increment the bytes accessed and the accesses per row 1071 bank.bytesAccessed += burstSize; 1072 ++bank.rowAccesses; 1073 1074 // if we reached the max, then issue with an auto-precharge 1075 bool auto_precharge = pageMgmt == Enums::close || 1076 bank.rowAccesses == maxAccessesPerRow; 1077 --- 40 unchanged lines hidden (view full) --- 1118 // 2) close_adaptive policy and we have not got any more hits 1119 auto_precharge = !got_more_hits && 1120 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1121 } 1122 1123 // if this access should use auto-precharge, then we are 1124 // closing the row 1125 if (auto_precharge) { |
1097 prechargeBank(bank, std::max(bank.freeAt, bank.tRASDoneAt) + tRP); | 1126 prechargeBank(bank, std::max(bank.freeAt, bank.preAllowedAt) + tRP); |
1098 1099 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1100 } 1101 1102 DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt); 1103 | 1127 1128 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1129 } 1130 1131 DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt); 1132 |
1104 // Update request parameters 1105 dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST; 1106 1107 DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \ 1108 "readytime is %lld busbusyuntil is %lld. " \ 1109 "Scheduling at readyTime\n", dram_pkt->addr, 1110 curTick(), accessLat, dram_pkt->readyTime, busBusyUntil); 1111 1112 // Make sure requests are not overlapping on the databus 1113 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 1114 | |
1115 // Update bus state 1116 busBusyUntil = dram_pkt->readyTime; 1117 1118 DPRINTF(DRAM,"Access time is %lld\n", 1119 dram_pkt->readyTime - dram_pkt->entryTime); 1120 1121 // Update the minimum timing between the requests, this is a 1122 // conservative estimate of when we have to schedule the next --- 284 unchanged lines hidden (view full) --- 1407 DPRINTF(DRAM, "Precharging all\n"); 1408 for (int i = 0; i < ranksPerChannel; i++) { 1409 for (int j = 0; j < banksPerRank; j++) { 1410 if (banks[i][j].openRow != Bank::NO_ROW) { 1411 // respect both causality and any existing bank 1412 // constraints 1413 Tick free_at = 1414 std::max(std::max(banks[i][j].freeAt, | 1133 // Update bus state 1134 busBusyUntil = dram_pkt->readyTime; 1135 1136 DPRINTF(DRAM,"Access time is %lld\n", 1137 dram_pkt->readyTime - dram_pkt->entryTime); 1138 1139 // Update the minimum timing between the requests, this is a 1140 // conservative estimate of when we have to schedule the next --- 284 unchanged lines hidden (view full) --- 1425 DPRINTF(DRAM, "Precharging all\n"); 1426 for (int i = 0; i < ranksPerChannel; i++) { 1427 for (int j = 0; j < banksPerRank; j++) { 1428 if (banks[i][j].openRow != Bank::NO_ROW) { 1429 // respect both causality and any existing bank 1430 // constraints 1431 Tick free_at = 1432 std::max(std::max(banks[i][j].freeAt, |
1415 banks[i][j].tRASDoneAt), | 1433 banks[i][j].preAllowedAt), |
1416 curTick()) + tRP; 1417 1418 prechargeBank(banks[i][j], free_at); 1419 } 1420 } 1421 } 1422 } else { 1423 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); --- 496 unchanged lines hidden --- | 1434 curTick()) + tRP; 1435 1436 prechargeBank(banks[i][j], free_at); 1437 } 1438 } 1439 } 1440 } else { 1441 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); --- 496 unchanged lines hidden --- |