1/* |
2 * Copyright (c) 2010-2018 ARM Limited |
3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated --- 68 unchanged lines hidden (view full) --- 79 bankGroupArch(p->bank_groups_per_rank > 0), 80 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 81 readBufferSize(p->read_buffer_size), 82 writeBufferSize(p->write_buffer_size), 83 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 84 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 85 minWritesPerSwitch(p->min_writes_per_switch), 86 writesThisTime(0), readsThisTime(0), |
87 tCK(p->tCK), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST), 88 tCCD_L_WR(p->tCCD_L_WR), |
89 tCCD_L(p->tCCD_L), tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), 90 tWR(p->tWR), tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 91 tRRD_L(p->tRRD_L), tXAW(p->tXAW), tXP(p->tXP), tXS(p->tXS), |
92 activationLimit(p->activation_limit), rankToRankDly(tCS + tBURST), 93 wrToRdDly(tCL + tBURST + p->tWTR), rdToWrDly(tRTW + tBURST), |
94 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 95 pageMgmt(p->page_policy), 96 maxAccessesPerRow(p->max_accesses_per_row), 97 frontendLatency(p->static_frontend_latency), 98 backendLatency(p->static_backend_latency), |
99 nextBurstAt(0), prevArrival(0), |
100 nextReqTime(0), activeRank(0), timeStampOffset(0), 101 lastStatsResetTick(0) 102{ 103 // sanity check the ranks since we rely on bit slicing for the 104 // address decoding 105 fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not " 106 "allowed, must be a power of two\n", ranksPerChannel); 107 --- 53 unchanged lines hidden (view full) --- 161 banksPerRank, bankGroupsPerRank); 162 } 163 // tCCD_L should be greater than minimal, back-to-back burst delay 164 if (tCCD_L <= tBURST) { 165 fatal("tCCD_L (%d) should be larger than tBURST (%d) when " 166 "bank groups per rank (%d) is greater than 1\n", 167 tCCD_L, tBURST, bankGroupsPerRank); 168 } |
169 // tCCD_L_WR should be greater than minimal, back-to-back burst delay 170 if (tCCD_L_WR <= tBURST) { 171 fatal("tCCD_L_WR (%d) should be larger than tBURST (%d) when " 172 "bank groups per rank (%d) is greater than 1\n", 173 tCCD_L_WR, tBURST, bankGroupsPerRank); 174 } |
175 // tRRD_L is greater than minimal, same bank group ACT-to-ACT delay 176 // some datasheets might specify it equal to tRRD 177 if (tRRD_L < tRRD) { 178 fatal("tRRD_L (%d) should be larger than tRRD (%d) when " 179 "bank groups per rank (%d) is greater than 1\n", 180 tRRD_L, tRRD, bankGroupsPerRank); 181 } 182 } --- 67 unchanged lines hidden (view full) --- 250 for (auto r : ranks) { 251 r->startup(curTick() + tREFI - tRP); 252 } 253 254 // shift the bus busy time sufficiently far ahead that we never 255 // have to worry about negative values when computing the time for 256 // the next request, this will add an insignificant bubble at the 257 // start of simulation |
258 nextBurstAt = curTick() + tRP + tRCD; |
259 } 260} 261 262Tick 263DRAMCtrl::recvAtomic(PacketPtr pkt) 264{ 265 DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 266 --- 509 unchanged lines hidden (view full) --- 776 panic("No scheduling policy chosen\n"); 777 return found_packet; 778} 779 780bool 781DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, Tick extra_col_delay) 782{ 783 // Only determine this if needed |
784 vector<uint32_t> earliest_banks(ranksPerChannel, 0); 785 786 // Has minBankPrep been called to populate earliest_banks? 787 bool filled_earliest_banks = false; 788 // can the PRE/ACT sequence be done without impacting utlization? |
789 bool hidden_bank_prep = false; 790 791 // search for seamless row hits first, if no seamless row hit is 792 // found then determine if there are other packets that can be issued 793 // without incurring additional bus delay due to bank timing 794 // Will select closed rows first to enable more open row possibilies 795 // in future selections 796 bool found_hidden_bank = false; --- 4 unchanged lines hidden (view full) --- 801 802 // if we have no row hit, prepped or not, and no seamless packet, 803 // just go for the earliest possible 804 bool found_earliest_pkt = false; 805 806 auto selected_pkt_it = queue.end(); 807 808 // time we need to issue a column command to be seamless |
809 const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick()); |
810 811 for (auto i = queue.begin(); i != queue.end() ; ++i) { 812 DRAMPacket* dram_pkt = *i; 813 const Bank& bank = dram_pkt->bankRef; |
814 const Tick col_allowed_at = dram_pkt->isRead ? bank.rdAllowedAt : 815 bank.wrAllowedAt; |
816 817 // check if rank is not doing a refresh and thus is available, if not, 818 // jump to the next packet 819 if (dram_pkt->rankRef.inRefIdleState()) { 820 // check if it is a row hit 821 if (bank.openRow == dram_pkt->row) { 822 // no additional rank-to-rank or same bank-group 823 // delays, or we switched read/write and might as well 824 // go for the row hit |
825 if (col_allowed_at <= min_col_at) { |
826 // FCFS within the hits, giving priority to 827 // commands that can issue seamlessly, without 828 // additional delay, such as same rank accesses 829 // and/or different bank-group accesses 830 DPRINTF(DRAM, "Seamless row buffer hit\n"); 831 selected_pkt_it = i; 832 // no need to look through the remaining queue entries 833 break; --- 4 unchanged lines hidden (view full) --- 838 // the current one 839 selected_pkt_it = i; 840 found_prepped_pkt = true; 841 DPRINTF(DRAM, "Prepped row buffer hit\n"); 842 } 843 } else if (!found_earliest_pkt) { 844 // if we have not initialised the bank status, do it 845 // now, and only once per scheduling decisions |
846 if (!filled_earliest_banks) { |
847 // determine entries with earliest bank delay |
848 std::tie(earliest_banks, hidden_bank_prep) = |
849 minBankPrep(queue, min_col_at); |
850 filled_earliest_banks = true; |
851 } 852 853 // bank is amongst first available banks 854 // minBankPrep will give priority to packets that can 855 // issue seamlessly |
856 if (bits(earliest_banks[dram_pkt->rank], 857 dram_pkt->bank, dram_pkt->bank)) { |
858 found_earliest_pkt = true; 859 found_hidden_bank = hidden_bank_prep; 860 861 // give priority to packets that can issue 862 // bank commands 'behind the scenes' 863 // any additional delay if any will be due to 864 // col-to-col command requirements 865 if (hidden_bank_prep || !found_prepped_pkt) --- 79 unchanged lines hidden (view full) --- 945 act_tick)); 946 947 DPRINTF(DRAMPower, "%llu,ACT,%d,%d\n", divCeil(act_tick, tCK) - 948 timeStampOffset, bank_ref.bank, rank_ref.rank); 949 950 // The next access has to respect tRAS for this bank 951 bank_ref.preAllowedAt = act_tick + tRAS; 952 |
953 // Respect the row-to-column command delay for both read and write cmds 954 bank_ref.rdAllowedAt = std::max(act_tick + tRCD, bank_ref.rdAllowedAt); 955 bank_ref.wrAllowedAt = std::max(act_tick + tRCD, bank_ref.wrAllowedAt); |
956 957 // start by enforcing tRRD 958 for (int i = 0; i < banksPerRank; i++) { 959 // next activate to any bank in this rank must not happen 960 // before tRRD 961 if (bankGroupArch && (bank_ref.bankgr == rank_ref.banks[i].bankgr)) { 962 // bank group architecture requires longer delays between 963 // ACT commands within the same bank group. Use tRRD_L --- 119 unchanged lines hidden (view full) --- 1083 } 1084 1085 // get the bank 1086 Bank& bank = dram_pkt->bankRef; 1087 1088 // for the state we need to track if it is a row hit or not 1089 bool row_hit = true; 1090 |
1091 // Determine the access latency and update the bank state 1092 if (bank.openRow == dram_pkt->row) { 1093 // nothing to do 1094 } else { 1095 row_hit = false; 1096 1097 // If there is a page open, precharge it. 1098 if (bank.openRow != Bank::NO_ROW) { 1099 prechargeBank(rank, bank, std::max(bank.preAllowedAt, curTick())); 1100 } 1101 1102 // next we need to account for the delay in activating the 1103 // page 1104 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 1105 1106 // Record the activation and deal with all the global timing 1107 // constraints caused be a new activation (tRRD and tXAW) 1108 activateBank(rank, bank, act_tick, dram_pkt->row); |
1109 } 1110 |
1111 // respect any constraints on the command (e.g. tRCD or tCCD) 1112 const Tick col_allowed_at = dram_pkt->isRead ? 1113 bank.rdAllowedAt : bank.wrAllowedAt; 1114 |
1115 // we need to wait until the bus is available before we can issue |
1116 // the command; need minimum of tBURST between commands 1117 Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()}); |
1118 1119 // update the packet ready time 1120 dram_pkt->readyTime = cmd_at + tCL + tBURST; 1121 |
1122 // update the time for the next read/write burst for each |
1123 // bank (add a max with tCCD/tCCD_L/tCCD_L_WR here) 1124 Tick dly_to_rd_cmd; 1125 Tick dly_to_wr_cmd; |
1126 for (int j = 0; j < ranksPerChannel; j++) { 1127 for (int i = 0; i < banksPerRank; i++) { 1128 // next burst to same bank group in this rank must not happen 1129 // before tCCD_L. Different bank group timing requirement is 1130 // tBURST; Add tCS for different ranks 1131 if (dram_pkt->rank == j) { 1132 if (bankGroupArch && 1133 (bank.bankgr == ranks[j]->banks[i].bankgr)) { 1134 // bank group architecture requires longer delays between 1135 // RD/WR burst commands to the same bank group. |
1136 // tCCD_L is default requirement for same BG timing 1137 // tCCD_L_WR is required for write-to-write 1138 // Need to also take bus turnaround delays into account 1139 dly_to_rd_cmd = dram_pkt->isRead ? 1140 tCCD_L : std::max(tCCD_L, wrToRdDly); 1141 dly_to_wr_cmd = dram_pkt->isRead ? 1142 std::max(tCCD_L, rdToWrDly) : tCCD_L_WR; |
1143 } else { |
1144 // tBURST is default requirement for diff BG timing 1145 // Need to also take bus turnaround delays into account 1146 dly_to_rd_cmd = dram_pkt->isRead ? tBURST : wrToRdDly; 1147 dly_to_wr_cmd = dram_pkt->isRead ? rdToWrDly : tBURST; |
1148 } 1149 } else { |
1150 // different rank is by default in a different bank group and 1151 // doesn't require longer tCCD or additional RTW, WTR delays 1152 // Need to account for rank-to-rank switching with tCS 1153 dly_to_wr_cmd = rankToRankDly; 1154 dly_to_rd_cmd = rankToRankDly; |
1155 } |
1156 ranks[j]->banks[i].rdAllowedAt = std::max(cmd_at + dly_to_rd_cmd, 1157 ranks[j]->banks[i].rdAllowedAt); 1158 ranks[j]->banks[i].wrAllowedAt = std::max(cmd_at + dly_to_wr_cmd, 1159 ranks[j]->banks[i].wrAllowedAt); |
1160 } 1161 } 1162 1163 // Save rank of current access 1164 activeRank = dram_pkt->rank; 1165 1166 // If this is a write, we also need to respect the write recovery 1167 // time before a precharge, in the case of a read, respect the --- 58 unchanged lines hidden (view full) --- 1226 1227 // DRAMPower trace command to be written 1228 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR"; 1229 1230 // MemCommand required for DRAMPower library 1231 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD : 1232 MemCommand::WR; 1233 |
1234 // Update bus state to reflect when previous command was issued 1235 nextBurstAt = cmd_at + tBURST; |
1236 |
1237 DPRINTF(DRAM, "Access to %lld, ready at %lld next burst at %lld.\n", 1238 dram_pkt->addr, dram_pkt->readyTime, nextBurstAt); |
1239 1240 dram_pkt->rankRef.cmdList.push_back(Command(command, dram_pkt->bank, 1241 cmd_at)); 1242 1243 DPRINTF(DRAMPower, "%llu,%s,%d,%d\n", divCeil(cmd_at, tCK) - 1244 timeStampOffset, mem_cmd, dram_pkt->bank, dram_pkt->rank); 1245 1246 // if this access should use auto-precharge, then we are --- 5 unchanged lines hidden (view full) --- 1252 1253 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1254 } 1255 1256 // Update the minimum timing between the requests, this is a 1257 // conservative estimate of when we have to schedule the next 1258 // request to not introduce any unecessary bubbles. In most cases 1259 // we will wake up sooner than we have to. |
1260 nextReqTime = nextBurstAt - (tRP + tRCD); |
1261 1262 // Update the stats and schedule the next request 1263 if (dram_pkt->isRead) { 1264 ++readsThisTime; 1265 if (row_hit) 1266 readRowHits++; 1267 bytesReadDRAM += burstSize; 1268 perBankRdBursts[dram_pkt->bankId]++; --- 116 unchanged lines hidden (view full) --- 1385 } else { 1386 // bool to check if there is a read to a free rank 1387 bool found_read = false; 1388 1389 // Figure out which read request goes next, and move it to the 1390 // front of the read queue 1391 // If we are changing command type, incorporate the minimum 1392 // bus turnaround delay which will be tCS (different rank) case |
1393 found_read = chooseNext(readQueue, switched_cmd_type ? tCS : 0); |
1394 1395 // if no read to an available rank is found then return 1396 // at this point. There could be writes to the available ranks 1397 // which are above the required threshold. However, to 1398 // avoid adding more complexity to the code, return and wait 1399 // for a refresh event to kick things into action again. 1400 if (!found_read) 1401 return; 1402 1403 DRAMPacket* dram_pkt = readQueue.front(); 1404 assert(dram_pkt->rankRef.inRefIdleState()); 1405 |
1406 doDRAMAccess(dram_pkt); 1407 1408 // At this point we're done dealing with the request 1409 readQueue.pop_front(); 1410 1411 // Every respQueue which will generate an event, increment count 1412 ++dram_pkt->rankRef.outstandingEvents; 1413 --- 43 unchanged lines hidden (view full) --- 1457 if (!found_write) 1458 return; 1459 1460 DRAMPacket* dram_pkt = writeQueue.front(); 1461 assert(dram_pkt->rankRef.inRefIdleState()); 1462 // sanity check 1463 assert(dram_pkt->size <= burstSize); 1464 |
1465 doDRAMAccess(dram_pkt); 1466 1467 writeQueue.pop_front(); 1468 1469 // removed write from queue, decrement count 1470 --dram_pkt->rankRef.writeEntries; 1471 1472 // Schedule write done event to decrement event count --- 42 unchanged lines hidden (view full) --- 1515 // cause a nextReqEvent to be scheduled before we do so as part of 1516 // the next request processing 1517 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1518 retryWrReq = false; 1519 port.sendRetryReq(); 1520 } 1521} 1522 |
1523pair<vector<uint32_t>, bool> |
1524DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue, 1525 Tick min_col_at) const 1526{ |
1527 Tick min_act_at = MaxTick; |
1528 vector<uint32_t> bank_mask(ranksPerChannel, 0); |
1529 1530 // latest Tick for which ACT can occur without incurring additoinal 1531 // delay on the data bus 1532 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick()); 1533 1534 // Flag condition when burst can issue back-to-back with previous burst 1535 bool found_seamless_bank = false; 1536 --- 23 unchanged lines hidden (view full) --- 1560 // simplistic approximation of when the bank can issue 1561 // an activate, ignoring any rank-to-rank switching 1562 // cost in this calculation 1563 Tick act_at = ranks[i]->banks[j].openRow == Bank::NO_ROW ? 1564 std::max(ranks[i]->banks[j].actAllowedAt, curTick()) : 1565 std::max(ranks[i]->banks[j].preAllowedAt, curTick()) + tRP; 1566 1567 // When is the earliest the R/W burst can issue? |
1568 const Tick col_allowed_at = (busState == READ) ? 1569 ranks[i]->banks[j].rdAllowedAt : 1570 ranks[i]->banks[j].wrAllowedAt; 1571 Tick col_at = std::max(col_allowed_at, act_at + tRCD); |
1572 1573 // bank can issue burst back-to-back (seamlessly) with 1574 // previous burst 1575 bool new_seamless_bank = col_at <= min_col_at; 1576 1577 // if we found a new seamless bank or we have no 1578 // seamless banks, and got a bank with an earlier 1579 // activate time, it should be added to the bit mask 1580 if (new_seamless_bank || 1581 (!found_seamless_bank && act_at <= min_act_at)) { 1582 // if we did not have a seamless bank before, and 1583 // we do now, reset the bank mask, also reset it 1584 // if we have not yet found a seamless bank and 1585 // the activate time is smaller than what we have 1586 // seen so far 1587 if (!found_seamless_bank && 1588 (new_seamless_bank || act_at < min_act_at)) { |
1589 std::fill(bank_mask.begin(), bank_mask.end(), 0); |
1590 } 1591 1592 found_seamless_bank |= new_seamless_bank; 1593 1594 // ACT can occur 'behind the scenes' 1595 hidden_bank_prep = act_at <= hidden_act_max; 1596 1597 // set the bit corresponding to the available bank |
1598 replaceBits(bank_mask[i], j, j, 1); |
1599 min_act_at = act_at; 1600 } 1601 } 1602 } 1603 } 1604 1605 return make_pair(bank_mask, hidden_bank_prep); 1606} --- 456 unchanged lines hidden (view full) --- 2063 // schedule wake-up with event to ensure entry has completed before 2064 // we try to wake-up 2065 schedule(wakeUpEvent, wake_up_tick); 2066 2067 for (auto &b : banks) { 2068 // respect both causality and any existing bank 2069 // constraints, some banks could already have a 2070 // (auto) precharge scheduled |
2071 b.wrAllowedAt = std::max(wake_up_tick + exit_delay, b.wrAllowedAt); 2072 b.rdAllowedAt = std::max(wake_up_tick + exit_delay, b.rdAllowedAt); |
2073 b.preAllowedAt = std::max(wake_up_tick + exit_delay, b.preAllowedAt); 2074 b.actAllowedAt = std::max(wake_up_tick + exit_delay, b.actAllowedAt); 2075 } 2076 // Transitioning out of low power state, clear flag 2077 inLowPowerState = false; 2078 2079 // push to DRAMPower 2080 // use pwrStateTrans for cases where we have a power event scheduled --- 695 unchanged lines hidden --- |