1/* 2 * Copyright (c) 2010-2018 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 39 unchanged lines hidden (view full) --- 48#include "mem/dram_ctrl.hh" 49 50#include "base/bitfield.hh" 51#include "base/trace.hh" 52#include "debug/DRAM.hh" 53#include "debug/DRAMPower.hh" 54#include "debug/DRAMState.hh" 55#include "debug/Drain.hh" |
56#include "debug/QOS.hh" |
57#include "sim/system.hh" 58 59using namespace std; 60using namespace Data; 61 62DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : |
63 QoS::MemCtrl(p), |
64 port(name() + ".port", *this), isTimingMode(false), 65 retryRdReq(false), retryWrReq(false), |
66 nextReqEvent([this]{ processNextReqEvent(); }, name()), 67 respondEvent([this]{ processRespondEvent(); }, name()), 68 deviceSize(p->device_size), 69 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 70 deviceRowBufferSize(p->device_rowbuffer_size), 71 devicesPerRank(p->devices_per_rank), 72 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 73 rowBufferSize(devicesPerRank * deviceRowBufferSize), --- 27 unchanged lines hidden (view full) --- 101{ 102 // sanity check the ranks since we rely on bit slicing for the 103 // address decoding 104 fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not " 105 "allowed, must be a power of two\n", ranksPerChannel); 106 107 fatal_if(!isPowerOf2(burstSize), "DRAM burst size %d is not allowed, " 108 "must be a power of two\n", burstSize); |
109 readQueue.resize(p->qos_priorities); 110 writeQueue.resize(p->qos_priorities); |
111 |
112 |
113 for (int i = 0; i < ranksPerChannel; i++) { 114 Rank* rank = new Rank(*this, p, i); 115 ranks.push_back(rank); 116 } 117 118 // perform a basic check of the write thresholds 119 if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 120 fatal("Write buffer low threshold %d must be smaller than the " --- 62 unchanged lines hidden (view full) --- 183 } 184 } 185 186} 187 188void 189DRAMCtrl::init() 190{ |
191 MemCtrl::init(); |
192 193 if (!port.isConnected()) { 194 fatal("DRAMCtrl %s is unconnected!\n", name()); 195 } else { 196 port.sendRangeChange(); 197 } 198 199 // a bit of sanity checks on the interleaving, save it for here to --- 80 unchanged lines hidden (view full) --- 280 } 281 return latency; 282} 283 284bool 285DRAMCtrl::readQueueFull(unsigned int neededEntries) const 286{ 287 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", |
288 readBufferSize, totalReadQueueSize + respQueue.size(), |
289 neededEntries); 290 |
291 auto rdsize_new = totalReadQueueSize + respQueue.size() + neededEntries; 292 return rdsize_new > readBufferSize; |
293} 294 295bool 296DRAMCtrl::writeQueueFull(unsigned int neededEntries) const 297{ 298 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", |
299 writeBufferSize, totalWriteQueueSize, neededEntries); 300 301 auto wrsize_new = (totalWriteQueueSize + neededEntries); 302 return wrsize_new > writeBufferSize; |
303} 304 305DRAMCtrl::DRAMPacket* 306DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 307 bool isRead) 308{ 309 // decode the address based on the address mapping scheme, with 310 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and --- 115 unchanged lines hidden (view full) --- 426 Addr addr = pkt->getAddr(); 427 unsigned pktsServicedByWrQ = 0; 428 BurstHelper* burst_helper = NULL; 429 for (int cnt = 0; cnt < pktCount; ++cnt) { 430 unsigned size = std::min((addr | (burstSize - 1)) + 1, 431 pkt->getAddr() + pkt->getSize()) - addr; 432 readPktSize[ceilLog2(size)]++; 433 readBursts++; |
434 masterReadAccesses[pkt->masterId()]++; |
435 436 // First check write buffer to see if the data is already at 437 // the controller 438 bool foundInWrQ = false; 439 Addr burst_addr = burstAlign(addr); 440 // if the burst address is not present then there is no need 441 // looking any further 442 if (isInWriteQueue.find(burst_addr) != isInWriteQueue.end()) { |
443 for (const auto& vec : writeQueue) { 444 for (const auto& p : vec) { 445 // check if the read is subsumed in the write queue 446 // packet we are looking at 447 if (p->addr <= addr && 448 ((addr + size) <= (p->addr + p->size))) { 449 450 foundInWrQ = true; 451 servicedByWrQ++; 452 pktsServicedByWrQ++; 453 DPRINTF(DRAM, 454 "Read to addr %lld with size %d serviced by " 455 "write queue\n", 456 addr, size); 457 bytesReadWrQ += burstSize; 458 break; 459 } |
460 } 461 } 462 } 463 464 // If not found in the write q, make a DRAM packet and 465 // push it onto the read queue 466 if (!foundInWrQ) { 467 468 // Make the burst helper for split packets 469 if (pktCount > 1 && burst_helper == NULL) { 470 DPRINTF(DRAM, "Read to addr %lld translates to %d " 471 "dram requests\n", pkt->getAddr(), pktCount); 472 burst_helper = new BurstHelper(pktCount); 473 } 474 475 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 476 dram_pkt->burstHelper = burst_helper; 477 478 assert(!readQueueFull(1)); |
479 rdQLenPdf[totalReadQueueSize + respQueue.size()]++; |
480 481 DPRINTF(DRAM, "Adding to read queue\n"); 482 |
483 readQueue[dram_pkt->qosValue()].push_back(dram_pkt); |
484 |
485 ++dram_pkt->rankRef.readEntries; 486 |
487 // log packet 488 logRequest(MemCtrl::READ, pkt->masterId(), pkt->qosValue(), 489 dram_pkt->addr, 1); 490 |
491 // Update stats |
492 avgRdQLen = totalReadQueueSize + respQueue.size(); |
493 } 494 495 // Starting address of next dram pkt (aligend to burstSize boundary) 496 addr = (addr | (burstSize - 1)) + 1; 497 } 498 499 // If all packets are serviced by write queue, we send the repsonse back 500 if (pktsServicedByWrQ == pktCount) { --- 23 unchanged lines hidden (view full) --- 524 // if the request size is larger than burst size, the pkt is split into 525 // multiple DRAM packets 526 Addr addr = pkt->getAddr(); 527 for (int cnt = 0; cnt < pktCount; ++cnt) { 528 unsigned size = std::min((addr | (burstSize - 1)) + 1, 529 pkt->getAddr() + pkt->getSize()) - addr; 530 writePktSize[ceilLog2(size)]++; 531 writeBursts++; |
532 masterWriteAccesses[pkt->masterId()]++; |
533 534 // see if we can merge with an existing item in the write 535 // queue and keep track of whether we have merged or not 536 bool merged = isInWriteQueue.find(burstAlign(addr)) != 537 isInWriteQueue.end(); 538 539 // if the item was not merged we need to create a new write 540 // and enqueue it 541 if (!merged) { 542 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 543 |
544 assert(totalWriteQueueSize < writeBufferSize); 545 wrQLenPdf[totalWriteQueueSize]++; |
546 547 DPRINTF(DRAM, "Adding to write queue\n"); 548 |
549 writeQueue[dram_pkt->qosValue()].push_back(dram_pkt); |
550 isInWriteQueue.insert(burstAlign(addr)); |
551 |
552 // log packet 553 logRequest(MemCtrl::WRITE, pkt->masterId(), pkt->qosValue(), 554 dram_pkt->addr, 1); 555 556 assert(totalWriteQueueSize == isInWriteQueue.size()); 557 |
558 // Update stats |
559 avgWrQLen = totalWriteQueueSize; |
560 561 // increment write entries of the rank 562 ++dram_pkt->rankRef.writeEntries; 563 } else { 564 DPRINTF(DRAM, "Merging write burst with existing queue entry\n"); 565 566 // keep track of the fact that this burst effectively 567 // disappeared as it was merged with an existing one --- 15 unchanged lines hidden (view full) --- 583 // queue, do so now 584 if (!nextReqEvent.scheduled()) { 585 DPRINTF(DRAM, "Request scheduled immediately\n"); 586 schedule(nextReqEvent, curTick()); 587 } 588} 589 590void |
591DRAMCtrl::printQs() const 592{ 593#if TRACING_ON |
594 DPRINTF(DRAM, "===READ QUEUE===\n\n"); |
595 for (const auto& queue : readQueue) { 596 for (const auto& packet : queue) { 597 DPRINTF(DRAM, "Read %lu\n", packet->addr); 598 } |
599 } |
600 |
601 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); |
602 for (const auto& packet : respQueue) { 603 DPRINTF(DRAM, "Response %lu\n", packet->addr); |
604 } |
605 |
606 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); |
607 for (const auto& queue : writeQueue) { 608 for (const auto& packet : queue) { 609 DPRINTF(DRAM, "Write %lu\n", packet->addr); 610 } |
611 } |
612#endif // TRACING_ON |
613} 614 615bool 616DRAMCtrl::recvTimingReq(PacketPtr pkt) 617{ 618 // This is where we enter from the outside world 619 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 620 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); --- 14 unchanged lines hidden (view full) --- 635 // Find out how many dram packets a pkt translates to 636 // If the burst size is equal or larger than the pkt size, then a pkt 637 // translates to only one dram packet. Otherwise, a pkt translates to 638 // multiple dram packets 639 unsigned size = pkt->getSize(); 640 unsigned offset = pkt->getAddr() & (burstSize - 1); 641 unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 642 |
643 // run the QoS scheduler and assign a QoS priority value to the packet 644 qosSchedule( { &readQueue, &writeQueue }, burstSize, pkt); 645 |
646 // check local buffers and do not accept if full 647 if (pkt->isRead()) { 648 assert(size != 0); 649 if (readQueueFull(dram_pkt_count)) { 650 DPRINTF(DRAM, "Read queue full, not accepting\n"); 651 // remember that we have to retry this port 652 retryRdReq = true; 653 numRdRetry++; --- 94 unchanged lines hidden (view full) --- 748 749 if (!respQueue.empty()) { 750 assert(respQueue.front()->readyTime >= curTick()); 751 assert(!respondEvent.scheduled()); 752 schedule(respondEvent, respQueue.front()->readyTime); 753 } else { 754 // if there is nothing left in any queue, signal a drain 755 if (drainState() == DrainState::Draining && |
756 !totalWriteQueueSize && !totalReadQueueSize && allRanksDrained()) { |
757 758 DPRINTF(Drain, "DRAM controller done draining\n"); 759 signalDrainDone(); 760 } 761 } 762 763 // We have made a location in the queue available at this point, 764 // so if there is a read that was forced to wait, retry now 765 if (retryRdReq) { 766 retryRdReq = false; 767 port.sendRetryReq(); 768 } 769} 770 |
771DRAMCtrl::DRAMPacketQueue::iterator 772DRAMCtrl::chooseNext(DRAMPacketQueue& queue, Tick extra_col_delay) |
773{ |
774 // This method does the arbitration between requests. |
775 |
776 DRAMCtrl::DRAMPacketQueue::iterator ret = queue.end(); |
777 |
778 if (!queue.empty()) { 779 if (queue.size() == 1) { 780 // available rank corresponds to state refresh idle 781 DRAMPacket* dram_pkt = *(queue.begin()); |
782 if (ranks[dram_pkt->rank]->inRefIdleState()) { |
783 ret = queue.begin(); 784 DPRINTF(DRAM, "Single request, going to a free rank\n"); 785 } else { 786 DPRINTF(DRAM, "Single request, going to a busy rank\n"); |
787 } |
788 } else if (memSchedPolicy == Enums::fcfs) { 789 // check if there is a packet going to a free rank 790 for (auto i = queue.begin(); i != queue.end(); ++i) { 791 DRAMPacket* dram_pkt = *i; 792 if (ranks[dram_pkt->rank]->inRefIdleState()) { 793 ret = i; 794 break; 795 } 796 } 797 } else if (memSchedPolicy == Enums::frfcfs) { 798 ret = chooseNextFRFCFS(queue, extra_col_delay); 799 } else { 800 panic("No scheduling policy chosen\n"); |
801 } |
802 } 803 return ret; |
804} 805 |
806DRAMCtrl::DRAMPacketQueue::iterator 807DRAMCtrl::chooseNextFRFCFS(DRAMPacketQueue& queue, Tick extra_col_delay) |
808{ 809 // Only determine this if needed 810 vector<uint32_t> earliest_banks(ranksPerChannel, 0); 811 812 // Has minBankPrep been called to populate earliest_banks? 813 bool filled_earliest_banks = false; 814 // can the PRE/ACT sequence be done without impacting utlization? 815 bool hidden_bank_prep = false; --- 16 unchanged lines hidden (view full) --- 832 auto selected_pkt_it = queue.end(); 833 834 // time we need to issue a column command to be seamless 835 const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick()); 836 837 for (auto i = queue.begin(); i != queue.end() ; ++i) { 838 DRAMPacket* dram_pkt = *i; 839 const Bank& bank = dram_pkt->bankRef; |
840 const Tick col_allowed_at = dram_pkt->isRead() ? bank.rdAllowedAt : 841 bank.wrAllowedAt; |
842 |
843 DPRINTF(DRAM, "%s checking packet in bank %d\n", 844 __func__, dram_pkt->bankRef.bank); 845 |
846 // check if rank is not doing a refresh and thus is available, if not, 847 // jump to the next packet 848 if (dram_pkt->rankRef.inRefIdleState()) { |
849 850 DPRINTF(DRAM, 851 "%s bank %d - Rank %d available\n", __func__, 852 dram_pkt->bankRef.bank, dram_pkt->rankRef.rank); 853 |
854 // check if it is a row hit 855 if (bank.openRow == dram_pkt->row) { 856 // no additional rank-to-rank or same bank-group 857 // delays, or we switched read/write and might as well 858 // go for the row hit 859 if (col_allowed_at <= min_col_at) { 860 // FCFS within the hits, giving priority to 861 // commands that can issue seamlessly, without 862 // additional delay, such as same rank accesses 863 // and/or different bank-group accesses |
864 DPRINTF(DRAM, "%s Seamless row buffer hit\n", __func__); |
865 selected_pkt_it = i; 866 // no need to look through the remaining queue entries 867 break; 868 } else if (!found_hidden_bank && !found_prepped_pkt) { 869 // if we did not find a packet to a closed row that can 870 // issue the bank commands without incurring delay, and 871 // did not yet find a packet to a prepped row, remember 872 // the current one 873 selected_pkt_it = i; 874 found_prepped_pkt = true; |
875 DPRINTF(DRAM, "%s Prepped row buffer hit\n", __func__); |
876 } 877 } else if (!found_earliest_pkt) { 878 // if we have not initialised the bank status, do it 879 // now, and only once per scheduling decisions 880 if (!filled_earliest_banks) { 881 // determine entries with earliest bank delay 882 std::tie(earliest_banks, hidden_bank_prep) = 883 minBankPrep(queue, min_col_at); --- 11 unchanged lines hidden (view full) --- 895 // give priority to packets that can issue 896 // bank commands 'behind the scenes' 897 // any additional delay if any will be due to 898 // col-to-col command requirements 899 if (hidden_bank_prep || !found_prepped_pkt) 900 selected_pkt_it = i; 901 } 902 } |
903 } else { 904 DPRINTF(DRAM, "%s bank %d - Rank %d not available\n", __func__, 905 dram_pkt->bankRef.bank, dram_pkt->rankRef.rank); |
906 } 907 } 908 |
909 if (selected_pkt_it == queue.end()) { 910 DPRINTF(DRAM, "%s no available ranks found\n", __func__); |
911 } 912 |
913 return selected_pkt_it; |
914} 915 916void 917DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 918{ 919 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 920 921 bool needsResponse = pkt->needsResponse(); --- 216 unchanged lines hidden (view full) --- 1138 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 1139 1140 // Record the activation and deal with all the global timing 1141 // constraints caused be a new activation (tRRD and tXAW) 1142 activateBank(rank, bank, act_tick, dram_pkt->row); 1143 } 1144 1145 // respect any constraints on the command (e.g. tRCD or tCCD) |
1146 const Tick col_allowed_at = dram_pkt->isRead() ? |
1147 bank.rdAllowedAt : bank.wrAllowedAt; 1148 1149 // we need to wait until the bus is available before we can issue 1150 // the command; need minimum of tBURST between commands 1151 Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()}); 1152 1153 // update the packet ready time 1154 dram_pkt->readyTime = cmd_at + tCL + tBURST; --- 10 unchanged lines hidden (view full) --- 1165 if (dram_pkt->rank == j) { 1166 if (bankGroupArch && 1167 (bank.bankgr == ranks[j]->banks[i].bankgr)) { 1168 // bank group architecture requires longer delays between 1169 // RD/WR burst commands to the same bank group. 1170 // tCCD_L is default requirement for same BG timing 1171 // tCCD_L_WR is required for write-to-write 1172 // Need to also take bus turnaround delays into account |
1173 dly_to_rd_cmd = dram_pkt->isRead() ? |
1174 tCCD_L : std::max(tCCD_L, wrToRdDly); |
1175 dly_to_wr_cmd = dram_pkt->isRead() ? |
1176 std::max(tCCD_L, rdToWrDly) : tCCD_L_WR; 1177 } else { 1178 // tBURST is default requirement for diff BG timing 1179 // Need to also take bus turnaround delays into account |
1180 dly_to_rd_cmd = dram_pkt->isRead() ? tBURST : wrToRdDly; 1181 dly_to_wr_cmd = dram_pkt->isRead() ? rdToWrDly : tBURST; |
1182 } 1183 } else { 1184 // different rank is by default in a different bank group and 1185 // doesn't require longer tCCD or additional RTW, WTR delays 1186 // Need to account for rank-to-rank switching with tCS 1187 dly_to_wr_cmd = rankToRankDly; 1188 dly_to_rd_cmd = rankToRankDly; 1189 } --- 6 unchanged lines hidden (view full) --- 1196 1197 // Save rank of current access 1198 activeRank = dram_pkt->rank; 1199 1200 // If this is a write, we also need to respect the write recovery 1201 // time before a precharge, in the case of a read, respect the 1202 // read to precharge constraint 1203 bank.preAllowedAt = std::max(bank.preAllowedAt, |
1204 dram_pkt->isRead() ? cmd_at + tRTP : |
1205 dram_pkt->readyTime + tWR); 1206 1207 // increment the bytes accessed and the accesses per row 1208 bank.bytesAccessed += burstSize; 1209 ++bank.rowAccesses; 1210 1211 // if we reached the max, then issue with an auto-precharge 1212 bool auto_precharge = pageMgmt == Enums::close || --- 11 unchanged lines hidden (view full) --- 1224 // 2) close_adaptive page policy does not blindly close the 1225 // page, but closes it only if there are no row hits in the queue. 1226 // In this case, only force an auto precharge when there 1227 // are no same page hits in the queue 1228 bool got_more_hits = false; 1229 bool got_bank_conflict = false; 1230 1231 // either look at the read queue or write queue |
1232 const std::vector<DRAMPacketQueue>& queue = 1233 dram_pkt->isRead() ? readQueue : writeQueue; |
1234 |
1235 for (uint8_t i = 0; i < numPriorities(); ++i) { 1236 auto p = queue[i].begin(); 1237 // keep on looking until we find a hit or reach the end of the queue 1238 // 1) if a hit is found, then both open and close adaptive policies keep 1239 // the page open 1240 // 2) if no hit is found, got_bank_conflict is set to true if a bank 1241 // conflict request is waiting in the queue 1242 // 3) make sure we are not considering the packet that we are 1243 // currently dealing with 1244 while (!got_more_hits && p != queue[i].end()) { 1245 if (dram_pkt != (*p)) { 1246 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 1247 (dram_pkt->bank == (*p)->bank); 1248 1249 bool same_row = dram_pkt->row == (*p)->row; 1250 got_more_hits |= same_rank_bank && same_row; 1251 got_bank_conflict |= same_rank_bank && !same_row; 1252 } 1253 ++p; 1254 } 1255 1256 if (got_more_hits) 1257 break; |
1258 } 1259 1260 // auto pre-charge when either 1261 // 1) open_adaptive policy, we have not got any more hits, and 1262 // have a bank conflict 1263 // 2) close_adaptive policy and we have not got any more hits 1264 auto_precharge = !got_more_hits && 1265 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1266 } 1267 1268 // DRAMPower trace command to be written |
1269 std::string mem_cmd = dram_pkt->isRead() ? "RD" : "WR"; |
1270 1271 // MemCommand required for DRAMPower library 1272 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD : 1273 MemCommand::WR; 1274 1275 // Update bus state to reflect when previous command was issued 1276 nextBurstAt = cmd_at + tBURST; 1277 --- 18 unchanged lines hidden (view full) --- 1296 1297 // Update the minimum timing between the requests, this is a 1298 // conservative estimate of when we have to schedule the next 1299 // request to not introduce any unecessary bubbles. In most cases 1300 // we will wake up sooner than we have to. 1301 nextReqTime = nextBurstAt - (tRP + tRCD); 1302 1303 // Update the stats and schedule the next request |
1304 if (dram_pkt->isRead()) { |
1305 ++readsThisTime; 1306 if (row_hit) 1307 readRowHits++; 1308 bytesReadDRAM += burstSize; 1309 perBankRdBursts[dram_pkt->bankId]++; 1310 1311 // Update latency stats 1312 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; |
1313 masterReadTotalLat[dram_pkt->masterId()] += 1314 dram_pkt->readyTime - dram_pkt->entryTime; 1315 |
1316 totBusLat += tBURST; 1317 totQLat += cmd_at - dram_pkt->entryTime; |
1318 masterReadBytes[dram_pkt->masterId()] += dram_pkt->size; |
1319 } else { 1320 ++writesThisTime; 1321 if (row_hit) 1322 writeRowHits++; 1323 bytesWritten += burstSize; 1324 perBankWrBursts[dram_pkt->bankId]++; |
1325 masterWriteBytes[dram_pkt->masterId()] += dram_pkt->size; 1326 masterWriteTotalLat[dram_pkt->masterId()] += 1327 dram_pkt->readyTime - dram_pkt->entryTime; |
1328 } 1329} 1330 1331void 1332DRAMCtrl::processNextReqEvent() 1333{ |
1334 // transition is handled by QoS algorithm if enabled 1335 if (turnPolicy) { 1336 // select bus state - only done if QoS algorithms are in use 1337 busStateNext = selectNextBusState(); 1338 } 1339 1340 // detect bus state change 1341 bool switched_cmd_type = (busState != busStateNext); 1342 // record stats 1343 recordTurnaroundStats(); 1344 1345 DPRINTF(DRAM, "QoS Turnarounds selected state %s %s\n", 1346 (busState==MemCtrl::READ)?"READ":"WRITE", 1347 switched_cmd_type?"[turnaround triggered]":""); 1348 1349 if (switched_cmd_type) { 1350 if (busState == READ) { 1351 DPRINTF(DRAM, 1352 "Switching to writes after %d reads with %d reads " 1353 "waiting\n", readsThisTime, totalReadQueueSize); 1354 rdPerTurnAround.sample(readsThisTime); 1355 readsThisTime = 0; 1356 } else { 1357 DPRINTF(DRAM, 1358 "Switching to reads after %d writes with %d writes " 1359 "waiting\n", writesThisTime, totalWriteQueueSize); 1360 wrPerTurnAround.sample(writesThisTime); 1361 writesThisTime = 0; 1362 } 1363 } 1364 1365 // updates current state 1366 busState = busStateNext; 1367 1368 // check ranks for refresh/wakeup - uses busStateNext, so done after turnaround 1369 // decisions |
1370 int busyRanks = 0; 1371 for (auto r : ranks) { 1372 if (!r->inRefIdleState()) { 1373 if (r->pwrState != PWR_SREF) { 1374 // rank is busy refreshing 1375 DPRINTF(DRAMState, "Rank %d is not available\n", r->rank); 1376 busyRanks++; 1377 --- 24 unchanged lines hidden (view full) --- 1402 1403 if (busyRanks == ranksPerChannel) { 1404 // if all ranks are refreshing wait for them to finish 1405 // and stall this state machine without taking any further 1406 // action, and do not schedule a new nextReqEvent 1407 return; 1408 } 1409 |
1410 // when we get here it is either a read or a write 1411 if (busState == READ) { 1412 1413 // track if we should switch or not 1414 bool switch_to_writes = false; 1415 |
1416 if (totalReadQueueSize == 0) { |
1417 // In the case there is no read request to go next, 1418 // trigger writes if we have passed the low threshold (or 1419 // if we are draining) |
1420 if (!(totalWriteQueueSize == 0) && |
1421 (drainState() == DrainState::Draining || |
1422 totalWriteQueueSize > writeLowThreshold)) { |
1423 |
1424 DPRINTF(DRAM, "Switching to writes due to read queue empty\n"); |
1425 switch_to_writes = true; 1426 } else { 1427 // check if we are drained 1428 // not done draining until in PWR_IDLE state 1429 // ensuring all banks are closed and 1430 // have exited low power states 1431 if (drainState() == DrainState::Draining && 1432 respQueue.empty() && allRanksDrained()) { 1433 1434 DPRINTF(Drain, "DRAM controller done draining\n"); 1435 signalDrainDone(); 1436 } 1437 1438 // nothing to do, not even any point in scheduling an 1439 // event for the next request 1440 return; 1441 } 1442 } else { |
1443 |
1444 bool read_found = false; 1445 DRAMPacketQueue::iterator to_read; 1446 uint8_t prio = numPriorities(); |
1447 |
1448 for (auto queue = readQueue.rbegin(); 1449 queue != readQueue.rend(); ++queue) { 1450 1451 prio--; 1452 1453 DPRINTF(QOS, 1454 "DRAM controller checking READ queue [%d] priority [%d elements]\n", 1455 prio, queue->size()); 1456 1457 // Figure out which read request goes next 1458 // If we are changing command type, incorporate the minimum 1459 // bus turnaround delay which will be tCS (different rank) case 1460 to_read = chooseNext((*queue), switched_cmd_type ? tCS : 0); 1461 1462 if (to_read != queue->end()) { 1463 // candidate read found 1464 read_found = true; 1465 break; 1466 } 1467 } 1468 |
1469 // if no read to an available rank is found then return 1470 // at this point. There could be writes to the available ranks 1471 // which are above the required threshold. However, to 1472 // avoid adding more complexity to the code, return and wait 1473 // for a refresh event to kick things into action again. |
1474 if (!read_found) { 1475 DPRINTF(DRAM, "No Reads Found - exiting\n"); |
1476 return; |
1477 } |
1478 |
1479 auto dram_pkt = *to_read; 1480 |
1481 assert(dram_pkt->rankRef.inRefIdleState()); 1482 1483 doDRAMAccess(dram_pkt); 1484 |
1485 // Every respQueue which will generate an event, increment count 1486 ++dram_pkt->rankRef.outstandingEvents; |
1487 // sanity check 1488 assert(dram_pkt->size <= burstSize); 1489 assert(dram_pkt->readyTime >= curTick()); 1490 |
1491 // log the response 1492 logResponse(MemCtrl::READ, (*to_read)->masterId(), 1493 dram_pkt->qosValue(), dram_pkt->getAddr(), 1, 1494 dram_pkt->readyTime - dram_pkt->entryTime); 1495 1496 |
1497 // Insert into response queue. It will be sent back to the |
1498 // requester at its readyTime |
1499 if (respQueue.empty()) { 1500 assert(!respondEvent.scheduled()); 1501 schedule(respondEvent, dram_pkt->readyTime); 1502 } else { 1503 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1504 assert(respondEvent.scheduled()); 1505 } 1506 1507 respQueue.push_back(dram_pkt); 1508 1509 // we have so many writes that we have to transition |
1510 if (totalWriteQueueSize > writeHighThreshold) { |
1511 switch_to_writes = true; 1512 } |
1513 1514 // remove the request from the queue - the iterator is no longer valid . 1515 readQueue[dram_pkt->qosValue()].erase(to_read); |
1516 } 1517 1518 // switching to writes, either because the read queue is empty 1519 // and the writes have passed the low threshold (or we are 1520 // draining), or because the writes hit the hight threshold 1521 if (switch_to_writes) { 1522 // transition to writing 1523 busStateNext = WRITE; 1524 } 1525 } else { |
1526 |
1527 bool write_found = false; 1528 DRAMPacketQueue::iterator to_write; 1529 uint8_t prio = numPriorities(); |
1530 |
1531 for (auto queue = writeQueue.rbegin(); 1532 queue != writeQueue.rend(); ++queue) { 1533 1534 prio--; 1535 1536 DPRINTF(QOS, 1537 "DRAM controller checking WRITE queue [%d] priority [%d elements]\n", 1538 prio, queue->size()); 1539 1540 // If we are changing command type, incorporate the minimum 1541 // bus turnaround delay 1542 to_write = chooseNext((*queue), 1543 switched_cmd_type ? std::min(tRTW, tCS) : 0); 1544 1545 if (to_write != queue->end()) { 1546 write_found = true; 1547 break; 1548 } 1549 } 1550 |
1551 // if there are no writes to a rank that is available to service 1552 // requests (i.e. rank is in refresh idle state) are found then 1553 // return. There could be reads to the available ranks. However, to 1554 // avoid adding more complexity to the code, return at this point and 1555 // wait for a refresh event to kick things into action again. |
1556 if (!write_found) { 1557 DPRINTF(DRAM, "No Writes Found - exiting\n"); |
1558 return; |
1559 } |
1560 |
1561 auto dram_pkt = *to_write; 1562 |
1563 assert(dram_pkt->rankRef.inRefIdleState()); 1564 // sanity check 1565 assert(dram_pkt->size <= burstSize); 1566 1567 doDRAMAccess(dram_pkt); 1568 |
1569 // removed write from queue, decrement count 1570 --dram_pkt->rankRef.writeEntries; 1571 1572 // Schedule write done event to decrement event count 1573 // after the readyTime has been reached 1574 // Only schedule latest write event to minimize events 1575 // required; only need to ensure that final event scheduled covers 1576 // the time that writes are outstanding and bus is active 1577 // to holdoff power-down entry events 1578 if (!dram_pkt->rankRef.writeDoneEvent.scheduled()) { 1579 schedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime); 1580 // New event, increment count 1581 ++dram_pkt->rankRef.outstandingEvents; 1582 1583 } else if (dram_pkt->rankRef.writeDoneEvent.when() < |
1584 dram_pkt->readyTime) { 1585 |
1586 reschedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime); 1587 } 1588 1589 isInWriteQueue.erase(burstAlign(dram_pkt->addr)); |
1590 1591 // log the response 1592 logResponse(MemCtrl::WRITE, dram_pkt->masterId(), 1593 dram_pkt->qosValue(), dram_pkt->getAddr(), 1, 1594 dram_pkt->readyTime - dram_pkt->entryTime); 1595 1596 1597 // remove the request from the queue - the iterator is no longer valid 1598 writeQueue[dram_pkt->qosValue()].erase(to_write); 1599 |
1600 delete dram_pkt; 1601 1602 // If we emptied the write queue, or got sufficiently below the 1603 // threshold (using the minWritesPerSwitch as the hysteresis) and 1604 // are not draining, or we have reads waiting and have done enough 1605 // writes, then switch to reads. |
1606 bool below_threshold = 1607 totalWriteQueueSize + minWritesPerSwitch < writeLowThreshold; 1608 1609 if (totalWriteQueueSize == 0 || 1610 (below_threshold && drainState() != DrainState::Draining) || 1611 (totalReadQueueSize && writesThisTime >= minWritesPerSwitch)) { 1612 |
1613 // turn the bus back around for reads again 1614 busStateNext = READ; 1615 1616 // note that the we switch back to reads also in the idle 1617 // case, which eventually will check for any draining and 1618 // also pause any further scheduling if there is really 1619 // nothing to do 1620 } 1621 } 1622 // It is possible that a refresh to another rank kicks things back into 1623 // action before reaching this point. 1624 if (!nextReqEvent.scheduled()) 1625 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1626 1627 // If there is space available and we have writes waiting then let 1628 // them retry. This is done here to ensure that the retry does not 1629 // cause a nextReqEvent to be scheduled before we do so as part of 1630 // the next request processing |
1631 if (retryWrReq && totalWriteQueueSize < writeBufferSize) { |
1632 retryWrReq = false; 1633 port.sendRetryReq(); 1634 } 1635} 1636 1637pair<vector<uint32_t>, bool> |
1638DRAMCtrl::minBankPrep(const DRAMPacketQueue& queue, |
1639 Tick min_col_at) const 1640{ 1641 Tick min_act_at = MaxTick; 1642 vector<uint32_t> bank_mask(ranksPerChannel, 0); 1643 1644 // latest Tick for which ACT can occur without incurring additoinal 1645 // delay on the data bus 1646 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick()); --- 848 unchanged lines hidden (view full) --- 2495 Stats::registerDumpCallback(new RankDumpCallback(this)); 2496 Stats::registerResetCallback(new RankResetCallback(this)); 2497} 2498void 2499DRAMCtrl::regStats() 2500{ 2501 using namespace Stats; 2502 |
2503 MemCtrl::regStats(); |
2504 2505 for (auto r : ranks) { 2506 r->regStats(); 2507 } 2508 2509 registerResetCallback(new MemResetCallback(this)); 2510 2511 readReqs --- 126 unchanged lines hidden (view full) --- 2638 .desc("What read queue length does an incoming req see"); 2639 2640 wrQLenPdf 2641 .init(writeBufferSize) 2642 .name(name() + ".wrQLenPdf") 2643 .desc("What write queue length does an incoming req see"); 2644 2645 bytesPerActivate |
2646 .init(maxAccessesPerRow ? maxAccessesPerRow : rowBufferSize) |
2647 .name(name() + ".bytesPerActivate") 2648 .desc("Bytes accessed per row activation") 2649 .flags(nozero); 2650 2651 rdPerTurnAround 2652 .init(readBufferSize) 2653 .name(name() + ".rdPerTurnAround") 2654 .desc("Reads before turning the bus around for writes") --- 94 unchanged lines hidden (view full) --- 2749 2750 pageHitRate 2751 .name(name() + ".pageHitRate") 2752 .desc("Row buffer hit rate, read and write combined") 2753 .precision(2); 2754 2755 pageHitRate = (writeRowHits + readRowHits) / 2756 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; |
2757 2758 // per-master bytes read and written to memory 2759 masterReadBytes 2760 .init(_system->maxMasters()) 2761 .name(name() + ".masterReadBytes") 2762 .desc("Per-master bytes read from memory") 2763 .flags(nozero | nonan); 2764 2765 masterWriteBytes 2766 .init(_system->maxMasters()) 2767 .name(name() + ".masterWriteBytes") 2768 .desc("Per-master bytes write to memory") 2769 .flags(nozero | nonan); 2770 2771 // per-master bytes read and written to memory rate 2772 masterReadRate.name(name() + ".masterReadRate") 2773 .desc("Per-master bytes read from memory rate (Bytes/sec)") 2774 .flags(nozero | nonan) 2775 .precision(12); 2776 2777 masterReadRate = masterReadBytes/simSeconds; 2778 2779 masterWriteRate 2780 .name(name() + ".masterWriteRate") 2781 .desc("Per-master bytes write to memory rate (Bytes/sec)") 2782 .flags(nozero | nonan) 2783 .precision(12); 2784 2785 masterWriteRate = masterWriteBytes/simSeconds; 2786 2787 masterReadAccesses 2788 .init(_system->maxMasters()) 2789 .name(name() + ".masterReadAccesses") 2790 .desc("Per-master read serviced memory accesses") 2791 .flags(nozero); 2792 2793 masterWriteAccesses 2794 .init(_system->maxMasters()) 2795 .name(name() + ".masterWriteAccesses") 2796 .desc("Per-master write serviced memory accesses") 2797 .flags(nozero); 2798 2799 2800 masterReadTotalLat 2801 .init(_system->maxMasters()) 2802 .name(name() + ".masterReadTotalLat") 2803 .desc("Per-master read total memory access latency") 2804 .flags(nozero | nonan); 2805 2806 masterReadAvgLat.name(name() + ".masterReadAvgLat") 2807 .desc("Per-master read average memory access latency") 2808 .flags(nonan) 2809 .precision(2); 2810 2811 masterReadAvgLat = masterReadTotalLat/masterReadAccesses; 2812 2813 masterWriteTotalLat 2814 .init(_system->maxMasters()) 2815 .name(name() + ".masterWriteTotalLat") 2816 .desc("Per-master write total memory access latency") 2817 .flags(nozero | nonan); 2818 2819 masterWriteAvgLat.name(name() + ".masterWriteAvgLat") 2820 .desc("Per-master write average memory access latency") 2821 .flags(nonan) 2822 .precision(2); 2823 2824 masterWriteAvgLat = masterWriteTotalLat/masterWriteAccesses; 2825 2826 for (int i = 0; i < _system->maxMasters(); i++) { 2827 const std::string master = _system->getMasterName(i); 2828 masterReadBytes.subname(i, master); 2829 masterReadRate.subname(i, master); 2830 masterWriteBytes.subname(i, master); 2831 masterWriteRate.subname(i, master); 2832 masterReadAccesses.subname(i, master); 2833 masterWriteAccesses.subname(i, master); 2834 masterReadTotalLat.subname(i, master); 2835 masterReadAvgLat.subname(i, master); 2836 masterWriteTotalLat.subname(i, master); 2837 masterWriteAvgLat.subname(i, master); 2838 } |
2839} 2840 2841void 2842DRAMCtrl::recvFunctional(PacketPtr pkt) 2843{ 2844 // rely on the abstract memory 2845 functionalAccess(pkt); 2846} --- 8 unchanged lines hidden (view full) --- 2855 } 2856} 2857 2858DrainState 2859DRAMCtrl::drain() 2860{ 2861 // if there is anything in any of our internal queues, keep track 2862 // of that as well |
2863 if (!(!totalWriteQueueSize && !totalReadQueueSize && respQueue.empty() && |
2864 allRanksDrained())) { 2865 2866 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," |
2867 " resp: %d\n", totalWriteQueueSize, totalReadQueueSize, |
2868 respQueue.size()); 2869 2870 // the only queue that is not drained automatically over time 2871 // is the write queue, thus kick things into action if needed |
2872 if (!totalWriteQueueSize && !nextReqEvent.scheduled()) { |
2873 schedule(nextReqEvent, curTick()); 2874 } 2875 2876 // also need to kick off events to exit self-refresh 2877 for (auto r : ranks) { 2878 // force self-refresh exit, which in turn will issue auto-refresh 2879 if (r->pwrState == PWR_SREF) { 2880 DPRINTF(DRAM,"Rank%d: Forcing self-refresh wakeup in drain\n", --- 91 unchanged lines hidden --- |