1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 39 unchanged lines hidden (view full) ---

48#include "mem/dram_ctrl.hh"
49
50#include "base/bitfield.hh"
51#include "base/trace.hh"
52#include "debug/DRAM.hh"
53#include "debug/DRAMPower.hh"
54#include "debug/DRAMState.hh"
55#include "debug/Drain.hh"
56#include "sim/system.hh"
57
58using namespace std;
59using namespace Data;
60
61DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) :
62 AbstractMemory(p),
63 port(name() + ".port", *this), isTimingMode(false),
64 retryRdReq(false), retryWrReq(false),
65 busState(READ),
66 busStateNext(READ),
67 nextReqEvent([this]{ processNextReqEvent(); }, name()),
68 respondEvent([this]{ processRespondEvent(); }, name()),
69 deviceSize(p->device_size),
70 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length),
71 deviceRowBufferSize(p->device_rowbuffer_size),
72 devicesPerRank(p->devices_per_rank),
73 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8),
74 rowBufferSize(devicesPerRank * deviceRowBufferSize),

--- 27 unchanged lines hidden (view full) ---

102{
103 // sanity check the ranks since we rely on bit slicing for the
104 // address decoding
105 fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not "
106 "allowed, must be a power of two\n", ranksPerChannel);
107
108 fatal_if(!isPowerOf2(burstSize), "DRAM burst size %d is not allowed, "
109 "must be a power of two\n", burstSize);
110
111 for (int i = 0; i < ranksPerChannel; i++) {
112 Rank* rank = new Rank(*this, p, i);
113 ranks.push_back(rank);
114 }
115
116 // perform a basic check of the write thresholds
117 if (p->write_low_thresh_perc >= p->write_high_thresh_perc)
118 fatal("Write buffer low threshold %d must be smaller than the "

--- 62 unchanged lines hidden (view full) ---

181 }
182 }
183
184}
185
186void
187DRAMCtrl::init()
188{
189 AbstractMemory::init();
190
191 if (!port.isConnected()) {
192 fatal("DRAMCtrl %s is unconnected!\n", name());
193 } else {
194 port.sendRangeChange();
195 }
196
197 // a bit of sanity checks on the interleaving, save it for here to

--- 80 unchanged lines hidden (view full) ---

278 }
279 return latency;
280}
281
282bool
283DRAMCtrl::readQueueFull(unsigned int neededEntries) const
284{
285 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n",
286 readBufferSize, readQueue.size() + respQueue.size(),
287 neededEntries);
288
289 return
290 (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize;
291}
292
293bool
294DRAMCtrl::writeQueueFull(unsigned int neededEntries) const
295{
296 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n",
297 writeBufferSize, writeQueue.size(), neededEntries);
298 return (writeQueue.size() + neededEntries) > writeBufferSize;
299}
300
301DRAMCtrl::DRAMPacket*
302DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size,
303 bool isRead)
304{
305 // decode the address based on the address mapping scheme, with
306 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and

--- 115 unchanged lines hidden (view full) ---

422 Addr addr = pkt->getAddr();
423 unsigned pktsServicedByWrQ = 0;
424 BurstHelper* burst_helper = NULL;
425 for (int cnt = 0; cnt < pktCount; ++cnt) {
426 unsigned size = std::min((addr | (burstSize - 1)) + 1,
427 pkt->getAddr() + pkt->getSize()) - addr;
428 readPktSize[ceilLog2(size)]++;
429 readBursts++;
430
431 // First check write buffer to see if the data is already at
432 // the controller
433 bool foundInWrQ = false;
434 Addr burst_addr = burstAlign(addr);
435 // if the burst address is not present then there is no need
436 // looking any further
437 if (isInWriteQueue.find(burst_addr) != isInWriteQueue.end()) {
438 for (const auto& p : writeQueue) {
439 // check if the read is subsumed in the write queue
440 // packet we are looking at
441 if (p->addr <= addr && (addr + size) <= (p->addr + p->size)) {
442 foundInWrQ = true;
443 servicedByWrQ++;
444 pktsServicedByWrQ++;
445 DPRINTF(DRAM, "Read to addr %lld with size %d serviced by "
446 "write queue\n", addr, size);
447 bytesReadWrQ += burstSize;
448 break;
449 }
450 }
451 }
452
453 // If not found in the write q, make a DRAM packet and
454 // push it onto the read queue
455 if (!foundInWrQ) {
456
457 // Make the burst helper for split packets
458 if (pktCount > 1 && burst_helper == NULL) {
459 DPRINTF(DRAM, "Read to addr %lld translates to %d "
460 "dram requests\n", pkt->getAddr(), pktCount);
461 burst_helper = new BurstHelper(pktCount);
462 }
463
464 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true);
465 dram_pkt->burstHelper = burst_helper;
466
467 assert(!readQueueFull(1));
468 rdQLenPdf[readQueue.size() + respQueue.size()]++;
469
470 DPRINTF(DRAM, "Adding to read queue\n");
471
472 readQueue.push_back(dram_pkt);
473
474 // increment read entries of the rank
475 ++dram_pkt->rankRef.readEntries;
476
477 // Update stats
478 avgRdQLen = readQueue.size() + respQueue.size();
479 }
480
481 // Starting address of next dram pkt (aligend to burstSize boundary)
482 addr = (addr | (burstSize - 1)) + 1;
483 }
484
485 // If all packets are serviced by write queue, we send the repsonse back
486 if (pktsServicedByWrQ == pktCount) {

--- 23 unchanged lines hidden (view full) ---

510 // if the request size is larger than burst size, the pkt is split into
511 // multiple DRAM packets
512 Addr addr = pkt->getAddr();
513 for (int cnt = 0; cnt < pktCount; ++cnt) {
514 unsigned size = std::min((addr | (burstSize - 1)) + 1,
515 pkt->getAddr() + pkt->getSize()) - addr;
516 writePktSize[ceilLog2(size)]++;
517 writeBursts++;
518
519 // see if we can merge with an existing item in the write
520 // queue and keep track of whether we have merged or not
521 bool merged = isInWriteQueue.find(burstAlign(addr)) !=
522 isInWriteQueue.end();
523
524 // if the item was not merged we need to create a new write
525 // and enqueue it
526 if (!merged) {
527 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false);
528
529 assert(writeQueue.size() < writeBufferSize);
530 wrQLenPdf[writeQueue.size()]++;
531
532 DPRINTF(DRAM, "Adding to write queue\n");
533
534 writeQueue.push_back(dram_pkt);
535 isInWriteQueue.insert(burstAlign(addr));
536 assert(writeQueue.size() == isInWriteQueue.size());
537
538 // Update stats
539 avgWrQLen = writeQueue.size();
540
541 // increment write entries of the rank
542 ++dram_pkt->rankRef.writeEntries;
543 } else {
544 DPRINTF(DRAM, "Merging write burst with existing queue entry\n");
545
546 // keep track of the fact that this burst effectively
547 // disappeared as it was merged with an existing one

--- 15 unchanged lines hidden (view full) ---

563 // queue, do so now
564 if (!nextReqEvent.scheduled()) {
565 DPRINTF(DRAM, "Request scheduled immediately\n");
566 schedule(nextReqEvent, curTick());
567 }
568}
569
570void
571DRAMCtrl::printQs() const {
572 DPRINTF(DRAM, "===READ QUEUE===\n\n");
573 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) {
574 DPRINTF(DRAM, "Read %lu\n", (*i)->addr);
575 }
576 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n");
577 for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) {
578 DPRINTF(DRAM, "Response %lu\n", (*i)->addr);
579 }
580 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n");
581 for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) {
582 DPRINTF(DRAM, "Write %lu\n", (*i)->addr);
583 }
584}
585
586bool
587DRAMCtrl::recvTimingReq(PacketPtr pkt)
588{
589 // This is where we enter from the outside world
590 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n",
591 pkt->cmdString(), pkt->getAddr(), pkt->getSize());

--- 14 unchanged lines hidden (view full) ---

606 // Find out how many dram packets a pkt translates to
607 // If the burst size is equal or larger than the pkt size, then a pkt
608 // translates to only one dram packet. Otherwise, a pkt translates to
609 // multiple dram packets
610 unsigned size = pkt->getSize();
611 unsigned offset = pkt->getAddr() & (burstSize - 1);
612 unsigned int dram_pkt_count = divCeil(offset + size, burstSize);
613
614 // check local buffers and do not accept if full
615 if (pkt->isRead()) {
616 assert(size != 0);
617 if (readQueueFull(dram_pkt_count)) {
618 DPRINTF(DRAM, "Read queue full, not accepting\n");
619 // remember that we have to retry this port
620 retryRdReq = true;
621 numRdRetry++;

--- 94 unchanged lines hidden (view full) ---

716
717 if (!respQueue.empty()) {
718 assert(respQueue.front()->readyTime >= curTick());
719 assert(!respondEvent.scheduled());
720 schedule(respondEvent, respQueue.front()->readyTime);
721 } else {
722 // if there is nothing left in any queue, signal a drain
723 if (drainState() == DrainState::Draining &&
724 writeQueue.empty() && readQueue.empty() && allRanksDrained()) {
725
726 DPRINTF(Drain, "DRAM controller done draining\n");
727 signalDrainDone();
728 }
729 }
730
731 // We have made a location in the queue available at this point,
732 // so if there is a read that was forced to wait, retry now
733 if (retryRdReq) {
734 retryRdReq = false;
735 port.sendRetryReq();
736 }
737}
738
739bool
740DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
741{
742 // This method does the arbitration between requests. The chosen
743 // packet is simply moved to the head of the queue. The other
744 // methods know that this is the place to look. For example, with
745 // FCFS, this method does nothing
746 assert(!queue.empty());
747
748 // bool to indicate if a packet to an available rank is found
749 bool found_packet = false;
750 if (queue.size() == 1) {
751 DRAMPacket* dram_pkt = queue.front();
752 // available rank corresponds to state refresh idle
753 if (ranks[dram_pkt->rank]->inRefIdleState()) {
754 found_packet = true;
755 DPRINTF(DRAM, "Single request, going to a free rank\n");
756 } else {
757 DPRINTF(DRAM, "Single request, going to a busy rank\n");
758 }
759 return found_packet;
760 }
761
762 if (memSchedPolicy == Enums::fcfs) {
763 // check if there is a packet going to a free rank
764 for (auto i = queue.begin(); i != queue.end() ; ++i) {
765 DRAMPacket* dram_pkt = *i;
766 if (ranks[dram_pkt->rank]->inRefIdleState()) {
767 queue.erase(i);
768 queue.push_front(dram_pkt);
769 found_packet = true;
770 break;
771 }
772 }
773 } else if (memSchedPolicy == Enums::frfcfs) {
774 found_packet = reorderQueue(queue, extra_col_delay);
775 } else
776 panic("No scheduling policy chosen\n");
777 return found_packet;
778}
779
780bool
781DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
782{
783 // Only determine this if needed
784 vector<uint32_t> earliest_banks(ranksPerChannel, 0);
785
786 // Has minBankPrep been called to populate earliest_banks?
787 bool filled_earliest_banks = false;
788 // can the PRE/ACT sequence be done without impacting utlization?
789 bool hidden_bank_prep = false;

--- 16 unchanged lines hidden (view full) ---

806 auto selected_pkt_it = queue.end();
807
808 // time we need to issue a column command to be seamless
809 const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick());
810
811 for (auto i = queue.begin(); i != queue.end() ; ++i) {
812 DRAMPacket* dram_pkt = *i;
813 const Bank& bank = dram_pkt->bankRef;
814 const Tick col_allowed_at = dram_pkt->isRead ? bank.rdAllowedAt :
815 bank.wrAllowedAt;
816
817 // check if rank is not doing a refresh and thus is available, if not,
818 // jump to the next packet
819 if (dram_pkt->rankRef.inRefIdleState()) {
820 // check if it is a row hit
821 if (bank.openRow == dram_pkt->row) {
822 // no additional rank-to-rank or same bank-group
823 // delays, or we switched read/write and might as well
824 // go for the row hit
825 if (col_allowed_at <= min_col_at) {
826 // FCFS within the hits, giving priority to
827 // commands that can issue seamlessly, without
828 // additional delay, such as same rank accesses
829 // and/or different bank-group accesses
830 DPRINTF(DRAM, "Seamless row buffer hit\n");
831 selected_pkt_it = i;
832 // no need to look through the remaining queue entries
833 break;
834 } else if (!found_hidden_bank && !found_prepped_pkt) {
835 // if we did not find a packet to a closed row that can
836 // issue the bank commands without incurring delay, and
837 // did not yet find a packet to a prepped row, remember
838 // the current one
839 selected_pkt_it = i;
840 found_prepped_pkt = true;
841 DPRINTF(DRAM, "Prepped row buffer hit\n");
842 }
843 } else if (!found_earliest_pkt) {
844 // if we have not initialised the bank status, do it
845 // now, and only once per scheduling decisions
846 if (!filled_earliest_banks) {
847 // determine entries with earliest bank delay
848 std::tie(earliest_banks, hidden_bank_prep) =
849 minBankPrep(queue, min_col_at);

--- 11 unchanged lines hidden (view full) ---

861 // give priority to packets that can issue
862 // bank commands 'behind the scenes'
863 // any additional delay if any will be due to
864 // col-to-col command requirements
865 if (hidden_bank_prep || !found_prepped_pkt)
866 selected_pkt_it = i;
867 }
868 }
869 }
870 }
871
872 if (selected_pkt_it != queue.end()) {
873 DRAMPacket* selected_pkt = *selected_pkt_it;
874 queue.erase(selected_pkt_it);
875 queue.push_front(selected_pkt);
876 return true;
877 }
878
879 return false;
880}
881
882void
883DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency)
884{
885 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr());
886
887 bool needsResponse = pkt->needsResponse();

--- 216 unchanged lines hidden (view full) ---

1104 Tick act_tick = std::max(bank.actAllowedAt, curTick());
1105
1106 // Record the activation and deal with all the global timing
1107 // constraints caused be a new activation (tRRD and tXAW)
1108 activateBank(rank, bank, act_tick, dram_pkt->row);
1109 }
1110
1111 // respect any constraints on the command (e.g. tRCD or tCCD)
1112 const Tick col_allowed_at = dram_pkt->isRead ?
1113 bank.rdAllowedAt : bank.wrAllowedAt;
1114
1115 // we need to wait until the bus is available before we can issue
1116 // the command; need minimum of tBURST between commands
1117 Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()});
1118
1119 // update the packet ready time
1120 dram_pkt->readyTime = cmd_at + tCL + tBURST;

--- 10 unchanged lines hidden (view full) ---

1131 if (dram_pkt->rank == j) {
1132 if (bankGroupArch &&
1133 (bank.bankgr == ranks[j]->banks[i].bankgr)) {
1134 // bank group architecture requires longer delays between
1135 // RD/WR burst commands to the same bank group.
1136 // tCCD_L is default requirement for same BG timing
1137 // tCCD_L_WR is required for write-to-write
1138 // Need to also take bus turnaround delays into account
1139 dly_to_rd_cmd = dram_pkt->isRead ?
1140 tCCD_L : std::max(tCCD_L, wrToRdDly);
1141 dly_to_wr_cmd = dram_pkt->isRead ?
1142 std::max(tCCD_L, rdToWrDly) : tCCD_L_WR;
1143 } else {
1144 // tBURST is default requirement for diff BG timing
1145 // Need to also take bus turnaround delays into account
1146 dly_to_rd_cmd = dram_pkt->isRead ? tBURST : wrToRdDly;
1147 dly_to_wr_cmd = dram_pkt->isRead ? rdToWrDly : tBURST;
1148 }
1149 } else {
1150 // different rank is by default in a different bank group and
1151 // doesn't require longer tCCD or additional RTW, WTR delays
1152 // Need to account for rank-to-rank switching with tCS
1153 dly_to_wr_cmd = rankToRankDly;
1154 dly_to_rd_cmd = rankToRankDly;
1155 }

--- 6 unchanged lines hidden (view full) ---

1162
1163 // Save rank of current access
1164 activeRank = dram_pkt->rank;
1165
1166 // If this is a write, we also need to respect the write recovery
1167 // time before a precharge, in the case of a read, respect the
1168 // read to precharge constraint
1169 bank.preAllowedAt = std::max(bank.preAllowedAt,
1170 dram_pkt->isRead ? cmd_at + tRTP :
1171 dram_pkt->readyTime + tWR);
1172
1173 // increment the bytes accessed and the accesses per row
1174 bank.bytesAccessed += burstSize;
1175 ++bank.rowAccesses;
1176
1177 // if we reached the max, then issue with an auto-precharge
1178 bool auto_precharge = pageMgmt == Enums::close ||

--- 11 unchanged lines hidden (view full) ---

1190 // 2) close_adaptive page policy does not blindly close the
1191 // page, but closes it only if there are no row hits in the queue.
1192 // In this case, only force an auto precharge when there
1193 // are no same page hits in the queue
1194 bool got_more_hits = false;
1195 bool got_bank_conflict = false;
1196
1197 // either look at the read queue or write queue
1198 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue :
1199 writeQueue;
1200 auto p = queue.begin();
1201 // make sure we are not considering the packet that we are
1202 // currently dealing with (which is the head of the queue)
1203 ++p;
1204
1205 // keep on looking until we find a hit or reach the end of the queue
1206 // 1) if a hit is found, then both open and close adaptive policies keep
1207 // the page open
1208 // 2) if no hit is found, got_bank_conflict is set to true if a bank
1209 // conflict request is waiting in the queue
1210 while (!got_more_hits && p != queue.end()) {
1211 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) &&
1212 (dram_pkt->bank == (*p)->bank);
1213 bool same_row = dram_pkt->row == (*p)->row;
1214 got_more_hits |= same_rank_bank && same_row;
1215 got_bank_conflict |= same_rank_bank && !same_row;
1216 ++p;
1217 }
1218
1219 // auto pre-charge when either
1220 // 1) open_adaptive policy, we have not got any more hits, and
1221 // have a bank conflict
1222 // 2) close_adaptive policy and we have not got any more hits
1223 auto_precharge = !got_more_hits &&
1224 (got_bank_conflict || pageMgmt == Enums::close_adaptive);
1225 }
1226
1227 // DRAMPower trace command to be written
1228 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR";
1229
1230 // MemCommand required for DRAMPower library
1231 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD :
1232 MemCommand::WR;
1233
1234 // Update bus state to reflect when previous command was issued
1235 nextBurstAt = cmd_at + tBURST;
1236

--- 18 unchanged lines hidden (view full) ---

1255
1256 // Update the minimum timing between the requests, this is a
1257 // conservative estimate of when we have to schedule the next
1258 // request to not introduce any unecessary bubbles. In most cases
1259 // we will wake up sooner than we have to.
1260 nextReqTime = nextBurstAt - (tRP + tRCD);
1261
1262 // Update the stats and schedule the next request
1263 if (dram_pkt->isRead) {
1264 ++readsThisTime;
1265 if (row_hit)
1266 readRowHits++;
1267 bytesReadDRAM += burstSize;
1268 perBankRdBursts[dram_pkt->bankId]++;
1269
1270 // Update latency stats
1271 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime;
1272 totBusLat += tBURST;
1273 totQLat += cmd_at - dram_pkt->entryTime;
1274 } else {
1275 ++writesThisTime;
1276 if (row_hit)
1277 writeRowHits++;
1278 bytesWritten += burstSize;
1279 perBankWrBursts[dram_pkt->bankId]++;
1280 }
1281}
1282
1283void
1284DRAMCtrl::processNextReqEvent()
1285{
1286 int busyRanks = 0;
1287 for (auto r : ranks) {
1288 if (!r->inRefIdleState()) {
1289 if (r->pwrState != PWR_SREF) {
1290 // rank is busy refreshing
1291 DPRINTF(DRAMState, "Rank %d is not available\n", r->rank);
1292 busyRanks++;
1293

--- 24 unchanged lines hidden (view full) ---

1318
1319 if (busyRanks == ranksPerChannel) {
1320 // if all ranks are refreshing wait for them to finish
1321 // and stall this state machine without taking any further
1322 // action, and do not schedule a new nextReqEvent
1323 return;
1324 }
1325
1326 // pre-emptively set to false. Overwrite if in transitioning to
1327 // a new state
1328 bool switched_cmd_type = false;
1329 if (busState != busStateNext) {
1330 if (busState == READ) {
1331 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads "
1332 "waiting\n", readsThisTime, readQueue.size());
1333
1334 // sample and reset the read-related stats as we are now
1335 // transitioning to writes, and all reads are done
1336 rdPerTurnAround.sample(readsThisTime);
1337 readsThisTime = 0;
1338
1339 // now proceed to do the actual writes
1340 switched_cmd_type = true;
1341 } else {
1342 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes "
1343 "waiting\n", writesThisTime, writeQueue.size());
1344
1345 wrPerTurnAround.sample(writesThisTime);
1346 writesThisTime = 0;
1347
1348 switched_cmd_type = true;
1349 }
1350 // update busState to match next state until next transition
1351 busState = busStateNext;
1352 }
1353
1354 // when we get here it is either a read or a write
1355 if (busState == READ) {
1356
1357 // track if we should switch or not
1358 bool switch_to_writes = false;
1359
1360 if (readQueue.empty()) {
1361 // In the case there is no read request to go next,
1362 // trigger writes if we have passed the low threshold (or
1363 // if we are draining)
1364 if (!writeQueue.empty() &&
1365 (drainState() == DrainState::Draining ||
1366 writeQueue.size() > writeLowThreshold)) {
1367
1368 switch_to_writes = true;
1369 } else {
1370 // check if we are drained
1371 // not done draining until in PWR_IDLE state
1372 // ensuring all banks are closed and
1373 // have exited low power states
1374 if (drainState() == DrainState::Draining &&
1375 respQueue.empty() && allRanksDrained()) {
1376
1377 DPRINTF(Drain, "DRAM controller done draining\n");
1378 signalDrainDone();
1379 }
1380
1381 // nothing to do, not even any point in scheduling an
1382 // event for the next request
1383 return;
1384 }
1385 } else {
1386 // bool to check if there is a read to a free rank
1387 bool found_read = false;
1388
1389 // Figure out which read request goes next, and move it to the
1390 // front of the read queue
1391 // If we are changing command type, incorporate the minimum
1392 // bus turnaround delay which will be tCS (different rank) case
1393 found_read = chooseNext(readQueue, switched_cmd_type ? tCS : 0);
1394
1395 // if no read to an available rank is found then return
1396 // at this point. There could be writes to the available ranks
1397 // which are above the required threshold. However, to
1398 // avoid adding more complexity to the code, return and wait
1399 // for a refresh event to kick things into action again.
1400 if (!found_read)
1401 return;
1402
1403 DRAMPacket* dram_pkt = readQueue.front();
1404 assert(dram_pkt->rankRef.inRefIdleState());
1405
1406 doDRAMAccess(dram_pkt);
1407
1408 // At this point we're done dealing with the request
1409 readQueue.pop_front();
1410
1411 // Every respQueue which will generate an event, increment count
1412 ++dram_pkt->rankRef.outstandingEvents;
1413
1414 // sanity check
1415 assert(dram_pkt->size <= burstSize);
1416 assert(dram_pkt->readyTime >= curTick());
1417
1418 // Insert into response queue. It will be sent back to the
1419 // requestor at its readyTime
1420 if (respQueue.empty()) {
1421 assert(!respondEvent.scheduled());
1422 schedule(respondEvent, dram_pkt->readyTime);
1423 } else {
1424 assert(respQueue.back()->readyTime <= dram_pkt->readyTime);
1425 assert(respondEvent.scheduled());
1426 }
1427
1428 respQueue.push_back(dram_pkt);
1429
1430 // we have so many writes that we have to transition
1431 if (writeQueue.size() > writeHighThreshold) {
1432 switch_to_writes = true;
1433 }
1434 }
1435
1436 // switching to writes, either because the read queue is empty
1437 // and the writes have passed the low threshold (or we are
1438 // draining), or because the writes hit the hight threshold
1439 if (switch_to_writes) {
1440 // transition to writing
1441 busStateNext = WRITE;
1442 }
1443 } else {
1444 // bool to check if write to free rank is found
1445 bool found_write = false;
1446
1447 // If we are changing command type, incorporate the minimum
1448 // bus turnaround delay
1449 found_write = chooseNext(writeQueue,
1450 switched_cmd_type ? std::min(tRTW, tCS) : 0);
1451
1452 // if there are no writes to a rank that is available to service
1453 // requests (i.e. rank is in refresh idle state) are found then
1454 // return. There could be reads to the available ranks. However, to
1455 // avoid adding more complexity to the code, return at this point and
1456 // wait for a refresh event to kick things into action again.
1457 if (!found_write)
1458 return;
1459
1460 DRAMPacket* dram_pkt = writeQueue.front();
1461 assert(dram_pkt->rankRef.inRefIdleState());
1462 // sanity check
1463 assert(dram_pkt->size <= burstSize);
1464
1465 doDRAMAccess(dram_pkt);
1466
1467 writeQueue.pop_front();
1468
1469 // removed write from queue, decrement count
1470 --dram_pkt->rankRef.writeEntries;
1471
1472 // Schedule write done event to decrement event count
1473 // after the readyTime has been reached
1474 // Only schedule latest write event to minimize events
1475 // required; only need to ensure that final event scheduled covers
1476 // the time that writes are outstanding and bus is active
1477 // to holdoff power-down entry events
1478 if (!dram_pkt->rankRef.writeDoneEvent.scheduled()) {
1479 schedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1480 // New event, increment count
1481 ++dram_pkt->rankRef.outstandingEvents;
1482
1483 } else if (dram_pkt->rankRef.writeDoneEvent.when() <
1484 dram_pkt-> readyTime) {
1485 reschedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1486 }
1487
1488 isInWriteQueue.erase(burstAlign(dram_pkt->addr));
1489 delete dram_pkt;
1490
1491 // If we emptied the write queue, or got sufficiently below the
1492 // threshold (using the minWritesPerSwitch as the hysteresis) and
1493 // are not draining, or we have reads waiting and have done enough
1494 // writes, then switch to reads.
1495 if (writeQueue.empty() ||
1496 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold &&
1497 drainState() != DrainState::Draining) ||
1498 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) {
1499 // turn the bus back around for reads again
1500 busStateNext = READ;
1501
1502 // note that the we switch back to reads also in the idle
1503 // case, which eventually will check for any draining and
1504 // also pause any further scheduling if there is really
1505 // nothing to do
1506 }
1507 }
1508 // It is possible that a refresh to another rank kicks things back into
1509 // action before reaching this point.
1510 if (!nextReqEvent.scheduled())
1511 schedule(nextReqEvent, std::max(nextReqTime, curTick()));
1512
1513 // If there is space available and we have writes waiting then let
1514 // them retry. This is done here to ensure that the retry does not
1515 // cause a nextReqEvent to be scheduled before we do so as part of
1516 // the next request processing
1517 if (retryWrReq && writeQueue.size() < writeBufferSize) {
1518 retryWrReq = false;
1519 port.sendRetryReq();
1520 }
1521}
1522
1523pair<vector<uint32_t>, bool>
1524DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue,
1525 Tick min_col_at) const
1526{
1527 Tick min_act_at = MaxTick;
1528 vector<uint32_t> bank_mask(ranksPerChannel, 0);
1529
1530 // latest Tick for which ACT can occur without incurring additoinal
1531 // delay on the data bus
1532 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick());

--- 848 unchanged lines hidden (view full) ---

2381 Stats::registerDumpCallback(new RankDumpCallback(this));
2382 Stats::registerResetCallback(new RankResetCallback(this));
2383}
2384void
2385DRAMCtrl::regStats()
2386{
2387 using namespace Stats;
2388
2389 AbstractMemory::regStats();
2390
2391 for (auto r : ranks) {
2392 r->regStats();
2393 }
2394
2395 registerResetCallback(new MemResetCallback(this));
2396
2397 readReqs

--- 126 unchanged lines hidden (view full) ---

2524 .desc("What read queue length does an incoming req see");
2525
2526 wrQLenPdf
2527 .init(writeBufferSize)
2528 .name(name() + ".wrQLenPdf")
2529 .desc("What write queue length does an incoming req see");
2530
2531 bytesPerActivate
2532 .init(maxAccessesPerRow)
2533 .name(name() + ".bytesPerActivate")
2534 .desc("Bytes accessed per row activation")
2535 .flags(nozero);
2536
2537 rdPerTurnAround
2538 .init(readBufferSize)
2539 .name(name() + ".rdPerTurnAround")
2540 .desc("Reads before turning the bus around for writes")

--- 94 unchanged lines hidden (view full) ---

2635
2636 pageHitRate
2637 .name(name() + ".pageHitRate")
2638 .desc("Row buffer hit rate, read and write combined")
2639 .precision(2);
2640
2641 pageHitRate = (writeRowHits + readRowHits) /
2642 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100;
2643}
2644
2645void
2646DRAMCtrl::recvFunctional(PacketPtr pkt)
2647{
2648 // rely on the abstract memory
2649 functionalAccess(pkt);
2650}

--- 8 unchanged lines hidden (view full) ---

2659 }
2660}
2661
2662DrainState
2663DRAMCtrl::drain()
2664{
2665 // if there is anything in any of our internal queues, keep track
2666 // of that as well
2667 if (!(writeQueue.empty() && readQueue.empty() && respQueue.empty() &&
2668 allRanksDrained())) {
2669
2670 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d,"
2671 " resp: %d\n", writeQueue.size(), readQueue.size(),
2672 respQueue.size());
2673
2674 // the only queue that is not drained automatically over time
2675 // is the write queue, thus kick things into action if needed
2676 if (!writeQueue.empty() && !nextReqEvent.scheduled()) {
2677 schedule(nextReqEvent, curTick());
2678 }
2679
2680 // also need to kick off events to exit self-refresh
2681 for (auto r : ranks) {
2682 // force self-refresh exit, which in turn will issue auto-refresh
2683 if (r->pwrState == PWR_SREF) {
2684 DPRINTF(DRAM,"Rank%d: Forcing self-refresh wakeup in drain\n",

--- 91 unchanged lines hidden ---