dram_ctrl.cc (12823:ba630bc7a36d) dram_ctrl.cc (12969:52de9d619ce6)
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 39 unchanged lines hidden (view full) ---

48#include "mem/dram_ctrl.hh"
49
50#include "base/bitfield.hh"
51#include "base/trace.hh"
52#include "debug/DRAM.hh"
53#include "debug/DRAMPower.hh"
54#include "debug/DRAMState.hh"
55#include "debug/Drain.hh"
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 39 unchanged lines hidden (view full) ---

48#include "mem/dram_ctrl.hh"
49
50#include "base/bitfield.hh"
51#include "base/trace.hh"
52#include "debug/DRAM.hh"
53#include "debug/DRAMPower.hh"
54#include "debug/DRAMState.hh"
55#include "debug/Drain.hh"
56#include "debug/QOS.hh"
56#include "sim/system.hh"
57
58using namespace std;
59using namespace Data;
60
61DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) :
57#include "sim/system.hh"
58
59using namespace std;
60using namespace Data;
61
62DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) :
62 AbstractMemory(p),
63 QoS::MemCtrl(p),
63 port(name() + ".port", *this), isTimingMode(false),
64 retryRdReq(false), retryWrReq(false),
64 port(name() + ".port", *this), isTimingMode(false),
65 retryRdReq(false), retryWrReq(false),
65 busState(READ),
66 busStateNext(READ),
67 nextReqEvent([this]{ processNextReqEvent(); }, name()),
68 respondEvent([this]{ processRespondEvent(); }, name()),
69 deviceSize(p->device_size),
70 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length),
71 deviceRowBufferSize(p->device_rowbuffer_size),
72 devicesPerRank(p->devices_per_rank),
73 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8),
74 rowBufferSize(devicesPerRank * deviceRowBufferSize),

--- 27 unchanged lines hidden (view full) ---

102{
103 // sanity check the ranks since we rely on bit slicing for the
104 // address decoding
105 fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not "
106 "allowed, must be a power of two\n", ranksPerChannel);
107
108 fatal_if(!isPowerOf2(burstSize), "DRAM burst size %d is not allowed, "
109 "must be a power of two\n", burstSize);
66 nextReqEvent([this]{ processNextReqEvent(); }, name()),
67 respondEvent([this]{ processRespondEvent(); }, name()),
68 deviceSize(p->device_size),
69 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length),
70 deviceRowBufferSize(p->device_rowbuffer_size),
71 devicesPerRank(p->devices_per_rank),
72 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8),
73 rowBufferSize(devicesPerRank * deviceRowBufferSize),

--- 27 unchanged lines hidden (view full) ---

101{
102 // sanity check the ranks since we rely on bit slicing for the
103 // address decoding
104 fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not "
105 "allowed, must be a power of two\n", ranksPerChannel);
106
107 fatal_if(!isPowerOf2(burstSize), "DRAM burst size %d is not allowed, "
108 "must be a power of two\n", burstSize);
109 readQueue.resize(p->qos_priorities);
110 writeQueue.resize(p->qos_priorities);
110
111
112
111 for (int i = 0; i < ranksPerChannel; i++) {
112 Rank* rank = new Rank(*this, p, i);
113 ranks.push_back(rank);
114 }
115
116 // perform a basic check of the write thresholds
117 if (p->write_low_thresh_perc >= p->write_high_thresh_perc)
118 fatal("Write buffer low threshold %d must be smaller than the "

--- 62 unchanged lines hidden (view full) ---

181 }
182 }
183
184}
185
186void
187DRAMCtrl::init()
188{
113 for (int i = 0; i < ranksPerChannel; i++) {
114 Rank* rank = new Rank(*this, p, i);
115 ranks.push_back(rank);
116 }
117
118 // perform a basic check of the write thresholds
119 if (p->write_low_thresh_perc >= p->write_high_thresh_perc)
120 fatal("Write buffer low threshold %d must be smaller than the "

--- 62 unchanged lines hidden (view full) ---

183 }
184 }
185
186}
187
188void
189DRAMCtrl::init()
190{
189 AbstractMemory::init();
191 MemCtrl::init();
190
191 if (!port.isConnected()) {
192 fatal("DRAMCtrl %s is unconnected!\n", name());
193 } else {
194 port.sendRangeChange();
195 }
196
197 // a bit of sanity checks on the interleaving, save it for here to

--- 80 unchanged lines hidden (view full) ---

278 }
279 return latency;
280}
281
282bool
283DRAMCtrl::readQueueFull(unsigned int neededEntries) const
284{
285 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n",
192
193 if (!port.isConnected()) {
194 fatal("DRAMCtrl %s is unconnected!\n", name());
195 } else {
196 port.sendRangeChange();
197 }
198
199 // a bit of sanity checks on the interleaving, save it for here to

--- 80 unchanged lines hidden (view full) ---

280 }
281 return latency;
282}
283
284bool
285DRAMCtrl::readQueueFull(unsigned int neededEntries) const
286{
287 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n",
286 readBufferSize, readQueue.size() + respQueue.size(),
288 readBufferSize, totalReadQueueSize + respQueue.size(),
287 neededEntries);
288
289 neededEntries);
290
289 return
290 (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize;
291 auto rdsize_new = totalReadQueueSize + respQueue.size() + neededEntries;
292 return rdsize_new > readBufferSize;
291}
292
293bool
294DRAMCtrl::writeQueueFull(unsigned int neededEntries) const
295{
296 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n",
293}
294
295bool
296DRAMCtrl::writeQueueFull(unsigned int neededEntries) const
297{
298 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n",
297 writeBufferSize, writeQueue.size(), neededEntries);
298 return (writeQueue.size() + neededEntries) > writeBufferSize;
299 writeBufferSize, totalWriteQueueSize, neededEntries);
300
301 auto wrsize_new = (totalWriteQueueSize + neededEntries);
302 return wrsize_new > writeBufferSize;
299}
300
301DRAMCtrl::DRAMPacket*
302DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size,
303 bool isRead)
304{
305 // decode the address based on the address mapping scheme, with
306 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and

--- 115 unchanged lines hidden (view full) ---

422 Addr addr = pkt->getAddr();
423 unsigned pktsServicedByWrQ = 0;
424 BurstHelper* burst_helper = NULL;
425 for (int cnt = 0; cnt < pktCount; ++cnt) {
426 unsigned size = std::min((addr | (burstSize - 1)) + 1,
427 pkt->getAddr() + pkt->getSize()) - addr;
428 readPktSize[ceilLog2(size)]++;
429 readBursts++;
303}
304
305DRAMCtrl::DRAMPacket*
306DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size,
307 bool isRead)
308{
309 // decode the address based on the address mapping scheme, with
310 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and

--- 115 unchanged lines hidden (view full) ---

426 Addr addr = pkt->getAddr();
427 unsigned pktsServicedByWrQ = 0;
428 BurstHelper* burst_helper = NULL;
429 for (int cnt = 0; cnt < pktCount; ++cnt) {
430 unsigned size = std::min((addr | (burstSize - 1)) + 1,
431 pkt->getAddr() + pkt->getSize()) - addr;
432 readPktSize[ceilLog2(size)]++;
433 readBursts++;
434 masterReadAccesses[pkt->masterId()]++;
430
431 // First check write buffer to see if the data is already at
432 // the controller
433 bool foundInWrQ = false;
434 Addr burst_addr = burstAlign(addr);
435 // if the burst address is not present then there is no need
436 // looking any further
437 if (isInWriteQueue.find(burst_addr) != isInWriteQueue.end()) {
435
436 // First check write buffer to see if the data is already at
437 // the controller
438 bool foundInWrQ = false;
439 Addr burst_addr = burstAlign(addr);
440 // if the burst address is not present then there is no need
441 // looking any further
442 if (isInWriteQueue.find(burst_addr) != isInWriteQueue.end()) {
438 for (const auto& p : writeQueue) {
439 // check if the read is subsumed in the write queue
440 // packet we are looking at
441 if (p->addr <= addr && (addr + size) <= (p->addr + p->size)) {
442 foundInWrQ = true;
443 servicedByWrQ++;
444 pktsServicedByWrQ++;
445 DPRINTF(DRAM, "Read to addr %lld with size %d serviced by "
446 "write queue\n", addr, size);
447 bytesReadWrQ += burstSize;
448 break;
443 for (const auto& vec : writeQueue) {
444 for (const auto& p : vec) {
445 // check if the read is subsumed in the write queue
446 // packet we are looking at
447 if (p->addr <= addr &&
448 ((addr + size) <= (p->addr + p->size))) {
449
450 foundInWrQ = true;
451 servicedByWrQ++;
452 pktsServicedByWrQ++;
453 DPRINTF(DRAM,
454 "Read to addr %lld with size %d serviced by "
455 "write queue\n",
456 addr, size);
457 bytesReadWrQ += burstSize;
458 break;
459 }
449 }
450 }
451 }
452
453 // If not found in the write q, make a DRAM packet and
454 // push it onto the read queue
455 if (!foundInWrQ) {
456
457 // Make the burst helper for split packets
458 if (pktCount > 1 && burst_helper == NULL) {
459 DPRINTF(DRAM, "Read to addr %lld translates to %d "
460 "dram requests\n", pkt->getAddr(), pktCount);
461 burst_helper = new BurstHelper(pktCount);
462 }
463
464 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true);
465 dram_pkt->burstHelper = burst_helper;
466
467 assert(!readQueueFull(1));
460 }
461 }
462 }
463
464 // If not found in the write q, make a DRAM packet and
465 // push it onto the read queue
466 if (!foundInWrQ) {
467
468 // Make the burst helper for split packets
469 if (pktCount > 1 && burst_helper == NULL) {
470 DPRINTF(DRAM, "Read to addr %lld translates to %d "
471 "dram requests\n", pkt->getAddr(), pktCount);
472 burst_helper = new BurstHelper(pktCount);
473 }
474
475 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true);
476 dram_pkt->burstHelper = burst_helper;
477
478 assert(!readQueueFull(1));
468 rdQLenPdf[readQueue.size() + respQueue.size()]++;
479 rdQLenPdf[totalReadQueueSize + respQueue.size()]++;
469
470 DPRINTF(DRAM, "Adding to read queue\n");
471
480
481 DPRINTF(DRAM, "Adding to read queue\n");
482
472 readQueue.push_back(dram_pkt);
483 readQueue[dram_pkt->qosValue()].push_back(dram_pkt);
473
484
474 // increment read entries of the rank
475 ++dram_pkt->rankRef.readEntries;
476
485 ++dram_pkt->rankRef.readEntries;
486
487 // log packet
488 logRequest(MemCtrl::READ, pkt->masterId(), pkt->qosValue(),
489 dram_pkt->addr, 1);
490
477 // Update stats
491 // Update stats
478 avgRdQLen = readQueue.size() + respQueue.size();
492 avgRdQLen = totalReadQueueSize + respQueue.size();
479 }
480
481 // Starting address of next dram pkt (aligend to burstSize boundary)
482 addr = (addr | (burstSize - 1)) + 1;
483 }
484
485 // If all packets are serviced by write queue, we send the repsonse back
486 if (pktsServicedByWrQ == pktCount) {

--- 23 unchanged lines hidden (view full) ---

510 // if the request size is larger than burst size, the pkt is split into
511 // multiple DRAM packets
512 Addr addr = pkt->getAddr();
513 for (int cnt = 0; cnt < pktCount; ++cnt) {
514 unsigned size = std::min((addr | (burstSize - 1)) + 1,
515 pkt->getAddr() + pkt->getSize()) - addr;
516 writePktSize[ceilLog2(size)]++;
517 writeBursts++;
493 }
494
495 // Starting address of next dram pkt (aligend to burstSize boundary)
496 addr = (addr | (burstSize - 1)) + 1;
497 }
498
499 // If all packets are serviced by write queue, we send the repsonse back
500 if (pktsServicedByWrQ == pktCount) {

--- 23 unchanged lines hidden (view full) ---

524 // if the request size is larger than burst size, the pkt is split into
525 // multiple DRAM packets
526 Addr addr = pkt->getAddr();
527 for (int cnt = 0; cnt < pktCount; ++cnt) {
528 unsigned size = std::min((addr | (burstSize - 1)) + 1,
529 pkt->getAddr() + pkt->getSize()) - addr;
530 writePktSize[ceilLog2(size)]++;
531 writeBursts++;
532 masterWriteAccesses[pkt->masterId()]++;
518
519 // see if we can merge with an existing item in the write
520 // queue and keep track of whether we have merged or not
521 bool merged = isInWriteQueue.find(burstAlign(addr)) !=
522 isInWriteQueue.end();
523
524 // if the item was not merged we need to create a new write
525 // and enqueue it
526 if (!merged) {
527 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false);
528
533
534 // see if we can merge with an existing item in the write
535 // queue and keep track of whether we have merged or not
536 bool merged = isInWriteQueue.find(burstAlign(addr)) !=
537 isInWriteQueue.end();
538
539 // if the item was not merged we need to create a new write
540 // and enqueue it
541 if (!merged) {
542 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false);
543
529 assert(writeQueue.size() < writeBufferSize);
530 wrQLenPdf[writeQueue.size()]++;
544 assert(totalWriteQueueSize < writeBufferSize);
545 wrQLenPdf[totalWriteQueueSize]++;
531
532 DPRINTF(DRAM, "Adding to write queue\n");
533
546
547 DPRINTF(DRAM, "Adding to write queue\n");
548
534 writeQueue.push_back(dram_pkt);
549 writeQueue[dram_pkt->qosValue()].push_back(dram_pkt);
535 isInWriteQueue.insert(burstAlign(addr));
550 isInWriteQueue.insert(burstAlign(addr));
536 assert(writeQueue.size() == isInWriteQueue.size());
537
551
552 // log packet
553 logRequest(MemCtrl::WRITE, pkt->masterId(), pkt->qosValue(),
554 dram_pkt->addr, 1);
555
556 assert(totalWriteQueueSize == isInWriteQueue.size());
557
538 // Update stats
558 // Update stats
539 avgWrQLen = writeQueue.size();
559 avgWrQLen = totalWriteQueueSize;
540
541 // increment write entries of the rank
542 ++dram_pkt->rankRef.writeEntries;
543 } else {
544 DPRINTF(DRAM, "Merging write burst with existing queue entry\n");
545
546 // keep track of the fact that this burst effectively
547 // disappeared as it was merged with an existing one

--- 15 unchanged lines hidden (view full) ---

563 // queue, do so now
564 if (!nextReqEvent.scheduled()) {
565 DPRINTF(DRAM, "Request scheduled immediately\n");
566 schedule(nextReqEvent, curTick());
567 }
568}
569
570void
560
561 // increment write entries of the rank
562 ++dram_pkt->rankRef.writeEntries;
563 } else {
564 DPRINTF(DRAM, "Merging write burst with existing queue entry\n");
565
566 // keep track of the fact that this burst effectively
567 // disappeared as it was merged with an existing one

--- 15 unchanged lines hidden (view full) ---

583 // queue, do so now
584 if (!nextReqEvent.scheduled()) {
585 DPRINTF(DRAM, "Request scheduled immediately\n");
586 schedule(nextReqEvent, curTick());
587 }
588}
589
590void
571DRAMCtrl::printQs() const {
591DRAMCtrl::printQs() const
592{
593#if TRACING_ON
572 DPRINTF(DRAM, "===READ QUEUE===\n\n");
594 DPRINTF(DRAM, "===READ QUEUE===\n\n");
573 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) {
574 DPRINTF(DRAM, "Read %lu\n", (*i)->addr);
595 for (const auto& queue : readQueue) {
596 for (const auto& packet : queue) {
597 DPRINTF(DRAM, "Read %lu\n", packet->addr);
598 }
575 }
599 }
600
576 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n");
601 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n");
577 for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) {
578 DPRINTF(DRAM, "Response %lu\n", (*i)->addr);
602 for (const auto& packet : respQueue) {
603 DPRINTF(DRAM, "Response %lu\n", packet->addr);
579 }
604 }
605
580 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n");
606 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n");
581 for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) {
582 DPRINTF(DRAM, "Write %lu\n", (*i)->addr);
607 for (const auto& queue : writeQueue) {
608 for (const auto& packet : queue) {
609 DPRINTF(DRAM, "Write %lu\n", packet->addr);
610 }
583 }
611 }
612#endif // TRACING_ON
584}
585
586bool
587DRAMCtrl::recvTimingReq(PacketPtr pkt)
588{
589 // This is where we enter from the outside world
590 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n",
591 pkt->cmdString(), pkt->getAddr(), pkt->getSize());

--- 14 unchanged lines hidden (view full) ---

606 // Find out how many dram packets a pkt translates to
607 // If the burst size is equal or larger than the pkt size, then a pkt
608 // translates to only one dram packet. Otherwise, a pkt translates to
609 // multiple dram packets
610 unsigned size = pkt->getSize();
611 unsigned offset = pkt->getAddr() & (burstSize - 1);
612 unsigned int dram_pkt_count = divCeil(offset + size, burstSize);
613
613}
614
615bool
616DRAMCtrl::recvTimingReq(PacketPtr pkt)
617{
618 // This is where we enter from the outside world
619 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n",
620 pkt->cmdString(), pkt->getAddr(), pkt->getSize());

--- 14 unchanged lines hidden (view full) ---

635 // Find out how many dram packets a pkt translates to
636 // If the burst size is equal or larger than the pkt size, then a pkt
637 // translates to only one dram packet. Otherwise, a pkt translates to
638 // multiple dram packets
639 unsigned size = pkt->getSize();
640 unsigned offset = pkt->getAddr() & (burstSize - 1);
641 unsigned int dram_pkt_count = divCeil(offset + size, burstSize);
642
643 // run the QoS scheduler and assign a QoS priority value to the packet
644 qosSchedule( { &readQueue, &writeQueue }, burstSize, pkt);
645
614 // check local buffers and do not accept if full
615 if (pkt->isRead()) {
616 assert(size != 0);
617 if (readQueueFull(dram_pkt_count)) {
618 DPRINTF(DRAM, "Read queue full, not accepting\n");
619 // remember that we have to retry this port
620 retryRdReq = true;
621 numRdRetry++;

--- 94 unchanged lines hidden (view full) ---

716
717 if (!respQueue.empty()) {
718 assert(respQueue.front()->readyTime >= curTick());
719 assert(!respondEvent.scheduled());
720 schedule(respondEvent, respQueue.front()->readyTime);
721 } else {
722 // if there is nothing left in any queue, signal a drain
723 if (drainState() == DrainState::Draining &&
646 // check local buffers and do not accept if full
647 if (pkt->isRead()) {
648 assert(size != 0);
649 if (readQueueFull(dram_pkt_count)) {
650 DPRINTF(DRAM, "Read queue full, not accepting\n");
651 // remember that we have to retry this port
652 retryRdReq = true;
653 numRdRetry++;

--- 94 unchanged lines hidden (view full) ---

748
749 if (!respQueue.empty()) {
750 assert(respQueue.front()->readyTime >= curTick());
751 assert(!respondEvent.scheduled());
752 schedule(respondEvent, respQueue.front()->readyTime);
753 } else {
754 // if there is nothing left in any queue, signal a drain
755 if (drainState() == DrainState::Draining &&
724 writeQueue.empty() && readQueue.empty() && allRanksDrained()) {
756 !totalWriteQueueSize && !totalReadQueueSize && allRanksDrained()) {
725
726 DPRINTF(Drain, "DRAM controller done draining\n");
727 signalDrainDone();
728 }
729 }
730
731 // We have made a location in the queue available at this point,
732 // so if there is a read that was forced to wait, retry now
733 if (retryRdReq) {
734 retryRdReq = false;
735 port.sendRetryReq();
736 }
737}
738
757
758 DPRINTF(Drain, "DRAM controller done draining\n");
759 signalDrainDone();
760 }
761 }
762
763 // We have made a location in the queue available at this point,
764 // so if there is a read that was forced to wait, retry now
765 if (retryRdReq) {
766 retryRdReq = false;
767 port.sendRetryReq();
768 }
769}
770
739bool
740DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
771DRAMCtrl::DRAMPacketQueue::iterator
772DRAMCtrl::chooseNext(DRAMPacketQueue& queue, Tick extra_col_delay)
741{
773{
742 // This method does the arbitration between requests. The chosen
743 // packet is simply moved to the head of the queue. The other
744 // methods know that this is the place to look. For example, with
745 // FCFS, this method does nothing
746 assert(!queue.empty());
774 // This method does the arbitration between requests.
747
775
748 // bool to indicate if a packet to an available rank is found
749 bool found_packet = false;
750 if (queue.size() == 1) {
751 DRAMPacket* dram_pkt = queue.front();
752 // available rank corresponds to state refresh idle
753 if (ranks[dram_pkt->rank]->inRefIdleState()) {
754 found_packet = true;
755 DPRINTF(DRAM, "Single request, going to a free rank\n");
756 } else {
757 DPRINTF(DRAM, "Single request, going to a busy rank\n");
758 }
759 return found_packet;
760 }
776 DRAMCtrl::DRAMPacketQueue::iterator ret = queue.end();
761
777
762 if (memSchedPolicy == Enums::fcfs) {
763 // check if there is a packet going to a free rank
764 for (auto i = queue.begin(); i != queue.end() ; ++i) {
765 DRAMPacket* dram_pkt = *i;
778 if (!queue.empty()) {
779 if (queue.size() == 1) {
780 // available rank corresponds to state refresh idle
781 DRAMPacket* dram_pkt = *(queue.begin());
766 if (ranks[dram_pkt->rank]->inRefIdleState()) {
782 if (ranks[dram_pkt->rank]->inRefIdleState()) {
767 queue.erase(i);
768 queue.push_front(dram_pkt);
769 found_packet = true;
770 break;
783 ret = queue.begin();
784 DPRINTF(DRAM, "Single request, going to a free rank\n");
785 } else {
786 DPRINTF(DRAM, "Single request, going to a busy rank\n");
771 }
787 }
788 } else if (memSchedPolicy == Enums::fcfs) {
789 // check if there is a packet going to a free rank
790 for (auto i = queue.begin(); i != queue.end(); ++i) {
791 DRAMPacket* dram_pkt = *i;
792 if (ranks[dram_pkt->rank]->inRefIdleState()) {
793 ret = i;
794 break;
795 }
796 }
797 } else if (memSchedPolicy == Enums::frfcfs) {
798 ret = chooseNextFRFCFS(queue, extra_col_delay);
799 } else {
800 panic("No scheduling policy chosen\n");
772 }
801 }
773 } else if (memSchedPolicy == Enums::frfcfs) {
774 found_packet = reorderQueue(queue, extra_col_delay);
775 } else
776 panic("No scheduling policy chosen\n");
777 return found_packet;
802 }
803 return ret;
778}
779
804}
805
780bool
781DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
806DRAMCtrl::DRAMPacketQueue::iterator
807DRAMCtrl::chooseNextFRFCFS(DRAMPacketQueue& queue, Tick extra_col_delay)
782{
783 // Only determine this if needed
784 vector<uint32_t> earliest_banks(ranksPerChannel, 0);
785
786 // Has minBankPrep been called to populate earliest_banks?
787 bool filled_earliest_banks = false;
788 // can the PRE/ACT sequence be done without impacting utlization?
789 bool hidden_bank_prep = false;

--- 16 unchanged lines hidden (view full) ---

806 auto selected_pkt_it = queue.end();
807
808 // time we need to issue a column command to be seamless
809 const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick());
810
811 for (auto i = queue.begin(); i != queue.end() ; ++i) {
812 DRAMPacket* dram_pkt = *i;
813 const Bank& bank = dram_pkt->bankRef;
808{
809 // Only determine this if needed
810 vector<uint32_t> earliest_banks(ranksPerChannel, 0);
811
812 // Has minBankPrep been called to populate earliest_banks?
813 bool filled_earliest_banks = false;
814 // can the PRE/ACT sequence be done without impacting utlization?
815 bool hidden_bank_prep = false;

--- 16 unchanged lines hidden (view full) ---

832 auto selected_pkt_it = queue.end();
833
834 // time we need to issue a column command to be seamless
835 const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick());
836
837 for (auto i = queue.begin(); i != queue.end() ; ++i) {
838 DRAMPacket* dram_pkt = *i;
839 const Bank& bank = dram_pkt->bankRef;
814 const Tick col_allowed_at = dram_pkt->isRead ? bank.rdAllowedAt :
815 bank.wrAllowedAt;
840 const Tick col_allowed_at = dram_pkt->isRead() ? bank.rdAllowedAt :
841 bank.wrAllowedAt;
816
842
843 DPRINTF(DRAM, "%s checking packet in bank %d\n",
844 __func__, dram_pkt->bankRef.bank);
845
817 // check if rank is not doing a refresh and thus is available, if not,
818 // jump to the next packet
819 if (dram_pkt->rankRef.inRefIdleState()) {
846 // check if rank is not doing a refresh and thus is available, if not,
847 // jump to the next packet
848 if (dram_pkt->rankRef.inRefIdleState()) {
849
850 DPRINTF(DRAM,
851 "%s bank %d - Rank %d available\n", __func__,
852 dram_pkt->bankRef.bank, dram_pkt->rankRef.rank);
853
820 // check if it is a row hit
821 if (bank.openRow == dram_pkt->row) {
822 // no additional rank-to-rank or same bank-group
823 // delays, or we switched read/write and might as well
824 // go for the row hit
825 if (col_allowed_at <= min_col_at) {
826 // FCFS within the hits, giving priority to
827 // commands that can issue seamlessly, without
828 // additional delay, such as same rank accesses
829 // and/or different bank-group accesses
854 // check if it is a row hit
855 if (bank.openRow == dram_pkt->row) {
856 // no additional rank-to-rank or same bank-group
857 // delays, or we switched read/write and might as well
858 // go for the row hit
859 if (col_allowed_at <= min_col_at) {
860 // FCFS within the hits, giving priority to
861 // commands that can issue seamlessly, without
862 // additional delay, such as same rank accesses
863 // and/or different bank-group accesses
830 DPRINTF(DRAM, "Seamless row buffer hit\n");
864 DPRINTF(DRAM, "%s Seamless row buffer hit\n", __func__);
831 selected_pkt_it = i;
832 // no need to look through the remaining queue entries
833 break;
834 } else if (!found_hidden_bank && !found_prepped_pkt) {
835 // if we did not find a packet to a closed row that can
836 // issue the bank commands without incurring delay, and
837 // did not yet find a packet to a prepped row, remember
838 // the current one
839 selected_pkt_it = i;
840 found_prepped_pkt = true;
865 selected_pkt_it = i;
866 // no need to look through the remaining queue entries
867 break;
868 } else if (!found_hidden_bank && !found_prepped_pkt) {
869 // if we did not find a packet to a closed row that can
870 // issue the bank commands without incurring delay, and
871 // did not yet find a packet to a prepped row, remember
872 // the current one
873 selected_pkt_it = i;
874 found_prepped_pkt = true;
841 DPRINTF(DRAM, "Prepped row buffer hit\n");
875 DPRINTF(DRAM, "%s Prepped row buffer hit\n", __func__);
842 }
843 } else if (!found_earliest_pkt) {
844 // if we have not initialised the bank status, do it
845 // now, and only once per scheduling decisions
846 if (!filled_earliest_banks) {
847 // determine entries with earliest bank delay
848 std::tie(earliest_banks, hidden_bank_prep) =
849 minBankPrep(queue, min_col_at);

--- 11 unchanged lines hidden (view full) ---

861 // give priority to packets that can issue
862 // bank commands 'behind the scenes'
863 // any additional delay if any will be due to
864 // col-to-col command requirements
865 if (hidden_bank_prep || !found_prepped_pkt)
866 selected_pkt_it = i;
867 }
868 }
876 }
877 } else if (!found_earliest_pkt) {
878 // if we have not initialised the bank status, do it
879 // now, and only once per scheduling decisions
880 if (!filled_earliest_banks) {
881 // determine entries with earliest bank delay
882 std::tie(earliest_banks, hidden_bank_prep) =
883 minBankPrep(queue, min_col_at);

--- 11 unchanged lines hidden (view full) ---

895 // give priority to packets that can issue
896 // bank commands 'behind the scenes'
897 // any additional delay if any will be due to
898 // col-to-col command requirements
899 if (hidden_bank_prep || !found_prepped_pkt)
900 selected_pkt_it = i;
901 }
902 }
903 } else {
904 DPRINTF(DRAM, "%s bank %d - Rank %d not available\n", __func__,
905 dram_pkt->bankRef.bank, dram_pkt->rankRef.rank);
869 }
870 }
871
906 }
907 }
908
872 if (selected_pkt_it != queue.end()) {
873 DRAMPacket* selected_pkt = *selected_pkt_it;
874 queue.erase(selected_pkt_it);
875 queue.push_front(selected_pkt);
876 return true;
909 if (selected_pkt_it == queue.end()) {
910 DPRINTF(DRAM, "%s no available ranks found\n", __func__);
877 }
878
911 }
912
879 return false;
913 return selected_pkt_it;
880}
881
882void
883DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency)
884{
885 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr());
886
887 bool needsResponse = pkt->needsResponse();

--- 216 unchanged lines hidden (view full) ---

1104 Tick act_tick = std::max(bank.actAllowedAt, curTick());
1105
1106 // Record the activation and deal with all the global timing
1107 // constraints caused be a new activation (tRRD and tXAW)
1108 activateBank(rank, bank, act_tick, dram_pkt->row);
1109 }
1110
1111 // respect any constraints on the command (e.g. tRCD or tCCD)
914}
915
916void
917DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency)
918{
919 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr());
920
921 bool needsResponse = pkt->needsResponse();

--- 216 unchanged lines hidden (view full) ---

1138 Tick act_tick = std::max(bank.actAllowedAt, curTick());
1139
1140 // Record the activation and deal with all the global timing
1141 // constraints caused be a new activation (tRRD and tXAW)
1142 activateBank(rank, bank, act_tick, dram_pkt->row);
1143 }
1144
1145 // respect any constraints on the command (e.g. tRCD or tCCD)
1112 const Tick col_allowed_at = dram_pkt->isRead ?
1146 const Tick col_allowed_at = dram_pkt->isRead() ?
1113 bank.rdAllowedAt : bank.wrAllowedAt;
1114
1115 // we need to wait until the bus is available before we can issue
1116 // the command; need minimum of tBURST between commands
1117 Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()});
1118
1119 // update the packet ready time
1120 dram_pkt->readyTime = cmd_at + tCL + tBURST;

--- 10 unchanged lines hidden (view full) ---

1131 if (dram_pkt->rank == j) {
1132 if (bankGroupArch &&
1133 (bank.bankgr == ranks[j]->banks[i].bankgr)) {
1134 // bank group architecture requires longer delays between
1135 // RD/WR burst commands to the same bank group.
1136 // tCCD_L is default requirement for same BG timing
1137 // tCCD_L_WR is required for write-to-write
1138 // Need to also take bus turnaround delays into account
1147 bank.rdAllowedAt : bank.wrAllowedAt;
1148
1149 // we need to wait until the bus is available before we can issue
1150 // the command; need minimum of tBURST between commands
1151 Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()});
1152
1153 // update the packet ready time
1154 dram_pkt->readyTime = cmd_at + tCL + tBURST;

--- 10 unchanged lines hidden (view full) ---

1165 if (dram_pkt->rank == j) {
1166 if (bankGroupArch &&
1167 (bank.bankgr == ranks[j]->banks[i].bankgr)) {
1168 // bank group architecture requires longer delays between
1169 // RD/WR burst commands to the same bank group.
1170 // tCCD_L is default requirement for same BG timing
1171 // tCCD_L_WR is required for write-to-write
1172 // Need to also take bus turnaround delays into account
1139 dly_to_rd_cmd = dram_pkt->isRead ?
1173 dly_to_rd_cmd = dram_pkt->isRead() ?
1140 tCCD_L : std::max(tCCD_L, wrToRdDly);
1174 tCCD_L : std::max(tCCD_L, wrToRdDly);
1141 dly_to_wr_cmd = dram_pkt->isRead ?
1175 dly_to_wr_cmd = dram_pkt->isRead() ?
1142 std::max(tCCD_L, rdToWrDly) : tCCD_L_WR;
1143 } else {
1144 // tBURST is default requirement for diff BG timing
1145 // Need to also take bus turnaround delays into account
1176 std::max(tCCD_L, rdToWrDly) : tCCD_L_WR;
1177 } else {
1178 // tBURST is default requirement for diff BG timing
1179 // Need to also take bus turnaround delays into account
1146 dly_to_rd_cmd = dram_pkt->isRead ? tBURST : wrToRdDly;
1147 dly_to_wr_cmd = dram_pkt->isRead ? rdToWrDly : tBURST;
1180 dly_to_rd_cmd = dram_pkt->isRead() ? tBURST : wrToRdDly;
1181 dly_to_wr_cmd = dram_pkt->isRead() ? rdToWrDly : tBURST;
1148 }
1149 } else {
1150 // different rank is by default in a different bank group and
1151 // doesn't require longer tCCD or additional RTW, WTR delays
1152 // Need to account for rank-to-rank switching with tCS
1153 dly_to_wr_cmd = rankToRankDly;
1154 dly_to_rd_cmd = rankToRankDly;
1155 }

--- 6 unchanged lines hidden (view full) ---

1162
1163 // Save rank of current access
1164 activeRank = dram_pkt->rank;
1165
1166 // If this is a write, we also need to respect the write recovery
1167 // time before a precharge, in the case of a read, respect the
1168 // read to precharge constraint
1169 bank.preAllowedAt = std::max(bank.preAllowedAt,
1182 }
1183 } else {
1184 // different rank is by default in a different bank group and
1185 // doesn't require longer tCCD or additional RTW, WTR delays
1186 // Need to account for rank-to-rank switching with tCS
1187 dly_to_wr_cmd = rankToRankDly;
1188 dly_to_rd_cmd = rankToRankDly;
1189 }

--- 6 unchanged lines hidden (view full) ---

1196
1197 // Save rank of current access
1198 activeRank = dram_pkt->rank;
1199
1200 // If this is a write, we also need to respect the write recovery
1201 // time before a precharge, in the case of a read, respect the
1202 // read to precharge constraint
1203 bank.preAllowedAt = std::max(bank.preAllowedAt,
1170 dram_pkt->isRead ? cmd_at + tRTP :
1204 dram_pkt->isRead() ? cmd_at + tRTP :
1171 dram_pkt->readyTime + tWR);
1172
1173 // increment the bytes accessed and the accesses per row
1174 bank.bytesAccessed += burstSize;
1175 ++bank.rowAccesses;
1176
1177 // if we reached the max, then issue with an auto-precharge
1178 bool auto_precharge = pageMgmt == Enums::close ||

--- 11 unchanged lines hidden (view full) ---

1190 // 2) close_adaptive page policy does not blindly close the
1191 // page, but closes it only if there are no row hits in the queue.
1192 // In this case, only force an auto precharge when there
1193 // are no same page hits in the queue
1194 bool got_more_hits = false;
1195 bool got_bank_conflict = false;
1196
1197 // either look at the read queue or write queue
1205 dram_pkt->readyTime + tWR);
1206
1207 // increment the bytes accessed and the accesses per row
1208 bank.bytesAccessed += burstSize;
1209 ++bank.rowAccesses;
1210
1211 // if we reached the max, then issue with an auto-precharge
1212 bool auto_precharge = pageMgmt == Enums::close ||

--- 11 unchanged lines hidden (view full) ---

1224 // 2) close_adaptive page policy does not blindly close the
1225 // page, but closes it only if there are no row hits in the queue.
1226 // In this case, only force an auto precharge when there
1227 // are no same page hits in the queue
1228 bool got_more_hits = false;
1229 bool got_bank_conflict = false;
1230
1231 // either look at the read queue or write queue
1198 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue :
1199 writeQueue;
1200 auto p = queue.begin();
1201 // make sure we are not considering the packet that we are
1202 // currently dealing with (which is the head of the queue)
1203 ++p;
1232 const std::vector<DRAMPacketQueue>& queue =
1233 dram_pkt->isRead() ? readQueue : writeQueue;
1204
1234
1205 // keep on looking until we find a hit or reach the end of the queue
1206 // 1) if a hit is found, then both open and close adaptive policies keep
1207 // the page open
1208 // 2) if no hit is found, got_bank_conflict is set to true if a bank
1209 // conflict request is waiting in the queue
1210 while (!got_more_hits && p != queue.end()) {
1211 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) &&
1212 (dram_pkt->bank == (*p)->bank);
1213 bool same_row = dram_pkt->row == (*p)->row;
1214 got_more_hits |= same_rank_bank && same_row;
1215 got_bank_conflict |= same_rank_bank && !same_row;
1216 ++p;
1235 for (uint8_t i = 0; i < numPriorities(); ++i) {
1236 auto p = queue[i].begin();
1237 // keep on looking until we find a hit or reach the end of the queue
1238 // 1) if a hit is found, then both open and close adaptive policies keep
1239 // the page open
1240 // 2) if no hit is found, got_bank_conflict is set to true if a bank
1241 // conflict request is waiting in the queue
1242 // 3) make sure we are not considering the packet that we are
1243 // currently dealing with
1244 while (!got_more_hits && p != queue[i].end()) {
1245 if (dram_pkt != (*p)) {
1246 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) &&
1247 (dram_pkt->bank == (*p)->bank);
1248
1249 bool same_row = dram_pkt->row == (*p)->row;
1250 got_more_hits |= same_rank_bank && same_row;
1251 got_bank_conflict |= same_rank_bank && !same_row;
1252 }
1253 ++p;
1254 }
1255
1256 if (got_more_hits)
1257 break;
1217 }
1218
1219 // auto pre-charge when either
1220 // 1) open_adaptive policy, we have not got any more hits, and
1221 // have a bank conflict
1222 // 2) close_adaptive policy and we have not got any more hits
1223 auto_precharge = !got_more_hits &&
1224 (got_bank_conflict || pageMgmt == Enums::close_adaptive);
1225 }
1226
1227 // DRAMPower trace command to be written
1258 }
1259
1260 // auto pre-charge when either
1261 // 1) open_adaptive policy, we have not got any more hits, and
1262 // have a bank conflict
1263 // 2) close_adaptive policy and we have not got any more hits
1264 auto_precharge = !got_more_hits &&
1265 (got_bank_conflict || pageMgmt == Enums::close_adaptive);
1266 }
1267
1268 // DRAMPower trace command to be written
1228 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR";
1269 std::string mem_cmd = dram_pkt->isRead() ? "RD" : "WR";
1229
1230 // MemCommand required for DRAMPower library
1231 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD :
1232 MemCommand::WR;
1233
1234 // Update bus state to reflect when previous command was issued
1235 nextBurstAt = cmd_at + tBURST;
1236

--- 18 unchanged lines hidden (view full) ---

1255
1256 // Update the minimum timing between the requests, this is a
1257 // conservative estimate of when we have to schedule the next
1258 // request to not introduce any unecessary bubbles. In most cases
1259 // we will wake up sooner than we have to.
1260 nextReqTime = nextBurstAt - (tRP + tRCD);
1261
1262 // Update the stats and schedule the next request
1270
1271 // MemCommand required for DRAMPower library
1272 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD :
1273 MemCommand::WR;
1274
1275 // Update bus state to reflect when previous command was issued
1276 nextBurstAt = cmd_at + tBURST;
1277

--- 18 unchanged lines hidden (view full) ---

1296
1297 // Update the minimum timing between the requests, this is a
1298 // conservative estimate of when we have to schedule the next
1299 // request to not introduce any unecessary bubbles. In most cases
1300 // we will wake up sooner than we have to.
1301 nextReqTime = nextBurstAt - (tRP + tRCD);
1302
1303 // Update the stats and schedule the next request
1263 if (dram_pkt->isRead) {
1304 if (dram_pkt->isRead()) {
1264 ++readsThisTime;
1265 if (row_hit)
1266 readRowHits++;
1267 bytesReadDRAM += burstSize;
1268 perBankRdBursts[dram_pkt->bankId]++;
1269
1270 // Update latency stats
1271 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime;
1305 ++readsThisTime;
1306 if (row_hit)
1307 readRowHits++;
1308 bytesReadDRAM += burstSize;
1309 perBankRdBursts[dram_pkt->bankId]++;
1310
1311 // Update latency stats
1312 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime;
1313 masterReadTotalLat[dram_pkt->masterId()] +=
1314 dram_pkt->readyTime - dram_pkt->entryTime;
1315
1272 totBusLat += tBURST;
1273 totQLat += cmd_at - dram_pkt->entryTime;
1316 totBusLat += tBURST;
1317 totQLat += cmd_at - dram_pkt->entryTime;
1318 masterReadBytes[dram_pkt->masterId()] += dram_pkt->size;
1274 } else {
1275 ++writesThisTime;
1276 if (row_hit)
1277 writeRowHits++;
1278 bytesWritten += burstSize;
1279 perBankWrBursts[dram_pkt->bankId]++;
1319 } else {
1320 ++writesThisTime;
1321 if (row_hit)
1322 writeRowHits++;
1323 bytesWritten += burstSize;
1324 perBankWrBursts[dram_pkt->bankId]++;
1325 masterWriteBytes[dram_pkt->masterId()] += dram_pkt->size;
1326 masterWriteTotalLat[dram_pkt->masterId()] +=
1327 dram_pkt->readyTime - dram_pkt->entryTime;
1280 }
1281}
1282
1283void
1284DRAMCtrl::processNextReqEvent()
1285{
1328 }
1329}
1330
1331void
1332DRAMCtrl::processNextReqEvent()
1333{
1334 // transition is handled by QoS algorithm if enabled
1335 if (turnPolicy) {
1336 // select bus state - only done if QoS algorithms are in use
1337 busStateNext = selectNextBusState();
1338 }
1339
1340 // detect bus state change
1341 bool switched_cmd_type = (busState != busStateNext);
1342 // record stats
1343 recordTurnaroundStats();
1344
1345 DPRINTF(DRAM, "QoS Turnarounds selected state %s %s\n",
1346 (busState==MemCtrl::READ)?"READ":"WRITE",
1347 switched_cmd_type?"[turnaround triggered]":"");
1348
1349 if (switched_cmd_type) {
1350 if (busState == READ) {
1351 DPRINTF(DRAM,
1352 "Switching to writes after %d reads with %d reads "
1353 "waiting\n", readsThisTime, totalReadQueueSize);
1354 rdPerTurnAround.sample(readsThisTime);
1355 readsThisTime = 0;
1356 } else {
1357 DPRINTF(DRAM,
1358 "Switching to reads after %d writes with %d writes "
1359 "waiting\n", writesThisTime, totalWriteQueueSize);
1360 wrPerTurnAround.sample(writesThisTime);
1361 writesThisTime = 0;
1362 }
1363 }
1364
1365 // updates current state
1366 busState = busStateNext;
1367
1368 // check ranks for refresh/wakeup - uses busStateNext, so done after turnaround
1369 // decisions
1286 int busyRanks = 0;
1287 for (auto r : ranks) {
1288 if (!r->inRefIdleState()) {
1289 if (r->pwrState != PWR_SREF) {
1290 // rank is busy refreshing
1291 DPRINTF(DRAMState, "Rank %d is not available\n", r->rank);
1292 busyRanks++;
1293

--- 24 unchanged lines hidden (view full) ---

1318
1319 if (busyRanks == ranksPerChannel) {
1320 // if all ranks are refreshing wait for them to finish
1321 // and stall this state machine without taking any further
1322 // action, and do not schedule a new nextReqEvent
1323 return;
1324 }
1325
1370 int busyRanks = 0;
1371 for (auto r : ranks) {
1372 if (!r->inRefIdleState()) {
1373 if (r->pwrState != PWR_SREF) {
1374 // rank is busy refreshing
1375 DPRINTF(DRAMState, "Rank %d is not available\n", r->rank);
1376 busyRanks++;
1377

--- 24 unchanged lines hidden (view full) ---

1402
1403 if (busyRanks == ranksPerChannel) {
1404 // if all ranks are refreshing wait for them to finish
1405 // and stall this state machine without taking any further
1406 // action, and do not schedule a new nextReqEvent
1407 return;
1408 }
1409
1326 // pre-emptively set to false. Overwrite if in transitioning to
1327 // a new state
1328 bool switched_cmd_type = false;
1329 if (busState != busStateNext) {
1330 if (busState == READ) {
1331 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads "
1332 "waiting\n", readsThisTime, readQueue.size());
1333
1334 // sample and reset the read-related stats as we are now
1335 // transitioning to writes, and all reads are done
1336 rdPerTurnAround.sample(readsThisTime);
1337 readsThisTime = 0;
1338
1339 // now proceed to do the actual writes
1340 switched_cmd_type = true;
1341 } else {
1342 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes "
1343 "waiting\n", writesThisTime, writeQueue.size());
1344
1345 wrPerTurnAround.sample(writesThisTime);
1346 writesThisTime = 0;
1347
1348 switched_cmd_type = true;
1349 }
1350 // update busState to match next state until next transition
1351 busState = busStateNext;
1352 }
1353
1354 // when we get here it is either a read or a write
1355 if (busState == READ) {
1356
1357 // track if we should switch or not
1358 bool switch_to_writes = false;
1359
1410 // when we get here it is either a read or a write
1411 if (busState == READ) {
1412
1413 // track if we should switch or not
1414 bool switch_to_writes = false;
1415
1360 if (readQueue.empty()) {
1416 if (totalReadQueueSize == 0) {
1361 // In the case there is no read request to go next,
1362 // trigger writes if we have passed the low threshold (or
1363 // if we are draining)
1417 // In the case there is no read request to go next,
1418 // trigger writes if we have passed the low threshold (or
1419 // if we are draining)
1364 if (!writeQueue.empty() &&
1420 if (!(totalWriteQueueSize == 0) &&
1365 (drainState() == DrainState::Draining ||
1421 (drainState() == DrainState::Draining ||
1366 writeQueue.size() > writeLowThreshold)) {
1422 totalWriteQueueSize > writeLowThreshold)) {
1367
1423
1424 DPRINTF(DRAM, "Switching to writes due to read queue empty\n");
1368 switch_to_writes = true;
1369 } else {
1370 // check if we are drained
1371 // not done draining until in PWR_IDLE state
1372 // ensuring all banks are closed and
1373 // have exited low power states
1374 if (drainState() == DrainState::Draining &&
1375 respQueue.empty() && allRanksDrained()) {
1376
1377 DPRINTF(Drain, "DRAM controller done draining\n");
1378 signalDrainDone();
1379 }
1380
1381 // nothing to do, not even any point in scheduling an
1382 // event for the next request
1383 return;
1384 }
1385 } else {
1425 switch_to_writes = true;
1426 } else {
1427 // check if we are drained
1428 // not done draining until in PWR_IDLE state
1429 // ensuring all banks are closed and
1430 // have exited low power states
1431 if (drainState() == DrainState::Draining &&
1432 respQueue.empty() && allRanksDrained()) {
1433
1434 DPRINTF(Drain, "DRAM controller done draining\n");
1435 signalDrainDone();
1436 }
1437
1438 // nothing to do, not even any point in scheduling an
1439 // event for the next request
1440 return;
1441 }
1442 } else {
1386 // bool to check if there is a read to a free rank
1387 bool found_read = false;
1388
1443
1389 // Figure out which read request goes next, and move it to the
1390 // front of the read queue
1391 // If we are changing command type, incorporate the minimum
1392 // bus turnaround delay which will be tCS (different rank) case
1393 found_read = chooseNext(readQueue, switched_cmd_type ? tCS : 0);
1444 bool read_found = false;
1445 DRAMPacketQueue::iterator to_read;
1446 uint8_t prio = numPriorities();
1394
1447
1448 for (auto queue = readQueue.rbegin();
1449 queue != readQueue.rend(); ++queue) {
1450
1451 prio--;
1452
1453 DPRINTF(QOS,
1454 "DRAM controller checking READ queue [%d] priority [%d elements]\n",
1455 prio, queue->size());
1456
1457 // Figure out which read request goes next
1458 // If we are changing command type, incorporate the minimum
1459 // bus turnaround delay which will be tCS (different rank) case
1460 to_read = chooseNext((*queue), switched_cmd_type ? tCS : 0);
1461
1462 if (to_read != queue->end()) {
1463 // candidate read found
1464 read_found = true;
1465 break;
1466 }
1467 }
1468
1395 // if no read to an available rank is found then return
1396 // at this point. There could be writes to the available ranks
1397 // which are above the required threshold. However, to
1398 // avoid adding more complexity to the code, return and wait
1399 // for a refresh event to kick things into action again.
1469 // if no read to an available rank is found then return
1470 // at this point. There could be writes to the available ranks
1471 // which are above the required threshold. However, to
1472 // avoid adding more complexity to the code, return and wait
1473 // for a refresh event to kick things into action again.
1400 if (!found_read)
1474 if (!read_found) {
1475 DPRINTF(DRAM, "No Reads Found - exiting\n");
1401 return;
1476 return;
1477 }
1402
1478
1403 DRAMPacket* dram_pkt = readQueue.front();
1479 auto dram_pkt = *to_read;
1480
1404 assert(dram_pkt->rankRef.inRefIdleState());
1405
1406 doDRAMAccess(dram_pkt);
1407
1481 assert(dram_pkt->rankRef.inRefIdleState());
1482
1483 doDRAMAccess(dram_pkt);
1484
1408 // At this point we're done dealing with the request
1409 readQueue.pop_front();
1410
1411 // Every respQueue which will generate an event, increment count
1412 ++dram_pkt->rankRef.outstandingEvents;
1485 // Every respQueue which will generate an event, increment count
1486 ++dram_pkt->rankRef.outstandingEvents;
1413
1414 // sanity check
1415 assert(dram_pkt->size <= burstSize);
1416 assert(dram_pkt->readyTime >= curTick());
1417
1487 // sanity check
1488 assert(dram_pkt->size <= burstSize);
1489 assert(dram_pkt->readyTime >= curTick());
1490
1491 // log the response
1492 logResponse(MemCtrl::READ, (*to_read)->masterId(),
1493 dram_pkt->qosValue(), dram_pkt->getAddr(), 1,
1494 dram_pkt->readyTime - dram_pkt->entryTime);
1495
1496
1418 // Insert into response queue. It will be sent back to the
1497 // Insert into response queue. It will be sent back to the
1419 // requestor at its readyTime
1498 // requester at its readyTime
1420 if (respQueue.empty()) {
1421 assert(!respondEvent.scheduled());
1422 schedule(respondEvent, dram_pkt->readyTime);
1423 } else {
1424 assert(respQueue.back()->readyTime <= dram_pkt->readyTime);
1425 assert(respondEvent.scheduled());
1426 }
1427
1428 respQueue.push_back(dram_pkt);
1429
1430 // we have so many writes that we have to transition
1499 if (respQueue.empty()) {
1500 assert(!respondEvent.scheduled());
1501 schedule(respondEvent, dram_pkt->readyTime);
1502 } else {
1503 assert(respQueue.back()->readyTime <= dram_pkt->readyTime);
1504 assert(respondEvent.scheduled());
1505 }
1506
1507 respQueue.push_back(dram_pkt);
1508
1509 // we have so many writes that we have to transition
1431 if (writeQueue.size() > writeHighThreshold) {
1510 if (totalWriteQueueSize > writeHighThreshold) {
1432 switch_to_writes = true;
1433 }
1511 switch_to_writes = true;
1512 }
1513
1514 // remove the request from the queue - the iterator is no longer valid .
1515 readQueue[dram_pkt->qosValue()].erase(to_read);
1434 }
1435
1436 // switching to writes, either because the read queue is empty
1437 // and the writes have passed the low threshold (or we are
1438 // draining), or because the writes hit the hight threshold
1439 if (switch_to_writes) {
1440 // transition to writing
1441 busStateNext = WRITE;
1442 }
1443 } else {
1516 }
1517
1518 // switching to writes, either because the read queue is empty
1519 // and the writes have passed the low threshold (or we are
1520 // draining), or because the writes hit the hight threshold
1521 if (switch_to_writes) {
1522 // transition to writing
1523 busStateNext = WRITE;
1524 }
1525 } else {
1444 // bool to check if write to free rank is found
1445 bool found_write = false;
1446
1526
1447 // If we are changing command type, incorporate the minimum
1448 // bus turnaround delay
1449 found_write = chooseNext(writeQueue,
1450 switched_cmd_type ? std::min(tRTW, tCS) : 0);
1527 bool write_found = false;
1528 DRAMPacketQueue::iterator to_write;
1529 uint8_t prio = numPriorities();
1451
1530
1531 for (auto queue = writeQueue.rbegin();
1532 queue != writeQueue.rend(); ++queue) {
1533
1534 prio--;
1535
1536 DPRINTF(QOS,
1537 "DRAM controller checking WRITE queue [%d] priority [%d elements]\n",
1538 prio, queue->size());
1539
1540 // If we are changing command type, incorporate the minimum
1541 // bus turnaround delay
1542 to_write = chooseNext((*queue),
1543 switched_cmd_type ? std::min(tRTW, tCS) : 0);
1544
1545 if (to_write != queue->end()) {
1546 write_found = true;
1547 break;
1548 }
1549 }
1550
1452 // if there are no writes to a rank that is available to service
1453 // requests (i.e. rank is in refresh idle state) are found then
1454 // return. There could be reads to the available ranks. However, to
1455 // avoid adding more complexity to the code, return at this point and
1456 // wait for a refresh event to kick things into action again.
1551 // if there are no writes to a rank that is available to service
1552 // requests (i.e. rank is in refresh idle state) are found then
1553 // return. There could be reads to the available ranks. However, to
1554 // avoid adding more complexity to the code, return at this point and
1555 // wait for a refresh event to kick things into action again.
1457 if (!found_write)
1556 if (!write_found) {
1557 DPRINTF(DRAM, "No Writes Found - exiting\n");
1458 return;
1558 return;
1559 }
1459
1560
1460 DRAMPacket* dram_pkt = writeQueue.front();
1561 auto dram_pkt = *to_write;
1562
1461 assert(dram_pkt->rankRef.inRefIdleState());
1462 // sanity check
1463 assert(dram_pkt->size <= burstSize);
1464
1465 doDRAMAccess(dram_pkt);
1466
1563 assert(dram_pkt->rankRef.inRefIdleState());
1564 // sanity check
1565 assert(dram_pkt->size <= burstSize);
1566
1567 doDRAMAccess(dram_pkt);
1568
1467 writeQueue.pop_front();
1468
1469 // removed write from queue, decrement count
1470 --dram_pkt->rankRef.writeEntries;
1471
1472 // Schedule write done event to decrement event count
1473 // after the readyTime has been reached
1474 // Only schedule latest write event to minimize events
1475 // required; only need to ensure that final event scheduled covers
1476 // the time that writes are outstanding and bus is active
1477 // to holdoff power-down entry events
1478 if (!dram_pkt->rankRef.writeDoneEvent.scheduled()) {
1479 schedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1480 // New event, increment count
1481 ++dram_pkt->rankRef.outstandingEvents;
1482
1483 } else if (dram_pkt->rankRef.writeDoneEvent.when() <
1569 // removed write from queue, decrement count
1570 --dram_pkt->rankRef.writeEntries;
1571
1572 // Schedule write done event to decrement event count
1573 // after the readyTime has been reached
1574 // Only schedule latest write event to minimize events
1575 // required; only need to ensure that final event scheduled covers
1576 // the time that writes are outstanding and bus is active
1577 // to holdoff power-down entry events
1578 if (!dram_pkt->rankRef.writeDoneEvent.scheduled()) {
1579 schedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1580 // New event, increment count
1581 ++dram_pkt->rankRef.outstandingEvents;
1582
1583 } else if (dram_pkt->rankRef.writeDoneEvent.when() <
1484 dram_pkt-> readyTime) {
1584 dram_pkt->readyTime) {
1585
1485 reschedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1486 }
1487
1488 isInWriteQueue.erase(burstAlign(dram_pkt->addr));
1586 reschedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime);
1587 }
1588
1589 isInWriteQueue.erase(burstAlign(dram_pkt->addr));
1590
1591 // log the response
1592 logResponse(MemCtrl::WRITE, dram_pkt->masterId(),
1593 dram_pkt->qosValue(), dram_pkt->getAddr(), 1,
1594 dram_pkt->readyTime - dram_pkt->entryTime);
1595
1596
1597 // remove the request from the queue - the iterator is no longer valid
1598 writeQueue[dram_pkt->qosValue()].erase(to_write);
1599
1489 delete dram_pkt;
1490
1491 // If we emptied the write queue, or got sufficiently below the
1492 // threshold (using the minWritesPerSwitch as the hysteresis) and
1493 // are not draining, or we have reads waiting and have done enough
1494 // writes, then switch to reads.
1600 delete dram_pkt;
1601
1602 // If we emptied the write queue, or got sufficiently below the
1603 // threshold (using the minWritesPerSwitch as the hysteresis) and
1604 // are not draining, or we have reads waiting and have done enough
1605 // writes, then switch to reads.
1495 if (writeQueue.empty() ||
1496 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold &&
1497 drainState() != DrainState::Draining) ||
1498 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) {
1606 bool below_threshold =
1607 totalWriteQueueSize + minWritesPerSwitch < writeLowThreshold;
1608
1609 if (totalWriteQueueSize == 0 ||
1610 (below_threshold && drainState() != DrainState::Draining) ||
1611 (totalReadQueueSize && writesThisTime >= minWritesPerSwitch)) {
1612
1499 // turn the bus back around for reads again
1500 busStateNext = READ;
1501
1502 // note that the we switch back to reads also in the idle
1503 // case, which eventually will check for any draining and
1504 // also pause any further scheduling if there is really
1505 // nothing to do
1506 }
1507 }
1508 // It is possible that a refresh to another rank kicks things back into
1509 // action before reaching this point.
1510 if (!nextReqEvent.scheduled())
1511 schedule(nextReqEvent, std::max(nextReqTime, curTick()));
1512
1513 // If there is space available and we have writes waiting then let
1514 // them retry. This is done here to ensure that the retry does not
1515 // cause a nextReqEvent to be scheduled before we do so as part of
1516 // the next request processing
1613 // turn the bus back around for reads again
1614 busStateNext = READ;
1615
1616 // note that the we switch back to reads also in the idle
1617 // case, which eventually will check for any draining and
1618 // also pause any further scheduling if there is really
1619 // nothing to do
1620 }
1621 }
1622 // It is possible that a refresh to another rank kicks things back into
1623 // action before reaching this point.
1624 if (!nextReqEvent.scheduled())
1625 schedule(nextReqEvent, std::max(nextReqTime, curTick()));
1626
1627 // If there is space available and we have writes waiting then let
1628 // them retry. This is done here to ensure that the retry does not
1629 // cause a nextReqEvent to be scheduled before we do so as part of
1630 // the next request processing
1517 if (retryWrReq && writeQueue.size() < writeBufferSize) {
1631 if (retryWrReq && totalWriteQueueSize < writeBufferSize) {
1518 retryWrReq = false;
1519 port.sendRetryReq();
1520 }
1521}
1522
1523pair<vector<uint32_t>, bool>
1632 retryWrReq = false;
1633 port.sendRetryReq();
1634 }
1635}
1636
1637pair<vector<uint32_t>, bool>
1524DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue,
1638DRAMCtrl::minBankPrep(const DRAMPacketQueue& queue,
1525 Tick min_col_at) const
1526{
1527 Tick min_act_at = MaxTick;
1528 vector<uint32_t> bank_mask(ranksPerChannel, 0);
1529
1530 // latest Tick for which ACT can occur without incurring additoinal
1531 // delay on the data bus
1532 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick());

--- 848 unchanged lines hidden (view full) ---

2381 Stats::registerDumpCallback(new RankDumpCallback(this));
2382 Stats::registerResetCallback(new RankResetCallback(this));
2383}
2384void
2385DRAMCtrl::regStats()
2386{
2387 using namespace Stats;
2388
1639 Tick min_col_at) const
1640{
1641 Tick min_act_at = MaxTick;
1642 vector<uint32_t> bank_mask(ranksPerChannel, 0);
1643
1644 // latest Tick for which ACT can occur without incurring additoinal
1645 // delay on the data bus
1646 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick());

--- 848 unchanged lines hidden (view full) ---

2495 Stats::registerDumpCallback(new RankDumpCallback(this));
2496 Stats::registerResetCallback(new RankResetCallback(this));
2497}
2498void
2499DRAMCtrl::regStats()
2500{
2501 using namespace Stats;
2502
2389 AbstractMemory::regStats();
2503 MemCtrl::regStats();
2390
2391 for (auto r : ranks) {
2392 r->regStats();
2393 }
2394
2395 registerResetCallback(new MemResetCallback(this));
2396
2397 readReqs

--- 126 unchanged lines hidden (view full) ---

2524 .desc("What read queue length does an incoming req see");
2525
2526 wrQLenPdf
2527 .init(writeBufferSize)
2528 .name(name() + ".wrQLenPdf")
2529 .desc("What write queue length does an incoming req see");
2530
2531 bytesPerActivate
2504
2505 for (auto r : ranks) {
2506 r->regStats();
2507 }
2508
2509 registerResetCallback(new MemResetCallback(this));
2510
2511 readReqs

--- 126 unchanged lines hidden (view full) ---

2638 .desc("What read queue length does an incoming req see");
2639
2640 wrQLenPdf
2641 .init(writeBufferSize)
2642 .name(name() + ".wrQLenPdf")
2643 .desc("What write queue length does an incoming req see");
2644
2645 bytesPerActivate
2532 .init(maxAccessesPerRow)
2646 .init(maxAccessesPerRow ? maxAccessesPerRow : rowBufferSize)
2533 .name(name() + ".bytesPerActivate")
2534 .desc("Bytes accessed per row activation")
2535 .flags(nozero);
2536
2537 rdPerTurnAround
2538 .init(readBufferSize)
2539 .name(name() + ".rdPerTurnAround")
2540 .desc("Reads before turning the bus around for writes")

--- 94 unchanged lines hidden (view full) ---

2635
2636 pageHitRate
2637 .name(name() + ".pageHitRate")
2638 .desc("Row buffer hit rate, read and write combined")
2639 .precision(2);
2640
2641 pageHitRate = (writeRowHits + readRowHits) /
2642 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100;
2647 .name(name() + ".bytesPerActivate")
2648 .desc("Bytes accessed per row activation")
2649 .flags(nozero);
2650
2651 rdPerTurnAround
2652 .init(readBufferSize)
2653 .name(name() + ".rdPerTurnAround")
2654 .desc("Reads before turning the bus around for writes")

--- 94 unchanged lines hidden (view full) ---

2749
2750 pageHitRate
2751 .name(name() + ".pageHitRate")
2752 .desc("Row buffer hit rate, read and write combined")
2753 .precision(2);
2754
2755 pageHitRate = (writeRowHits + readRowHits) /
2756 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100;
2757
2758 // per-master bytes read and written to memory
2759 masterReadBytes
2760 .init(_system->maxMasters())
2761 .name(name() + ".masterReadBytes")
2762 .desc("Per-master bytes read from memory")
2763 .flags(nozero | nonan);
2764
2765 masterWriteBytes
2766 .init(_system->maxMasters())
2767 .name(name() + ".masterWriteBytes")
2768 .desc("Per-master bytes write to memory")
2769 .flags(nozero | nonan);
2770
2771 // per-master bytes read and written to memory rate
2772 masterReadRate.name(name() + ".masterReadRate")
2773 .desc("Per-master bytes read from memory rate (Bytes/sec)")
2774 .flags(nozero | nonan)
2775 .precision(12);
2776
2777 masterReadRate = masterReadBytes/simSeconds;
2778
2779 masterWriteRate
2780 .name(name() + ".masterWriteRate")
2781 .desc("Per-master bytes write to memory rate (Bytes/sec)")
2782 .flags(nozero | nonan)
2783 .precision(12);
2784
2785 masterWriteRate = masterWriteBytes/simSeconds;
2786
2787 masterReadAccesses
2788 .init(_system->maxMasters())
2789 .name(name() + ".masterReadAccesses")
2790 .desc("Per-master read serviced memory accesses")
2791 .flags(nozero);
2792
2793 masterWriteAccesses
2794 .init(_system->maxMasters())
2795 .name(name() + ".masterWriteAccesses")
2796 .desc("Per-master write serviced memory accesses")
2797 .flags(nozero);
2798
2799
2800 masterReadTotalLat
2801 .init(_system->maxMasters())
2802 .name(name() + ".masterReadTotalLat")
2803 .desc("Per-master read total memory access latency")
2804 .flags(nozero | nonan);
2805
2806 masterReadAvgLat.name(name() + ".masterReadAvgLat")
2807 .desc("Per-master read average memory access latency")
2808 .flags(nonan)
2809 .precision(2);
2810
2811 masterReadAvgLat = masterReadTotalLat/masterReadAccesses;
2812
2813 masterWriteTotalLat
2814 .init(_system->maxMasters())
2815 .name(name() + ".masterWriteTotalLat")
2816 .desc("Per-master write total memory access latency")
2817 .flags(nozero | nonan);
2818
2819 masterWriteAvgLat.name(name() + ".masterWriteAvgLat")
2820 .desc("Per-master write average memory access latency")
2821 .flags(nonan)
2822 .precision(2);
2823
2824 masterWriteAvgLat = masterWriteTotalLat/masterWriteAccesses;
2825
2826 for (int i = 0; i < _system->maxMasters(); i++) {
2827 const std::string master = _system->getMasterName(i);
2828 masterReadBytes.subname(i, master);
2829 masterReadRate.subname(i, master);
2830 masterWriteBytes.subname(i, master);
2831 masterWriteRate.subname(i, master);
2832 masterReadAccesses.subname(i, master);
2833 masterWriteAccesses.subname(i, master);
2834 masterReadTotalLat.subname(i, master);
2835 masterReadAvgLat.subname(i, master);
2836 masterWriteTotalLat.subname(i, master);
2837 masterWriteAvgLat.subname(i, master);
2838 }
2643}
2644
2645void
2646DRAMCtrl::recvFunctional(PacketPtr pkt)
2647{
2648 // rely on the abstract memory
2649 functionalAccess(pkt);
2650}

--- 8 unchanged lines hidden (view full) ---

2659 }
2660}
2661
2662DrainState
2663DRAMCtrl::drain()
2664{
2665 // if there is anything in any of our internal queues, keep track
2666 // of that as well
2839}
2840
2841void
2842DRAMCtrl::recvFunctional(PacketPtr pkt)
2843{
2844 // rely on the abstract memory
2845 functionalAccess(pkt);
2846}

--- 8 unchanged lines hidden (view full) ---

2855 }
2856}
2857
2858DrainState
2859DRAMCtrl::drain()
2860{
2861 // if there is anything in any of our internal queues, keep track
2862 // of that as well
2667 if (!(writeQueue.empty() && readQueue.empty() && respQueue.empty() &&
2863 if (!(!totalWriteQueueSize && !totalReadQueueSize && respQueue.empty() &&
2668 allRanksDrained())) {
2669
2670 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d,"
2864 allRanksDrained())) {
2865
2866 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d,"
2671 " resp: %d\n", writeQueue.size(), readQueue.size(),
2867 " resp: %d\n", totalWriteQueueSize, totalReadQueueSize,
2672 respQueue.size());
2673
2674 // the only queue that is not drained automatically over time
2675 // is the write queue, thus kick things into action if needed
2868 respQueue.size());
2869
2870 // the only queue that is not drained automatically over time
2871 // is the write queue, thus kick things into action if needed
2676 if (!writeQueue.empty() && !nextReqEvent.scheduled()) {
2872 if (!totalWriteQueueSize && !nextReqEvent.scheduled()) {
2677 schedule(nextReqEvent, curTick());
2678 }
2679
2680 // also need to kick off events to exit self-refresh
2681 for (auto r : ranks) {
2682 // force self-refresh exit, which in turn will issue auto-refresh
2683 if (r->pwrState == PWR_SREF) {
2684 DPRINTF(DRAM,"Rank%d: Forcing self-refresh wakeup in drain\n",

--- 91 unchanged lines hidden --- 		2873 schedule(nextReqEvent, curTick());
2874 }
2875
2876 // also need to kick off events to exit self-refresh
2877 for (auto r : ranks) {
2878 // force self-refresh exit, which in turn will issue auto-refresh
2879 if (r->pwrState == PWR_SREF) {
2880 DPRINTF(DRAM,"Rank%d: Forcing self-refresh wakeup in drain\n",

--- 91 unchanged lines hidden ---