dram_ctrl.cc revision 11284:b3926db25371
114153Sgiacomo.travaglini@arm.com/* 27090SN/A * Copyright (c) 2010-2015 ARM Limited 37090SN/A * All rights reserved 47090SN/A * 57090SN/A * The license below extends only to copyright in the software and shall 67090SN/A * not be construed as granting a license to any other intellectual 77090SN/A * property including but not limited to intellectual property relating 87090SN/A * to a hardware implementation of the functionality of the software 97090SN/A * licensed hereunder. You may use the software subject to the license 107090SN/A * terms below provided that you ensure that this notice is replicated 117090SN/A * unmodified and in its entirety in all distributions of the software, 127090SN/A * modified or unmodified, in source code or in binary form. 134486SN/A * 144486SN/A * Copyright (c) 2013 Amin Farmahini-Farahani 154486SN/A * All rights reserved. 164486SN/A * 174486SN/A * Redistribution and use in source and binary forms, with or without 184486SN/A * modification, are permitted provided that the following conditions are 194486SN/A * met: redistributions of source code must retain the above copyright 204486SN/A * notice, this list of conditions and the following disclaimer; 214486SN/A * redistributions in binary form must reproduce the above copyright 224486SN/A * notice, this list of conditions and the following disclaimer in the 234486SN/A * documentation and/or other materials provided with the distribution; 244486SN/A * neither the name of the copyright holders nor the names of its 254486SN/A * contributors may be used to endorse or promote products derived from 264486SN/A * this software without specific prior written permission. 274486SN/A * 284486SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 294486SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 304486SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 314486SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 324486SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 334486SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 344486SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 354486SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 364486SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 374486SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 384486SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 397584SAli.Saidi@arm.com * 407584SAli.Saidi@arm.com * Authors: Andreas Hansson 417754SWilliam.Wang@arm.com * Ani Udipi 4212472Sglenn.bergmans@arm.com * Neha Agarwal 434486SN/A * Omar Naji 4412472Sglenn.bergmans@arm.com */ 453630SN/A 463630SN/A#include "base/bitfield.hh" 4712472Sglenn.bergmans@arm.com#include "base/trace.hh" 4813665Sandreas.sandberg@arm.com#include "debug/DRAM.hh" 4913665Sandreas.sandberg@arm.com#include "debug/DRAMPower.hh" 5013665Sandreas.sandberg@arm.com#include "debug/DRAMState.hh" 5113665Sandreas.sandberg@arm.com#include "debug/Drain.hh" 5213665Sandreas.sandberg@arm.com#include "mem/dram_ctrl.hh" 5313665Sandreas.sandberg@arm.com#include "sim/system.hh" 5413665Sandreas.sandberg@arm.com 5513665Sandreas.sandberg@arm.comusing namespace std; 5613665Sandreas.sandberg@arm.comusing namespace Data; 5713665Sandreas.sandberg@arm.com 5813665Sandreas.sandberg@arm.comDRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 5913665Sandreas.sandberg@arm.com AbstractMemory(p), 6013665Sandreas.sandberg@arm.com port(name() + ".port", *this), isTimingMode(false), 6113665Sandreas.sandberg@arm.com retryRdReq(false), retryWrReq(false), 6213665Sandreas.sandberg@arm.com busState(READ), 6313665Sandreas.sandberg@arm.com nextReqEvent(this), respondEvent(this), 6413665Sandreas.sandberg@arm.com deviceSize(p->device_size), 6513665Sandreas.sandberg@arm.com deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 6613665Sandreas.sandberg@arm.com deviceRowBufferSize(p->device_rowbuffer_size), 6713665Sandreas.sandberg@arm.com devicesPerRank(p->devices_per_rank), 6814283Sgiacomo.travaglini@arm.com burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 693630SN/A rowBufferSize(devicesPerRank * deviceRowBufferSize), 7011841Sandreas.sandberg@arm.com columnsPerRowBuffer(rowBufferSize / burstSize), 7111841Sandreas.sandberg@arm.com columnsPerStripe(range.interleaved() ? range.granularity() / burstSize : 1), 7211841Sandreas.sandberg@arm.com ranksPerChannel(p->ranks_per_channel), 7311841Sandreas.sandberg@arm.com bankGroupsPerRank(p->bank_groups_per_rank), 7413665Sandreas.sandberg@arm.com bankGroupArch(p->bank_groups_per_rank > 0), 7511841Sandreas.sandberg@arm.com banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 7611841Sandreas.sandberg@arm.com readBufferSize(p->read_buffer_size), 7711841Sandreas.sandberg@arm.com writeBufferSize(p->write_buffer_size), 7811841Sandreas.sandberg@arm.com writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 7913505Sgiacomo.travaglini@arm.com writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 8011841Sandreas.sandberg@arm.com minWritesPerSwitch(p->min_writes_per_switch), 8111841Sandreas.sandberg@arm.com writesThisTime(0), readsThisTime(0), 829806Sstever@gmail.com tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST), 839806Sstever@gmail.com tCCD_L(p->tCCD_L), tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), 847584SAli.Saidi@arm.com tWR(p->tWR), tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 859338SAndreas.Sandberg@arm.com tRRD_L(p->tRRD_L), tXAW(p->tXAW), activationLimit(p->activation_limit), 867584SAli.Saidi@arm.com memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 873898SN/A pageMgmt(p->page_policy), 889806Sstever@gmail.com maxAccessesPerRow(p->max_accesses_per_row), 897950SAli.Saidi@ARM.com frontendLatency(p->static_frontend_latency), 907950SAli.Saidi@ARM.com backendLatency(p->static_backend_latency), 919338SAndreas.Sandberg@arm.com busBusyUntil(0), prevArrival(0), 929525SAndreas.Sandberg@ARM.com nextReqTime(0), activeRank(0), timeStampOffset(0) 937950SAli.Saidi@ARM.com{ 947950SAli.Saidi@ARM.com // sanity check the ranks since we rely on bit slicing for the 957950SAli.Saidi@ARM.com // address decoding 967950SAli.Saidi@ARM.com fatal_if(!isPowerOf2(ranksPerChannel), "DRAM rank count of %d is not " 977587SAli.Saidi@arm.com "allowed, must be a power of two\n", ranksPerChannel); 987587SAli.Saidi@arm.com 997587SAli.Saidi@arm.com fatal_if(!isPowerOf2(burstSize), "DRAM burst size %d is not allowed, " 1009338SAndreas.Sandberg@arm.com "must be a power of two\n", burstSize); 1017753SWilliam.Wang@arm.com 1027753SWilliam.Wang@arm.com for (int i = 0; i < ranksPerChannel; i++) { 1039525SAndreas.Sandberg@ARM.com Rank* rank = new Rank(*this, p); 1047753SWilliam.Wang@arm.com ranks.push_back(rank); 1057587SAli.Saidi@arm.com 1067587SAli.Saidi@arm.com rank->actTicks.resize(activationLimit, 0); 1078282SAli.Saidi@ARM.com rank->banks.resize(banksPerRank); 1088282SAli.Saidi@ARM.com rank->rank = i; 1099338SAndreas.Sandberg@arm.com 1108282SAli.Saidi@ARM.com for (int b = 0; b < banksPerRank; b++) { 11111296Sandreas.sandberg@arm.com rank->banks[b].bank = b; 11211296Sandreas.sandberg@arm.com // GDDR addressing of banks to BG is linear. 11311296Sandreas.sandberg@arm.com // Here we assume that all DRAM generations address bank groups as 11411296Sandreas.sandberg@arm.com // follows: 11511296Sandreas.sandberg@arm.com if (bankGroupArch) { 11611296Sandreas.sandberg@arm.com // Simply assign lower bits to bank group in order to 11711296Sandreas.sandberg@arm.com // rotate across bank groups as banks are incremented 11811296Sandreas.sandberg@arm.com // e.g. with 4 banks per bank group and 16 banks total: 11911296Sandreas.sandberg@arm.com // banks 0,4,8,12 are in bank group 0 12011296Sandreas.sandberg@arm.com // banks 1,5,9,13 are in bank group 1 12111296Sandreas.sandberg@arm.com // banks 2,6,10,14 are in bank group 2 12211296Sandreas.sandberg@arm.com // banks 3,7,11,15 are in bank group 3 12311296Sandreas.sandberg@arm.com rank->banks[b].bankgr = b % bankGroupsPerRank; 12411296Sandreas.sandberg@arm.com } else { 12513805Sgiacomo.travaglini@arm.com // No bank groups; simply assign to bank number 12613805Sgiacomo.travaglini@arm.com rank->banks[b].bankgr = b; 12713805Sgiacomo.travaglini@arm.com } 12813805Sgiacomo.travaglini@arm.com } 12912474Sglenn.bergmans@arm.com } 13014153Sgiacomo.travaglini@arm.com 13114153Sgiacomo.travaglini@arm.com // perform a basic check of the write thresholds 13214153Sgiacomo.travaglini@arm.com if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 13312474Sglenn.bergmans@arm.com fatal("Write buffer low threshold %d must be smaller than the " 13412474Sglenn.bergmans@arm.com "high threshold %d\n", p->write_low_thresh_perc, 13512474Sglenn.bergmans@arm.com p->write_high_thresh_perc); 13612474Sglenn.bergmans@arm.com 13712474Sglenn.bergmans@arm.com // determine the rows per bank by looking at the total capacity 13812474Sglenn.bergmans@arm.com uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 13912474Sglenn.bergmans@arm.com 14012474Sglenn.bergmans@arm.com // determine the dram actual capacity from the DRAM config in Mbytes 14112474Sglenn.bergmans@arm.com uint64_t deviceCapacity = deviceSize / (1024 * 1024) * devicesPerRank * 14212474Sglenn.bergmans@arm.com ranksPerChannel; 14312474Sglenn.bergmans@arm.com 14412474Sglenn.bergmans@arm.com // if actual DRAM size does not match memory capacity in system warn! 14512474Sglenn.bergmans@arm.com if (deviceCapacity != capacity / (1024 * 1024)) 14612474Sglenn.bergmans@arm.com warn("DRAM device capacity (%d Mbytes) does not match the " 14712474Sglenn.bergmans@arm.com "address range assigned (%d Mbytes)\n", deviceCapacity, 14814153Sgiacomo.travaglini@arm.com capacity / (1024 * 1024)); 14912474Sglenn.bergmans@arm.com 15012474Sglenn.bergmans@arm.com DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 15112474Sglenn.bergmans@arm.com AbstractMemory::size()); 15212474Sglenn.bergmans@arm.com 15312474Sglenn.bergmans@arm.com DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 15412474Sglenn.bergmans@arm.com rowBufferSize, columnsPerRowBuffer); 15512474Sglenn.bergmans@arm.com 15612474Sglenn.bergmans@arm.com rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 15712474Sglenn.bergmans@arm.com 15812474Sglenn.bergmans@arm.com // some basic sanity checks 15912474Sglenn.bergmans@arm.com if (tREFI <= tRP || tREFI <= tRFC) { 16012474Sglenn.bergmans@arm.com fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 16112474Sglenn.bergmans@arm.com tREFI, tRP, tRFC); 16212474Sglenn.bergmans@arm.com } 16312474Sglenn.bergmans@arm.com 16412474Sglenn.bergmans@arm.com // basic bank group architecture checks -> 16512474Sglenn.bergmans@arm.com if (bankGroupArch) { 16612474Sglenn.bergmans@arm.com // must have at least one bank per bank group 16712474Sglenn.bergmans@arm.com if (bankGroupsPerRank > banksPerRank) { 16812474Sglenn.bergmans@arm.com fatal("banks per rank (%d) must be equal to or larger than " 16912474Sglenn.bergmans@arm.com "banks groups per rank (%d)\n", 17012474Sglenn.bergmans@arm.com banksPerRank, bankGroupsPerRank); 17112474Sglenn.bergmans@arm.com } 17214153Sgiacomo.travaglini@arm.com // must have same number of banks in each bank group 17314153Sgiacomo.travaglini@arm.com if ((banksPerRank % bankGroupsPerRank) != 0) { 17412474Sglenn.bergmans@arm.com fatal("Banks per rank (%d) must be evenly divisible by bank groups " 17512474Sglenn.bergmans@arm.com "per rank (%d) for equal banks per bank group\n", 17614153Sgiacomo.travaglini@arm.com banksPerRank, bankGroupsPerRank); 17714153Sgiacomo.travaglini@arm.com } 17814153Sgiacomo.travaglini@arm.com // tCCD_L should be greater than minimal, back-to-back burst delay 17914153Sgiacomo.travaglini@arm.com if (tCCD_L <= tBURST) { 18014153Sgiacomo.travaglini@arm.com fatal("tCCD_L (%d) should be larger than tBURST (%d) when " 18114153Sgiacomo.travaglini@arm.com "bank groups per rank (%d) is greater than 1\n", 18214153Sgiacomo.travaglini@arm.com tCCD_L, tBURST, bankGroupsPerRank); 18312474Sglenn.bergmans@arm.com } 18414153Sgiacomo.travaglini@arm.com // tRRD_L is greater than minimal, same bank group ACT-to-ACT delay 18514153Sgiacomo.travaglini@arm.com // some datasheets might specify it equal to tRRD 18614153Sgiacomo.travaglini@arm.com if (tRRD_L < tRRD) { 18712474Sglenn.bergmans@arm.com fatal("tRRD_L (%d) should be larger than tRRD (%d) when " 18814153Sgiacomo.travaglini@arm.com "bank groups per rank (%d) is greater than 1\n", 18914153Sgiacomo.travaglini@arm.com tRRD_L, tRRD, bankGroupsPerRank); 19012474Sglenn.bergmans@arm.com } 19112474Sglenn.bergmans@arm.com } 19212474Sglenn.bergmans@arm.com 19312474Sglenn.bergmans@arm.com} 19412474Sglenn.bergmans@arm.com 19512474Sglenn.bergmans@arm.comvoid 19612474Sglenn.bergmans@arm.comDRAMCtrl::init() 19712474Sglenn.bergmans@arm.com{ 19812474Sglenn.bergmans@arm.com AbstractMemory::init(); 19912474Sglenn.bergmans@arm.com 20012474Sglenn.bergmans@arm.com if (!port.isConnected()) { 20112474Sglenn.bergmans@arm.com fatal("DRAMCtrl %s is unconnected!\n", name()); 20212474Sglenn.bergmans@arm.com } else { 20313805Sgiacomo.travaglini@arm.com port.sendRangeChange(); 20413805Sgiacomo.travaglini@arm.com } 20512474Sglenn.bergmans@arm.com 20612474Sglenn.bergmans@arm.com // a bit of sanity checks on the interleaving, save it for here to 20712474Sglenn.bergmans@arm.com // ensure that the system pointer is initialised 2087584SAli.Saidi@arm.com if (range.interleaved()) { 2097584SAli.Saidi@arm.com if (channels != range.stripes()) 2109338SAndreas.Sandberg@arm.com fatal("%s has %d interleaved address stripes but %d channel(s)\n", 2118524SAli.Saidi@ARM.com name(), range.stripes(), channels); 2128524SAli.Saidi@ARM.com 2138299Schander.sudanthi@arm.com if (addrMapping == Enums::RoRaBaChCo) { 2147584SAli.Saidi@arm.com if (rowBufferSize != range.granularity()) { 21512472Sglenn.bergmans@arm.com fatal("Channel interleaving of %s doesn't match RoRaBaChCo " 21612472Sglenn.bergmans@arm.com "address map\n", name()); 21712472Sglenn.bergmans@arm.com } 21812472Sglenn.bergmans@arm.com } else if (addrMapping == Enums::RoRaBaCoCh || 21912472Sglenn.bergmans@arm.com addrMapping == Enums::RoCoRaBaCh) { 22012472Sglenn.bergmans@arm.com // for the interleavings with channel bits in the bottom, 22112472Sglenn.bergmans@arm.com // if the system uses a channel striping granularity that 22212472Sglenn.bergmans@arm.com // is larger than the DRAM burst size, then map the 22312472Sglenn.bergmans@arm.com // sequential accesses within a stripe to a number of 22412472Sglenn.bergmans@arm.com // columns in the DRAM, effectively placing some of the 22512472Sglenn.bergmans@arm.com // lower-order column bits as the least-significant bits 22612472Sglenn.bergmans@arm.com // of the address (above the ones denoting the burst size) 22711011SAndreas.Sandberg@ARM.com assert(columnsPerStripe >= 1); 22811011SAndreas.Sandberg@ARM.com 22911011SAndreas.Sandberg@ARM.com // channel striping has to be done at a granularity that 23011011SAndreas.Sandberg@ARM.com // is equal or larger to a cache line 23111011SAndreas.Sandberg@ARM.com if (system()->cacheLineSize() > range.granularity()) { 23211011SAndreas.Sandberg@ARM.com fatal("Channel interleaving of %s must be at least as large " 23311011SAndreas.Sandberg@ARM.com "as the cache line size\n", name()); 23411011SAndreas.Sandberg@ARM.com } 23511011SAndreas.Sandberg@ARM.com 23611011SAndreas.Sandberg@ARM.com // ...and equal or smaller than the row-buffer size 23711011SAndreas.Sandberg@ARM.com if (rowBufferSize < range.granularity()) { 23811011SAndreas.Sandberg@ARM.com fatal("Channel interleaving of %s must be at most as large " 23911011SAndreas.Sandberg@ARM.com "as the row-buffer size\n", name()); 24011011SAndreas.Sandberg@ARM.com } 24111011SAndreas.Sandberg@ARM.com // this is essentially the check above, so just to be sure 24211011SAndreas.Sandberg@ARM.com assert(columnsPerStripe <= columnsPerRowBuffer); 24311011SAndreas.Sandberg@ARM.com } 24411011SAndreas.Sandberg@ARM.com } 24511011SAndreas.Sandberg@ARM.com} 24611011SAndreas.Sandberg@ARM.com 24711011SAndreas.Sandberg@ARM.comvoid 24811011SAndreas.Sandberg@ARM.comDRAMCtrl::startup() 24912472Sglenn.bergmans@arm.com{ 25012472Sglenn.bergmans@arm.com // remember the memory system mode of operation 25112472Sglenn.bergmans@arm.com isTimingMode = system()->isTimingMode(); 25212472Sglenn.bergmans@arm.com 25312472Sglenn.bergmans@arm.com if (isTimingMode) { 25412472Sglenn.bergmans@arm.com // timestamp offset should be in clock cycles for DRAMPower 25512472Sglenn.bergmans@arm.com timeStampOffset = divCeil(curTick(), tCK); 25612472Sglenn.bergmans@arm.com 25712472Sglenn.bergmans@arm.com // update the start tick for the precharge accounting to the 25812472Sglenn.bergmans@arm.com // current tick 25912472Sglenn.bergmans@arm.com for (auto r : ranks) { 26012472Sglenn.bergmans@arm.com r->startup(curTick() + tREFI - tRP); 26112472Sglenn.bergmans@arm.com } 26212472Sglenn.bergmans@arm.com 26311421Sdavid.guillen@arm.com // shift the bus busy time sufficiently far ahead that we never 26411421Sdavid.guillen@arm.com // have to worry about negative values when computing the time for 26511421Sdavid.guillen@arm.com // the next request, this will add an insignificant bubble at the 26611421Sdavid.guillen@arm.com // start of simulation 26711421Sdavid.guillen@arm.com busBusyUntil = curTick() + tRP + tRCD + tCL; 26811421Sdavid.guillen@arm.com } 26911421Sdavid.guillen@arm.com} 27011421Sdavid.guillen@arm.com 27111421Sdavid.guillen@arm.comTick 27211421Sdavid.guillen@arm.comDRAMCtrl::recvAtomic(PacketPtr pkt) 27311421Sdavid.guillen@arm.com{ 27411421Sdavid.guillen@arm.com DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 27511421Sdavid.guillen@arm.com 27611421Sdavid.guillen@arm.com // do the actual memory access and turn the packet into a response 27711421Sdavid.guillen@arm.com access(pkt); 27811421Sdavid.guillen@arm.com 27911236Sandreas.sandberg@arm.com Tick latency = 0; 28011236Sandreas.sandberg@arm.com if (!pkt->cacheResponding() && pkt->hasData()) { 28111236Sandreas.sandberg@arm.com // this value is not supposed to be accurate, just enough to 28211236Sandreas.sandberg@arm.com // keep things going, mimic a closed page 28311236Sandreas.sandberg@arm.com latency = tRP + tRCD + tCL; 28411236Sandreas.sandberg@arm.com } 28511236Sandreas.sandberg@arm.com return latency; 28611236Sandreas.sandberg@arm.com} 28711236Sandreas.sandberg@arm.com 28811011SAndreas.Sandberg@ARM.combool 28911011SAndreas.Sandberg@ARM.comDRAMCtrl::readQueueFull(unsigned int neededEntries) const 29011421Sdavid.guillen@arm.com{ 29111421Sdavid.guillen@arm.com DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 29211421Sdavid.guillen@arm.com readBufferSize, readQueue.size() + respQueue.size(), 29311236Sandreas.sandberg@arm.com neededEntries); 29411236Sandreas.sandberg@arm.com 29511236Sandreas.sandberg@arm.com return 29611236Sandreas.sandberg@arm.com (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 29711236Sandreas.sandberg@arm.com} 29811421Sdavid.guillen@arm.com 29911421Sdavid.guillen@arm.combool 30011421Sdavid.guillen@arm.comDRAMCtrl::writeQueueFull(unsigned int neededEntries) const 30112472Sglenn.bergmans@arm.com{ 30212472Sglenn.bergmans@arm.com DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 30312472Sglenn.bergmans@arm.com writeBufferSize, writeQueue.size(), neededEntries); 30412472Sglenn.bergmans@arm.com return (writeQueue.size() + neededEntries) > writeBufferSize; 30512472Sglenn.bergmans@arm.com} 30612472Sglenn.bergmans@arm.com 30712472Sglenn.bergmans@arm.comDRAMCtrl::DRAMPacket* 30812472Sglenn.bergmans@arm.comDRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 30912472Sglenn.bergmans@arm.com bool isRead) 31012472Sglenn.bergmans@arm.com{ 31112472Sglenn.bergmans@arm.com // decode the address based on the address mapping scheme, with 31212472Sglenn.bergmans@arm.com // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 31312472Sglenn.bergmans@arm.com // channel, respectively 31412472Sglenn.bergmans@arm.com uint8_t rank; 31511236Sandreas.sandberg@arm.com uint8_t bank; 31611236Sandreas.sandberg@arm.com // use a 64-bit unsigned during the computations as the row is 31711236Sandreas.sandberg@arm.com // always the top bits, and check before creating the DRAMPacket 31811236Sandreas.sandberg@arm.com uint64_t row; 31911236Sandreas.sandberg@arm.com 32011236Sandreas.sandberg@arm.com // truncate the address to a DRAM burst, which makes it unique to 32111236Sandreas.sandberg@arm.com // a specific column, row, bank, rank and channel 32211236Sandreas.sandberg@arm.com Addr addr = dramPktAddr / burstSize; 32311236Sandreas.sandberg@arm.com 32411011SAndreas.Sandberg@ARM.com // we have removed the lowest order address bits that denote the 32511011SAndreas.Sandberg@ARM.com // position within the column 32611236Sandreas.sandberg@arm.com if (addrMapping == Enums::RoRaBaChCo) { 32711236Sandreas.sandberg@arm.com // the lowest order bits denote the column to ensure that 32811236Sandreas.sandberg@arm.com // sequential cache lines occupy the same row 32911236Sandreas.sandberg@arm.com addr = addr / columnsPerRowBuffer; 33011236Sandreas.sandberg@arm.com 33111236Sandreas.sandberg@arm.com // take out the channel part of the address 33211236Sandreas.sandberg@arm.com addr = addr / channels; 33311011SAndreas.Sandberg@ARM.com 33412472Sglenn.bergmans@arm.com // after the channel bits, get the bank bits to interleave 33512472Sglenn.bergmans@arm.com // over the banks 33612472Sglenn.bergmans@arm.com bank = addr % banksPerRank; 33712472Sglenn.bergmans@arm.com addr = addr / banksPerRank; 33812472Sglenn.bergmans@arm.com 33912472Sglenn.bergmans@arm.com // after the bank, we get the rank bits which thus interleaves 34012472Sglenn.bergmans@arm.com // over the ranks 34112472Sglenn.bergmans@arm.com rank = addr % ranksPerChannel; 34212472Sglenn.bergmans@arm.com addr = addr / ranksPerChannel; 34312472Sglenn.bergmans@arm.com 34412472Sglenn.bergmans@arm.com // lastly, get the row bits, no need to remove them from addr 34512472Sglenn.bergmans@arm.com row = addr % rowsPerBank; 34612472Sglenn.bergmans@arm.com } else if (addrMapping == Enums::RoRaBaCoCh) { 3479806Sstever@gmail.com // take out the lower-order column bits 3487584SAli.Saidi@arm.com addr = addr / columnsPerStripe; 3499338SAndreas.Sandberg@arm.com 3507584SAli.Saidi@arm.com // take out the channel part of the address 3517584SAli.Saidi@arm.com addr = addr / channels; 3527584SAli.Saidi@arm.com 3537584SAli.Saidi@arm.com // next, the higher-order column bites 3547584SAli.Saidi@arm.com addr = addr / (columnsPerRowBuffer / columnsPerStripe); 3559338SAndreas.Sandberg@arm.com 3569525SAndreas.Sandberg@ARM.com // after the column bits, we get the bank bits to interleave 3577584SAli.Saidi@arm.com // over the banks 3587584SAli.Saidi@arm.com bank = addr % banksPerRank; 3597584SAli.Saidi@arm.com addr = addr / banksPerRank; 3607584SAli.Saidi@arm.com 36112472Sglenn.bergmans@arm.com // after the bank, we get the rank bits which thus interleaves 36212472Sglenn.bergmans@arm.com // over the ranks 36312472Sglenn.bergmans@arm.com rank = addr % ranksPerChannel; 36412472Sglenn.bergmans@arm.com addr = addr / ranksPerChannel; 36512472Sglenn.bergmans@arm.com 36612472Sglenn.bergmans@arm.com // lastly, get the row bits, no need to remove them from addr 36712472Sglenn.bergmans@arm.com row = addr % rowsPerBank; 36812472Sglenn.bergmans@arm.com } else if (addrMapping == Enums::RoCoRaBaCh) { 36912472Sglenn.bergmans@arm.com // optimise for closed page mode and utilise maximum 37012472Sglenn.bergmans@arm.com // parallelism of the DRAM (at the cost of power) 37112472Sglenn.bergmans@arm.com 37212472Sglenn.bergmans@arm.com // take out the lower-order column bits 37312472Sglenn.bergmans@arm.com addr = addr / columnsPerStripe; 37412472Sglenn.bergmans@arm.com 3759806Sstever@gmail.com // take out the channel part of the address, not that this has 3767584SAli.Saidi@arm.com // to match with how accesses are interleaved between the 3779338SAndreas.Sandberg@arm.com // controllers in the address mapping 3789525SAndreas.Sandberg@ARM.com addr = addr / channels; 3797584SAli.Saidi@arm.com 3807584SAli.Saidi@arm.com // start with the bank bits, as this provides the maximum 3817584SAli.Saidi@arm.com // opportunity for parallelism between requests 3827584SAli.Saidi@arm.com bank = addr % banksPerRank; 3837584SAli.Saidi@arm.com addr = addr / banksPerRank; 3847584SAli.Saidi@arm.com 38512077Sgedare@rtems.org // next get the rank bits 38612077Sgedare@rtems.org rank = addr % ranksPerChannel; 38712077Sgedare@rtems.org addr = addr / ranksPerChannel; 38812077Sgedare@rtems.org 38912077Sgedare@rtems.org // next, the higher-order column bites 39012077Sgedare@rtems.org addr = addr / (columnsPerRowBuffer / columnsPerStripe); 3918512Sgeoffrey.blake@arm.com 3928512Sgeoffrey.blake@arm.com // lastly, get the row bits, no need to remove them from addr 3939338SAndreas.Sandberg@arm.com row = addr % rowsPerBank; 39413106Sgiacomo.travaglini@arm.com } else 39513106Sgiacomo.travaglini@arm.com panic("Unknown address mapping policy chosen!"); 3968512Sgeoffrey.blake@arm.com 39712467SCurtis.Dunham@arm.com assert(rank < ranksPerChannel); 39810037SARM gem5 Developers assert(bank < banksPerRank); 39910037SARM gem5 Developers assert(row < rowsPerBank); 40011668Sandreas.sandberg@arm.com assert(row < Bank::NO_ROW); 40112975Sgiacomo.travaglini@arm.com 40212975Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 40312975Sgiacomo.travaglini@arm.com dramPktAddr, rank, bank, row); 40412975Sgiacomo.travaglini@arm.com 40510037SARM gem5 Developers // create the corresponding DRAM packet with the entry time and 40612472Sglenn.bergmans@arm.com // ready time set to the current tick, the latter will be updated 40712472Sglenn.bergmans@arm.com // later 40812472Sglenn.bergmans@arm.com uint16_t bank_id = banksPerRank * rank + bank; 40912472Sglenn.bergmans@arm.com return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 41012472Sglenn.bergmans@arm.com size, ranks[rank]->banks[bank], *ranks[rank]); 41112472Sglenn.bergmans@arm.com} 41212733Sandreas.sandberg@arm.com 41312975Sgiacomo.travaglini@arm.comvoid 41412975Sgiacomo.travaglini@arm.comDRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 41512975Sgiacomo.travaglini@arm.com{ 41612975Sgiacomo.travaglini@arm.com // only add to the read queue here. whenever the request is 41712733Sandreas.sandberg@arm.com // eventually done, set the readyTime, and call schedule() 41812472Sglenn.bergmans@arm.com assert(!pkt->isWrite()); 41912472Sglenn.bergmans@arm.com 42012472Sglenn.bergmans@arm.com assert(pktCount != 0); 42112472Sglenn.bergmans@arm.com 42212472Sglenn.bergmans@arm.com // if the request size is larger than burst size, the pkt is split into 42310847Sandreas.sandberg@arm.com // multiple DRAM packets 42410847Sandreas.sandberg@arm.com // Note if the pkt starting address is not aligened to burst size, the 42510847Sandreas.sandberg@arm.com // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 42610847Sandreas.sandberg@arm.com // are aligned to burst size boundaries. This is to ensure we accurately 42710847Sandreas.sandberg@arm.com // check read packets against packets in write queue. 42810847Sandreas.sandberg@arm.com Addr addr = pkt->getAddr(); 42912975Sgiacomo.travaglini@arm.com unsigned pktsServicedByWrQ = 0; 43012975Sgiacomo.travaglini@arm.com BurstHelper* burst_helper = NULL; 43110847Sandreas.sandberg@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 4328870SAli.Saidi@ARM.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 4338870SAli.Saidi@ARM.com pkt->getAddr() + pkt->getSize()) - addr; 4349338SAndreas.Sandberg@arm.com readPktSize[ceilLog2(size)]++; 4358870SAli.Saidi@ARM.com readBursts++; 4368870SAli.Saidi@ARM.com 4378870SAli.Saidi@ARM.com // First check write buffer to see if the data is already at 43812472Sglenn.bergmans@arm.com // the controller 43912472Sglenn.bergmans@arm.com bool foundInWrQ = false; 44012472Sglenn.bergmans@arm.com Addr burst_addr = burstAlign(addr); 44112472Sglenn.bergmans@arm.com // if the burst address is not present then there is no need 44212472Sglenn.bergmans@arm.com // looking any further 44312472Sglenn.bergmans@arm.com if (isInWriteQueue.find(burst_addr) != isInWriteQueue.end()) { 44412472Sglenn.bergmans@arm.com for (const auto& p : writeQueue) { 44512472Sglenn.bergmans@arm.com // check if the read is subsumed in the write queue 44612472Sglenn.bergmans@arm.com // packet we are looking at 44712472Sglenn.bergmans@arm.com if (p->addr <= addr && (addr + size) <= (p->addr + p->size)) { 4487950SAli.Saidi@ARM.com foundInWrQ = true; 4497754SWilliam.Wang@arm.com servicedByWrQ++; 4509338SAndreas.Sandberg@arm.com pktsServicedByWrQ++; 4517754SWilliam.Wang@arm.com DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 4527754SWilliam.Wang@arm.com "write queue\n", addr, size); 45312659Sandreas.sandberg@arm.com bytesReadWrQ += burstSize; 45412659Sandreas.sandberg@arm.com break; 45512472Sglenn.bergmans@arm.com } 45612472Sglenn.bergmans@arm.com } 45712472Sglenn.bergmans@arm.com } 45812472Sglenn.bergmans@arm.com 45912472Sglenn.bergmans@arm.com // If not found in the write q, make a DRAM packet and 46012472Sglenn.bergmans@arm.com // push it onto the read queue 46112472Sglenn.bergmans@arm.com if (!foundInWrQ) { 46212472Sglenn.bergmans@arm.com 46312472Sglenn.bergmans@arm.com // Make the burst helper for split packets 46412472Sglenn.bergmans@arm.com if (pktCount > 1 && burst_helper == NULL) { 4657753SWilliam.Wang@arm.com DPRINTF(DRAM, "Read to addr %lld translates to %d " 4667753SWilliam.Wang@arm.com "dram requests\n", pkt->getAddr(), pktCount); 4679338SAndreas.Sandberg@arm.com burst_helper = new BurstHelper(pktCount); 4689394Sandreas.hansson@arm.com } 4699330Schander.sudanthi@arm.com 4707753SWilliam.Wang@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 4719939Sdam.sunwoo@arm.com dram_pkt->burstHelper = burst_helper; 4729939Sdam.sunwoo@arm.com 4739646SChris.Emmons@arm.com assert(!readQueueFull(1)); 4749646SChris.Emmons@arm.com rdQLenPdf[readQueue.size() + respQueue.size()]++; 4759646SChris.Emmons@arm.com 4769646SChris.Emmons@arm.com DPRINTF(DRAM, "Adding to read queue\n"); 4779646SChris.Emmons@arm.com 4789646SChris.Emmons@arm.com readQueue.push_back(dram_pkt); 47911237Sandreas.sandberg@arm.com 48010840Sandreas.sandberg@arm.com // Update stats 48111090Sandreas.sandberg@arm.com avgRdQLen = readQueue.size() + respQueue.size(); 48211090Sandreas.sandberg@arm.com } 48312232Sgiacomo.travaglini@arm.com 48412232Sgiacomo.travaglini@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 48512232Sgiacomo.travaglini@arm.com addr = (addr | (burstSize - 1)) + 1; 48612232Sgiacomo.travaglini@arm.com } 4879646SChris.Emmons@arm.com 48811090Sandreas.sandberg@arm.com // If all packets are serviced by write queue, we send the repsonse back 48911090Sandreas.sandberg@arm.com if (pktsServicedByWrQ == pktCount) { 49011090Sandreas.sandberg@arm.com accessAndRespond(pkt, frontendLatency); 49111090Sandreas.sandberg@arm.com return; 49211898Ssudhanshu.jha@arm.com } 49311898Ssudhanshu.jha@arm.com 49411090Sandreas.sandberg@arm.com // Update how many split packets are serviced by write queue 49514283Sgiacomo.travaglini@arm.com if (burst_helper != NULL) 49614283Sgiacomo.travaglini@arm.com burst_helper->burstsServiced = pktsServicedByWrQ; 49714283Sgiacomo.travaglini@arm.com 49814283Sgiacomo.travaglini@arm.com // If we are not already scheduled to get a request out of the 49914283Sgiacomo.travaglini@arm.com // queue, do so now 50012472Sglenn.bergmans@arm.com if (!nextReqEvent.scheduled()) { 50114283Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 50214283Sgiacomo.travaglini@arm.com schedule(nextReqEvent, curTick()); 50314283Sgiacomo.travaglini@arm.com } 50414283Sgiacomo.travaglini@arm.com} 50514283Sgiacomo.travaglini@arm.com 50614283Sgiacomo.travaglini@arm.comvoid 50714283Sgiacomo.travaglini@arm.comDRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 50814283Sgiacomo.travaglini@arm.com{ 50914283Sgiacomo.travaglini@arm.com // only add to the write queue here. whenever the request is 51014283Sgiacomo.travaglini@arm.com // eventually done, set the readyTime, and call schedule() 51114283Sgiacomo.travaglini@arm.com assert(pkt->isWrite()); 51214283Sgiacomo.travaglini@arm.com 51314283Sgiacomo.travaglini@arm.com // if the request size is larger than burst size, the pkt is split into 51414283Sgiacomo.travaglini@arm.com // multiple DRAM packets 51514283Sgiacomo.travaglini@arm.com Addr addr = pkt->getAddr(); 51614283Sgiacomo.travaglini@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 51714283Sgiacomo.travaglini@arm.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 51812472Sglenn.bergmans@arm.com pkt->getAddr() + pkt->getSize()) - addr; 51912472Sglenn.bergmans@arm.com writePktSize[ceilLog2(size)]++; 52012472Sglenn.bergmans@arm.com writeBursts++; 52112472Sglenn.bergmans@arm.com 52212472Sglenn.bergmans@arm.com // see if we can merge with an existing item in the write 52312472Sglenn.bergmans@arm.com // queue and keep track of whether we have merged or not 52412472Sglenn.bergmans@arm.com bool merged = isInWriteQueue.find(burstAlign(addr)) != 52512472Sglenn.bergmans@arm.com isInWriteQueue.end(); 52612472Sglenn.bergmans@arm.com 52712472Sglenn.bergmans@arm.com // if the item was not merged we need to create a new write 52812472Sglenn.bergmans@arm.com // and enqueue it 52912472Sglenn.bergmans@arm.com if (!merged) { 53012472Sglenn.bergmans@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 53112472Sglenn.bergmans@arm.com 53214274Sgiacomo.travaglini@arm.com assert(writeQueue.size() < writeBufferSize); 53314274Sgiacomo.travaglini@arm.com wrQLenPdf[writeQueue.size()]++; 53414283Sgiacomo.travaglini@arm.com 53514283Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Adding to write queue\n"); 53612472Sglenn.bergmans@arm.com 53712472Sglenn.bergmans@arm.com writeQueue.push_back(dram_pkt); 5387584SAli.Saidi@arm.com isInWriteQueue.insert(burstAlign(addr)); 5397584SAli.Saidi@arm.com assert(writeQueue.size() == isInWriteQueue.size()); 5409338SAndreas.Sandberg@arm.com 5413630SN/A // Update stats 54213636Sgiacomo.travaglini@arm.com avgWrQLen = writeQueue.size(); 5438870SAli.Saidi@ARM.com } else { 54411297Sandreas.sandberg@arm.com DPRINTF(DRAM, "Merging write burst with existing queue entry\n"); 54511297Sandreas.sandberg@arm.com 54611297Sandreas.sandberg@arm.com // keep track of the fact that this burst effectively 54711297Sandreas.sandberg@arm.com // disappeared as it was merged with an existing one 54811297Sandreas.sandberg@arm.com mergedWrBursts++; 54911297Sandreas.sandberg@arm.com } 55011297Sandreas.sandberg@arm.com 55111297Sandreas.sandberg@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 55211597Sandreas.sandberg@arm.com addr = (addr | (burstSize - 1)) + 1; 55311597Sandreas.sandberg@arm.com } 55411597Sandreas.sandberg@arm.com 55511597Sandreas.sandberg@arm.com // we do not wait for the writes to be send to the actual memory, 55611597Sandreas.sandberg@arm.com // but instead take responsibility for the consistency here and 55711597Sandreas.sandberg@arm.com // snoop the write queue for any upcoming reads 55811597Sandreas.sandberg@arm.com // @todo, if a pkt size is larger than burst size, we might need a 55911597Sandreas.sandberg@arm.com // different front end latency 56011597Sandreas.sandberg@arm.com accessAndRespond(pkt, frontendLatency); 56111597Sandreas.sandberg@arm.com 56211297Sandreas.sandberg@arm.com // If we are not already scheduled to get a request out of the 56311597Sandreas.sandberg@arm.com // queue, do so now 56411297Sandreas.sandberg@arm.com if (!nextReqEvent.scheduled()) { 56511297Sandreas.sandberg@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 56611297Sandreas.sandberg@arm.com schedule(nextReqEvent, curTick()); 56711297Sandreas.sandberg@arm.com } 56811297Sandreas.sandberg@arm.com} 56911297Sandreas.sandberg@arm.com 57010353SGeoffrey.Blake@arm.comvoid 57110353SGeoffrey.Blake@arm.comDRAMCtrl::printQs() const { 57210353SGeoffrey.Blake@arm.com DPRINTF(DRAM, "===READ QUEUE===\n\n"); 57310353SGeoffrey.Blake@arm.com for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 57410353SGeoffrey.Blake@arm.com DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 57510353SGeoffrey.Blake@arm.com } 57610353SGeoffrey.Blake@arm.com DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 57711297Sandreas.sandberg@arm.com for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 57810353SGeoffrey.Blake@arm.com DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 57910353SGeoffrey.Blake@arm.com } 58011297Sandreas.sandberg@arm.com DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 58111297Sandreas.sandberg@arm.com for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 58212069Snikos.nikoleris@arm.com DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 58312069Snikos.nikoleris@arm.com } 58411297Sandreas.sandberg@arm.com} 58511297Sandreas.sandberg@arm.com 58611297Sandreas.sandberg@arm.combool 58711597Sandreas.sandberg@arm.comDRAMCtrl::recvTimingReq(PacketPtr pkt) 58811597Sandreas.sandberg@arm.com{ 58911297Sandreas.sandberg@arm.com // This is where we enter from the outside world 5908870SAli.Saidi@ARM.com DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 59112598Snikos.nikoleris@arm.com pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 59212598Snikos.nikoleris@arm.com 59312598Snikos.nikoleris@arm.com // if a cache is responding, sink the packet without further action 59412598Snikos.nikoleris@arm.com if (pkt->cacheResponding()) { 59512598Snikos.nikoleris@arm.com pendingDelete.reset(pkt); 5968870SAli.Saidi@ARM.com return true; 59710037SARM gem5 Developers } 59810037SARM gem5 Developers 5998870SAli.Saidi@ARM.com // Calc avg gap between requests 60012472Sglenn.bergmans@arm.com if (prevArrival != 0) { 60112472Sglenn.bergmans@arm.com totGap += curTick() - prevArrival; 60212472Sglenn.bergmans@arm.com } 60312472Sglenn.bergmans@arm.com prevArrival = curTick(); 60412472Sglenn.bergmans@arm.com 60512785Sandreas.sandberg@arm.com 60612785Sandreas.sandberg@arm.com // Find out how many dram packets a pkt translates to 60712472Sglenn.bergmans@arm.com // If the burst size is equal or larger than the pkt size, then a pkt 60812472Sglenn.bergmans@arm.com // translates to only one dram packet. Otherwise, a pkt translates to 60912472Sglenn.bergmans@arm.com // multiple dram packets 61012472Sglenn.bergmans@arm.com unsigned size = pkt->getSize(); 61112472Sglenn.bergmans@arm.com unsigned offset = pkt->getAddr() & (burstSize - 1); 61212472Sglenn.bergmans@arm.com unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 61312472Sglenn.bergmans@arm.com 6143630SN/A // check local buffers and do not accept if full 6157753SWilliam.Wang@arm.com if (pkt->isRead()) { 6167753SWilliam.Wang@arm.com assert(size != 0); 6177753SWilliam.Wang@arm.com if (readQueueFull(dram_pkt_count)) { 6187584SAli.Saidi@arm.com DPRINTF(DRAM, "Read queue full, not accepting\n"); 6197584SAli.Saidi@arm.com // remember that we have to retry this port 62011236Sandreas.sandberg@arm.com retryRdReq = true; 62111236Sandreas.sandberg@arm.com numRdRetry++; 62211236Sandreas.sandberg@arm.com return false; 62313505Sgiacomo.travaglini@arm.com } else { 62411244Sandreas.sandberg@arm.com addToReadQueue(pkt, dram_pkt_count); 62511244Sandreas.sandberg@arm.com readReqs++; 62611244Sandreas.sandberg@arm.com bytesReadSys += size; 6277584SAli.Saidi@arm.com } 6287584SAli.Saidi@arm.com } else if (pkt->isWrite()) { 62912077Sgedare@rtems.org assert(size != 0); 63013106Sgiacomo.travaglini@arm.com if (writeQueueFull(dram_pkt_count)) { 63113106Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Write queue full, not accepting\n"); 63212077Sgedare@rtems.org // remember that we have to retry this port 6337753SWilliam.Wang@arm.com retryWrReq = true; 63412659Sandreas.sandberg@arm.com numWrRetry++; 63512659Sandreas.sandberg@arm.com return false; 6368282SAli.Saidi@ARM.com } else { 6378525SAli.Saidi@ARM.com addToWriteQueue(pkt, dram_pkt_count); 6388212SAli.Saidi@ARM.com writeReqs++; 6398212SAli.Saidi@ARM.com bytesWrittenSys += size; 6408212SAli.Saidi@ARM.com } 6418212SAli.Saidi@ARM.com } else { 6428212SAli.Saidi@ARM.com DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 6437584SAli.Saidi@arm.com neitherReadNorWrite++; 6447731SAli.Saidi@ARM.com accessAndRespond(pkt, 1); 6458461SAli.Saidi@ARM.com } 6468461SAli.Saidi@ARM.com 6477696SAli.Saidi@ARM.com return true; 6487696SAli.Saidi@ARM.com} 6497696SAli.Saidi@ARM.com 6507696SAli.Saidi@ARM.comvoid 6517696SAli.Saidi@ARM.comDRAMCtrl::processRespondEvent() 6527696SAli.Saidi@ARM.com{ 6537696SAli.Saidi@ARM.com DPRINTF(DRAM, 6547696SAli.Saidi@ARM.com "processRespondEvent(): Some req has reached its readyTime\n"); 6557696SAli.Saidi@ARM.com 6567696SAli.Saidi@ARM.com DRAMPacket* dram_pkt = respQueue.front(); 6577696SAli.Saidi@ARM.com 6587696SAli.Saidi@ARM.com if (dram_pkt->burstHelper) { 6597696SAli.Saidi@ARM.com // it is a split packet 6607696SAli.Saidi@ARM.com dram_pkt->burstHelper->burstsServiced++; 6618906Skoansin.tan@gmail.com if (dram_pkt->burstHelper->burstsServiced == 66210397Sstephan.diestelhorst@arm.com dram_pkt->burstHelper->burstCount) { 6637696SAli.Saidi@ARM.com // we have now serviced all children packets of a system packet 6647696SAli.Saidi@ARM.com // so we can now respond to the requester 6658713Sandreas.hansson@arm.com // @todo we probably want to have a different front end and back 6668713Sandreas.hansson@arm.com // end latency for split packets 6678713Sandreas.hansson@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 6688839Sandreas.hansson@arm.com delete dram_pkt->burstHelper; 6698839Sandreas.hansson@arm.com dram_pkt->burstHelper = NULL; 6708839Sandreas.hansson@arm.com } 67112077Sgedare@rtems.org } else { 6728839Sandreas.hansson@arm.com // it is not a split packet 6738713Sandreas.hansson@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 6748713Sandreas.hansson@arm.com } 6758713Sandreas.hansson@arm.com 6768713Sandreas.hansson@arm.com delete respQueue.front(); 6778870SAli.Saidi@ARM.com respQueue.pop_front(); 6788870SAli.Saidi@ARM.com 6798870SAli.Saidi@ARM.com if (!respQueue.empty()) { 6807696SAli.Saidi@ARM.com assert(respQueue.front()->readyTime >= curTick()); 68110353SGeoffrey.Blake@arm.com assert(!respondEvent.scheduled()); 68210353SGeoffrey.Blake@arm.com schedule(respondEvent, respQueue.front()->readyTime); 68310353SGeoffrey.Blake@arm.com } else { 68410353SGeoffrey.Blake@arm.com // if there is nothing left in any queue, signal a drain 68510353SGeoffrey.Blake@arm.com if (drainState() == DrainState::Draining && 68610353SGeoffrey.Blake@arm.com writeQueue.empty() && readQueue.empty()) { 68710353SGeoffrey.Blake@arm.com 68810353SGeoffrey.Blake@arm.com DPRINTF(Drain, "DRAM controller done draining\n"); 6897696SAli.Saidi@ARM.com signalDrainDone(); 6907696SAli.Saidi@ARM.com } 6917696SAli.Saidi@ARM.com } 6927696SAli.Saidi@ARM.com 6938839Sandreas.hansson@arm.com // We have made a location in the queue available at this point, 6948839Sandreas.hansson@arm.com // so if there is a read that was forced to wait, retry now 69511244Sandreas.sandberg@arm.com if (retryRdReq) { 6968839Sandreas.hansson@arm.com retryRdReq = false; 6978839Sandreas.hansson@arm.com port.sendRetryReq(); 6988839Sandreas.hansson@arm.com } 6998839Sandreas.hansson@arm.com} 7008839Sandreas.hansson@arm.com 7018839Sandreas.hansson@arm.combool 7028839Sandreas.hansson@arm.comDRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, Tick extra_col_delay) 7038839Sandreas.hansson@arm.com{ 7048839Sandreas.hansson@arm.com // This method does the arbitration between requests. The chosen 7058839Sandreas.hansson@arm.com // packet is simply moved to the head of the queue. The other 7068839Sandreas.hansson@arm.com // methods know that this is the place to look. For example, with 7078839Sandreas.hansson@arm.com // FCFS, this method does nothing 7088839Sandreas.hansson@arm.com assert(!queue.empty()); 7098839Sandreas.hansson@arm.com 7108839Sandreas.hansson@arm.com // bool to indicate if a packet to an available rank is found 7118839Sandreas.hansson@arm.com bool found_packet = false; 7128839Sandreas.hansson@arm.com if (queue.size() == 1) { 7138839Sandreas.hansson@arm.com DRAMPacket* dram_pkt = queue.front(); 7148839Sandreas.hansson@arm.com // available rank corresponds to state refresh idle 7158839Sandreas.hansson@arm.com if (ranks[dram_pkt->rank]->isAvailable()) { 7168839Sandreas.hansson@arm.com found_packet = true; 7178839Sandreas.hansson@arm.com DPRINTF(DRAM, "Single request, going to a free rank\n"); 7188906Skoansin.tan@gmail.com } else { 7198839Sandreas.hansson@arm.com DPRINTF(DRAM, "Single request, going to a busy rank\n"); 72010397Sstephan.diestelhorst@arm.com } 7217696SAli.Saidi@ARM.com return found_packet; 72210353SGeoffrey.Blake@arm.com } 72310353SGeoffrey.Blake@arm.com 72410353SGeoffrey.Blake@arm.com if (memSchedPolicy == Enums::fcfs) { 72510353SGeoffrey.Blake@arm.com // check if there is a packet going to a free rank 72610353SGeoffrey.Blake@arm.com for(auto i = queue.begin(); i != queue.end() ; ++i) { 72710353SGeoffrey.Blake@arm.com DRAMPacket* dram_pkt = *i; 72810353SGeoffrey.Blake@arm.com if (ranks[dram_pkt->rank]->isAvailable()) { 72910353SGeoffrey.Blake@arm.com queue.erase(i); 73010353SGeoffrey.Blake@arm.com queue.push_front(dram_pkt); 73110353SGeoffrey.Blake@arm.com found_packet = true; 73210353SGeoffrey.Blake@arm.com break; 73310353SGeoffrey.Blake@arm.com } 73410353SGeoffrey.Blake@arm.com } 73510353SGeoffrey.Blake@arm.com } else if (memSchedPolicy == Enums::frfcfs) { 73610353SGeoffrey.Blake@arm.com found_packet = reorderQueue(queue, extra_col_delay); 73710353SGeoffrey.Blake@arm.com } else 73810353SGeoffrey.Blake@arm.com panic("No scheduling policy chosen\n"); 73910353SGeoffrey.Blake@arm.com return found_packet; 74010353SGeoffrey.Blake@arm.com} 74110353SGeoffrey.Blake@arm.com 74210353SGeoffrey.Blake@arm.combool 74310353SGeoffrey.Blake@arm.comDRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, Tick extra_col_delay) 74410353SGeoffrey.Blake@arm.com{ 74510353SGeoffrey.Blake@arm.com // Only determine this if needed 74610353SGeoffrey.Blake@arm.com uint64_t earliest_banks = 0; 74710353SGeoffrey.Blake@arm.com bool hidden_bank_prep = false; 74810353SGeoffrey.Blake@arm.com 74910397Sstephan.diestelhorst@arm.com // search for seamless row hits first, if no seamless row hit is 75010353SGeoffrey.Blake@arm.com // found then determine if there are other packets that can be issued 7518870SAli.Saidi@ARM.com // without incurring additional bus delay due to bank timing 75213636Sgiacomo.travaglini@arm.com // Will select closed rows first to enable more open row possibilies 75312069Snikos.nikoleris@arm.com // in future selections 75412069Snikos.nikoleris@arm.com bool found_hidden_bank = false; 75512069Snikos.nikoleris@arm.com 75612069Snikos.nikoleris@arm.com // remember if we found a row hit, not seamless, but bank prepped 75712069Snikos.nikoleris@arm.com // and ready 75812069Snikos.nikoleris@arm.com bool found_prepped_pkt = false; 75912069Snikos.nikoleris@arm.com 76012069Snikos.nikoleris@arm.com // if we have no row hit, prepped or not, and no seamless packet, 76112069Snikos.nikoleris@arm.com // just go for the earliest possible 76212069Snikos.nikoleris@arm.com bool found_earliest_pkt = false; 76312069Snikos.nikoleris@arm.com 76412069Snikos.nikoleris@arm.com auto selected_pkt_it = queue.end(); 76512069Snikos.nikoleris@arm.com 76611236Sandreas.sandberg@arm.com // time we need to issue a column command to be seamless 76711236Sandreas.sandberg@arm.com const Tick min_col_at = std::max(busBusyUntil - tCL + extra_col_delay, 76812069Snikos.nikoleris@arm.com curTick()); 76912069Snikos.nikoleris@arm.com 77013505Sgiacomo.travaglini@arm.com for (auto i = queue.begin(); i != queue.end() ; ++i) { 77113814Sgiacomo.travaglini@arm.com DRAMPacket* dram_pkt = *i; 77212069Snikos.nikoleris@arm.com const Bank& bank = dram_pkt->bankRef; 77313106Sgiacomo.travaglini@arm.com 77413106Sgiacomo.travaglini@arm.com // check if rank is available, if not, jump to the next packet 77512069Snikos.nikoleris@arm.com if (dram_pkt->rankRef.isAvailable()) { 77612069Snikos.nikoleris@arm.com // check if it is a row hit 77712069Snikos.nikoleris@arm.com if (bank.openRow == dram_pkt->row) { 77812069Snikos.nikoleris@arm.com // no additional rank-to-rank or same bank-group 77912069Snikos.nikoleris@arm.com // delays, or we switched read/write and might as well 78012069Snikos.nikoleris@arm.com // go for the row hit 78112069Snikos.nikoleris@arm.com if (bank.colAllowedAt <= min_col_at) { 78212069Snikos.nikoleris@arm.com // FCFS within the hits, giving priority to 78312069Snikos.nikoleris@arm.com // commands that can issue seamlessly, without 78412069Snikos.nikoleris@arm.com // additional delay, such as same rank accesses 78512069Snikos.nikoleris@arm.com // and/or different bank-group accesses 78612069Snikos.nikoleris@arm.com DPRINTF(DRAM, "Seamless row buffer hit\n"); 78712069Snikos.nikoleris@arm.com selected_pkt_it = i; 78812069Snikos.nikoleris@arm.com // no need to look through the remaining queue entries 78912069Snikos.nikoleris@arm.com break; 79012069Snikos.nikoleris@arm.com } else if (!found_hidden_bank && !found_prepped_pkt) { 79112069Snikos.nikoleris@arm.com // if we did not find a packet to a closed row that can 79212069Snikos.nikoleris@arm.com // issue the bank commands without incurring delay, and 79311244Sandreas.sandberg@arm.com // did not yet find a packet to a prepped row, remember 79411244Sandreas.sandberg@arm.com // the current one 79511244Sandreas.sandberg@arm.com selected_pkt_it = i; 79612069Snikos.nikoleris@arm.com found_prepped_pkt = true; 79712975Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Prepped row buffer hit\n"); 79812975Sgiacomo.travaglini@arm.com } 79912975Sgiacomo.travaglini@arm.com } else if (!found_earliest_pkt) { 80012975Sgiacomo.travaglini@arm.com // if we have not initialised the bank status, do it 80112975Sgiacomo.travaglini@arm.com // now, and only once per scheduling decisions 8029185SAli.Saidi@ARM.com if (earliest_banks == 0) { 8039185SAli.Saidi@ARM.com // determine entries with earliest bank delay 8048870SAli.Saidi@ARM.com pair<uint64_t, bool> bankStatus = 80512659Sandreas.sandberg@arm.com minBankPrep(queue, min_col_at); 80612659Sandreas.sandberg@arm.com earliest_banks = bankStatus.first; 8078870SAli.Saidi@ARM.com hidden_bank_prep = bankStatus.second; 8088870SAli.Saidi@ARM.com } 8098870SAli.Saidi@ARM.com 8108870SAli.Saidi@ARM.com // bank is amongst first available banks 8118870SAli.Saidi@ARM.com // minBankPrep will give priority to packets that can 8129052Sgeoffrey.blake@arm.com // issue seamlessly 8139835Sandreas.hansson@arm.com if (bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 8149835Sandreas.hansson@arm.com found_earliest_pkt = true; 8158870SAli.Saidi@ARM.com found_hidden_bank = hidden_bank_prep; 8168870SAli.Saidi@ARM.com 8178870SAli.Saidi@ARM.com // give priority to packets that can issue 8188870SAli.Saidi@ARM.com // bank commands 'behind the scenes' 8198870SAli.Saidi@ARM.com // any additional delay if any will be due to 8208870SAli.Saidi@ARM.com // col-to-col command requirements 8218870SAli.Saidi@ARM.com if (hidden_bank_prep || !found_prepped_pkt) 8228870SAli.Saidi@ARM.com selected_pkt_it = i; 8238870SAli.Saidi@ARM.com } 8248870SAli.Saidi@ARM.com } 8258870SAli.Saidi@ARM.com } 8268870SAli.Saidi@ARM.com } 82710397Sstephan.diestelhorst@arm.com 8288870SAli.Saidi@ARM.com if (selected_pkt_it != queue.end()) { 82912069Snikos.nikoleris@arm.com DRAMPacket* selected_pkt = *selected_pkt_it; 83012069Snikos.nikoleris@arm.com queue.erase(selected_pkt_it); 83112069Snikos.nikoleris@arm.com queue.push_front(selected_pkt); 83212069Snikos.nikoleris@arm.com return true; 83312069Snikos.nikoleris@arm.com } 83412069Snikos.nikoleris@arm.com 83512069Snikos.nikoleris@arm.com return false; 83612069Snikos.nikoleris@arm.com} 83712069Snikos.nikoleris@arm.com 83812069Snikos.nikoleris@arm.comvoid 83912069Snikos.nikoleris@arm.comDRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 84012069Snikos.nikoleris@arm.com{ 84112069Snikos.nikoleris@arm.com DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 84212069Snikos.nikoleris@arm.com 84312069Snikos.nikoleris@arm.com bool needsResponse = pkt->needsResponse(); 84412069Snikos.nikoleris@arm.com // do the actual memory access which also turns the packet into a 84512069Snikos.nikoleris@arm.com // response 84612069Snikos.nikoleris@arm.com access(pkt); 84712069Snikos.nikoleris@arm.com 84812069Snikos.nikoleris@arm.com // turn packet around to go back to requester if response expected 84912069Snikos.nikoleris@arm.com if (needsResponse) { 85012069Snikos.nikoleris@arm.com // access already turned the packet into a response 85112069Snikos.nikoleris@arm.com assert(pkt->isResponse()); 85212069Snikos.nikoleris@arm.com // response_time consumes the static latency and is charged also 85312069Snikos.nikoleris@arm.com // with headerDelay that takes into account the delay provided by 85412069Snikos.nikoleris@arm.com // the xbar and also the payloadDelay that takes into account the 85512069Snikos.nikoleris@arm.com // number of data beats. 85612069Snikos.nikoleris@arm.com Tick response_time = curTick() + static_latency + pkt->headerDelay + 85712069Snikos.nikoleris@arm.com pkt->payloadDelay; 85812069Snikos.nikoleris@arm.com // Here we reset the timing of the packet before sending it out. 85912069Snikos.nikoleris@arm.com pkt->headerDelay = pkt->payloadDelay = 0; 86012069Snikos.nikoleris@arm.com 86110353SGeoffrey.Blake@arm.com // queue the packet in the response queue to be sent out after 86210353SGeoffrey.Blake@arm.com // the static latency has passed 86310353SGeoffrey.Blake@arm.com port.schedTimingResp(pkt, response_time, true); 86410353SGeoffrey.Blake@arm.com } else { 86510353SGeoffrey.Blake@arm.com // @todo the packet is going to be deleted, and the DRAMPacket 86610353SGeoffrey.Blake@arm.com // is still having a pointer to it 86710353SGeoffrey.Blake@arm.com pendingDelete.reset(pkt); 86810353SGeoffrey.Blake@arm.com } 86913505Sgiacomo.travaglini@arm.com 87013505Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Done\n"); 87110353SGeoffrey.Blake@arm.com 87210353SGeoffrey.Blake@arm.com return; 87310353SGeoffrey.Blake@arm.com} 8748870SAli.Saidi@ARM.com 87512598Snikos.nikoleris@arm.comvoid 87612598Snikos.nikoleris@arm.comDRAMCtrl::activateBank(Rank& rank_ref, Bank& bank_ref, 87712598Snikos.nikoleris@arm.com Tick act_tick, uint32_t row) 87812598Snikos.nikoleris@arm.com{ 87912116Sjose.marinho@arm.com assert(rank_ref.actTicks.size() == activationLimit); 88012116Sjose.marinho@arm.com 88110037SARM gem5 Developers DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 88210037SARM gem5 Developers 8838870SAli.Saidi@ARM.com // update the open row 88410037SARM gem5 Developers assert(bank_ref.openRow == Bank::NO_ROW); 88510358SAli.Saidi@ARM.com bank_ref.openRow = row; 88613636Sgiacomo.travaglini@arm.com 88713636Sgiacomo.travaglini@arm.com // start counting anew, this covers both the case when we 88813636Sgiacomo.travaglini@arm.com // auto-precharged, and when this access is forced to 88911244Sandreas.sandberg@arm.com // precharge 89011244Sandreas.sandberg@arm.com bank_ref.bytesAccessed = 0; 89111244Sandreas.sandberg@arm.com bank_ref.rowAccesses = 0; 89211244Sandreas.sandberg@arm.com 89310037SARM gem5 Developers ++rank_ref.numBanksActive; 89412598Snikos.nikoleris@arm.com assert(rank_ref.numBanksActive <= banksPerRank); 89512598Snikos.nikoleris@arm.com 89612598Snikos.nikoleris@arm.com DPRINTF(DRAM, "Activate bank %d, rank %d at tick %lld, now got %d active\n", 89712598Snikos.nikoleris@arm.com bank_ref.bank, rank_ref.rank, act_tick, 89812116Sjose.marinho@arm.com ranks[rank_ref.rank]->numBanksActive); 89912116Sjose.marinho@arm.com 90010037SARM gem5 Developers rank_ref.power.powerlib.doCommand(MemCommand::ACT, bank_ref.bank, 90110037SARM gem5 Developers divCeil(act_tick, tCK) - 90210037SARM gem5 Developers timeStampOffset); 90313532Sjairo.balart@metempsy.com 90411297Sandreas.sandberg@arm.com DPRINTF(DRAMPower, "%llu,ACT,%d,%d\n", divCeil(act_tick, tCK) - 90511297Sandreas.sandberg@arm.com timeStampOffset, bank_ref.bank, rank_ref.rank); 90611297Sandreas.sandberg@arm.com 90711297Sandreas.sandberg@arm.com // The next access has to respect tRAS for this bank 90811297Sandreas.sandberg@arm.com bank_ref.preAllowedAt = act_tick + tRAS; 90911297Sandreas.sandberg@arm.com 91011297Sandreas.sandberg@arm.com // Respect the row-to-column command delay 91111297Sandreas.sandberg@arm.com bank_ref.colAllowedAt = std::max(act_tick + tRCD, bank_ref.colAllowedAt); 91211297Sandreas.sandberg@arm.com 91311297Sandreas.sandberg@arm.com // start by enforcing tRRD 91411297Sandreas.sandberg@arm.com for(int i = 0; i < banksPerRank; i++) { 91511297Sandreas.sandberg@arm.com // next activate to any bank in this rank must not happen 91611297Sandreas.sandberg@arm.com // before tRRD 91711297Sandreas.sandberg@arm.com if (bankGroupArch && (bank_ref.bankgr == rank_ref.banks[i].bankgr)) { 91811297Sandreas.sandberg@arm.com // bank group architecture requires longer delays between 91911297Sandreas.sandberg@arm.com // ACT commands within the same bank group. Use tRRD_L 92011297Sandreas.sandberg@arm.com // in this case 92111297Sandreas.sandberg@arm.com rank_ref.banks[i].actAllowedAt = std::max(act_tick + tRRD_L, 92211297Sandreas.sandberg@arm.com rank_ref.banks[i].actAllowedAt); 92311297Sandreas.sandberg@arm.com } else { 92411297Sandreas.sandberg@arm.com // use shorter tRRD value when either 92511297Sandreas.sandberg@arm.com // 1) bank group architecture is not supportted 92611297Sandreas.sandberg@arm.com // 2) bank is in a different bank group 92711297Sandreas.sandberg@arm.com rank_ref.banks[i].actAllowedAt = std::max(act_tick + tRRD, 92811297Sandreas.sandberg@arm.com rank_ref.banks[i].actAllowedAt); 92911297Sandreas.sandberg@arm.com } 93011297Sandreas.sandberg@arm.com } 93111297Sandreas.sandberg@arm.com 93212006Sandreas.sandberg@arm.com // next, we deal with tXAW, if the activation limit is disabled 93311297Sandreas.sandberg@arm.com // then we directly schedule an activate power event 93411297Sandreas.sandberg@arm.com if (!rank_ref.actTicks.empty()) { 93511297Sandreas.sandberg@arm.com // sanity check 93611297Sandreas.sandberg@arm.com if (rank_ref.actTicks.back() && 93711297Sandreas.sandberg@arm.com (act_tick - rank_ref.actTicks.back()) < tXAW) { 93811297Sandreas.sandberg@arm.com panic("Got %d activates in window %d (%llu - %llu) which " 93911297Sandreas.sandberg@arm.com "is smaller than %llu\n", activationLimit, act_tick - 94011297Sandreas.sandberg@arm.com rank_ref.actTicks.back(), act_tick, 94111297Sandreas.sandberg@arm.com rank_ref.actTicks.back(), tXAW); 94211297Sandreas.sandberg@arm.com } 94311297Sandreas.sandberg@arm.com 94412741Sandreas.sandberg@arm.com // shift the times used for the book keeping, the last element 94512741Sandreas.sandberg@arm.com // (highest index) is the oldest one and hence the lowest value 94611297Sandreas.sandberg@arm.com rank_ref.actTicks.pop_back(); 94711297Sandreas.sandberg@arm.com 94811297Sandreas.sandberg@arm.com // record an new activation (in the future) 94911297Sandreas.sandberg@arm.com rank_ref.actTicks.push_front(act_tick); 95011297Sandreas.sandberg@arm.com 95111297Sandreas.sandberg@arm.com // cannot activate more than X times in time window tXAW, push the 95212896Sandreas.sandberg@arm.com // next one (the X + 1'st activate) to be tXAW away from the 95311297Sandreas.sandberg@arm.com // oldest in our window of X 95411297Sandreas.sandberg@arm.com if (rank_ref.actTicks.back() && 95511297Sandreas.sandberg@arm.com (act_tick - rank_ref.actTicks.back()) < tXAW) { 95611297Sandreas.sandberg@arm.com DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate " 95711297Sandreas.sandberg@arm.com "no earlier than %llu\n", activationLimit, 95811297Sandreas.sandberg@arm.com rank_ref.actTicks.back() + tXAW); 95911297Sandreas.sandberg@arm.com for(int j = 0; j < banksPerRank; j++) 96011297Sandreas.sandberg@arm.com // next activate must not happen before end of window 96111297Sandreas.sandberg@arm.com rank_ref.banks[j].actAllowedAt = 96211297Sandreas.sandberg@arm.com std::max(rank_ref.actTicks.back() + tXAW, 96311297Sandreas.sandberg@arm.com rank_ref.banks[j].actAllowedAt); 96411297Sandreas.sandberg@arm.com } 96511297Sandreas.sandberg@arm.com } 96611297Sandreas.sandberg@arm.com 96711297Sandreas.sandberg@arm.com // at the point when this activate takes place, make sure we 96811297Sandreas.sandberg@arm.com // transition to the active power state 96911297Sandreas.sandberg@arm.com if (!rank_ref.activateEvent.scheduled()) 97011297Sandreas.sandberg@arm.com schedule(rank_ref.activateEvent, act_tick); 97111297Sandreas.sandberg@arm.com else if (rank_ref.activateEvent.when() > act_tick) 97211297Sandreas.sandberg@arm.com // move it sooner in time 97311297Sandreas.sandberg@arm.com reschedule(rank_ref.activateEvent, act_tick); 97411297Sandreas.sandberg@arm.com} 97511297Sandreas.sandberg@arm.com 97611297Sandreas.sandberg@arm.comvoid 97711297Sandreas.sandberg@arm.comDRAMCtrl::prechargeBank(Rank& rank_ref, Bank& bank, Tick pre_at, bool trace) 97811297Sandreas.sandberg@arm.com{ 97911297Sandreas.sandberg@arm.com // make sure the bank has an open row 98011297Sandreas.sandberg@arm.com assert(bank.openRow != Bank::NO_ROW); 98111297Sandreas.sandberg@arm.com 98211297Sandreas.sandberg@arm.com // sample the bytes per activate here since we are closing 98311297Sandreas.sandberg@arm.com // the page 98411297Sandreas.sandberg@arm.com bytesPerActivate.sample(bank.bytesAccessed); 98511297Sandreas.sandberg@arm.com 98611297Sandreas.sandberg@arm.com bank.openRow = Bank::NO_ROW; 98711297Sandreas.sandberg@arm.com 98811297Sandreas.sandberg@arm.com // no precharge allowed before this one 98911297Sandreas.sandberg@arm.com bank.preAllowedAt = pre_at; 99012741Sandreas.sandberg@arm.com 99112741Sandreas.sandberg@arm.com Tick pre_done_at = pre_at + tRP; 99211297Sandreas.sandberg@arm.com 99311297Sandreas.sandberg@arm.com bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 99411297Sandreas.sandberg@arm.com 99511297Sandreas.sandberg@arm.com assert(rank_ref.numBanksActive != 0); 99611297Sandreas.sandberg@arm.com --rank_ref.numBanksActive; 99711297Sandreas.sandberg@arm.com 99811297Sandreas.sandberg@arm.com DPRINTF(DRAM, "Precharging bank %d, rank %d at tick %lld, now got " 99911297Sandreas.sandberg@arm.com "%d active\n", bank.bank, rank_ref.rank, pre_at, 100011297Sandreas.sandberg@arm.com rank_ref.numBanksActive); 100113636Sgiacomo.travaglini@arm.com 100211297Sandreas.sandberg@arm.com if (trace) { 100311297Sandreas.sandberg@arm.com 100411297Sandreas.sandberg@arm.com rank_ref.power.powerlib.doCommand(MemCommand::PRE, bank.bank, 100511297Sandreas.sandberg@arm.com divCeil(pre_at, tCK) - 100611297Sandreas.sandberg@arm.com timeStampOffset); 100711297Sandreas.sandberg@arm.com DPRINTF(DRAMPower, "%llu,PRE,%d,%d\n", divCeil(pre_at, tCK) - 100811297Sandreas.sandberg@arm.com timeStampOffset, bank.bank, rank_ref.rank); 100911297Sandreas.sandberg@arm.com } 101011297Sandreas.sandberg@arm.com // if we look at the current number of active banks we might be 101111297Sandreas.sandberg@arm.com // tempted to think the DRAM is now idle, however this can be 101211297Sandreas.sandberg@arm.com // undone by an activate that is scheduled to happen before we 101311297Sandreas.sandberg@arm.com // would have reached the idle state, so schedule an event and 101411297Sandreas.sandberg@arm.com // rather check once we actually make it to the point in time when 101511297Sandreas.sandberg@arm.com // the (last) precharge takes place 101611297Sandreas.sandberg@arm.com if (!rank_ref.prechargeEvent.scheduled()) 101712975Sgiacomo.travaglini@arm.com schedule(rank_ref.prechargeEvent, pre_done_at); 101812975Sgiacomo.travaglini@arm.com else if (rank_ref.prechargeEvent.when() < pre_done_at) 101912975Sgiacomo.travaglini@arm.com reschedule(rank_ref.prechargeEvent, pre_done_at); 102012975Sgiacomo.travaglini@arm.com} 102111297Sandreas.sandberg@arm.com 102211297Sandreas.sandberg@arm.comvoid 102311297Sandreas.sandberg@arm.comDRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 102411297Sandreas.sandberg@arm.com{ 102511297Sandreas.sandberg@arm.com DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 102611297Sandreas.sandberg@arm.com dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 102711297Sandreas.sandberg@arm.com 102812472Sglenn.bergmans@arm.com // get the rank 102912472Sglenn.bergmans@arm.com Rank& rank = dram_pkt->rankRef; 103012472Sglenn.bergmans@arm.com 103113015Sciro.santilli@arm.com // get the bank 103213015Sciro.santilli@arm.com Bank& bank = dram_pkt->bankRef; 103313015Sciro.santilli@arm.com 103411297Sandreas.sandberg@arm.com // for the state we need to track if it is a row hit or not 103512659Sandreas.sandberg@arm.com bool row_hit = true; 103612659Sandreas.sandberg@arm.com 103711297Sandreas.sandberg@arm.com // respect any constraints on the command (e.g. tRCD or tCCD) 103811297Sandreas.sandberg@arm.com Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 103911297Sandreas.sandberg@arm.com 104011297Sandreas.sandberg@arm.com // Determine the access latency and update the bank state 104111297Sandreas.sandberg@arm.com if (bank.openRow == dram_pkt->row) { 104211297Sandreas.sandberg@arm.com // nothing to do 104311297Sandreas.sandberg@arm.com } else { 104411297Sandreas.sandberg@arm.com row_hit = false; 104511297Sandreas.sandberg@arm.com 104611297Sandreas.sandberg@arm.com // If there is a page open, precharge it. 104711297Sandreas.sandberg@arm.com if (bank.openRow != Bank::NO_ROW) { 104812741Sandreas.sandberg@arm.com prechargeBank(rank, bank, std::max(bank.preAllowedAt, curTick())); 104912741Sandreas.sandberg@arm.com } 105012741Sandreas.sandberg@arm.com 105112741Sandreas.sandberg@arm.com // next we need to account for the delay in activating the 105212741Sandreas.sandberg@arm.com // page 105312741Sandreas.sandberg@arm.com Tick act_tick = std::max(bank.actAllowedAt, curTick()); 105411297Sandreas.sandberg@arm.com 105511297Sandreas.sandberg@arm.com // Record the activation and deal with all the global timing 105611297Sandreas.sandberg@arm.com // constraints caused be a new activation (tRRD and tXAW) 105711297Sandreas.sandberg@arm.com activateBank(rank, bank, act_tick, dram_pkt->row); 105813015Sciro.santilli@arm.com 105912472Sglenn.bergmans@arm.com // issue the command as early as possible 106012472Sglenn.bergmans@arm.com cmd_at = bank.colAllowedAt; 106111297Sandreas.sandberg@arm.com } 106211297Sandreas.sandberg@arm.com 106311297Sandreas.sandberg@arm.com // we need to wait until the bus is available before we can issue 106412472Sglenn.bergmans@arm.com // the command 106512741Sandreas.sandberg@arm.com cmd_at = std::max(cmd_at, busBusyUntil - tCL); 106612741Sandreas.sandberg@arm.com 106711297Sandreas.sandberg@arm.com // update the packet ready time 106811297Sandreas.sandberg@arm.com dram_pkt->readyTime = cmd_at + tCL + tBURST; 106911597Sandreas.sandberg@arm.com 107011297Sandreas.sandberg@arm.com // only one burst can use the bus at any one point in time 107111597Sandreas.sandberg@arm.com assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 107211297Sandreas.sandberg@arm.com 107311297Sandreas.sandberg@arm.com // update the time for the next read/write burst for each 107412598Snikos.nikoleris@arm.com // bank (add a max with tCCD/tCCD_L here) 107512598Snikos.nikoleris@arm.com Tick cmd_dly; 107612598Snikos.nikoleris@arm.com for(int j = 0; j < ranksPerChannel; j++) { 107712598Snikos.nikoleris@arm.com for(int i = 0; i < banksPerRank; i++) { 107812116Sjose.marinho@arm.com // next burst to same bank group in this rank must not happen 107912116Sjose.marinho@arm.com // before tCCD_L. Different bank group timing requirement is 108011297Sandreas.sandberg@arm.com // tBURST; Add tCS for different ranks 108111297Sandreas.sandberg@arm.com if (dram_pkt->rank == j) { 108212006Sandreas.sandberg@arm.com if (bankGroupArch && 108312006Sandreas.sandberg@arm.com (bank.bankgr == ranks[j]->banks[i].bankgr)) { 108412006Sandreas.sandberg@arm.com // bank group architecture requires longer delays between 108512006Sandreas.sandberg@arm.com // RD/WR burst commands to the same bank group. 108612006Sandreas.sandberg@arm.com // Use tCCD_L in this case 108712472Sglenn.bergmans@arm.com cmd_dly = tCCD_L; 108812472Sglenn.bergmans@arm.com } else { 108912472Sglenn.bergmans@arm.com // use tBURST (equivalent to tCCD_S), the shorter 109013532Sjairo.balart@metempsy.com // cas-to-cas delay value, when either: 109112472Sglenn.bergmans@arm.com // 1) bank group architecture is not supportted 109212472Sglenn.bergmans@arm.com // 2) bank is in a different bank group 109312472Sglenn.bergmans@arm.com cmd_dly = tBURST; 109412472Sglenn.bergmans@arm.com } 109512472Sglenn.bergmans@arm.com } else { 109612472Sglenn.bergmans@arm.com // different rank is by default in a different bank group 109712472Sglenn.bergmans@arm.com // use tBURST (equivalent to tCCD_S), which is the shorter 109812472Sglenn.bergmans@arm.com // cas-to-cas delay in this case 109912472Sglenn.bergmans@arm.com // Add tCS to account for rank-to-rank bus delay requirements 110012472Sglenn.bergmans@arm.com cmd_dly = tBURST + tCS; 110112760Srohit.kurup@arm.com } 110213532Sjairo.balart@metempsy.com ranks[j]->banks[i].colAllowedAt = std::max(cmd_at + cmd_dly, 110313532Sjairo.balart@metempsy.com ranks[j]->banks[i].colAllowedAt); 110413532Sjairo.balart@metempsy.com } 110513814Sgiacomo.travaglini@arm.com } 110613532Sjairo.balart@metempsy.com 110713532Sjairo.balart@metempsy.com // Save rank of current access 110813532Sjairo.balart@metempsy.com activeRank = dram_pkt->rank; 110913532Sjairo.balart@metempsy.com 111013532Sjairo.balart@metempsy.com // If this is a write, we also need to respect the write recovery 111113532Sjairo.balart@metempsy.com // time before a precharge, in the case of a read, respect the 111213532Sjairo.balart@metempsy.com // read to precharge constraint 111313532Sjairo.balart@metempsy.com bank.preAllowedAt = std::max(bank.preAllowedAt, 111413532Sjairo.balart@metempsy.com dram_pkt->isRead ? cmd_at + tRTP : 111512760Srohit.kurup@arm.com dram_pkt->readyTime + tWR); 111612760Srohit.kurup@arm.com 111712760Srohit.kurup@arm.com // increment the bytes accessed and the accesses per row 111812760Srohit.kurup@arm.com bank.bytesAccessed += burstSize; 111912760Srohit.kurup@arm.com ++bank.rowAccesses; 112012760Srohit.kurup@arm.com 112112760Srohit.kurup@arm.com // if we reached the max, then issue with an auto-precharge 112212760Srohit.kurup@arm.com bool auto_precharge = pageMgmt == Enums::close || 112312760Srohit.kurup@arm.com bank.rowAccesses == maxAccessesPerRow; 112413532Sjairo.balart@metempsy.com 112513532Sjairo.balart@metempsy.com // if we did not hit the limit, we might still want to 112613880Sgiacomo.travaglini@arm.com // auto-precharge 112713996Sgiacomo.travaglini@arm.com if (!auto_precharge && 112814225Sadrian.herrera@arm.com (pageMgmt == Enums::open_adaptive || 112913532Sjairo.balart@metempsy.com pageMgmt == Enums::close_adaptive)) { 113013879Sgiacomo.travaglini@arm.com // a twist on the open and close page policies: 113113879Sgiacomo.travaglini@arm.com // 1) open_adaptive page policy does not blindly keep the 113213879Sgiacomo.travaglini@arm.com // page open, but close it if there are no row hits, and there 113313532Sjairo.balart@metempsy.com // are bank conflicts in the queue 113413532Sjairo.balart@metempsy.com // 2) close_adaptive page policy does not blindly close the 113513996Sgiacomo.travaglini@arm.com // page, but closes it only if there are no row hits in the queue. 113613532Sjairo.balart@metempsy.com // In this case, only force an auto precharge when there 113713532Sjairo.balart@metempsy.com // are no same page hits in the queue 113813532Sjairo.balart@metempsy.com bool got_more_hits = false; 113913532Sjairo.balart@metempsy.com bool got_bank_conflict = false; 114013532Sjairo.balart@metempsy.com 114113532Sjairo.balart@metempsy.com // either look at the read queue or write queue 114213532Sjairo.balart@metempsy.com const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 114313532Sjairo.balart@metempsy.com writeQueue; 114413532Sjairo.balart@metempsy.com auto p = queue.begin(); 114513532Sjairo.balart@metempsy.com // make sure we are not considering the packet that we are 114613532Sjairo.balart@metempsy.com // currently dealing with (which is the head of the queue) 114713532Sjairo.balart@metempsy.com ++p; 114813532Sjairo.balart@metempsy.com 114913532Sjairo.balart@metempsy.com // keep on looking until we find a hit or reach the end of the queue 115013532Sjairo.balart@metempsy.com // 1) if a hit is found, then both open and close adaptive policies keep 1151 // the page open 1152 // 2) if no hit is found, got_bank_conflict is set to true if a bank 1153 // conflict request is waiting in the queue 1154 while (!got_more_hits && p != queue.end()) { 1155 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 1156 (dram_pkt->bank == (*p)->bank); 1157 bool same_row = dram_pkt->row == (*p)->row; 1158 got_more_hits |= same_rank_bank && same_row; 1159 got_bank_conflict |= same_rank_bank && !same_row; 1160 ++p; 1161 } 1162 1163 // auto pre-charge when either 1164 // 1) open_adaptive policy, we have not got any more hits, and 1165 // have a bank conflict 1166 // 2) close_adaptive policy and we have not got any more hits 1167 auto_precharge = !got_more_hits && 1168 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1169 } 1170 1171 // DRAMPower trace command to be written 1172 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR"; 1173 1174 // MemCommand required for DRAMPower library 1175 MemCommand::cmds command = (mem_cmd == "RD") ? MemCommand::RD : 1176 MemCommand::WR; 1177 1178 // if this access should use auto-precharge, then we are 1179 // closing the row 1180 if (auto_precharge) { 1181 // if auto-precharge push a PRE command at the correct tick to the 1182 // list used by DRAMPower library to calculate power 1183 prechargeBank(rank, bank, std::max(curTick(), bank.preAllowedAt)); 1184 1185 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1186 } 1187 1188 // Update bus state 1189 busBusyUntil = dram_pkt->readyTime; 1190 1191 DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 1192 dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 1193 1194 dram_pkt->rankRef.power.powerlib.doCommand(command, dram_pkt->bank, 1195 divCeil(cmd_at, tCK) - 1196 timeStampOffset); 1197 1198 DPRINTF(DRAMPower, "%llu,%s,%d,%d\n", divCeil(cmd_at, tCK) - 1199 timeStampOffset, mem_cmd, dram_pkt->bank, dram_pkt->rank); 1200 1201 // Update the minimum timing between the requests, this is a 1202 // conservative estimate of when we have to schedule the next 1203 // request to not introduce any unecessary bubbles. In most cases 1204 // we will wake up sooner than we have to. 1205 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1206 1207 // Update the stats and schedule the next request 1208 if (dram_pkt->isRead) { 1209 ++readsThisTime; 1210 if (row_hit) 1211 readRowHits++; 1212 bytesReadDRAM += burstSize; 1213 perBankRdBursts[dram_pkt->bankId]++; 1214 1215 // Update latency stats 1216 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 1217 totBusLat += tBURST; 1218 totQLat += cmd_at - dram_pkt->entryTime; 1219 } else { 1220 ++writesThisTime; 1221 if (row_hit) 1222 writeRowHits++; 1223 bytesWritten += burstSize; 1224 perBankWrBursts[dram_pkt->bankId]++; 1225 } 1226} 1227 1228void 1229DRAMCtrl::processNextReqEvent() 1230{ 1231 int busyRanks = 0; 1232 for (auto r : ranks) { 1233 if (!r->isAvailable()) { 1234 // rank is busy refreshing 1235 busyRanks++; 1236 1237 // let the rank know that if it was waiting to drain, it 1238 // is now done and ready to proceed 1239 r->checkDrainDone(); 1240 } 1241 } 1242 1243 if (busyRanks == ranksPerChannel) { 1244 // if all ranks are refreshing wait for them to finish 1245 // and stall this state machine without taking any further 1246 // action, and do not schedule a new nextReqEvent 1247 return; 1248 } 1249 1250 // pre-emptively set to false. Overwrite if in READ_TO_WRITE 1251 // or WRITE_TO_READ state 1252 bool switched_cmd_type = false; 1253 if (busState == READ_TO_WRITE) { 1254 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1255 "waiting\n", readsThisTime, readQueue.size()); 1256 1257 // sample and reset the read-related stats as we are now 1258 // transitioning to writes, and all reads are done 1259 rdPerTurnAround.sample(readsThisTime); 1260 readsThisTime = 0; 1261 1262 // now proceed to do the actual writes 1263 busState = WRITE; 1264 switched_cmd_type = true; 1265 } else if (busState == WRITE_TO_READ) { 1266 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1267 "waiting\n", writesThisTime, writeQueue.size()); 1268 1269 wrPerTurnAround.sample(writesThisTime); 1270 writesThisTime = 0; 1271 1272 busState = READ; 1273 switched_cmd_type = true; 1274 } 1275 1276 // when we get here it is either a read or a write 1277 if (busState == READ) { 1278 1279 // track if we should switch or not 1280 bool switch_to_writes = false; 1281 1282 if (readQueue.empty()) { 1283 // In the case there is no read request to go next, 1284 // trigger writes if we have passed the low threshold (or 1285 // if we are draining) 1286 if (!writeQueue.empty() && 1287 (drainState() == DrainState::Draining || 1288 writeQueue.size() > writeLowThreshold)) { 1289 1290 switch_to_writes = true; 1291 } else { 1292 // check if we are drained 1293 if (drainState() == DrainState::Draining && 1294 respQueue.empty()) { 1295 1296 DPRINTF(Drain, "DRAM controller done draining\n"); 1297 signalDrainDone(); 1298 } 1299 1300 // nothing to do, not even any point in scheduling an 1301 // event for the next request 1302 return; 1303 } 1304 } else { 1305 // bool to check if there is a read to a free rank 1306 bool found_read = false; 1307 1308 // Figure out which read request goes next, and move it to the 1309 // front of the read queue 1310 // If we are changing command type, incorporate the minimum 1311 // bus turnaround delay which will be tCS (different rank) case 1312 found_read = chooseNext(readQueue, 1313 switched_cmd_type ? tCS : 0); 1314 1315 // if no read to an available rank is found then return 1316 // at this point. There could be writes to the available ranks 1317 // which are above the required threshold. However, to 1318 // avoid adding more complexity to the code, return and wait 1319 // for a refresh event to kick things into action again. 1320 if (!found_read) 1321 return; 1322 1323 DRAMPacket* dram_pkt = readQueue.front(); 1324 assert(dram_pkt->rankRef.isAvailable()); 1325 // here we get a bit creative and shift the bus busy time not 1326 // just the tWTR, but also a CAS latency to capture the fact 1327 // that we are allowed to prepare a new bank, but not issue a 1328 // read command until after tWTR, in essence we capture a 1329 // bubble on the data bus that is tWTR + tCL 1330 if (switched_cmd_type && dram_pkt->rank == activeRank) { 1331 busBusyUntil += tWTR + tCL; 1332 } 1333 1334 doDRAMAccess(dram_pkt); 1335 1336 // At this point we're done dealing with the request 1337 readQueue.pop_front(); 1338 1339 // sanity check 1340 assert(dram_pkt->size <= burstSize); 1341 assert(dram_pkt->readyTime >= curTick()); 1342 1343 // Insert into response queue. It will be sent back to the 1344 // requestor at its readyTime 1345 if (respQueue.empty()) { 1346 assert(!respondEvent.scheduled()); 1347 schedule(respondEvent, dram_pkt->readyTime); 1348 } else { 1349 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1350 assert(respondEvent.scheduled()); 1351 } 1352 1353 respQueue.push_back(dram_pkt); 1354 1355 // we have so many writes that we have to transition 1356 if (writeQueue.size() > writeHighThreshold) { 1357 switch_to_writes = true; 1358 } 1359 } 1360 1361 // switching to writes, either because the read queue is empty 1362 // and the writes have passed the low threshold (or we are 1363 // draining), or because the writes hit the hight threshold 1364 if (switch_to_writes) { 1365 // transition to writing 1366 busState = READ_TO_WRITE; 1367 } 1368 } else { 1369 // bool to check if write to free rank is found 1370 bool found_write = false; 1371 1372 // If we are changing command type, incorporate the minimum 1373 // bus turnaround delay 1374 found_write = chooseNext(writeQueue, 1375 switched_cmd_type ? std::min(tRTW, tCS) : 0); 1376 1377 // if no writes to an available rank are found then return. 1378 // There could be reads to the available ranks. However, to avoid 1379 // adding more complexity to the code, return at this point and wait 1380 // for a refresh event to kick things into action again. 1381 if (!found_write) 1382 return; 1383 1384 DRAMPacket* dram_pkt = writeQueue.front(); 1385 assert(dram_pkt->rankRef.isAvailable()); 1386 // sanity check 1387 assert(dram_pkt->size <= burstSize); 1388 1389 // add a bubble to the data bus, as defined by the 1390 // tRTW when access is to the same rank as previous burst 1391 // Different rank timing is handled with tCS, which is 1392 // applied to colAllowedAt 1393 if (switched_cmd_type && dram_pkt->rank == activeRank) { 1394 busBusyUntil += tRTW; 1395 } 1396 1397 doDRAMAccess(dram_pkt); 1398 1399 writeQueue.pop_front(); 1400 isInWriteQueue.erase(burstAlign(dram_pkt->addr)); 1401 delete dram_pkt; 1402 1403 // If we emptied the write queue, or got sufficiently below the 1404 // threshold (using the minWritesPerSwitch as the hysteresis) and 1405 // are not draining, or we have reads waiting and have done enough 1406 // writes, then switch to reads. 1407 if (writeQueue.empty() || 1408 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1409 drainState() != DrainState::Draining) || 1410 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1411 // turn the bus back around for reads again 1412 busState = WRITE_TO_READ; 1413 1414 // note that the we switch back to reads also in the idle 1415 // case, which eventually will check for any draining and 1416 // also pause any further scheduling if there is really 1417 // nothing to do 1418 } 1419 } 1420 // It is possible that a refresh to another rank kicks things back into 1421 // action before reaching this point. 1422 if (!nextReqEvent.scheduled()) 1423 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1424 1425 // If there is space available and we have writes waiting then let 1426 // them retry. This is done here to ensure that the retry does not 1427 // cause a nextReqEvent to be scheduled before we do so as part of 1428 // the next request processing 1429 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1430 retryWrReq = false; 1431 port.sendRetryReq(); 1432 } 1433} 1434 1435pair<uint64_t, bool> 1436DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue, 1437 Tick min_col_at) const 1438{ 1439 uint64_t bank_mask = 0; 1440 Tick min_act_at = MaxTick; 1441 1442 // latest Tick for which ACT can occur without incurring additoinal 1443 // delay on the data bus 1444 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick()); 1445 1446 // Flag condition when burst can issue back-to-back with previous burst 1447 bool found_seamless_bank = false; 1448 1449 // Flag condition when bank can be opened without incurring additional 1450 // delay on the data bus 1451 bool hidden_bank_prep = false; 1452 1453 // determine if we have queued transactions targetting the 1454 // bank in question 1455 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 1456 for (const auto& p : queue) { 1457 if(p->rankRef.isAvailable()) 1458 got_waiting[p->bankId] = true; 1459 } 1460 1461 // Find command with optimal bank timing 1462 // Will prioritize commands that can issue seamlessly. 1463 for (int i = 0; i < ranksPerChannel; i++) { 1464 for (int j = 0; j < banksPerRank; j++) { 1465 uint16_t bank_id = i * banksPerRank + j; 1466 1467 // if we have waiting requests for the bank, and it is 1468 // amongst the first available, update the mask 1469 if (got_waiting[bank_id]) { 1470 // make sure this rank is not currently refreshing. 1471 assert(ranks[i]->isAvailable()); 1472 // simplistic approximation of when the bank can issue 1473 // an activate, ignoring any rank-to-rank switching 1474 // cost in this calculation 1475 Tick act_at = ranks[i]->banks[j].openRow == Bank::NO_ROW ? 1476 std::max(ranks[i]->banks[j].actAllowedAt, curTick()) : 1477 std::max(ranks[i]->banks[j].preAllowedAt, curTick()) + tRP; 1478 1479 // When is the earliest the R/W burst can issue? 1480 Tick col_at = std::max(ranks[i]->banks[j].colAllowedAt, 1481 act_at + tRCD); 1482 1483 // bank can issue burst back-to-back (seamlessly) with 1484 // previous burst 1485 bool new_seamless_bank = col_at <= min_col_at; 1486 1487 // if we found a new seamless bank or we have no 1488 // seamless banks, and got a bank with an earlier 1489 // activate time, it should be added to the bit mask 1490 if (new_seamless_bank || 1491 (!found_seamless_bank && act_at <= min_act_at)) { 1492 // if we did not have a seamless bank before, and 1493 // we do now, reset the bank mask, also reset it 1494 // if we have not yet found a seamless bank and 1495 // the activate time is smaller than what we have 1496 // seen so far 1497 if (!found_seamless_bank && 1498 (new_seamless_bank || act_at < min_act_at)) { 1499 bank_mask = 0; 1500 } 1501 1502 found_seamless_bank |= new_seamless_bank; 1503 1504 // ACT can occur 'behind the scenes' 1505 hidden_bank_prep = act_at <= hidden_act_max; 1506 1507 // set the bit corresponding to the available bank 1508 replaceBits(bank_mask, bank_id, bank_id, 1); 1509 min_act_at = act_at; 1510 } 1511 } 1512 } 1513 } 1514 1515 return make_pair(bank_mask, hidden_bank_prep); 1516} 1517 1518DRAMCtrl::Rank::Rank(DRAMCtrl& _memory, const DRAMCtrlParams* _p) 1519 : EventManager(&_memory), memory(_memory), 1520 pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), pwrStateTick(0), 1521 refreshState(REF_IDLE), refreshDueAt(0), 1522 power(_p, false), numBanksActive(0), 1523 activateEvent(*this), prechargeEvent(*this), 1524 refreshEvent(*this), powerEvent(*this) 1525{ } 1526 1527void 1528DRAMCtrl::Rank::startup(Tick ref_tick) 1529{ 1530 assert(ref_tick > curTick()); 1531 1532 pwrStateTick = curTick(); 1533 1534 // kick off the refresh, and give ourselves enough time to 1535 // precharge 1536 schedule(refreshEvent, ref_tick); 1537} 1538 1539void 1540DRAMCtrl::Rank::suspend() 1541{ 1542 deschedule(refreshEvent); 1543} 1544 1545void 1546DRAMCtrl::Rank::checkDrainDone() 1547{ 1548 // if this rank was waiting to drain it is now able to proceed to 1549 // precharge 1550 if (refreshState == REF_DRAIN) { 1551 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1552 1553 refreshState = REF_PRE; 1554 1555 // hand control back to the refresh event loop 1556 schedule(refreshEvent, curTick()); 1557 } 1558} 1559 1560void 1561DRAMCtrl::Rank::processActivateEvent() 1562{ 1563 // we should transition to the active state as soon as any bank is active 1564 if (pwrState != PWR_ACT) 1565 // note that at this point numBanksActive could be back at 1566 // zero again due to a precharge scheduled in the future 1567 schedulePowerEvent(PWR_ACT, curTick()); 1568} 1569 1570void 1571DRAMCtrl::Rank::processPrechargeEvent() 1572{ 1573 // if we reached zero, then special conditions apply as we track 1574 // if all banks are precharged for the power models 1575 if (numBanksActive == 0) { 1576 // we should transition to the idle state when the last bank 1577 // is precharged 1578 schedulePowerEvent(PWR_IDLE, curTick()); 1579 } 1580} 1581 1582void 1583DRAMCtrl::Rank::processRefreshEvent() 1584{ 1585 // when first preparing the refresh, remember when it was due 1586 if (refreshState == REF_IDLE) { 1587 // remember when the refresh is due 1588 refreshDueAt = curTick(); 1589 1590 // proceed to drain 1591 refreshState = REF_DRAIN; 1592 1593 DPRINTF(DRAM, "Refresh due\n"); 1594 } 1595 1596 // let any scheduled read or write to the same rank go ahead, 1597 // after which it will 1598 // hand control back to this event loop 1599 if (refreshState == REF_DRAIN) { 1600 // if a request is at the moment being handled and this request is 1601 // accessing the current rank then wait for it to finish 1602 if ((rank == memory.activeRank) 1603 && (memory.nextReqEvent.scheduled())) { 1604 // hand control over to the request loop until it is 1605 // evaluated next 1606 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1607 1608 return; 1609 } else { 1610 refreshState = REF_PRE; 1611 } 1612 } 1613 1614 // at this point, ensure that all banks are precharged 1615 if (refreshState == REF_PRE) { 1616 // precharge any active bank if we are not already in the idle 1617 // state 1618 if (pwrState != PWR_IDLE) { 1619 // at the moment, we use a precharge all even if there is 1620 // only a single bank open 1621 DPRINTF(DRAM, "Precharging all\n"); 1622 1623 // first determine when we can precharge 1624 Tick pre_at = curTick(); 1625 1626 for (auto &b : banks) { 1627 // respect both causality and any existing bank 1628 // constraints, some banks could already have a 1629 // (auto) precharge scheduled 1630 pre_at = std::max(b.preAllowedAt, pre_at); 1631 } 1632 1633 // make sure all banks per rank are precharged, and for those that 1634 // already are, update their availability 1635 Tick act_allowed_at = pre_at + memory.tRP; 1636 1637 for (auto &b : banks) { 1638 if (b.openRow != Bank::NO_ROW) { 1639 memory.prechargeBank(*this, b, pre_at, false); 1640 } else { 1641 b.actAllowedAt = std::max(b.actAllowedAt, act_allowed_at); 1642 b.preAllowedAt = std::max(b.preAllowedAt, pre_at); 1643 } 1644 } 1645 1646 // precharge all banks in rank 1647 power.powerlib.doCommand(MemCommand::PREA, 0, 1648 divCeil(pre_at, memory.tCK) - 1649 memory.timeStampOffset); 1650 1651 DPRINTF(DRAMPower, "%llu,PREA,0,%d\n", 1652 divCeil(pre_at, memory.tCK) - 1653 memory.timeStampOffset, rank); 1654 } else { 1655 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1656 1657 // go ahead and kick the power state machine into gear if 1658 // we are already idle 1659 schedulePowerEvent(PWR_REF, curTick()); 1660 } 1661 1662 refreshState = REF_RUN; 1663 assert(numBanksActive == 0); 1664 1665 // wait for all banks to be precharged, at which point the 1666 // power state machine will transition to the idle state, and 1667 // automatically move to a refresh, at that point it will also 1668 // call this method to get the refresh event loop going again 1669 return; 1670 } 1671 1672 // last but not least we perform the actual refresh 1673 if (refreshState == REF_RUN) { 1674 // should never get here with any banks active 1675 assert(numBanksActive == 0); 1676 assert(pwrState == PWR_REF); 1677 1678 Tick ref_done_at = curTick() + memory.tRFC; 1679 1680 for (auto &b : banks) { 1681 b.actAllowedAt = ref_done_at; 1682 } 1683 1684 // at the moment this affects all ranks 1685 power.powerlib.doCommand(MemCommand::REF, 0, 1686 divCeil(curTick(), memory.tCK) - 1687 memory.timeStampOffset); 1688 1689 // at the moment sort the list of commands and update the counters 1690 // for DRAMPower libray when doing a refresh 1691 sort(power.powerlib.cmdList.begin(), 1692 power.powerlib.cmdList.end(), DRAMCtrl::sortTime); 1693 1694 // update the counters for DRAMPower, passing false to 1695 // indicate that this is not the last command in the 1696 // list. DRAMPower requires this information for the 1697 // correct calculation of the background energy at the end 1698 // of the simulation. Ideally we would want to call this 1699 // function with true once at the end of the 1700 // simulation. However, the discarded energy is extremly 1701 // small and does not effect the final results. 1702 power.powerlib.updateCounters(false); 1703 1704 // call the energy function 1705 power.powerlib.calcEnergy(); 1706 1707 // Update the stats 1708 updatePowerStats(); 1709 1710 DPRINTF(DRAMPower, "%llu,REF,0,%d\n", divCeil(curTick(), memory.tCK) - 1711 memory.timeStampOffset, rank); 1712 1713 // make sure we did not wait so long that we cannot make up 1714 // for it 1715 if (refreshDueAt + memory.tREFI < ref_done_at) { 1716 fatal("Refresh was delayed so long we cannot catch up\n"); 1717 } 1718 1719 // compensate for the delay in actually performing the refresh 1720 // when scheduling the next one 1721 schedule(refreshEvent, refreshDueAt + memory.tREFI - memory.tRP); 1722 1723 assert(!powerEvent.scheduled()); 1724 1725 // move to the idle power state once the refresh is done, this 1726 // will also move the refresh state machine to the refresh 1727 // idle state 1728 schedulePowerEvent(PWR_IDLE, ref_done_at); 1729 1730 DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 1731 ref_done_at, refreshDueAt + memory.tREFI); 1732 } 1733} 1734 1735void 1736DRAMCtrl::Rank::schedulePowerEvent(PowerState pwr_state, Tick tick) 1737{ 1738 // respect causality 1739 assert(tick >= curTick()); 1740 1741 if (!powerEvent.scheduled()) { 1742 DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 1743 tick, pwr_state); 1744 1745 // insert the new transition 1746 pwrStateTrans = pwr_state; 1747 1748 schedule(powerEvent, tick); 1749 } else { 1750 panic("Scheduled power event at %llu to state %d, " 1751 "with scheduled event at %llu to %d\n", tick, pwr_state, 1752 powerEvent.when(), pwrStateTrans); 1753 } 1754} 1755 1756void 1757DRAMCtrl::Rank::processPowerEvent() 1758{ 1759 // remember where we were, and for how long 1760 Tick duration = curTick() - pwrStateTick; 1761 PowerState prev_state = pwrState; 1762 1763 // update the accounting 1764 pwrStateTime[prev_state] += duration; 1765 1766 pwrState = pwrStateTrans; 1767 pwrStateTick = curTick(); 1768 1769 if (pwrState == PWR_IDLE) { 1770 DPRINTF(DRAMState, "All banks precharged\n"); 1771 1772 // if we were refreshing, make sure we start scheduling requests again 1773 if (prev_state == PWR_REF) { 1774 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 1775 assert(pwrState == PWR_IDLE); 1776 1777 // kick things into action again 1778 refreshState = REF_IDLE; 1779 // a request event could be already scheduled by the state 1780 // machine of the other rank 1781 if (!memory.nextReqEvent.scheduled()) 1782 schedule(memory.nextReqEvent, curTick()); 1783 } else { 1784 assert(prev_state == PWR_ACT); 1785 1786 // if we have a pending refresh, and are now moving to 1787 // the idle state, direclty transition to a refresh 1788 if (refreshState == REF_RUN) { 1789 // there should be nothing waiting at this point 1790 assert(!powerEvent.scheduled()); 1791 1792 // update the state in zero time and proceed below 1793 pwrState = PWR_REF; 1794 } 1795 } 1796 } 1797 1798 // we transition to the refresh state, let the refresh state 1799 // machine know of this state update and let it deal with the 1800 // scheduling of the next power state transition as well as the 1801 // following refresh 1802 if (pwrState == PWR_REF) { 1803 DPRINTF(DRAMState, "Refreshing\n"); 1804 // kick the refresh event loop into action again, and that 1805 // in turn will schedule a transition to the idle power 1806 // state once the refresh is done 1807 assert(refreshState == REF_RUN); 1808 processRefreshEvent(); 1809 } 1810} 1811 1812void 1813DRAMCtrl::Rank::updatePowerStats() 1814{ 1815 // Get the energy and power from DRAMPower 1816 Data::MemoryPowerModel::Energy energy = 1817 power.powerlib.getEnergy(); 1818 Data::MemoryPowerModel::Power rank_power = 1819 power.powerlib.getPower(); 1820 1821 actEnergy = energy.act_energy * memory.devicesPerRank; 1822 preEnergy = energy.pre_energy * memory.devicesPerRank; 1823 readEnergy = energy.read_energy * memory.devicesPerRank; 1824 writeEnergy = energy.write_energy * memory.devicesPerRank; 1825 refreshEnergy = energy.ref_energy * memory.devicesPerRank; 1826 actBackEnergy = energy.act_stdby_energy * memory.devicesPerRank; 1827 preBackEnergy = energy.pre_stdby_energy * memory.devicesPerRank; 1828 totalEnergy = energy.total_energy * memory.devicesPerRank; 1829 averagePower = rank_power.average_power * memory.devicesPerRank; 1830} 1831 1832void 1833DRAMCtrl::Rank::regStats() 1834{ 1835 using namespace Stats; 1836 1837 pwrStateTime 1838 .init(5) 1839 .name(name() + ".memoryStateTime") 1840 .desc("Time in different power states"); 1841 pwrStateTime.subname(0, "IDLE"); 1842 pwrStateTime.subname(1, "REF"); 1843 pwrStateTime.subname(2, "PRE_PDN"); 1844 pwrStateTime.subname(3, "ACT"); 1845 pwrStateTime.subname(4, "ACT_PDN"); 1846 1847 actEnergy 1848 .name(name() + ".actEnergy") 1849 .desc("Energy for activate commands per rank (pJ)"); 1850 1851 preEnergy 1852 .name(name() + ".preEnergy") 1853 .desc("Energy for precharge commands per rank (pJ)"); 1854 1855 readEnergy 1856 .name(name() + ".readEnergy") 1857 .desc("Energy for read commands per rank (pJ)"); 1858 1859 writeEnergy 1860 .name(name() + ".writeEnergy") 1861 .desc("Energy for write commands per rank (pJ)"); 1862 1863 refreshEnergy 1864 .name(name() + ".refreshEnergy") 1865 .desc("Energy for refresh commands per rank (pJ)"); 1866 1867 actBackEnergy 1868 .name(name() + ".actBackEnergy") 1869 .desc("Energy for active background per rank (pJ)"); 1870 1871 preBackEnergy 1872 .name(name() + ".preBackEnergy") 1873 .desc("Energy for precharge background per rank (pJ)"); 1874 1875 totalEnergy 1876 .name(name() + ".totalEnergy") 1877 .desc("Total energy per rank (pJ)"); 1878 1879 averagePower 1880 .name(name() + ".averagePower") 1881 .desc("Core power per rank (mW)"); 1882} 1883void 1884DRAMCtrl::regStats() 1885{ 1886 using namespace Stats; 1887 1888 AbstractMemory::regStats(); 1889 1890 for (auto r : ranks) { 1891 r->regStats(); 1892 } 1893 1894 readReqs 1895 .name(name() + ".readReqs") 1896 .desc("Number of read requests accepted"); 1897 1898 writeReqs 1899 .name(name() + ".writeReqs") 1900 .desc("Number of write requests accepted"); 1901 1902 readBursts 1903 .name(name() + ".readBursts") 1904 .desc("Number of DRAM read bursts, " 1905 "including those serviced by the write queue"); 1906 1907 writeBursts 1908 .name(name() + ".writeBursts") 1909 .desc("Number of DRAM write bursts, " 1910 "including those merged in the write queue"); 1911 1912 servicedByWrQ 1913 .name(name() + ".servicedByWrQ") 1914 .desc("Number of DRAM read bursts serviced by the write queue"); 1915 1916 mergedWrBursts 1917 .name(name() + ".mergedWrBursts") 1918 .desc("Number of DRAM write bursts merged with an existing one"); 1919 1920 neitherReadNorWrite 1921 .name(name() + ".neitherReadNorWriteReqs") 1922 .desc("Number of requests that are neither read nor write"); 1923 1924 perBankRdBursts 1925 .init(banksPerRank * ranksPerChannel) 1926 .name(name() + ".perBankRdBursts") 1927 .desc("Per bank write bursts"); 1928 1929 perBankWrBursts 1930 .init(banksPerRank * ranksPerChannel) 1931 .name(name() + ".perBankWrBursts") 1932 .desc("Per bank write bursts"); 1933 1934 avgRdQLen 1935 .name(name() + ".avgRdQLen") 1936 .desc("Average read queue length when enqueuing") 1937 .precision(2); 1938 1939 avgWrQLen 1940 .name(name() + ".avgWrQLen") 1941 .desc("Average write queue length when enqueuing") 1942 .precision(2); 1943 1944 totQLat 1945 .name(name() + ".totQLat") 1946 .desc("Total ticks spent queuing"); 1947 1948 totBusLat 1949 .name(name() + ".totBusLat") 1950 .desc("Total ticks spent in databus transfers"); 1951 1952 totMemAccLat 1953 .name(name() + ".totMemAccLat") 1954 .desc("Total ticks spent from burst creation until serviced " 1955 "by the DRAM"); 1956 1957 avgQLat 1958 .name(name() + ".avgQLat") 1959 .desc("Average queueing delay per DRAM burst") 1960 .precision(2); 1961 1962 avgQLat = totQLat / (readBursts - servicedByWrQ); 1963 1964 avgBusLat 1965 .name(name() + ".avgBusLat") 1966 .desc("Average bus latency per DRAM burst") 1967 .precision(2); 1968 1969 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1970 1971 avgMemAccLat 1972 .name(name() + ".avgMemAccLat") 1973 .desc("Average memory access latency per DRAM burst") 1974 .precision(2); 1975 1976 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1977 1978 numRdRetry 1979 .name(name() + ".numRdRetry") 1980 .desc("Number of times read queue was full causing retry"); 1981 1982 numWrRetry 1983 .name(name() + ".numWrRetry") 1984 .desc("Number of times write queue was full causing retry"); 1985 1986 readRowHits 1987 .name(name() + ".readRowHits") 1988 .desc("Number of row buffer hits during reads"); 1989 1990 writeRowHits 1991 .name(name() + ".writeRowHits") 1992 .desc("Number of row buffer hits during writes"); 1993 1994 readRowHitRate 1995 .name(name() + ".readRowHitRate") 1996 .desc("Row buffer hit rate for reads") 1997 .precision(2); 1998 1999 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 2000 2001 writeRowHitRate 2002 .name(name() + ".writeRowHitRate") 2003 .desc("Row buffer hit rate for writes") 2004 .precision(2); 2005 2006 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 2007 2008 readPktSize 2009 .init(ceilLog2(burstSize) + 1) 2010 .name(name() + ".readPktSize") 2011 .desc("Read request sizes (log2)"); 2012 2013 writePktSize 2014 .init(ceilLog2(burstSize) + 1) 2015 .name(name() + ".writePktSize") 2016 .desc("Write request sizes (log2)"); 2017 2018 rdQLenPdf 2019 .init(readBufferSize) 2020 .name(name() + ".rdQLenPdf") 2021 .desc("What read queue length does an incoming req see"); 2022 2023 wrQLenPdf 2024 .init(writeBufferSize) 2025 .name(name() + ".wrQLenPdf") 2026 .desc("What write queue length does an incoming req see"); 2027 2028 bytesPerActivate 2029 .init(maxAccessesPerRow) 2030 .name(name() + ".bytesPerActivate") 2031 .desc("Bytes accessed per row activation") 2032 .flags(nozero); 2033 2034 rdPerTurnAround 2035 .init(readBufferSize) 2036 .name(name() + ".rdPerTurnAround") 2037 .desc("Reads before turning the bus around for writes") 2038 .flags(nozero); 2039 2040 wrPerTurnAround 2041 .init(writeBufferSize) 2042 .name(name() + ".wrPerTurnAround") 2043 .desc("Writes before turning the bus around for reads") 2044 .flags(nozero); 2045 2046 bytesReadDRAM 2047 .name(name() + ".bytesReadDRAM") 2048 .desc("Total number of bytes read from DRAM"); 2049 2050 bytesReadWrQ 2051 .name(name() + ".bytesReadWrQ") 2052 .desc("Total number of bytes read from write queue"); 2053 2054 bytesWritten 2055 .name(name() + ".bytesWritten") 2056 .desc("Total number of bytes written to DRAM"); 2057 2058 bytesReadSys 2059 .name(name() + ".bytesReadSys") 2060 .desc("Total read bytes from the system interface side"); 2061 2062 bytesWrittenSys 2063 .name(name() + ".bytesWrittenSys") 2064 .desc("Total written bytes from the system interface side"); 2065 2066 avgRdBW 2067 .name(name() + ".avgRdBW") 2068 .desc("Average DRAM read bandwidth in MiByte/s") 2069 .precision(2); 2070 2071 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 2072 2073 avgWrBW 2074 .name(name() + ".avgWrBW") 2075 .desc("Average achieved write bandwidth in MiByte/s") 2076 .precision(2); 2077 2078 avgWrBW = (bytesWritten / 1000000) / simSeconds; 2079 2080 avgRdBWSys 2081 .name(name() + ".avgRdBWSys") 2082 .desc("Average system read bandwidth in MiByte/s") 2083 .precision(2); 2084 2085 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 2086 2087 avgWrBWSys 2088 .name(name() + ".avgWrBWSys") 2089 .desc("Average system write bandwidth in MiByte/s") 2090 .precision(2); 2091 2092 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 2093 2094 peakBW 2095 .name(name() + ".peakBW") 2096 .desc("Theoretical peak bandwidth in MiByte/s") 2097 .precision(2); 2098 2099 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 2100 2101 busUtil 2102 .name(name() + ".busUtil") 2103 .desc("Data bus utilization in percentage") 2104 .precision(2); 2105 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 2106 2107 totGap 2108 .name(name() + ".totGap") 2109 .desc("Total gap between requests"); 2110 2111 avgGap 2112 .name(name() + ".avgGap") 2113 .desc("Average gap between requests") 2114 .precision(2); 2115 2116 avgGap = totGap / (readReqs + writeReqs); 2117 2118 // Stats for DRAM Power calculation based on Micron datasheet 2119 busUtilRead 2120 .name(name() + ".busUtilRead") 2121 .desc("Data bus utilization in percentage for reads") 2122 .precision(2); 2123 2124 busUtilRead = avgRdBW / peakBW * 100; 2125 2126 busUtilWrite 2127 .name(name() + ".busUtilWrite") 2128 .desc("Data bus utilization in percentage for writes") 2129 .precision(2); 2130 2131 busUtilWrite = avgWrBW / peakBW * 100; 2132 2133 pageHitRate 2134 .name(name() + ".pageHitRate") 2135 .desc("Row buffer hit rate, read and write combined") 2136 .precision(2); 2137 2138 pageHitRate = (writeRowHits + readRowHits) / 2139 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 2140} 2141 2142void 2143DRAMCtrl::recvFunctional(PacketPtr pkt) 2144{ 2145 // rely on the abstract memory 2146 functionalAccess(pkt); 2147} 2148 2149BaseSlavePort& 2150DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 2151{ 2152 if (if_name != "port") { 2153 return MemObject::getSlavePort(if_name, idx); 2154 } else { 2155 return port; 2156 } 2157} 2158 2159DrainState 2160DRAMCtrl::drain() 2161{ 2162 // if there is anything in any of our internal queues, keep track 2163 // of that as well 2164 if (!(writeQueue.empty() && readQueue.empty() && respQueue.empty())) { 2165 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 2166 " resp: %d\n", writeQueue.size(), readQueue.size(), 2167 respQueue.size()); 2168 2169 // the only part that is not drained automatically over time 2170 // is the write queue, thus kick things into action if needed 2171 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 2172 schedule(nextReqEvent, curTick()); 2173 } 2174 return DrainState::Draining; 2175 } else { 2176 return DrainState::Drained; 2177 } 2178} 2179 2180void 2181DRAMCtrl::drainResume() 2182{ 2183 if (!isTimingMode && system()->isTimingMode()) { 2184 // if we switched to timing mode, kick things into action, 2185 // and behave as if we restored from a checkpoint 2186 startup(); 2187 } else if (isTimingMode && !system()->isTimingMode()) { 2188 // if we switch from timing mode, stop the refresh events to 2189 // not cause issues with KVM 2190 for (auto r : ranks) { 2191 r->suspend(); 2192 } 2193 } 2194 2195 // update the mode 2196 isTimingMode = system()->isTimingMode(); 2197} 2198 2199DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 2200 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 2201 memory(_memory) 2202{ } 2203 2204AddrRangeList 2205DRAMCtrl::MemoryPort::getAddrRanges() const 2206{ 2207 AddrRangeList ranges; 2208 ranges.push_back(memory.getAddrRange()); 2209 return ranges; 2210} 2211 2212void 2213DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 2214{ 2215 pkt->pushLabel(memory.name()); 2216 2217 if (!queue.checkFunctional(pkt)) { 2218 // Default implementation of SimpleTimingPort::recvFunctional() 2219 // calls recvAtomic() and throws away the latency; we can save a 2220 // little here by just not calculating the latency. 2221 memory.recvFunctional(pkt); 2222 } 2223 2224 pkt->popLabel(); 2225} 2226 2227Tick 2228DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 2229{ 2230 return memory.recvAtomic(pkt); 2231} 2232 2233bool 2234DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 2235{ 2236 // pass it to the memory controller 2237 return memory.recvTimingReq(pkt); 2238} 2239 2240DRAMCtrl* 2241DRAMCtrlParams::create() 2242{ 2243 return new DRAMCtrl(this); 2244} 2245