dram_ctrl.cc revision 10286
114039Sstacze01@arm.com/* 214039Sstacze01@arm.com * Copyright (c) 2010-2014 ARM Limited 314039Sstacze01@arm.com * All rights reserved 414039Sstacze01@arm.com * 514039Sstacze01@arm.com * The license below extends only to copyright in the software and shall 614039Sstacze01@arm.com * not be construed as granting a license to any other intellectual 714039Sstacze01@arm.com * property including but not limited to intellectual property relating 814039Sstacze01@arm.com * to a hardware implementation of the functionality of the software 914039Sstacze01@arm.com * licensed hereunder. You may use the software subject to the license 1014039Sstacze01@arm.com * terms below provided that you ensure that this notice is replicated 1114039Sstacze01@arm.com * unmodified and in its entirety in all distributions of the software, 1214039Sstacze01@arm.com * modified or unmodified, in source code or in binary form. 1314039Sstacze01@arm.com * 1414039Sstacze01@arm.com * Copyright (c) 2013 Amin Farmahini-Farahani 1514039Sstacze01@arm.com * All rights reserved. 1614039Sstacze01@arm.com * 1714039Sstacze01@arm.com * Redistribution and use in source and binary forms, with or without 1814039Sstacze01@arm.com * modification, are permitted provided that the following conditions are 1914039Sstacze01@arm.com * met: redistributions of source code must retain the above copyright 2014039Sstacze01@arm.com * notice, this list of conditions and the following disclaimer; 2114039Sstacze01@arm.com * redistributions in binary form must reproduce the above copyright 2214039Sstacze01@arm.com * notice, this list of conditions and the following disclaimer in the 2314039Sstacze01@arm.com * documentation and/or other materials provided with the distribution; 2414039Sstacze01@arm.com * neither the name of the copyright holders nor the names of its 2514039Sstacze01@arm.com * contributors may be used to endorse or promote products derived from 2614039Sstacze01@arm.com * this software without specific prior written permission. 2714039Sstacze01@arm.com * 2814039Sstacze01@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2914039Sstacze01@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3014039Sstacze01@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3114039Sstacze01@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3214039Sstacze01@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3314039Sstacze01@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3414039Sstacze01@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3514039Sstacze01@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3614039Sstacze01@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3714039Sstacze01@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3814039Sstacze01@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3914039Sstacze01@arm.com * 4014039Sstacze01@arm.com * Authors: Andreas Hansson 4114039Sstacze01@arm.com * Ani Udipi 4214039Sstacze01@arm.com * Neha Agarwal 4314039Sstacze01@arm.com */ 4414039Sstacze01@arm.com 4514039Sstacze01@arm.com#include "base/bitfield.hh" 4614039Sstacze01@arm.com#include "base/trace.hh" 4714039Sstacze01@arm.com#include "debug/DRAM.hh" 4814039Sstacze01@arm.com#include "debug/DRAMPower.hh" 4914039Sstacze01@arm.com#include "debug/DRAMState.hh" 5014039Sstacze01@arm.com#include "debug/Drain.hh" 5114039Sstacze01@arm.com#include "mem/dram_ctrl.hh" 5214039Sstacze01@arm.com#include "sim/system.hh" 5314039Sstacze01@arm.com 5414039Sstacze01@arm.comusing namespace std; 5514039Sstacze01@arm.com 5614039Sstacze01@arm.comDRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 5714039Sstacze01@arm.com AbstractMemory(p), 5814039Sstacze01@arm.com port(name() + ".port", *this), 5914039Sstacze01@arm.com retryRdReq(false), retryWrReq(false), 6014039Sstacze01@arm.com busState(READ), 6114039Sstacze01@arm.com nextReqEvent(this), respondEvent(this), activateEvent(this), 6214039Sstacze01@arm.com prechargeEvent(this), refreshEvent(this), powerEvent(this), 6314039Sstacze01@arm.com drainManager(NULL), 6414039Sstacze01@arm.com deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 6514039Sstacze01@arm.com deviceRowBufferSize(p->device_rowbuffer_size), 6614039Sstacze01@arm.com devicesPerRank(p->devices_per_rank), 6714039Sstacze01@arm.com burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 6814039Sstacze01@arm.com rowBufferSize(devicesPerRank * deviceRowBufferSize), 6914039Sstacze01@arm.com columnsPerRowBuffer(rowBufferSize / burstSize), 7014039Sstacze01@arm.com columnsPerStripe(range.granularity() / burstSize), 7114039Sstacze01@arm.com ranksPerChannel(p->ranks_per_channel), 7214039Sstacze01@arm.com banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 7314039Sstacze01@arm.com readBufferSize(p->read_buffer_size), 7414039Sstacze01@arm.com writeBufferSize(p->write_buffer_size), 7514039Sstacze01@arm.com writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 7614039Sstacze01@arm.com writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 7714039Sstacze01@arm.com minWritesPerSwitch(p->min_writes_per_switch), 7814039Sstacze01@arm.com writesThisTime(0), readsThisTime(0), 7914039Sstacze01@arm.com tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), 8014039Sstacze01@arm.com tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR), 8114039Sstacze01@arm.com tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 8214039Sstacze01@arm.com tXAW(p->tXAW), activationLimit(p->activation_limit), 8314039Sstacze01@arm.com memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 8414039Sstacze01@arm.com pageMgmt(p->page_policy), 8514039Sstacze01@arm.com maxAccessesPerRow(p->max_accesses_per_row), 8614039Sstacze01@arm.com frontendLatency(p->static_frontend_latency), 8714039Sstacze01@arm.com backendLatency(p->static_backend_latency), 8814039Sstacze01@arm.com busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), 8914039Sstacze01@arm.com pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), prevArrival(0), 9014039Sstacze01@arm.com nextReqTime(0), pwrStateTick(0), numBanksActive(0) 9114039Sstacze01@arm.com{ 9214039Sstacze01@arm.com // create the bank states based on the dimensions of the ranks and 9314039Sstacze01@arm.com // banks 9414039Sstacze01@arm.com banks.resize(ranksPerChannel); 9514039Sstacze01@arm.com actTicks.resize(ranksPerChannel); 9614039Sstacze01@arm.com for (size_t c = 0; c < ranksPerChannel; ++c) { 9714039Sstacze01@arm.com banks[c].resize(banksPerRank); 9814039Sstacze01@arm.com actTicks[c].resize(activationLimit, 0); 9914039Sstacze01@arm.com } 10014039Sstacze01@arm.com 10114039Sstacze01@arm.com // set the bank indices 10214039Sstacze01@arm.com for (int r = 0; r < ranksPerChannel; r++) { 10314039Sstacze01@arm.com for (int b = 0; b < banksPerRank; b++) { 10414039Sstacze01@arm.com banks[r][b].rank = r; 10514039Sstacze01@arm.com banks[r][b].bank = b; 10614039Sstacze01@arm.com } 10714039Sstacze01@arm.com } 10814039Sstacze01@arm.com 10914039Sstacze01@arm.com // perform a basic check of the write thresholds 11014039Sstacze01@arm.com if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 11114039Sstacze01@arm.com fatal("Write buffer low threshold %d must be smaller than the " 11214039Sstacze01@arm.com "high threshold %d\n", p->write_low_thresh_perc, 11314039Sstacze01@arm.com p->write_high_thresh_perc); 11414039Sstacze01@arm.com 11514039Sstacze01@arm.com // determine the rows per bank by looking at the total capacity 11614039Sstacze01@arm.com uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 11714039Sstacze01@arm.com 11814039Sstacze01@arm.com DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 11914039Sstacze01@arm.com AbstractMemory::size()); 12014039Sstacze01@arm.com 12114039Sstacze01@arm.com DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 12214039Sstacze01@arm.com rowBufferSize, columnsPerRowBuffer); 12314039Sstacze01@arm.com 12414039Sstacze01@arm.com rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 12514039Sstacze01@arm.com 12614039Sstacze01@arm.com // a bit of sanity checks on the interleaving 12714039Sstacze01@arm.com if (range.interleaved()) { 12814039Sstacze01@arm.com if (channels != range.stripes()) 12914039Sstacze01@arm.com fatal("%s has %d interleaved address stripes but %d channel(s)\n", 13014039Sstacze01@arm.com name(), range.stripes(), channels); 13114039Sstacze01@arm.com 13214039Sstacze01@arm.com if (addrMapping == Enums::RoRaBaChCo) { 13314039Sstacze01@arm.com if (rowBufferSize != range.granularity()) { 13414039Sstacze01@arm.com fatal("Channel interleaving of %s doesn't match RoRaBaChCo " 13514039Sstacze01@arm.com "address map\n", name()); 13614039Sstacze01@arm.com } 13714039Sstacze01@arm.com } else if (addrMapping == Enums::RoRaBaCoCh || 13814039Sstacze01@arm.com addrMapping == Enums::RoCoRaBaCh) { 13914039Sstacze01@arm.com // for the interleavings with channel bits in the bottom, 14014039Sstacze01@arm.com // if the system uses a channel striping granularity that 14114039Sstacze01@arm.com // is larger than the DRAM burst size, then map the 14214039Sstacze01@arm.com // sequential accesses within a stripe to a number of 14314039Sstacze01@arm.com // columns in the DRAM, effectively placing some of the 14414039Sstacze01@arm.com // lower-order column bits as the least-significant bits 14514039Sstacze01@arm.com // of the address (above the ones denoting the burst size) 14614039Sstacze01@arm.com assert(columnsPerStripe >= 1); 14714039Sstacze01@arm.com 14814039Sstacze01@arm.com // channel striping has to be done at a granularity that 14914039Sstacze01@arm.com // is equal or larger to a cache line 15014039Sstacze01@arm.com if (system()->cacheLineSize() > range.granularity()) { 15114039Sstacze01@arm.com fatal("Channel interleaving of %s must be at least as large " 15214039Sstacze01@arm.com "as the cache line size\n", name()); 15314039Sstacze01@arm.com } 15414039Sstacze01@arm.com 15514039Sstacze01@arm.com // ...and equal or smaller than the row-buffer size 15614039Sstacze01@arm.com if (rowBufferSize < range.granularity()) { 15714039Sstacze01@arm.com fatal("Channel interleaving of %s must be at most as large " 15814039Sstacze01@arm.com "as the row-buffer size\n", name()); 15914039Sstacze01@arm.com } 16014039Sstacze01@arm.com // this is essentially the check above, so just to be sure 16114039Sstacze01@arm.com assert(columnsPerStripe <= columnsPerRowBuffer); 16214039Sstacze01@arm.com } 16314039Sstacze01@arm.com } 16414039Sstacze01@arm.com 16514039Sstacze01@arm.com // some basic sanity checks 16614039Sstacze01@arm.com if (tREFI <= tRP || tREFI <= tRFC) { 16714039Sstacze01@arm.com fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 16814039Sstacze01@arm.com tREFI, tRP, tRFC); 16914039Sstacze01@arm.com } 17014039Sstacze01@arm.com} 17114039Sstacze01@arm.com 17214039Sstacze01@arm.comvoid 17314039Sstacze01@arm.comDRAMCtrl::init() 17414039Sstacze01@arm.com{ 17514039Sstacze01@arm.com if (!port.isConnected()) { 17614039Sstacze01@arm.com fatal("DRAMCtrl %s is unconnected!\n", name()); 17714039Sstacze01@arm.com } else { 17814039Sstacze01@arm.com port.sendRangeChange(); 17914039Sstacze01@arm.com } 18014039Sstacze01@arm.com} 18114039Sstacze01@arm.com 18214039Sstacze01@arm.comvoid 18314039Sstacze01@arm.comDRAMCtrl::startup() 18414039Sstacze01@arm.com{ 18514039Sstacze01@arm.com // update the start tick for the precharge accounting to the 18614039Sstacze01@arm.com // current tick 18714039Sstacze01@arm.com pwrStateTick = curTick(); 18814039Sstacze01@arm.com 18914039Sstacze01@arm.com // shift the bus busy time sufficiently far ahead that we never 19014039Sstacze01@arm.com // have to worry about negative values when computing the time for 19114039Sstacze01@arm.com // the next request, this will add an insignificant bubble at the 19214039Sstacze01@arm.com // start of simulation 19314039Sstacze01@arm.com busBusyUntil = curTick() + tRP + tRCD + tCL; 19414039Sstacze01@arm.com 19514039Sstacze01@arm.com // kick off the refresh, and give ourselves enough time to 19614039Sstacze01@arm.com // precharge 19714039Sstacze01@arm.com schedule(refreshEvent, curTick() + tREFI - tRP); 19814039Sstacze01@arm.com} 19914039Sstacze01@arm.com 20014039Sstacze01@arm.comTick 20114039Sstacze01@arm.comDRAMCtrl::recvAtomic(PacketPtr pkt) 20214039Sstacze01@arm.com{ 20314039Sstacze01@arm.com DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 20414039Sstacze01@arm.com 20514039Sstacze01@arm.com // do the actual memory access and turn the packet into a response 20614039Sstacze01@arm.com access(pkt); 20714039Sstacze01@arm.com 20814039Sstacze01@arm.com Tick latency = 0; 20914039Sstacze01@arm.com if (!pkt->memInhibitAsserted() && pkt->hasData()) { 21014039Sstacze01@arm.com // this value is not supposed to be accurate, just enough to 21114039Sstacze01@arm.com // keep things going, mimic a closed page 21214039Sstacze01@arm.com latency = tRP + tRCD + tCL; 21314039Sstacze01@arm.com } 21414039Sstacze01@arm.com return latency; 21514039Sstacze01@arm.com} 21614039Sstacze01@arm.com 21714039Sstacze01@arm.combool 21814039Sstacze01@arm.comDRAMCtrl::readQueueFull(unsigned int neededEntries) const 21914039Sstacze01@arm.com{ 22014039Sstacze01@arm.com DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 22114039Sstacze01@arm.com readBufferSize, readQueue.size() + respQueue.size(), 22214039Sstacze01@arm.com neededEntries); 22314039Sstacze01@arm.com 22414039Sstacze01@arm.com return 22514039Sstacze01@arm.com (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 22614039Sstacze01@arm.com} 22714039Sstacze01@arm.com 22814039Sstacze01@arm.combool 22914039Sstacze01@arm.comDRAMCtrl::writeQueueFull(unsigned int neededEntries) const 23014039Sstacze01@arm.com{ 23114039Sstacze01@arm.com DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 23214039Sstacze01@arm.com writeBufferSize, writeQueue.size(), neededEntries); 23314039Sstacze01@arm.com return (writeQueue.size() + neededEntries) > writeBufferSize; 23414039Sstacze01@arm.com} 23514039Sstacze01@arm.com 23614039Sstacze01@arm.comDRAMCtrl::DRAMPacket* 23714039Sstacze01@arm.comDRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 23814039Sstacze01@arm.com bool isRead) 23914039Sstacze01@arm.com{ 24014039Sstacze01@arm.com // decode the address based on the address mapping scheme, with 24114039Sstacze01@arm.com // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 24214039Sstacze01@arm.com // channel, respectively 24314039Sstacze01@arm.com uint8_t rank; 24414039Sstacze01@arm.com uint8_t bank; 24514039Sstacze01@arm.com // use a 64-bit unsigned during the computations as the row is 24614039Sstacze01@arm.com // always the top bits, and check before creating the DRAMPacket 24714039Sstacze01@arm.com uint64_t row; 24814039Sstacze01@arm.com 24914039Sstacze01@arm.com // truncate the address to a DRAM burst, which makes it unique to 25014039Sstacze01@arm.com // a specific column, row, bank, rank and channel 25114039Sstacze01@arm.com Addr addr = dramPktAddr / burstSize; 25214039Sstacze01@arm.com 25314039Sstacze01@arm.com // we have removed the lowest order address bits that denote the 25414039Sstacze01@arm.com // position within the column 25514039Sstacze01@arm.com if (addrMapping == Enums::RoRaBaChCo) { 25614039Sstacze01@arm.com // the lowest order bits denote the column to ensure that 25714039Sstacze01@arm.com // sequential cache lines occupy the same row 25814039Sstacze01@arm.com addr = addr / columnsPerRowBuffer; 25914039Sstacze01@arm.com 26014039Sstacze01@arm.com // take out the channel part of the address 26114039Sstacze01@arm.com addr = addr / channels; 26214039Sstacze01@arm.com 26314039Sstacze01@arm.com // after the channel bits, get the bank bits to interleave 26414039Sstacze01@arm.com // over the banks 26514039Sstacze01@arm.com bank = addr % banksPerRank; 26614039Sstacze01@arm.com addr = addr / banksPerRank; 26714039Sstacze01@arm.com 26814039Sstacze01@arm.com // after the bank, we get the rank bits which thus interleaves 26914039Sstacze01@arm.com // over the ranks 27014039Sstacze01@arm.com rank = addr % ranksPerChannel; 27114039Sstacze01@arm.com addr = addr / ranksPerChannel; 27214039Sstacze01@arm.com 27314039Sstacze01@arm.com // lastly, get the row bits 27414039Sstacze01@arm.com row = addr % rowsPerBank; 27514039Sstacze01@arm.com addr = addr / rowsPerBank; 27614039Sstacze01@arm.com } else if (addrMapping == Enums::RoRaBaCoCh) { 27714039Sstacze01@arm.com // take out the lower-order column bits 27814039Sstacze01@arm.com addr = addr / columnsPerStripe; 27914039Sstacze01@arm.com 28014039Sstacze01@arm.com // take out the channel part of the address 28114039Sstacze01@arm.com addr = addr / channels; 28214039Sstacze01@arm.com 28314039Sstacze01@arm.com // next, the higher-order column bites 28414039Sstacze01@arm.com addr = addr / (columnsPerRowBuffer / columnsPerStripe); 28514039Sstacze01@arm.com 28614039Sstacze01@arm.com // after the column bits, we get the bank bits to interleave 28714039Sstacze01@arm.com // over the banks 28814039Sstacze01@arm.com bank = addr % banksPerRank; 28914039Sstacze01@arm.com addr = addr / banksPerRank; 29014039Sstacze01@arm.com 29114039Sstacze01@arm.com // after the bank, we get the rank bits which thus interleaves 29214039Sstacze01@arm.com // over the ranks 29314039Sstacze01@arm.com rank = addr % ranksPerChannel; 29414039Sstacze01@arm.com addr = addr / ranksPerChannel; 29514039Sstacze01@arm.com 29614039Sstacze01@arm.com // lastly, get the row bits 29714039Sstacze01@arm.com row = addr % rowsPerBank; 29814039Sstacze01@arm.com addr = addr / rowsPerBank; 29914039Sstacze01@arm.com } else if (addrMapping == Enums::RoCoRaBaCh) { 30014039Sstacze01@arm.com // optimise for closed page mode and utilise maximum 30114039Sstacze01@arm.com // parallelism of the DRAM (at the cost of power) 30214039Sstacze01@arm.com 30314039Sstacze01@arm.com // take out the lower-order column bits 30414039Sstacze01@arm.com addr = addr / columnsPerStripe; 30514039Sstacze01@arm.com 30614039Sstacze01@arm.com // take out the channel part of the address, not that this has 30714039Sstacze01@arm.com // to match with how accesses are interleaved between the 30814039Sstacze01@arm.com // controllers in the address mapping 30914039Sstacze01@arm.com addr = addr / channels; 31014039Sstacze01@arm.com 31114039Sstacze01@arm.com // start with the bank bits, as this provides the maximum 31214039Sstacze01@arm.com // opportunity for parallelism between requests 31314039Sstacze01@arm.com bank = addr % banksPerRank; 31414039Sstacze01@arm.com addr = addr / banksPerRank; 31514039Sstacze01@arm.com 31614039Sstacze01@arm.com // next get the rank bits 31714039Sstacze01@arm.com rank = addr % ranksPerChannel; 31814039Sstacze01@arm.com addr = addr / ranksPerChannel; 31914039Sstacze01@arm.com 32014039Sstacze01@arm.com // next, the higher-order column bites 32114039Sstacze01@arm.com addr = addr / (columnsPerRowBuffer / columnsPerStripe); 32214039Sstacze01@arm.com 32314039Sstacze01@arm.com // lastly, get the row bits 32414039Sstacze01@arm.com row = addr % rowsPerBank; 32514039Sstacze01@arm.com addr = addr / rowsPerBank; 32614039Sstacze01@arm.com } else 32714039Sstacze01@arm.com panic("Unknown address mapping policy chosen!"); 32814039Sstacze01@arm.com 32914039Sstacze01@arm.com assert(rank < ranksPerChannel); 33014039Sstacze01@arm.com assert(bank < banksPerRank); 33114039Sstacze01@arm.com assert(row < rowsPerBank); 33214039Sstacze01@arm.com assert(row < Bank::NO_ROW); 33314039Sstacze01@arm.com 33414039Sstacze01@arm.com DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 33514039Sstacze01@arm.com dramPktAddr, rank, bank, row); 33614039Sstacze01@arm.com 33714039Sstacze01@arm.com // create the corresponding DRAM packet with the entry time and 33814039Sstacze01@arm.com // ready time set to the current tick, the latter will be updated 33914039Sstacze01@arm.com // later 34014039Sstacze01@arm.com uint16_t bank_id = banksPerRank * rank + bank; 34114039Sstacze01@arm.com return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 34214039Sstacze01@arm.com size, banks[rank][bank]); 34314039Sstacze01@arm.com} 34414039Sstacze01@arm.com 34514039Sstacze01@arm.comvoid 34614039Sstacze01@arm.comDRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 34714039Sstacze01@arm.com{ 34814039Sstacze01@arm.com // only add to the read queue here. whenever the request is 34914039Sstacze01@arm.com // eventually done, set the readyTime, and call schedule() 35014039Sstacze01@arm.com assert(!pkt->isWrite()); 35114039Sstacze01@arm.com 35214039Sstacze01@arm.com assert(pktCount != 0); 35314039Sstacze01@arm.com 35414039Sstacze01@arm.com // if the request size is larger than burst size, the pkt is split into 35514039Sstacze01@arm.com // multiple DRAM packets 35614039Sstacze01@arm.com // Note if the pkt starting address is not aligened to burst size, the 35714039Sstacze01@arm.com // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 35814039Sstacze01@arm.com // are aligned to burst size boundaries. This is to ensure we accurately 35914039Sstacze01@arm.com // check read packets against packets in write queue. 36014039Sstacze01@arm.com Addr addr = pkt->getAddr(); 36114039Sstacze01@arm.com unsigned pktsServicedByWrQ = 0; 36214039Sstacze01@arm.com BurstHelper* burst_helper = NULL; 36314039Sstacze01@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 36414039Sstacze01@arm.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 36514039Sstacze01@arm.com pkt->getAddr() + pkt->getSize()) - addr; 36614039Sstacze01@arm.com readPktSize[ceilLog2(size)]++; 36714039Sstacze01@arm.com readBursts++; 36814039Sstacze01@arm.com 36914039Sstacze01@arm.com // First check write buffer to see if the data is already at 37014039Sstacze01@arm.com // the controller 37114039Sstacze01@arm.com bool foundInWrQ = false; 37214039Sstacze01@arm.com for (auto i = writeQueue.begin(); i != writeQueue.end(); ++i) { 37314039Sstacze01@arm.com // check if the read is subsumed in the write entry we are 37414039Sstacze01@arm.com // looking at 37514039Sstacze01@arm.com if ((*i)->addr <= addr && 37614039Sstacze01@arm.com (addr + size) <= ((*i)->addr + (*i)->size)) { 37714039Sstacze01@arm.com foundInWrQ = true; 37814039Sstacze01@arm.com servicedByWrQ++; 37914039Sstacze01@arm.com pktsServicedByWrQ++; 38014039Sstacze01@arm.com DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 38114039Sstacze01@arm.com "write queue\n", addr, size); 38214039Sstacze01@arm.com bytesReadWrQ += burstSize; 38314039Sstacze01@arm.com break; 38414039Sstacze01@arm.com } 38514039Sstacze01@arm.com } 38614116Sgiacomo.travaglini@arm.com 38714039Sstacze01@arm.com // If not found in the write q, make a DRAM packet and 38814039Sstacze01@arm.com // push it onto the read queue 38914039Sstacze01@arm.com if (!foundInWrQ) { 39014039Sstacze01@arm.com 39114039Sstacze01@arm.com // Make the burst helper for split packets 39214039Sstacze01@arm.com if (pktCount > 1 && burst_helper == NULL) { 39314039Sstacze01@arm.com DPRINTF(DRAM, "Read to addr %lld translates to %d " 39414039Sstacze01@arm.com "dram requests\n", pkt->getAddr(), pktCount); 39514116Sgiacomo.travaglini@arm.com burst_helper = new BurstHelper(pktCount); 39614116Sgiacomo.travaglini@arm.com } 39714116Sgiacomo.travaglini@arm.com 39814132Sgiacomo.travaglini@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 39914132Sgiacomo.travaglini@arm.com dram_pkt->burstHelper = burst_helper; 40014132Sgiacomo.travaglini@arm.com 40114132Sgiacomo.travaglini@arm.com assert(!readQueueFull(1)); 40214132Sgiacomo.travaglini@arm.com rdQLenPdf[readQueue.size() + respQueue.size()]++; 40314039Sstacze01@arm.com 40414116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Adding to read queue\n"); 40514039Sstacze01@arm.com 40614116Sgiacomo.travaglini@arm.com readQueue.push_back(dram_pkt); 40714116Sgiacomo.travaglini@arm.com 40814116Sgiacomo.travaglini@arm.com // Update stats 40914116Sgiacomo.travaglini@arm.com avgRdQLen = readQueue.size() + respQueue.size(); 41014116Sgiacomo.travaglini@arm.com } 41114116Sgiacomo.travaglini@arm.com 41214116Sgiacomo.travaglini@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 41314132Sgiacomo.travaglini@arm.com addr = (addr | (burstSize - 1)) + 1; 41414132Sgiacomo.travaglini@arm.com } 41514132Sgiacomo.travaglini@arm.com 41614132Sgiacomo.travaglini@arm.com // If all packets are serviced by write queue, we send the repsonse back 41714132Sgiacomo.travaglini@arm.com if (pktsServicedByWrQ == pktCount) { 41814116Sgiacomo.travaglini@arm.com accessAndRespond(pkt, frontendLatency); 41914116Sgiacomo.travaglini@arm.com return; 42014116Sgiacomo.travaglini@arm.com } 42114116Sgiacomo.travaglini@arm.com 42214132Sgiacomo.travaglini@arm.com // Update how many split packets are serviced by write queue 42314116Sgiacomo.travaglini@arm.com if (burst_helper != NULL) 42414132Sgiacomo.travaglini@arm.com burst_helper->burstsServiced = pktsServicedByWrQ; 42514132Sgiacomo.travaglini@arm.com 42614132Sgiacomo.travaglini@arm.com // If we are not already scheduled to get a request out of the 42714132Sgiacomo.travaglini@arm.com // queue, do so now 42814132Sgiacomo.travaglini@arm.com if (!nextReqEvent.scheduled()) { 42914132Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 43014116Sgiacomo.travaglini@arm.com schedule(nextReqEvent, curTick()); 43114039Sstacze01@arm.com } 43214116Sgiacomo.travaglini@arm.com} 43314039Sstacze01@arm.com 43414116Sgiacomo.travaglini@arm.comvoid 43514116Sgiacomo.travaglini@arm.comDRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 43614116Sgiacomo.travaglini@arm.com{ 43714116Sgiacomo.travaglini@arm.com // only add to the write queue here. whenever the request is 43814132Sgiacomo.travaglini@arm.com // eventually done, set the readyTime, and call schedule() 43914132Sgiacomo.travaglini@arm.com assert(pkt->isWrite()); 44014132Sgiacomo.travaglini@arm.com 44114132Sgiacomo.travaglini@arm.com // if the request size is larger than burst size, the pkt is split into 44214132Sgiacomo.travaglini@arm.com // multiple DRAM packets 44314132Sgiacomo.travaglini@arm.com Addr addr = pkt->getAddr(); 44414132Sgiacomo.travaglini@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 44514039Sstacze01@arm.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 44614116Sgiacomo.travaglini@arm.com pkt->getAddr() + pkt->getSize()) - addr; 44714039Sstacze01@arm.com writePktSize[ceilLog2(size)]++; 44814116Sgiacomo.travaglini@arm.com writeBursts++; 44914116Sgiacomo.travaglini@arm.com 45014116Sgiacomo.travaglini@arm.com // see if we can merge with an existing item in the write 45114132Sgiacomo.travaglini@arm.com // queue and keep track of whether we have merged or not so we 45214132Sgiacomo.travaglini@arm.com // can stop at that point and also avoid enqueueing a new 45314132Sgiacomo.travaglini@arm.com // request 45414132Sgiacomo.travaglini@arm.com bool merged = false; 45514132Sgiacomo.travaglini@arm.com auto w = writeQueue.begin(); 45614039Sstacze01@arm.com 45714116Sgiacomo.travaglini@arm.com while(!merged && w != writeQueue.end()) { 45814039Sstacze01@arm.com // either of the two could be first, if they are the same 45914116Sgiacomo.travaglini@arm.com // it does not matter which way we go 46014116Sgiacomo.travaglini@arm.com if ((*w)->addr >= addr) { 46114116Sgiacomo.travaglini@arm.com // the existing one starts after the new one, figure 46214116Sgiacomo.travaglini@arm.com // out where the new one ends with respect to the 46314116Sgiacomo.travaglini@arm.com // existing one 46414116Sgiacomo.travaglini@arm.com if ((addr + size) >= ((*w)->addr + (*w)->size)) { 46514116Sgiacomo.travaglini@arm.com // check if the existing one is completely 46614116Sgiacomo.travaglini@arm.com // subsumed in the new one 46714116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Merging write covering existing burst\n"); 46814116Sgiacomo.travaglini@arm.com merged = true; 46914116Sgiacomo.travaglini@arm.com // update both the address and the size 47014116Sgiacomo.travaglini@arm.com (*w)->addr = addr; 47114116Sgiacomo.travaglini@arm.com (*w)->size = size; 47214116Sgiacomo.travaglini@arm.com } else if ((addr + size) >= (*w)->addr && 47314116Sgiacomo.travaglini@arm.com ((*w)->addr + (*w)->size - addr) <= burstSize) { 47414116Sgiacomo.travaglini@arm.com // the new one is just before or partially 47514116Sgiacomo.travaglini@arm.com // overlapping with the existing one, and together 47614116Sgiacomo.travaglini@arm.com // they fit within a burst 47714116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Merging write before existing burst\n"); 47814116Sgiacomo.travaglini@arm.com merged = true; 47914116Sgiacomo.travaglini@arm.com // the existing queue item needs to be adjusted with 48014116Sgiacomo.travaglini@arm.com // respect to both address and size 48114116Sgiacomo.travaglini@arm.com (*w)->size = (*w)->addr + (*w)->size - addr; 48214116Sgiacomo.travaglini@arm.com (*w)->addr = addr; 48314116Sgiacomo.travaglini@arm.com } 48414116Sgiacomo.travaglini@arm.com } else { 48514116Sgiacomo.travaglini@arm.com // the new one starts after the current one, figure 48614116Sgiacomo.travaglini@arm.com // out where the existing one ends with respect to the 48714116Sgiacomo.travaglini@arm.com // new one 48814116Sgiacomo.travaglini@arm.com if (((*w)->addr + (*w)->size) >= (addr + size)) { 48914116Sgiacomo.travaglini@arm.com // check if the new one is completely subsumed in the 49014116Sgiacomo.travaglini@arm.com // existing one 49114116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Merging write into existing burst\n"); 49214116Sgiacomo.travaglini@arm.com merged = true; 49314116Sgiacomo.travaglini@arm.com // no adjustments necessary 49414116Sgiacomo.travaglini@arm.com } else if (((*w)->addr + (*w)->size) >= addr && 49514116Sgiacomo.travaglini@arm.com (addr + size - (*w)->addr) <= burstSize) { 49614116Sgiacomo.travaglini@arm.com // the existing one is just before or partially 49714116Sgiacomo.travaglini@arm.com // overlapping with the new one, and together 49814116Sgiacomo.travaglini@arm.com // they fit within a burst 49914116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Merging write after existing burst\n"); 50014116Sgiacomo.travaglini@arm.com merged = true; 50114116Sgiacomo.travaglini@arm.com // the address is right, and only the size has 50214116Sgiacomo.travaglini@arm.com // to be adjusted 50314116Sgiacomo.travaglini@arm.com (*w)->size = addr + size - (*w)->addr; 50414116Sgiacomo.travaglini@arm.com } 50514116Sgiacomo.travaglini@arm.com } 50614116Sgiacomo.travaglini@arm.com ++w; 50714116Sgiacomo.travaglini@arm.com } 50814116Sgiacomo.travaglini@arm.com 50914116Sgiacomo.travaglini@arm.com // if the item was not merged we need to create a new write 51014116Sgiacomo.travaglini@arm.com // and enqueue it 51114116Sgiacomo.travaglini@arm.com if (!merged) { 51214116Sgiacomo.travaglini@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 51314116Sgiacomo.travaglini@arm.com 51414116Sgiacomo.travaglini@arm.com assert(writeQueue.size() < writeBufferSize); 51514116Sgiacomo.travaglini@arm.com wrQLenPdf[writeQueue.size()]++; 51614116Sgiacomo.travaglini@arm.com 51714116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Adding to write queue\n"); 51814116Sgiacomo.travaglini@arm.com 51914116Sgiacomo.travaglini@arm.com writeQueue.push_back(dram_pkt); 52014116Sgiacomo.travaglini@arm.com 52114116Sgiacomo.travaglini@arm.com // Update stats 52214116Sgiacomo.travaglini@arm.com avgWrQLen = writeQueue.size(); 52314116Sgiacomo.travaglini@arm.com } else { 52414116Sgiacomo.travaglini@arm.com // keep track of the fact that this burst effectively 52514116Sgiacomo.travaglini@arm.com // disappeared as it was merged with an existing one 52614116Sgiacomo.travaglini@arm.com mergedWrBursts++; 52714116Sgiacomo.travaglini@arm.com } 52814116Sgiacomo.travaglini@arm.com 52914116Sgiacomo.travaglini@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 53014116Sgiacomo.travaglini@arm.com addr = (addr | (burstSize - 1)) + 1; 53114116Sgiacomo.travaglini@arm.com } 53214116Sgiacomo.travaglini@arm.com 53314116Sgiacomo.travaglini@arm.com // we do not wait for the writes to be send to the actual memory, 53414116Sgiacomo.travaglini@arm.com // but instead take responsibility for the consistency here and 53514116Sgiacomo.travaglini@arm.com // snoop the write queue for any upcoming reads 53614116Sgiacomo.travaglini@arm.com // @todo, if a pkt size is larger than burst size, we might need a 53714116Sgiacomo.travaglini@arm.com // different front end latency 53814116Sgiacomo.travaglini@arm.com accessAndRespond(pkt, frontendLatency); 53914116Sgiacomo.travaglini@arm.com 54014116Sgiacomo.travaglini@arm.com // If we are not already scheduled to get a request out of the 54114116Sgiacomo.travaglini@arm.com // queue, do so now 54214116Sgiacomo.travaglini@arm.com if (!nextReqEvent.scheduled()) { 54314116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 54414116Sgiacomo.travaglini@arm.com schedule(nextReqEvent, curTick()); 54514039Sstacze01@arm.com } 54614039Sstacze01@arm.com} 54714039Sstacze01@arm.com 54814039Sstacze01@arm.comvoid 54914039Sstacze01@arm.comDRAMCtrl::printQs() const { 55014039Sstacze01@arm.com DPRINTF(DRAM, "===READ QUEUE===\n\n"); 55114039Sstacze01@arm.com for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 55214039Sstacze01@arm.com DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 55314116Sgiacomo.travaglini@arm.com } 55414039Sstacze01@arm.com DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 55514116Sgiacomo.travaglini@arm.com for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 55614116Sgiacomo.travaglini@arm.com DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 55714039Sstacze01@arm.com } 55814039Sstacze01@arm.com DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 55914039Sstacze01@arm.com for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 56014039Sstacze01@arm.com DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 56114116Sgiacomo.travaglini@arm.com } 56214039Sstacze01@arm.com} 56314039Sstacze01@arm.com 56414039Sstacze01@arm.combool 56514039Sstacze01@arm.comDRAMCtrl::recvTimingReq(PacketPtr pkt) 56614039Sstacze01@arm.com{ 56714039Sstacze01@arm.com /// @todo temporary hack to deal with memory corruption issues until 56814039Sstacze01@arm.com /// 4-phase transactions are complete 56914039Sstacze01@arm.com for (int x = 0; x < pendingDelete.size(); x++) 57014098Smichiel.vantol@arm.com delete pendingDelete[x]; 57114039Sstacze01@arm.com pendingDelete.clear(); 57214039Sstacze01@arm.com 57314039Sstacze01@arm.com // This is where we enter from the outside world 57414039Sstacze01@arm.com DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 57514098Smichiel.vantol@arm.com pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 57614039Sstacze01@arm.com 57714039Sstacze01@arm.com // simply drop inhibited packets for now 57814039Sstacze01@arm.com if (pkt->memInhibitAsserted()) { 57914039Sstacze01@arm.com DPRINTF(DRAM, "Inhibited packet -- Dropping it now\n"); 58014039Sstacze01@arm.com pendingDelete.push_back(pkt); 58114039Sstacze01@arm.com return true; 58214039Sstacze01@arm.com } 58314039Sstacze01@arm.com 58414039Sstacze01@arm.com // Calc avg gap between requests 58514039Sstacze01@arm.com if (prevArrival != 0) { 58614039Sstacze01@arm.com totGap += curTick() - prevArrival; 58714039Sstacze01@arm.com } 58814039Sstacze01@arm.com prevArrival = curTick(); 58914039Sstacze01@arm.com 59014039Sstacze01@arm.com 59114039Sstacze01@arm.com // Find out how many dram packets a pkt translates to 59214039Sstacze01@arm.com // If the burst size is equal or larger than the pkt size, then a pkt 59314039Sstacze01@arm.com // translates to only one dram packet. Otherwise, a pkt translates to 59414039Sstacze01@arm.com // multiple dram packets 59514039Sstacze01@arm.com unsigned size = pkt->getSize(); 59614039Sstacze01@arm.com unsigned offset = pkt->getAddr() & (burstSize - 1); 59714039Sstacze01@arm.com unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 59814039Sstacze01@arm.com 59914039Sstacze01@arm.com // check local buffers and do not accept if full 60014039Sstacze01@arm.com if (pkt->isRead()) { 60114039Sstacze01@arm.com assert(size != 0); 60214039Sstacze01@arm.com if (readQueueFull(dram_pkt_count)) { 60314039Sstacze01@arm.com DPRINTF(DRAM, "Read queue full, not accepting\n"); 60414039Sstacze01@arm.com // remember that we have to retry this port 60514039Sstacze01@arm.com retryRdReq = true; 60614039Sstacze01@arm.com numRdRetry++; 60714039Sstacze01@arm.com return false; 60814039Sstacze01@arm.com } else { 60914039Sstacze01@arm.com addToReadQueue(pkt, dram_pkt_count); 61014039Sstacze01@arm.com readReqs++; 61114039Sstacze01@arm.com bytesReadSys += size; 61214039Sstacze01@arm.com } 61314039Sstacze01@arm.com } else if (pkt->isWrite()) { 61414039Sstacze01@arm.com assert(size != 0); 61514039Sstacze01@arm.com if (writeQueueFull(dram_pkt_count)) { 61614039Sstacze01@arm.com DPRINTF(DRAM, "Write queue full, not accepting\n"); 61714039Sstacze01@arm.com // remember that we have to retry this port 61814039Sstacze01@arm.com retryWrReq = true; 61914039Sstacze01@arm.com numWrRetry++; 62014039Sstacze01@arm.com return false; 62114039Sstacze01@arm.com } else { 62214039Sstacze01@arm.com addToWriteQueue(pkt, dram_pkt_count); 62314039Sstacze01@arm.com writeReqs++; 62414039Sstacze01@arm.com bytesWrittenSys += size; 62514039Sstacze01@arm.com } 62614039Sstacze01@arm.com } else { 62714039Sstacze01@arm.com DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 62814039Sstacze01@arm.com neitherReadNorWrite++; 62914039Sstacze01@arm.com accessAndRespond(pkt, 1); 63014039Sstacze01@arm.com } 63114039Sstacze01@arm.com 63214039Sstacze01@arm.com return true; 63314039Sstacze01@arm.com} 63414039Sstacze01@arm.com 63514039Sstacze01@arm.comvoid 63614039Sstacze01@arm.comDRAMCtrl::processRespondEvent() 63714039Sstacze01@arm.com{ 63814039Sstacze01@arm.com DPRINTF(DRAM, 63914039Sstacze01@arm.com "processRespondEvent(): Some req has reached its readyTime\n"); 64014039Sstacze01@arm.com 64114039Sstacze01@arm.com DRAMPacket* dram_pkt = respQueue.front(); 64214039Sstacze01@arm.com 64314103Sgiacomo.travaglini@arm.com if (dram_pkt->burstHelper) { 64414103Sgiacomo.travaglini@arm.com // it is a split packet 64514103Sgiacomo.travaglini@arm.com dram_pkt->burstHelper->burstsServiced++; 64614103Sgiacomo.travaglini@arm.com if (dram_pkt->burstHelper->burstsServiced == 64714103Sgiacomo.travaglini@arm.com dram_pkt->burstHelper->burstCount) { 64814103Sgiacomo.travaglini@arm.com // we have now serviced all children packets of a system packet 64914103Sgiacomo.travaglini@arm.com // so we can now respond to the requester 65014103Sgiacomo.travaglini@arm.com // @todo we probably want to have a different front end and back 65114103Sgiacomo.travaglini@arm.com // end latency for split packets 65214103Sgiacomo.travaglini@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 65314039Sstacze01@arm.com delete dram_pkt->burstHelper; 65414039Sstacze01@arm.com dram_pkt->burstHelper = NULL; 65514039Sstacze01@arm.com } 65614039Sstacze01@arm.com } else { 65714039Sstacze01@arm.com // it is not a split packet 65814039Sstacze01@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 65914039Sstacze01@arm.com } 66014039Sstacze01@arm.com 66114039Sstacze01@arm.com delete respQueue.front(); 66214039Sstacze01@arm.com respQueue.pop_front(); 66314039Sstacze01@arm.com 66414039Sstacze01@arm.com if (!respQueue.empty()) { 66514039Sstacze01@arm.com assert(respQueue.front()->readyTime >= curTick()); 66614039Sstacze01@arm.com assert(!respondEvent.scheduled()); 66714039Sstacze01@arm.com schedule(respondEvent, respQueue.front()->readyTime); 66814039Sstacze01@arm.com } else { 66914103Sgiacomo.travaglini@arm.com // if there is nothing left in any queue, signal a drain 67014103Sgiacomo.travaglini@arm.com if (writeQueue.empty() && readQueue.empty() && 67114103Sgiacomo.travaglini@arm.com drainManager) { 67214103Sgiacomo.travaglini@arm.com drainManager->signalDrainDone(); 67314103Sgiacomo.travaglini@arm.com drainManager = NULL; 67414103Sgiacomo.travaglini@arm.com } 67514103Sgiacomo.travaglini@arm.com } 67614103Sgiacomo.travaglini@arm.com 67714039Sstacze01@arm.com // We have made a location in the queue available at this point, 67814039Sstacze01@arm.com // so if there is a read that was forced to wait, retry now 67914039Sstacze01@arm.com if (retryRdReq) { 68014039Sstacze01@arm.com retryRdReq = false; 68114039Sstacze01@arm.com port.sendRetry(); 68214039Sstacze01@arm.com } 68314039Sstacze01@arm.com} 68414039Sstacze01@arm.com 68514039Sstacze01@arm.comvoid 68614039Sstacze01@arm.comDRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue) 68714039Sstacze01@arm.com{ 68814039Sstacze01@arm.com // This method does the arbitration between requests. The chosen 68914039Sstacze01@arm.com // packet is simply moved to the head of the queue. The other 69014039Sstacze01@arm.com // methods know that this is the place to look. For example, with 69114039Sstacze01@arm.com // FCFS, this method does nothing 69214039Sstacze01@arm.com assert(!queue.empty()); 69314039Sstacze01@arm.com 69414039Sstacze01@arm.com if (queue.size() == 1) { 69514039Sstacze01@arm.com DPRINTF(DRAM, "Single request, nothing to do\n"); 69614039Sstacze01@arm.com return; 69714039Sstacze01@arm.com } 69814039Sstacze01@arm.com 69914039Sstacze01@arm.com if (memSchedPolicy == Enums::fcfs) { 70014039Sstacze01@arm.com // Do nothing, since the correct request is already head 70114039Sstacze01@arm.com } else if (memSchedPolicy == Enums::frfcfs) { 70214039Sstacze01@arm.com reorderQueue(queue); 70314039Sstacze01@arm.com } else 70414039Sstacze01@arm.com panic("No scheduling policy chosen\n"); 70514039Sstacze01@arm.com} 70614039Sstacze01@arm.com 70714039Sstacze01@arm.comvoid 70814039Sstacze01@arm.comDRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue) 70914039Sstacze01@arm.com{ 71014039Sstacze01@arm.com // Only determine this when needed 71114039Sstacze01@arm.com uint64_t earliest_banks = 0; 71214039Sstacze01@arm.com 71314039Sstacze01@arm.com // Search for row hits first, if no row hit is found then schedule the 71414039Sstacze01@arm.com // packet to one of the earliest banks available 71514039Sstacze01@arm.com bool found_earliest_pkt = false; 71614039Sstacze01@arm.com auto selected_pkt_it = queue.begin(); 71714039Sstacze01@arm.com 71814039Sstacze01@arm.com for (auto i = queue.begin(); i != queue.end() ; ++i) { 71914039Sstacze01@arm.com DRAMPacket* dram_pkt = *i; 72014039Sstacze01@arm.com const Bank& bank = dram_pkt->bankRef; 72114039Sstacze01@arm.com // Check if it is a row hit 72214039Sstacze01@arm.com if (bank.openRow == dram_pkt->row) { 72314039Sstacze01@arm.com // FCFS within the hits 72414039Sstacze01@arm.com DPRINTF(DRAM, "Row buffer hit\n"); 72514039Sstacze01@arm.com selected_pkt_it = i; 72614039Sstacze01@arm.com break; 72714039Sstacze01@arm.com } else if (!found_earliest_pkt) { 72814039Sstacze01@arm.com // No row hit, go for first ready 72914039Sstacze01@arm.com if (earliest_banks == 0) 73014039Sstacze01@arm.com earliest_banks = minBankActAt(queue); 73114039Sstacze01@arm.com 73214039Sstacze01@arm.com // simplistic approximation of when the bank can issue an 73314039Sstacze01@arm.com // activate, this is calculated in minBankActAt and could 73414039Sstacze01@arm.com // be cached 73514039Sstacze01@arm.com Tick act_at = bank.openRow == Bank::NO_ROW ? 73614039Sstacze01@arm.com bank.actAllowedAt : 73714039Sstacze01@arm.com std::max(bank.preAllowedAt, curTick()) + tRP; 73814039Sstacze01@arm.com 73914039Sstacze01@arm.com // Bank is ready or is the first available bank 74014039Sstacze01@arm.com if (act_at <= curTick() || 74114039Sstacze01@arm.com bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 74214039Sstacze01@arm.com // Remember the packet to be scheduled to one of the earliest 74314039Sstacze01@arm.com // banks available, FCFS amongst the earliest banks 74414039Sstacze01@arm.com selected_pkt_it = i; 74514039Sstacze01@arm.com found_earliest_pkt = true; 74614039Sstacze01@arm.com } 74714039Sstacze01@arm.com } 74814039Sstacze01@arm.com } 74914039Sstacze01@arm.com 75014039Sstacze01@arm.com DRAMPacket* selected_pkt = *selected_pkt_it; 75114039Sstacze01@arm.com queue.erase(selected_pkt_it); 75214039Sstacze01@arm.com queue.push_front(selected_pkt); 75314039Sstacze01@arm.com} 75414039Sstacze01@arm.com 75514039Sstacze01@arm.comvoid 75614039Sstacze01@arm.comDRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 75714039Sstacze01@arm.com{ 75814039Sstacze01@arm.com DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 75914039Sstacze01@arm.com 76014039Sstacze01@arm.com bool needsResponse = pkt->needsResponse(); 76114039Sstacze01@arm.com // do the actual memory access which also turns the packet into a 76214039Sstacze01@arm.com // response 76314039Sstacze01@arm.com access(pkt); 76414039Sstacze01@arm.com 76514039Sstacze01@arm.com // turn packet around to go back to requester if response expected 76614039Sstacze01@arm.com if (needsResponse) { 76714039Sstacze01@arm.com // access already turned the packet into a response 76814039Sstacze01@arm.com assert(pkt->isResponse()); 76914039Sstacze01@arm.com 77014039Sstacze01@arm.com // @todo someone should pay for this 77114039Sstacze01@arm.com pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 77214039Sstacze01@arm.com 77314039Sstacze01@arm.com // queue the packet in the response queue to be sent out after 77414039Sstacze01@arm.com // the static latency has passed 77514039Sstacze01@arm.com port.schedTimingResp(pkt, curTick() + static_latency); 77614039Sstacze01@arm.com } else { 77714039Sstacze01@arm.com // @todo the packet is going to be deleted, and the DRAMPacket 77814039Sstacze01@arm.com // is still having a pointer to it 77914039Sstacze01@arm.com pendingDelete.push_back(pkt); 78014039Sstacze01@arm.com } 78114039Sstacze01@arm.com 78214039Sstacze01@arm.com DPRINTF(DRAM, "Done\n"); 78314039Sstacze01@arm.com 78414039Sstacze01@arm.com return; 78514039Sstacze01@arm.com} 78614039Sstacze01@arm.com 78714039Sstacze01@arm.comvoid 78814039Sstacze01@arm.comDRAMCtrl::activateBank(Bank& bank, Tick act_tick, uint32_t row) 78914039Sstacze01@arm.com{ 79014039Sstacze01@arm.com // get the rank index from the bank 79114039Sstacze01@arm.com uint8_t rank = bank.rank; 79214039Sstacze01@arm.com 79314039Sstacze01@arm.com assert(actTicks[rank].size() == activationLimit); 79414039Sstacze01@arm.com 79514064Sadrian.herrera@arm.com DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 79614064Sadrian.herrera@arm.com 79714064Sadrian.herrera@arm.com // update the open row 79814064Sadrian.herrera@arm.com assert(bank.openRow == Bank::NO_ROW); 79914064Sadrian.herrera@arm.com bank.openRow = row; 80014039Sstacze01@arm.com 80114039Sstacze01@arm.com // start counting anew, this covers both the case when we 80214039Sstacze01@arm.com // auto-precharged, and when this access is forced to 80314039Sstacze01@arm.com // precharge 80414039Sstacze01@arm.com bank.bytesAccessed = 0; 80514039Sstacze01@arm.com bank.rowAccesses = 0; 80614039Sstacze01@arm.com 80714039Sstacze01@arm.com ++numBanksActive; 80814039Sstacze01@arm.com assert(numBanksActive <= banksPerRank * ranksPerChannel); 80914039Sstacze01@arm.com 81014039Sstacze01@arm.com DPRINTF(DRAM, "Activate bank %d, rank %d at tick %lld, now got %d active\n", 81114039Sstacze01@arm.com bank.bank, bank.rank, act_tick, numBanksActive); 81214039Sstacze01@arm.com 81314039Sstacze01@arm.com DPRINTF(DRAMPower, "%llu,ACT,%d,%d\n", divCeil(act_tick, tCK), bank.bank, 81414039Sstacze01@arm.com bank.rank); 81514039Sstacze01@arm.com 81614039Sstacze01@arm.com // The next access has to respect tRAS for this bank 81714039Sstacze01@arm.com bank.preAllowedAt = act_tick + tRAS; 81814039Sstacze01@arm.com 81914039Sstacze01@arm.com // Respect the row-to-column command delay 82014039Sstacze01@arm.com bank.colAllowedAt = act_tick + tRCD; 82114039Sstacze01@arm.com 82214039Sstacze01@arm.com // start by enforcing tRRD 82314039Sstacze01@arm.com for(int i = 0; i < banksPerRank; i++) { 82414039Sstacze01@arm.com // next activate to any bank in this rank must not happen 82514039Sstacze01@arm.com // before tRRD 82614039Sstacze01@arm.com banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 82714039Sstacze01@arm.com banks[rank][i].actAllowedAt); 82814039Sstacze01@arm.com } 82914039Sstacze01@arm.com 83014039Sstacze01@arm.com // next, we deal with tXAW, if the activation limit is disabled 83114039Sstacze01@arm.com // then we are done 83214039Sstacze01@arm.com if (actTicks[rank].empty()) 83314039Sstacze01@arm.com return; 83414039Sstacze01@arm.com 83514039Sstacze01@arm.com // sanity check 83614039Sstacze01@arm.com if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 837 panic("Got %d activates in window %d (%llu - %llu) which is smaller " 838 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 839 act_tick, actTicks[rank].back(), tXAW); 840 } 841 842 // shift the times used for the book keeping, the last element 843 // (highest index) is the oldest one and hence the lowest value 844 actTicks[rank].pop_back(); 845 846 // record an new activation (in the future) 847 actTicks[rank].push_front(act_tick); 848 849 // cannot activate more than X times in time window tXAW, push the 850 // next one (the X + 1'st activate) to be tXAW away from the 851 // oldest in our window of X 852 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 853 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " 854 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); 855 for(int j = 0; j < banksPerRank; j++) 856 // next activate must not happen before end of window 857 banks[rank][j].actAllowedAt = 858 std::max(actTicks[rank].back() + tXAW, 859 banks[rank][j].actAllowedAt); 860 } 861 862 // at the point when this activate takes place, make sure we 863 // transition to the active power state 864 if (!activateEvent.scheduled()) 865 schedule(activateEvent, act_tick); 866 else if (activateEvent.when() > act_tick) 867 // move it sooner in time 868 reschedule(activateEvent, act_tick); 869} 870 871void 872DRAMCtrl::processActivateEvent() 873{ 874 // we should transition to the active state as soon as any bank is active 875 if (pwrState != PWR_ACT) 876 // note that at this point numBanksActive could be back at 877 // zero again due to a precharge scheduled in the future 878 schedulePowerEvent(PWR_ACT, curTick()); 879} 880 881void 882DRAMCtrl::prechargeBank(Bank& bank, Tick pre_at, bool trace) 883{ 884 // make sure the bank has an open row 885 assert(bank.openRow != Bank::NO_ROW); 886 887 // sample the bytes per activate here since we are closing 888 // the page 889 bytesPerActivate.sample(bank.bytesAccessed); 890 891 bank.openRow = Bank::NO_ROW; 892 893 // no precharge allowed before this one 894 bank.preAllowedAt = pre_at; 895 896 Tick pre_done_at = pre_at + tRP; 897 898 bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 899 900 assert(numBanksActive != 0); 901 --numBanksActive; 902 903 DPRINTF(DRAM, "Precharging bank %d, rank %d at tick %lld, now got " 904 "%d active\n", bank.bank, bank.rank, pre_at, numBanksActive); 905 906 if (trace) 907 DPRINTF(DRAMPower, "%llu,PRE,%d,%d\n", divCeil(pre_at, tCK), 908 bank.bank, bank.rank); 909 910 // if we look at the current number of active banks we might be 911 // tempted to think the DRAM is now idle, however this can be 912 // undone by an activate that is scheduled to happen before we 913 // would have reached the idle state, so schedule an event and 914 // rather check once we actually make it to the point in time when 915 // the (last) precharge takes place 916 if (!prechargeEvent.scheduled()) 917 schedule(prechargeEvent, pre_done_at); 918 else if (prechargeEvent.when() < pre_done_at) 919 reschedule(prechargeEvent, pre_done_at); 920} 921 922void 923DRAMCtrl::processPrechargeEvent() 924{ 925 // if we reached zero, then special conditions apply as we track 926 // if all banks are precharged for the power models 927 if (numBanksActive == 0) { 928 // we should transition to the idle state when the last bank 929 // is precharged 930 schedulePowerEvent(PWR_IDLE, curTick()); 931 } 932} 933 934void 935DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 936{ 937 DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 938 dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 939 940 // get the bank 941 Bank& bank = dram_pkt->bankRef; 942 943 // for the state we need to track if it is a row hit or not 944 bool row_hit = true; 945 946 // respect any constraints on the command (e.g. tRCD or tCCD) 947 Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 948 949 // Determine the access latency and update the bank state 950 if (bank.openRow == dram_pkt->row) { 951 // nothing to do 952 } else { 953 row_hit = false; 954 955 // If there is a page open, precharge it. 956 if (bank.openRow != Bank::NO_ROW) { 957 prechargeBank(bank, std::max(bank.preAllowedAt, curTick())); 958 } 959 960 // next we need to account for the delay in activating the 961 // page 962 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 963 964 // Record the activation and deal with all the global timing 965 // constraints caused be a new activation (tRRD and tXAW) 966 activateBank(bank, act_tick, dram_pkt->row); 967 968 // issue the command as early as possible 969 cmd_at = bank.colAllowedAt; 970 } 971 972 // we need to wait until the bus is available before we can issue 973 // the command 974 cmd_at = std::max(cmd_at, busBusyUntil - tCL); 975 976 // update the packet ready time 977 dram_pkt->readyTime = cmd_at + tCL + tBURST; 978 979 // only one burst can use the bus at any one point in time 980 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 981 982 // not strictly necessary, but update the time for the next 983 // read/write (add a max with tCCD here) 984 bank.colAllowedAt = cmd_at + tBURST; 985 986 // If this is a write, we also need to respect the write recovery 987 // time before a precharge, in the case of a read, respect the 988 // read to precharge constraint 989 bank.preAllowedAt = std::max(bank.preAllowedAt, 990 dram_pkt->isRead ? cmd_at + tRTP : 991 dram_pkt->readyTime + tWR); 992 993 // increment the bytes accessed and the accesses per row 994 bank.bytesAccessed += burstSize; 995 ++bank.rowAccesses; 996 997 // if we reached the max, then issue with an auto-precharge 998 bool auto_precharge = pageMgmt == Enums::close || 999 bank.rowAccesses == maxAccessesPerRow; 1000 1001 // if we did not hit the limit, we might still want to 1002 // auto-precharge 1003 if (!auto_precharge && 1004 (pageMgmt == Enums::open_adaptive || 1005 pageMgmt == Enums::close_adaptive)) { 1006 // a twist on the open and close page policies: 1007 // 1) open_adaptive page policy does not blindly keep the 1008 // page open, but close it if there are no row hits, and there 1009 // are bank conflicts in the queue 1010 // 2) close_adaptive page policy does not blindly close the 1011 // page, but closes it only if there are no row hits in the queue. 1012 // In this case, only force an auto precharge when there 1013 // are no same page hits in the queue 1014 bool got_more_hits = false; 1015 bool got_bank_conflict = false; 1016 1017 // either look at the read queue or write queue 1018 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 1019 writeQueue; 1020 auto p = queue.begin(); 1021 // make sure we are not considering the packet that we are 1022 // currently dealing with (which is the head of the queue) 1023 ++p; 1024 1025 // keep on looking until we have found required condition or 1026 // reached the end 1027 while (!(got_more_hits && 1028 (got_bank_conflict || pageMgmt == Enums::close_adaptive)) && 1029 p != queue.end()) { 1030 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 1031 (dram_pkt->bank == (*p)->bank); 1032 bool same_row = dram_pkt->row == (*p)->row; 1033 got_more_hits |= same_rank_bank && same_row; 1034 got_bank_conflict |= same_rank_bank && !same_row; 1035 ++p; 1036 } 1037 1038 // auto pre-charge when either 1039 // 1) open_adaptive policy, we have not got any more hits, and 1040 // have a bank conflict 1041 // 2) close_adaptive policy and we have not got any more hits 1042 auto_precharge = !got_more_hits && 1043 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1044 } 1045 1046 // DRAMPower trace command to be written 1047 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR"; 1048 1049 // if this access should use auto-precharge, then we are 1050 // closing the row 1051 if (auto_precharge) { 1052 prechargeBank(bank, std::max(curTick(), bank.preAllowedAt), false); 1053 1054 mem_cmd.append("A"); 1055 1056 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1057 } 1058 1059 // Update bus state 1060 busBusyUntil = dram_pkt->readyTime; 1061 1062 DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 1063 dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 1064 1065 DPRINTF(DRAMPower, "%llu,%s,%d,%d\n", divCeil(cmd_at, tCK), mem_cmd, 1066 dram_pkt->bank, dram_pkt->rank); 1067 1068 // Update the minimum timing between the requests, this is a 1069 // conservative estimate of when we have to schedule the next 1070 // request to not introduce any unecessary bubbles. In most cases 1071 // we will wake up sooner than we have to. 1072 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1073 1074 // Update the stats and schedule the next request 1075 if (dram_pkt->isRead) { 1076 ++readsThisTime; 1077 if (row_hit) 1078 readRowHits++; 1079 bytesReadDRAM += burstSize; 1080 perBankRdBursts[dram_pkt->bankId]++; 1081 1082 // Update latency stats 1083 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 1084 totBusLat += tBURST; 1085 totQLat += cmd_at - dram_pkt->entryTime; 1086 } else { 1087 ++writesThisTime; 1088 if (row_hit) 1089 writeRowHits++; 1090 bytesWritten += burstSize; 1091 perBankWrBursts[dram_pkt->bankId]++; 1092 } 1093} 1094 1095void 1096DRAMCtrl::processNextReqEvent() 1097{ 1098 if (busState == READ_TO_WRITE) { 1099 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1100 "waiting\n", readsThisTime, readQueue.size()); 1101 1102 // sample and reset the read-related stats as we are now 1103 // transitioning to writes, and all reads are done 1104 rdPerTurnAround.sample(readsThisTime); 1105 readsThisTime = 0; 1106 1107 // now proceed to do the actual writes 1108 busState = WRITE; 1109 } else if (busState == WRITE_TO_READ) { 1110 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1111 "waiting\n", writesThisTime, writeQueue.size()); 1112 1113 wrPerTurnAround.sample(writesThisTime); 1114 writesThisTime = 0; 1115 1116 busState = READ; 1117 } 1118 1119 if (refreshState != REF_IDLE) { 1120 // if a refresh waiting for this event loop to finish, then hand 1121 // over now, and do not schedule a new nextReqEvent 1122 if (refreshState == REF_DRAIN) { 1123 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1124 1125 refreshState = REF_PRE; 1126 1127 // hand control back to the refresh event loop 1128 schedule(refreshEvent, curTick()); 1129 } 1130 1131 // let the refresh finish before issuing any further requests 1132 return; 1133 } 1134 1135 // when we get here it is either a read or a write 1136 if (busState == READ) { 1137 1138 // track if we should switch or not 1139 bool switch_to_writes = false; 1140 1141 if (readQueue.empty()) { 1142 // In the case there is no read request to go next, 1143 // trigger writes if we have passed the low threshold (or 1144 // if we are draining) 1145 if (!writeQueue.empty() && 1146 (drainManager || writeQueue.size() > writeLowThreshold)) { 1147 1148 switch_to_writes = true; 1149 } else { 1150 // check if we are drained 1151 if (respQueue.empty () && drainManager) { 1152 drainManager->signalDrainDone(); 1153 drainManager = NULL; 1154 } 1155 1156 // nothing to do, not even any point in scheduling an 1157 // event for the next request 1158 return; 1159 } 1160 } else { 1161 // Figure out which read request goes next, and move it to the 1162 // front of the read queue 1163 chooseNext(readQueue); 1164 1165 DRAMPacket* dram_pkt = readQueue.front(); 1166 1167 doDRAMAccess(dram_pkt); 1168 1169 // At this point we're done dealing with the request 1170 readQueue.pop_front(); 1171 1172 // sanity check 1173 assert(dram_pkt->size <= burstSize); 1174 assert(dram_pkt->readyTime >= curTick()); 1175 1176 // Insert into response queue. It will be sent back to the 1177 // requestor at its readyTime 1178 if (respQueue.empty()) { 1179 assert(!respondEvent.scheduled()); 1180 schedule(respondEvent, dram_pkt->readyTime); 1181 } else { 1182 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1183 assert(respondEvent.scheduled()); 1184 } 1185 1186 respQueue.push_back(dram_pkt); 1187 1188 // we have so many writes that we have to transition 1189 if (writeQueue.size() > writeHighThreshold) { 1190 switch_to_writes = true; 1191 } 1192 } 1193 1194 // switching to writes, either because the read queue is empty 1195 // and the writes have passed the low threshold (or we are 1196 // draining), or because the writes hit the hight threshold 1197 if (switch_to_writes) { 1198 // transition to writing 1199 busState = READ_TO_WRITE; 1200 1201 // add a bubble to the data bus, as defined by the 1202 // tRTW parameter 1203 busBusyUntil += tRTW; 1204 1205 // update the minimum timing between the requests, 1206 // this shifts us back in time far enough to do any 1207 // bank preparation 1208 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1209 } 1210 } else { 1211 chooseNext(writeQueue); 1212 DRAMPacket* dram_pkt = writeQueue.front(); 1213 // sanity check 1214 assert(dram_pkt->size <= burstSize); 1215 doDRAMAccess(dram_pkt); 1216 1217 writeQueue.pop_front(); 1218 delete dram_pkt; 1219 1220 // If we emptied the write queue, or got sufficiently below the 1221 // threshold (using the minWritesPerSwitch as the hysteresis) and 1222 // are not draining, or we have reads waiting and have done enough 1223 // writes, then switch to reads. 1224 if (writeQueue.empty() || 1225 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1226 !drainManager) || 1227 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1228 // turn the bus back around for reads again 1229 busState = WRITE_TO_READ; 1230 1231 // note that the we switch back to reads also in the idle 1232 // case, which eventually will check for any draining and 1233 // also pause any further scheduling if there is really 1234 // nothing to do 1235 1236 // here we get a bit creative and shift the bus busy time not 1237 // just the tWTR, but also a CAS latency to capture the fact 1238 // that we are allowed to prepare a new bank, but not issue a 1239 // read command until after tWTR, in essence we capture a 1240 // bubble on the data bus that is tWTR + tCL 1241 busBusyUntil += tWTR + tCL; 1242 1243 // update the minimum timing between the requests, this shifts 1244 // us back in time far enough to do any bank preparation 1245 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1246 } 1247 } 1248 1249 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1250 1251 // If there is space available and we have writes waiting then let 1252 // them retry. This is done here to ensure that the retry does not 1253 // cause a nextReqEvent to be scheduled before we do so as part of 1254 // the next request processing 1255 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1256 retryWrReq = false; 1257 port.sendRetry(); 1258 } 1259} 1260 1261uint64_t 1262DRAMCtrl::minBankActAt(const deque<DRAMPacket*>& queue) const 1263{ 1264 uint64_t bank_mask = 0; 1265 Tick min_act_at = MaxTick; 1266 1267 // deterimne if we have queued transactions targetting a 1268 // bank in question 1269 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 1270 for (auto p = queue.begin(); p != queue.end(); ++p) { 1271 got_waiting[(*p)->bankId] = true; 1272 } 1273 1274 for (int i = 0; i < ranksPerChannel; i++) { 1275 for (int j = 0; j < banksPerRank; j++) { 1276 uint8_t bank_id = i * banksPerRank + j; 1277 1278 // if we have waiting requests for the bank, and it is 1279 // amongst the first available, update the mask 1280 if (got_waiting[bank_id]) { 1281 // simplistic approximation of when the bank can issue 1282 // an activate, ignoring any rank-to-rank switching 1283 // cost 1284 Tick act_at = banks[i][j].openRow == Bank::NO_ROW ? 1285 banks[i][j].actAllowedAt : 1286 std::max(banks[i][j].preAllowedAt, curTick()) + tRP; 1287 1288 if (act_at <= min_act_at) { 1289 // reset bank mask if new minimum is found 1290 if (act_at < min_act_at) 1291 bank_mask = 0; 1292 // set the bit corresponding to the available bank 1293 replaceBits(bank_mask, bank_id, bank_id, 1); 1294 min_act_at = act_at; 1295 } 1296 } 1297 } 1298 } 1299 1300 return bank_mask; 1301} 1302 1303void 1304DRAMCtrl::processRefreshEvent() 1305{ 1306 // when first preparing the refresh, remember when it was due 1307 if (refreshState == REF_IDLE) { 1308 // remember when the refresh is due 1309 refreshDueAt = curTick(); 1310 1311 // proceed to drain 1312 refreshState = REF_DRAIN; 1313 1314 DPRINTF(DRAM, "Refresh due\n"); 1315 } 1316 1317 // let any scheduled read or write go ahead, after which it will 1318 // hand control back to this event loop 1319 if (refreshState == REF_DRAIN) { 1320 if (nextReqEvent.scheduled()) { 1321 // hand control over to the request loop until it is 1322 // evaluated next 1323 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1324 1325 return; 1326 } else { 1327 refreshState = REF_PRE; 1328 } 1329 } 1330 1331 // at this point, ensure that all banks are precharged 1332 if (refreshState == REF_PRE) { 1333 // precharge any active bank if we are not already in the idle 1334 // state 1335 if (pwrState != PWR_IDLE) { 1336 // at the moment, we use a precharge all even if there is 1337 // only a single bank open 1338 DPRINTF(DRAM, "Precharging all\n"); 1339 1340 // first determine when we can precharge 1341 Tick pre_at = curTick(); 1342 for (int i = 0; i < ranksPerChannel; i++) { 1343 for (int j = 0; j < banksPerRank; j++) { 1344 // respect both causality and any existing bank 1345 // constraints, some banks could already have a 1346 // (auto) precharge scheduled 1347 pre_at = std::max(banks[i][j].preAllowedAt, pre_at); 1348 } 1349 } 1350 1351 // make sure all banks are precharged, and for those that 1352 // already are, update their availability 1353 Tick act_allowed_at = pre_at + tRP; 1354 1355 for (int i = 0; i < ranksPerChannel; i++) { 1356 for (int j = 0; j < banksPerRank; j++) { 1357 if (banks[i][j].openRow != Bank::NO_ROW) { 1358 prechargeBank(banks[i][j], pre_at, false); 1359 } else { 1360 banks[i][j].actAllowedAt = 1361 std::max(banks[i][j].actAllowedAt, act_allowed_at); 1362 banks[i][j].preAllowedAt = 1363 std::max(banks[i][j].preAllowedAt, pre_at); 1364 } 1365 } 1366 1367 // at the moment this affects all ranks 1368 DPRINTF(DRAMPower, "%llu,PREA,0,%d\n", divCeil(pre_at, tCK), 1369 i); 1370 } 1371 } else { 1372 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1373 1374 // go ahead and kick the power state machine into gear if 1375 // we are already idle 1376 schedulePowerEvent(PWR_REF, curTick()); 1377 } 1378 1379 refreshState = REF_RUN; 1380 assert(numBanksActive == 0); 1381 1382 // wait for all banks to be precharged, at which point the 1383 // power state machine will transition to the idle state, and 1384 // automatically move to a refresh, at that point it will also 1385 // call this method to get the refresh event loop going again 1386 return; 1387 } 1388 1389 // last but not least we perform the actual refresh 1390 if (refreshState == REF_RUN) { 1391 // should never get here with any banks active 1392 assert(numBanksActive == 0); 1393 assert(pwrState == PWR_REF); 1394 1395 Tick ref_done_at = curTick() + tRFC; 1396 1397 for (int i = 0; i < ranksPerChannel; i++) { 1398 for (int j = 0; j < banksPerRank; j++) { 1399 banks[i][j].actAllowedAt = ref_done_at; 1400 } 1401 1402 // at the moment this affects all ranks 1403 DPRINTF(DRAMPower, "%llu,REF,0,%d\n", divCeil(curTick(), tCK), i); 1404 } 1405 1406 // make sure we did not wait so long that we cannot make up 1407 // for it 1408 if (refreshDueAt + tREFI < ref_done_at) { 1409 fatal("Refresh was delayed so long we cannot catch up\n"); 1410 } 1411 1412 // compensate for the delay in actually performing the refresh 1413 // when scheduling the next one 1414 schedule(refreshEvent, refreshDueAt + tREFI - tRP); 1415 1416 assert(!powerEvent.scheduled()); 1417 1418 // move to the idle power state once the refresh is done, this 1419 // will also move the refresh state machine to the refresh 1420 // idle state 1421 schedulePowerEvent(PWR_IDLE, ref_done_at); 1422 1423 DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 1424 ref_done_at, refreshDueAt + tREFI); 1425 } 1426} 1427 1428void 1429DRAMCtrl::schedulePowerEvent(PowerState pwr_state, Tick tick) 1430{ 1431 // respect causality 1432 assert(tick >= curTick()); 1433 1434 if (!powerEvent.scheduled()) { 1435 DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 1436 tick, pwr_state); 1437 1438 // insert the new transition 1439 pwrStateTrans = pwr_state; 1440 1441 schedule(powerEvent, tick); 1442 } else { 1443 panic("Scheduled power event at %llu to state %d, " 1444 "with scheduled event at %llu to %d\n", tick, pwr_state, 1445 powerEvent.when(), pwrStateTrans); 1446 } 1447} 1448 1449void 1450DRAMCtrl::processPowerEvent() 1451{ 1452 // remember where we were, and for how long 1453 Tick duration = curTick() - pwrStateTick; 1454 PowerState prev_state = pwrState; 1455 1456 // update the accounting 1457 pwrStateTime[prev_state] += duration; 1458 1459 pwrState = pwrStateTrans; 1460 pwrStateTick = curTick(); 1461 1462 if (pwrState == PWR_IDLE) { 1463 DPRINTF(DRAMState, "All banks precharged\n"); 1464 1465 // if we were refreshing, make sure we start scheduling requests again 1466 if (prev_state == PWR_REF) { 1467 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 1468 assert(pwrState == PWR_IDLE); 1469 1470 // kick things into action again 1471 refreshState = REF_IDLE; 1472 assert(!nextReqEvent.scheduled()); 1473 schedule(nextReqEvent, curTick()); 1474 } else { 1475 assert(prev_state == PWR_ACT); 1476 1477 // if we have a pending refresh, and are now moving to 1478 // the idle state, direclty transition to a refresh 1479 if (refreshState == REF_RUN) { 1480 // there should be nothing waiting at this point 1481 assert(!powerEvent.scheduled()); 1482 1483 // update the state in zero time and proceed below 1484 pwrState = PWR_REF; 1485 } 1486 } 1487 } 1488 1489 // we transition to the refresh state, let the refresh state 1490 // machine know of this state update and let it deal with the 1491 // scheduling of the next power state transition as well as the 1492 // following refresh 1493 if (pwrState == PWR_REF) { 1494 DPRINTF(DRAMState, "Refreshing\n"); 1495 // kick the refresh event loop into action again, and that 1496 // in turn will schedule a transition to the idle power 1497 // state once the refresh is done 1498 assert(refreshState == REF_RUN); 1499 processRefreshEvent(); 1500 } 1501} 1502 1503void 1504DRAMCtrl::regStats() 1505{ 1506 using namespace Stats; 1507 1508 AbstractMemory::regStats(); 1509 1510 readReqs 1511 .name(name() + ".readReqs") 1512 .desc("Number of read requests accepted"); 1513 1514 writeReqs 1515 .name(name() + ".writeReqs") 1516 .desc("Number of write requests accepted"); 1517 1518 readBursts 1519 .name(name() + ".readBursts") 1520 .desc("Number of DRAM read bursts, " 1521 "including those serviced by the write queue"); 1522 1523 writeBursts 1524 .name(name() + ".writeBursts") 1525 .desc("Number of DRAM write bursts, " 1526 "including those merged in the write queue"); 1527 1528 servicedByWrQ 1529 .name(name() + ".servicedByWrQ") 1530 .desc("Number of DRAM read bursts serviced by the write queue"); 1531 1532 mergedWrBursts 1533 .name(name() + ".mergedWrBursts") 1534 .desc("Number of DRAM write bursts merged with an existing one"); 1535 1536 neitherReadNorWrite 1537 .name(name() + ".neitherReadNorWriteReqs") 1538 .desc("Number of requests that are neither read nor write"); 1539 1540 perBankRdBursts 1541 .init(banksPerRank * ranksPerChannel) 1542 .name(name() + ".perBankRdBursts") 1543 .desc("Per bank write bursts"); 1544 1545 perBankWrBursts 1546 .init(banksPerRank * ranksPerChannel) 1547 .name(name() + ".perBankWrBursts") 1548 .desc("Per bank write bursts"); 1549 1550 avgRdQLen 1551 .name(name() + ".avgRdQLen") 1552 .desc("Average read queue length when enqueuing") 1553 .precision(2); 1554 1555 avgWrQLen 1556 .name(name() + ".avgWrQLen") 1557 .desc("Average write queue length when enqueuing") 1558 .precision(2); 1559 1560 totQLat 1561 .name(name() + ".totQLat") 1562 .desc("Total ticks spent queuing"); 1563 1564 totBusLat 1565 .name(name() + ".totBusLat") 1566 .desc("Total ticks spent in databus transfers"); 1567 1568 totMemAccLat 1569 .name(name() + ".totMemAccLat") 1570 .desc("Total ticks spent from burst creation until serviced " 1571 "by the DRAM"); 1572 1573 avgQLat 1574 .name(name() + ".avgQLat") 1575 .desc("Average queueing delay per DRAM burst") 1576 .precision(2); 1577 1578 avgQLat = totQLat / (readBursts - servicedByWrQ); 1579 1580 avgBusLat 1581 .name(name() + ".avgBusLat") 1582 .desc("Average bus latency per DRAM burst") 1583 .precision(2); 1584 1585 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1586 1587 avgMemAccLat 1588 .name(name() + ".avgMemAccLat") 1589 .desc("Average memory access latency per DRAM burst") 1590 .precision(2); 1591 1592 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1593 1594 numRdRetry 1595 .name(name() + ".numRdRetry") 1596 .desc("Number of times read queue was full causing retry"); 1597 1598 numWrRetry 1599 .name(name() + ".numWrRetry") 1600 .desc("Number of times write queue was full causing retry"); 1601 1602 readRowHits 1603 .name(name() + ".readRowHits") 1604 .desc("Number of row buffer hits during reads"); 1605 1606 writeRowHits 1607 .name(name() + ".writeRowHits") 1608 .desc("Number of row buffer hits during writes"); 1609 1610 readRowHitRate 1611 .name(name() + ".readRowHitRate") 1612 .desc("Row buffer hit rate for reads") 1613 .precision(2); 1614 1615 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 1616 1617 writeRowHitRate 1618 .name(name() + ".writeRowHitRate") 1619 .desc("Row buffer hit rate for writes") 1620 .precision(2); 1621 1622 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 1623 1624 readPktSize 1625 .init(ceilLog2(burstSize) + 1) 1626 .name(name() + ".readPktSize") 1627 .desc("Read request sizes (log2)"); 1628 1629 writePktSize 1630 .init(ceilLog2(burstSize) + 1) 1631 .name(name() + ".writePktSize") 1632 .desc("Write request sizes (log2)"); 1633 1634 rdQLenPdf 1635 .init(readBufferSize) 1636 .name(name() + ".rdQLenPdf") 1637 .desc("What read queue length does an incoming req see"); 1638 1639 wrQLenPdf 1640 .init(writeBufferSize) 1641 .name(name() + ".wrQLenPdf") 1642 .desc("What write queue length does an incoming req see"); 1643 1644 bytesPerActivate 1645 .init(maxAccessesPerRow) 1646 .name(name() + ".bytesPerActivate") 1647 .desc("Bytes accessed per row activation") 1648 .flags(nozero); 1649 1650 rdPerTurnAround 1651 .init(readBufferSize) 1652 .name(name() + ".rdPerTurnAround") 1653 .desc("Reads before turning the bus around for writes") 1654 .flags(nozero); 1655 1656 wrPerTurnAround 1657 .init(writeBufferSize) 1658 .name(name() + ".wrPerTurnAround") 1659 .desc("Writes before turning the bus around for reads") 1660 .flags(nozero); 1661 1662 bytesReadDRAM 1663 .name(name() + ".bytesReadDRAM") 1664 .desc("Total number of bytes read from DRAM"); 1665 1666 bytesReadWrQ 1667 .name(name() + ".bytesReadWrQ") 1668 .desc("Total number of bytes read from write queue"); 1669 1670 bytesWritten 1671 .name(name() + ".bytesWritten") 1672 .desc("Total number of bytes written to DRAM"); 1673 1674 bytesReadSys 1675 .name(name() + ".bytesReadSys") 1676 .desc("Total read bytes from the system interface side"); 1677 1678 bytesWrittenSys 1679 .name(name() + ".bytesWrittenSys") 1680 .desc("Total written bytes from the system interface side"); 1681 1682 avgRdBW 1683 .name(name() + ".avgRdBW") 1684 .desc("Average DRAM read bandwidth in MiByte/s") 1685 .precision(2); 1686 1687 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 1688 1689 avgWrBW 1690 .name(name() + ".avgWrBW") 1691 .desc("Average achieved write bandwidth in MiByte/s") 1692 .precision(2); 1693 1694 avgWrBW = (bytesWritten / 1000000) / simSeconds; 1695 1696 avgRdBWSys 1697 .name(name() + ".avgRdBWSys") 1698 .desc("Average system read bandwidth in MiByte/s") 1699 .precision(2); 1700 1701 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 1702 1703 avgWrBWSys 1704 .name(name() + ".avgWrBWSys") 1705 .desc("Average system write bandwidth in MiByte/s") 1706 .precision(2); 1707 1708 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 1709 1710 peakBW 1711 .name(name() + ".peakBW") 1712 .desc("Theoretical peak bandwidth in MiByte/s") 1713 .precision(2); 1714 1715 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 1716 1717 busUtil 1718 .name(name() + ".busUtil") 1719 .desc("Data bus utilization in percentage") 1720 .precision(2); 1721 1722 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 1723 1724 totGap 1725 .name(name() + ".totGap") 1726 .desc("Total gap between requests"); 1727 1728 avgGap 1729 .name(name() + ".avgGap") 1730 .desc("Average gap between requests") 1731 .precision(2); 1732 1733 avgGap = totGap / (readReqs + writeReqs); 1734 1735 // Stats for DRAM Power calculation based on Micron datasheet 1736 busUtilRead 1737 .name(name() + ".busUtilRead") 1738 .desc("Data bus utilization in percentage for reads") 1739 .precision(2); 1740 1741 busUtilRead = avgRdBW / peakBW * 100; 1742 1743 busUtilWrite 1744 .name(name() + ".busUtilWrite") 1745 .desc("Data bus utilization in percentage for writes") 1746 .precision(2); 1747 1748 busUtilWrite = avgWrBW / peakBW * 100; 1749 1750 pageHitRate 1751 .name(name() + ".pageHitRate") 1752 .desc("Row buffer hit rate, read and write combined") 1753 .precision(2); 1754 1755 pageHitRate = (writeRowHits + readRowHits) / 1756 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 1757 1758 pwrStateTime 1759 .init(5) 1760 .name(name() + ".memoryStateTime") 1761 .desc("Time in different power states"); 1762 pwrStateTime.subname(0, "IDLE"); 1763 pwrStateTime.subname(1, "REF"); 1764 pwrStateTime.subname(2, "PRE_PDN"); 1765 pwrStateTime.subname(3, "ACT"); 1766 pwrStateTime.subname(4, "ACT_PDN"); 1767} 1768 1769void 1770DRAMCtrl::recvFunctional(PacketPtr pkt) 1771{ 1772 // rely on the abstract memory 1773 functionalAccess(pkt); 1774} 1775 1776BaseSlavePort& 1777DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 1778{ 1779 if (if_name != "port") { 1780 return MemObject::getSlavePort(if_name, idx); 1781 } else { 1782 return port; 1783 } 1784} 1785 1786unsigned int 1787DRAMCtrl::drain(DrainManager *dm) 1788{ 1789 unsigned int count = port.drain(dm); 1790 1791 // if there is anything in any of our internal queues, keep track 1792 // of that as well 1793 if (!(writeQueue.empty() && readQueue.empty() && 1794 respQueue.empty())) { 1795 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 1796 " resp: %d\n", writeQueue.size(), readQueue.size(), 1797 respQueue.size()); 1798 ++count; 1799 drainManager = dm; 1800 1801 // the only part that is not drained automatically over time 1802 // is the write queue, thus kick things into action if needed 1803 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 1804 schedule(nextReqEvent, curTick()); 1805 } 1806 } 1807 1808 if (count) 1809 setDrainState(Drainable::Draining); 1810 else 1811 setDrainState(Drainable::Drained); 1812 return count; 1813} 1814 1815DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 1816 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 1817 memory(_memory) 1818{ } 1819 1820AddrRangeList 1821DRAMCtrl::MemoryPort::getAddrRanges() const 1822{ 1823 AddrRangeList ranges; 1824 ranges.push_back(memory.getAddrRange()); 1825 return ranges; 1826} 1827 1828void 1829DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 1830{ 1831 pkt->pushLabel(memory.name()); 1832 1833 if (!queue.checkFunctional(pkt)) { 1834 // Default implementation of SimpleTimingPort::recvFunctional() 1835 // calls recvAtomic() and throws away the latency; we can save a 1836 // little here by just not calculating the latency. 1837 memory.recvFunctional(pkt); 1838 } 1839 1840 pkt->popLabel(); 1841} 1842 1843Tick 1844DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 1845{ 1846 return memory.recvAtomic(pkt); 1847} 1848 1849bool 1850DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 1851{ 1852 // pass it to the memory controller 1853 return memory.recvTimingReq(pkt); 1854} 1855 1856DRAMCtrl* 1857DRAMCtrlParams::create() 1858{ 1859 return new DRAMCtrl(this); 1860} 1861