dram_ctrl.cc revision 10215
110388SAndreas.Sandberg@ARM.com/* 210388SAndreas.Sandberg@ARM.com * Copyright (c) 2010-2014 ARM Limited 310388SAndreas.Sandberg@ARM.com * All rights reserved 410388SAndreas.Sandberg@ARM.com * 510388SAndreas.Sandberg@ARM.com * The license below extends only to copyright in the software and shall 610388SAndreas.Sandberg@ARM.com * not be construed as granting a license to any other intellectual 710388SAndreas.Sandberg@ARM.com * property including but not limited to intellectual property relating 810388SAndreas.Sandberg@ARM.com * to a hardware implementation of the functionality of the software 910388SAndreas.Sandberg@ARM.com * licensed hereunder. You may use the software subject to the license 1010388SAndreas.Sandberg@ARM.com * terms below provided that you ensure that this notice is replicated 1110388SAndreas.Sandberg@ARM.com * unmodified and in its entirety in all distributions of the software, 1210388SAndreas.Sandberg@ARM.com * modified or unmodified, in source code or in binary form. 1310388SAndreas.Sandberg@ARM.com * 1410388SAndreas.Sandberg@ARM.com * Copyright (c) 2013 Amin Farmahini-Farahani 1510388SAndreas.Sandberg@ARM.com * All rights reserved. 1610388SAndreas.Sandberg@ARM.com * 1710388SAndreas.Sandberg@ARM.com * Redistribution and use in source and binary forms, with or without 1810388SAndreas.Sandberg@ARM.com * modification, are permitted provided that the following conditions are 1910388SAndreas.Sandberg@ARM.com * met: redistributions of source code must retain the above copyright 2010388SAndreas.Sandberg@ARM.com * notice, this list of conditions and the following disclaimer; 2110388SAndreas.Sandberg@ARM.com * redistributions in binary form must reproduce the above copyright 2210388SAndreas.Sandberg@ARM.com * notice, this list of conditions and the following disclaimer in the 2310388SAndreas.Sandberg@ARM.com * documentation and/or other materials provided with the distribution; 2410388SAndreas.Sandberg@ARM.com * neither the name of the copyright holders nor the names of its 2510388SAndreas.Sandberg@ARM.com * contributors may be used to endorse or promote products derived from 2610388SAndreas.Sandberg@ARM.com * this software without specific prior written permission. 2710388SAndreas.Sandberg@ARM.com * 2810388SAndreas.Sandberg@ARM.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2910388SAndreas.Sandberg@ARM.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3010388SAndreas.Sandberg@ARM.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3110388SAndreas.Sandberg@ARM.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3210388SAndreas.Sandberg@ARM.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3310388SAndreas.Sandberg@ARM.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3410388SAndreas.Sandberg@ARM.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3510388SAndreas.Sandberg@ARM.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3610388SAndreas.Sandberg@ARM.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3710388SAndreas.Sandberg@ARM.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3810388SAndreas.Sandberg@ARM.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3910388SAndreas.Sandberg@ARM.com * 4010388SAndreas.Sandberg@ARM.com * Authors: Andreas Hansson 4110388SAndreas.Sandberg@ARM.com * Ani Udipi 4210388SAndreas.Sandberg@ARM.com * Neha Agarwal 4310388SAndreas.Sandberg@ARM.com */ 4410388SAndreas.Sandberg@ARM.com 4510388SAndreas.Sandberg@ARM.com#include "base/bitfield.hh" 4610559Sandreas.hansson@arm.com#include "base/trace.hh" 4710559Sandreas.hansson@arm.com#include "debug/DRAM.hh" 4810388SAndreas.Sandberg@ARM.com#include "debug/DRAMState.hh" 4910388SAndreas.Sandberg@ARM.com#include "debug/Drain.hh" 5010388SAndreas.Sandberg@ARM.com#include "mem/dram_ctrl.hh" 5110388SAndreas.Sandberg@ARM.com#include "sim/system.hh" 5210388SAndreas.Sandberg@ARM.com 5310388SAndreas.Sandberg@ARM.comusing namespace std; 5410388SAndreas.Sandberg@ARM.com 5510388SAndreas.Sandberg@ARM.comDRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 5610388SAndreas.Sandberg@ARM.com AbstractMemory(p), 5710388SAndreas.Sandberg@ARM.com port(name() + ".port", *this), 5810388SAndreas.Sandberg@ARM.com retryRdReq(false), retryWrReq(false), 5910388SAndreas.Sandberg@ARM.com busState(READ), 6010388SAndreas.Sandberg@ARM.com nextReqEvent(this), respondEvent(this), activateEvent(this), 6110388SAndreas.Sandberg@ARM.com prechargeEvent(this), refreshEvent(this), powerEvent(this), 6210388SAndreas.Sandberg@ARM.com drainManager(NULL), 6310388SAndreas.Sandberg@ARM.com deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 6410388SAndreas.Sandberg@ARM.com deviceRowBufferSize(p->device_rowbuffer_size), 6510388SAndreas.Sandberg@ARM.com devicesPerRank(p->devices_per_rank), 6610388SAndreas.Sandberg@ARM.com burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 6710388SAndreas.Sandberg@ARM.com rowBufferSize(devicesPerRank * deviceRowBufferSize), 6810388SAndreas.Sandberg@ARM.com columnsPerRowBuffer(rowBufferSize / burstSize), 6910388SAndreas.Sandberg@ARM.com ranksPerChannel(p->ranks_per_channel), 7010388SAndreas.Sandberg@ARM.com banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 7110388SAndreas.Sandberg@ARM.com readBufferSize(p->read_buffer_size), 7210388SAndreas.Sandberg@ARM.com writeBufferSize(p->write_buffer_size), 7310388SAndreas.Sandberg@ARM.com writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 7410388SAndreas.Sandberg@ARM.com writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 7510388SAndreas.Sandberg@ARM.com minWritesPerSwitch(p->min_writes_per_switch), 7610388SAndreas.Sandberg@ARM.com writesThisTime(0), readsThisTime(0), 7710388SAndreas.Sandberg@ARM.com tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), 7810388SAndreas.Sandberg@ARM.com tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR), 7910388SAndreas.Sandberg@ARM.com tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 8010388SAndreas.Sandberg@ARM.com tXAW(p->tXAW), activationLimit(p->activation_limit), 8110388SAndreas.Sandberg@ARM.com memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 8210388SAndreas.Sandberg@ARM.com pageMgmt(p->page_policy), 8310388SAndreas.Sandberg@ARM.com maxAccessesPerRow(p->max_accesses_per_row), 8410388SAndreas.Sandberg@ARM.com frontendLatency(p->static_frontend_latency), 8510388SAndreas.Sandberg@ARM.com backendLatency(p->static_backend_latency), 8610388SAndreas.Sandberg@ARM.com busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), 8710388SAndreas.Sandberg@ARM.com pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), prevArrival(0), 8810388SAndreas.Sandberg@ARM.com nextReqTime(0), pwrStateTick(0), numBanksActive(0) 8910388SAndreas.Sandberg@ARM.com{ 9010388SAndreas.Sandberg@ARM.com // create the bank states based on the dimensions of the ranks and 9110388SAndreas.Sandberg@ARM.com // banks 9210388SAndreas.Sandberg@ARM.com banks.resize(ranksPerChannel); 9310388SAndreas.Sandberg@ARM.com actTicks.resize(ranksPerChannel); 9410388SAndreas.Sandberg@ARM.com for (size_t c = 0; c < ranksPerChannel; ++c) { 9510388SAndreas.Sandberg@ARM.com banks[c].resize(banksPerRank); 9610388SAndreas.Sandberg@ARM.com actTicks[c].resize(activationLimit, 0); 9710388SAndreas.Sandberg@ARM.com } 9810388SAndreas.Sandberg@ARM.com 9910388SAndreas.Sandberg@ARM.com // perform a basic check of the write thresholds 10010388SAndreas.Sandberg@ARM.com if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 10110388SAndreas.Sandberg@ARM.com fatal("Write buffer low threshold %d must be smaller than the " 10210388SAndreas.Sandberg@ARM.com "high threshold %d\n", p->write_low_thresh_perc, 10310388SAndreas.Sandberg@ARM.com p->write_high_thresh_perc); 10410388SAndreas.Sandberg@ARM.com 10510388SAndreas.Sandberg@ARM.com // determine the rows per bank by looking at the total capacity 10610388SAndreas.Sandberg@ARM.com uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 10710388SAndreas.Sandberg@ARM.com 10810388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 10910388SAndreas.Sandberg@ARM.com AbstractMemory::size()); 11010388SAndreas.Sandberg@ARM.com 11110388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 11210388SAndreas.Sandberg@ARM.com rowBufferSize, columnsPerRowBuffer); 11310388SAndreas.Sandberg@ARM.com 11410388SAndreas.Sandberg@ARM.com rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 11510388SAndreas.Sandberg@ARM.com 11610388SAndreas.Sandberg@ARM.com if (range.interleaved()) { 11710388SAndreas.Sandberg@ARM.com if (channels != range.stripes()) 11810388SAndreas.Sandberg@ARM.com fatal("%s has %d interleaved address stripes but %d channel(s)\n", 11910388SAndreas.Sandberg@ARM.com name(), range.stripes(), channels); 12010388SAndreas.Sandberg@ARM.com 12110388SAndreas.Sandberg@ARM.com if (addrMapping == Enums::RoRaBaChCo) { 12210388SAndreas.Sandberg@ARM.com if (rowBufferSize != range.granularity()) { 12310388SAndreas.Sandberg@ARM.com fatal("Interleaving of %s doesn't match RoRaBaChCo " 12410388SAndreas.Sandberg@ARM.com "address map\n", name()); 12510388SAndreas.Sandberg@ARM.com } 12610388SAndreas.Sandberg@ARM.com } else if (addrMapping == Enums::RoRaBaCoCh) { 12710388SAndreas.Sandberg@ARM.com if (system()->cacheLineSize() != range.granularity()) { 12810388SAndreas.Sandberg@ARM.com fatal("Interleaving of %s doesn't match RoRaBaCoCh " 12910388SAndreas.Sandberg@ARM.com "address map\n", name()); 13010388SAndreas.Sandberg@ARM.com } 13110388SAndreas.Sandberg@ARM.com } else if (addrMapping == Enums::RoCoRaBaCh) { 13210388SAndreas.Sandberg@ARM.com if (system()->cacheLineSize() != range.granularity()) 13310388SAndreas.Sandberg@ARM.com fatal("Interleaving of %s doesn't match RoCoRaBaCh " 13410388SAndreas.Sandberg@ARM.com "address map\n", name()); 13510388SAndreas.Sandberg@ARM.com } 13610388SAndreas.Sandberg@ARM.com } 13710388SAndreas.Sandberg@ARM.com 13810388SAndreas.Sandberg@ARM.com // some basic sanity checks 13910388SAndreas.Sandberg@ARM.com if (tREFI <= tRP || tREFI <= tRFC) { 14010388SAndreas.Sandberg@ARM.com fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 14110388SAndreas.Sandberg@ARM.com tREFI, tRP, tRFC); 14210388SAndreas.Sandberg@ARM.com } 14310388SAndreas.Sandberg@ARM.com} 14410388SAndreas.Sandberg@ARM.com 14510388SAndreas.Sandberg@ARM.comvoid 14610388SAndreas.Sandberg@ARM.comDRAMCtrl::init() 14710388SAndreas.Sandberg@ARM.com{ 14810388SAndreas.Sandberg@ARM.com if (!port.isConnected()) { 14910388SAndreas.Sandberg@ARM.com fatal("DRAMCtrl %s is unconnected!\n", name()); 15010388SAndreas.Sandberg@ARM.com } else { 15110388SAndreas.Sandberg@ARM.com port.sendRangeChange(); 15210388SAndreas.Sandberg@ARM.com } 15310388SAndreas.Sandberg@ARM.com} 15410388SAndreas.Sandberg@ARM.com 15510388SAndreas.Sandberg@ARM.comvoid 15610388SAndreas.Sandberg@ARM.comDRAMCtrl::startup() 15710388SAndreas.Sandberg@ARM.com{ 15810388SAndreas.Sandberg@ARM.com // update the start tick for the precharge accounting to the 15910388SAndreas.Sandberg@ARM.com // current tick 16010388SAndreas.Sandberg@ARM.com pwrStateTick = curTick(); 16110388SAndreas.Sandberg@ARM.com 16210388SAndreas.Sandberg@ARM.com // shift the bus busy time sufficiently far ahead that we never 16310388SAndreas.Sandberg@ARM.com // have to worry about negative values when computing the time for 16410388SAndreas.Sandberg@ARM.com // the next request, this will add an insignificant bubble at the 16510388SAndreas.Sandberg@ARM.com // start of simulation 16610388SAndreas.Sandberg@ARM.com busBusyUntil = curTick() + tRP + tRCD + tCL; 16710388SAndreas.Sandberg@ARM.com 16810388SAndreas.Sandberg@ARM.com // kick off the refresh, and give ourselves enough time to 16910388SAndreas.Sandberg@ARM.com // precharge 17010388SAndreas.Sandberg@ARM.com schedule(refreshEvent, curTick() + tREFI - tRP); 17110388SAndreas.Sandberg@ARM.com} 17210388SAndreas.Sandberg@ARM.com 17310388SAndreas.Sandberg@ARM.comTick 17410388SAndreas.Sandberg@ARM.comDRAMCtrl::recvAtomic(PacketPtr pkt) 17510388SAndreas.Sandberg@ARM.com{ 17610388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 17710388SAndreas.Sandberg@ARM.com 17810388SAndreas.Sandberg@ARM.com // do the actual memory access and turn the packet into a response 17910388SAndreas.Sandberg@ARM.com access(pkt); 18010388SAndreas.Sandberg@ARM.com 18110388SAndreas.Sandberg@ARM.com Tick latency = 0; 18210388SAndreas.Sandberg@ARM.com if (!pkt->memInhibitAsserted() && pkt->hasData()) { 18310388SAndreas.Sandberg@ARM.com // this value is not supposed to be accurate, just enough to 18410388SAndreas.Sandberg@ARM.com // keep things going, mimic a closed page 18510388SAndreas.Sandberg@ARM.com latency = tRP + tRCD + tCL; 18610388SAndreas.Sandberg@ARM.com } 18710388SAndreas.Sandberg@ARM.com return latency; 18810388SAndreas.Sandberg@ARM.com} 18910388SAndreas.Sandberg@ARM.com 19010388SAndreas.Sandberg@ARM.combool 19110388SAndreas.Sandberg@ARM.comDRAMCtrl::readQueueFull(unsigned int neededEntries) const 19210388SAndreas.Sandberg@ARM.com{ 19310388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 19410388SAndreas.Sandberg@ARM.com readBufferSize, readQueue.size() + respQueue.size(), 19510388SAndreas.Sandberg@ARM.com neededEntries); 19610388SAndreas.Sandberg@ARM.com 19710388SAndreas.Sandberg@ARM.com return 19810388SAndreas.Sandberg@ARM.com (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 19910388SAndreas.Sandberg@ARM.com} 20010388SAndreas.Sandberg@ARM.com 20110388SAndreas.Sandberg@ARM.combool 20210388SAndreas.Sandberg@ARM.comDRAMCtrl::writeQueueFull(unsigned int neededEntries) const 20310388SAndreas.Sandberg@ARM.com{ 20410388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 20510388SAndreas.Sandberg@ARM.com writeBufferSize, writeQueue.size(), neededEntries); 20610388SAndreas.Sandberg@ARM.com return (writeQueue.size() + neededEntries) > writeBufferSize; 20710388SAndreas.Sandberg@ARM.com} 20810388SAndreas.Sandberg@ARM.com 20910388SAndreas.Sandberg@ARM.comDRAMCtrl::DRAMPacket* 21010388SAndreas.Sandberg@ARM.comDRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 21110388SAndreas.Sandberg@ARM.com bool isRead) 21210388SAndreas.Sandberg@ARM.com{ 21310388SAndreas.Sandberg@ARM.com // decode the address based on the address mapping scheme, with 21410388SAndreas.Sandberg@ARM.com // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 21510388SAndreas.Sandberg@ARM.com // channel, respectively 21610388SAndreas.Sandberg@ARM.com uint8_t rank; 21710388SAndreas.Sandberg@ARM.com uint8_t bank; 21810388SAndreas.Sandberg@ARM.com uint16_t row; 21910388SAndreas.Sandberg@ARM.com 22010388SAndreas.Sandberg@ARM.com // truncate the address to the access granularity 22110388SAndreas.Sandberg@ARM.com Addr addr = dramPktAddr / burstSize; 22210388SAndreas.Sandberg@ARM.com 22310388SAndreas.Sandberg@ARM.com // we have removed the lowest order address bits that denote the 22410388SAndreas.Sandberg@ARM.com // position within the column 22510388SAndreas.Sandberg@ARM.com if (addrMapping == Enums::RoRaBaChCo) { 22610388SAndreas.Sandberg@ARM.com // the lowest order bits denote the column to ensure that 22710388SAndreas.Sandberg@ARM.com // sequential cache lines occupy the same row 22810388SAndreas.Sandberg@ARM.com addr = addr / columnsPerRowBuffer; 22910388SAndreas.Sandberg@ARM.com 23010388SAndreas.Sandberg@ARM.com // take out the channel part of the address 23110388SAndreas.Sandberg@ARM.com addr = addr / channels; 23210388SAndreas.Sandberg@ARM.com 23310388SAndreas.Sandberg@ARM.com // after the channel bits, get the bank bits to interleave 23410388SAndreas.Sandberg@ARM.com // over the banks 23510388SAndreas.Sandberg@ARM.com bank = addr % banksPerRank; 23610905Sandreas.sandberg@arm.com addr = addr / banksPerRank; 23710388SAndreas.Sandberg@ARM.com 23810388SAndreas.Sandberg@ARM.com // after the bank, we get the rank bits which thus interleaves 23910388SAndreas.Sandberg@ARM.com // over the ranks 24010388SAndreas.Sandberg@ARM.com rank = addr % ranksPerChannel; 24110388SAndreas.Sandberg@ARM.com addr = addr / ranksPerChannel; 24210388SAndreas.Sandberg@ARM.com 24310905Sandreas.sandberg@arm.com // lastly, get the row bits 24410388SAndreas.Sandberg@ARM.com row = addr % rowsPerBank; 24510388SAndreas.Sandberg@ARM.com addr = addr / rowsPerBank; 24610388SAndreas.Sandberg@ARM.com } else if (addrMapping == Enums::RoRaBaCoCh) { 24710905Sandreas.sandberg@arm.com // take out the channel part of the address 24810388SAndreas.Sandberg@ARM.com addr = addr / channels; 24910388SAndreas.Sandberg@ARM.com 25010388SAndreas.Sandberg@ARM.com // next, the column 25110388SAndreas.Sandberg@ARM.com addr = addr / columnsPerRowBuffer; 25210388SAndreas.Sandberg@ARM.com 25310388SAndreas.Sandberg@ARM.com // after the column bits, we get the bank bits to interleave 25410388SAndreas.Sandberg@ARM.com // over the banks 25510388SAndreas.Sandberg@ARM.com bank = addr % banksPerRank; 25610388SAndreas.Sandberg@ARM.com addr = addr / banksPerRank; 25710388SAndreas.Sandberg@ARM.com 25810388SAndreas.Sandberg@ARM.com // after the bank, we get the rank bits which thus interleaves 25910388SAndreas.Sandberg@ARM.com // over the ranks 26010388SAndreas.Sandberg@ARM.com rank = addr % ranksPerChannel; 26110388SAndreas.Sandberg@ARM.com addr = addr / ranksPerChannel; 26210388SAndreas.Sandberg@ARM.com 26310388SAndreas.Sandberg@ARM.com // lastly, get the row bits 26410388SAndreas.Sandberg@ARM.com row = addr % rowsPerBank; 26510388SAndreas.Sandberg@ARM.com addr = addr / rowsPerBank; 26610388SAndreas.Sandberg@ARM.com } else if (addrMapping == Enums::RoCoRaBaCh) { 26710388SAndreas.Sandberg@ARM.com // optimise for closed page mode and utilise maximum 26810388SAndreas.Sandberg@ARM.com // parallelism of the DRAM (at the cost of power) 26910388SAndreas.Sandberg@ARM.com 27010388SAndreas.Sandberg@ARM.com // take out the channel part of the address, not that this has 27110388SAndreas.Sandberg@ARM.com // to match with how accesses are interleaved between the 27210388SAndreas.Sandberg@ARM.com // controllers in the address mapping 27310388SAndreas.Sandberg@ARM.com addr = addr / channels; 27410388SAndreas.Sandberg@ARM.com 27510388SAndreas.Sandberg@ARM.com // start with the bank bits, as this provides the maximum 27610388SAndreas.Sandberg@ARM.com // opportunity for parallelism between requests 27710388SAndreas.Sandberg@ARM.com bank = addr % banksPerRank; 27810388SAndreas.Sandberg@ARM.com addr = addr / banksPerRank; 27910388SAndreas.Sandberg@ARM.com 28010388SAndreas.Sandberg@ARM.com // next get the rank bits 28110388SAndreas.Sandberg@ARM.com rank = addr % ranksPerChannel; 28210388SAndreas.Sandberg@ARM.com addr = addr / ranksPerChannel; 28310388SAndreas.Sandberg@ARM.com 28410388SAndreas.Sandberg@ARM.com // next the column bits which we do not need to keep track of 28510388SAndreas.Sandberg@ARM.com // and simply skip past 28610388SAndreas.Sandberg@ARM.com addr = addr / columnsPerRowBuffer; 28710388SAndreas.Sandberg@ARM.com 28810388SAndreas.Sandberg@ARM.com // lastly, get the row bits 28910388SAndreas.Sandberg@ARM.com row = addr % rowsPerBank; 29010388SAndreas.Sandberg@ARM.com addr = addr / rowsPerBank; 29110388SAndreas.Sandberg@ARM.com } else 29210388SAndreas.Sandberg@ARM.com panic("Unknown address mapping policy chosen!"); 29310388SAndreas.Sandberg@ARM.com 29410388SAndreas.Sandberg@ARM.com assert(rank < ranksPerChannel); 29510388SAndreas.Sandberg@ARM.com assert(bank < banksPerRank); 29610388SAndreas.Sandberg@ARM.com assert(row < rowsPerBank); 29710388SAndreas.Sandberg@ARM.com 29810388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 29910388SAndreas.Sandberg@ARM.com dramPktAddr, rank, bank, row); 30010388SAndreas.Sandberg@ARM.com 30110388SAndreas.Sandberg@ARM.com // create the corresponding DRAM packet with the entry time and 30210388SAndreas.Sandberg@ARM.com // ready time set to the current tick, the latter will be updated 30310388SAndreas.Sandberg@ARM.com // later 30410388SAndreas.Sandberg@ARM.com uint16_t bank_id = banksPerRank * rank + bank; 30510388SAndreas.Sandberg@ARM.com return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 30610388SAndreas.Sandberg@ARM.com size, banks[rank][bank]); 30710388SAndreas.Sandberg@ARM.com} 30810388SAndreas.Sandberg@ARM.com 30910388SAndreas.Sandberg@ARM.comvoid 31010388SAndreas.Sandberg@ARM.comDRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 31110388SAndreas.Sandberg@ARM.com{ 31210388SAndreas.Sandberg@ARM.com // only add to the read queue here. whenever the request is 31310388SAndreas.Sandberg@ARM.com // eventually done, set the readyTime, and call schedule() 31410388SAndreas.Sandberg@ARM.com assert(!pkt->isWrite()); 31510388SAndreas.Sandberg@ARM.com 31610388SAndreas.Sandberg@ARM.com assert(pktCount != 0); 31710388SAndreas.Sandberg@ARM.com 31810388SAndreas.Sandberg@ARM.com // if the request size is larger than burst size, the pkt is split into 31910388SAndreas.Sandberg@ARM.com // multiple DRAM packets 32010388SAndreas.Sandberg@ARM.com // Note if the pkt starting address is not aligened to burst size, the 32110388SAndreas.Sandberg@ARM.com // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 32210388SAndreas.Sandberg@ARM.com // are aligned to burst size boundaries. This is to ensure we accurately 32310388SAndreas.Sandberg@ARM.com // check read packets against packets in write queue. 32410388SAndreas.Sandberg@ARM.com Addr addr = pkt->getAddr(); 32510388SAndreas.Sandberg@ARM.com unsigned pktsServicedByWrQ = 0; 32610388SAndreas.Sandberg@ARM.com BurstHelper* burst_helper = NULL; 32710388SAndreas.Sandberg@ARM.com for (int cnt = 0; cnt < pktCount; ++cnt) { 32810388SAndreas.Sandberg@ARM.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 32910388SAndreas.Sandberg@ARM.com pkt->getAddr() + pkt->getSize()) - addr; 33010388SAndreas.Sandberg@ARM.com readPktSize[ceilLog2(size)]++; 33110388SAndreas.Sandberg@ARM.com readBursts++; 33210388SAndreas.Sandberg@ARM.com 33310388SAndreas.Sandberg@ARM.com // First check write buffer to see if the data is already at 33410388SAndreas.Sandberg@ARM.com // the controller 33510388SAndreas.Sandberg@ARM.com bool foundInWrQ = false; 33610388SAndreas.Sandberg@ARM.com for (auto i = writeQueue.begin(); i != writeQueue.end(); ++i) { 33710388SAndreas.Sandberg@ARM.com // check if the read is subsumed in the write entry we are 33810388SAndreas.Sandberg@ARM.com // looking at 33910905Sandreas.sandberg@arm.com if ((*i)->addr <= addr && 34010388SAndreas.Sandberg@ARM.com (addr + size) <= ((*i)->addr + (*i)->size)) { 34110388SAndreas.Sandberg@ARM.com foundInWrQ = true; 34210905Sandreas.sandberg@arm.com servicedByWrQ++; 34310388SAndreas.Sandberg@ARM.com pktsServicedByWrQ++; 34410905Sandreas.sandberg@arm.com DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 34510905Sandreas.sandberg@arm.com "write queue\n", addr, size); 34610388SAndreas.Sandberg@ARM.com bytesReadWrQ += burstSize; 34710388SAndreas.Sandberg@ARM.com break; 34810388SAndreas.Sandberg@ARM.com } 34910905Sandreas.sandberg@arm.com } 35010388SAndreas.Sandberg@ARM.com 35110388SAndreas.Sandberg@ARM.com // If not found in the write q, make a DRAM packet and 35210905Sandreas.sandberg@arm.com // push it onto the read queue 35310388SAndreas.Sandberg@ARM.com if (!foundInWrQ) { 35410388SAndreas.Sandberg@ARM.com 35510905Sandreas.sandberg@arm.com // Make the burst helper for split packets 35610388SAndreas.Sandberg@ARM.com if (pktCount > 1 && burst_helper == NULL) { 35710388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Read to addr %lld translates to %d " 35810388SAndreas.Sandberg@ARM.com "dram requests\n", pkt->getAddr(), pktCount); 35910388SAndreas.Sandberg@ARM.com burst_helper = new BurstHelper(pktCount); 36010388SAndreas.Sandberg@ARM.com } 36110388SAndreas.Sandberg@ARM.com 36210388SAndreas.Sandberg@ARM.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 36310388SAndreas.Sandberg@ARM.com dram_pkt->burstHelper = burst_helper; 36410388SAndreas.Sandberg@ARM.com 36510388SAndreas.Sandberg@ARM.com assert(!readQueueFull(1)); 36610388SAndreas.Sandberg@ARM.com rdQLenPdf[readQueue.size() + respQueue.size()]++; 36710388SAndreas.Sandberg@ARM.com 36810388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Adding to read queue\n"); 36910388SAndreas.Sandberg@ARM.com 37010388SAndreas.Sandberg@ARM.com readQueue.push_back(dram_pkt); 37110388SAndreas.Sandberg@ARM.com 37210388SAndreas.Sandberg@ARM.com // Update stats 37310388SAndreas.Sandberg@ARM.com avgRdQLen = readQueue.size() + respQueue.size(); 37410388SAndreas.Sandberg@ARM.com } 37510388SAndreas.Sandberg@ARM.com 37610388SAndreas.Sandberg@ARM.com // Starting address of next dram pkt (aligend to burstSize boundary) 37710388SAndreas.Sandberg@ARM.com addr = (addr | (burstSize - 1)) + 1; 37810388SAndreas.Sandberg@ARM.com } 37910388SAndreas.Sandberg@ARM.com 38010388SAndreas.Sandberg@ARM.com // If all packets are serviced by write queue, we send the repsonse back 38110388SAndreas.Sandberg@ARM.com if (pktsServicedByWrQ == pktCount) { 38210388SAndreas.Sandberg@ARM.com accessAndRespond(pkt, frontendLatency); 38310388SAndreas.Sandberg@ARM.com return; 38410388SAndreas.Sandberg@ARM.com } 38510388SAndreas.Sandberg@ARM.com 38610388SAndreas.Sandberg@ARM.com // Update how many split packets are serviced by write queue 38710388SAndreas.Sandberg@ARM.com if (burst_helper != NULL) 38810388SAndreas.Sandberg@ARM.com burst_helper->burstsServiced = pktsServicedByWrQ; 38910388SAndreas.Sandberg@ARM.com 39010388SAndreas.Sandberg@ARM.com // If we are not already scheduled to get a request out of the 39110388SAndreas.Sandberg@ARM.com // queue, do so now 39210388SAndreas.Sandberg@ARM.com if (!nextReqEvent.scheduled()) { 39310388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Request scheduled immediately\n"); 39410388SAndreas.Sandberg@ARM.com schedule(nextReqEvent, curTick()); 39510388SAndreas.Sandberg@ARM.com } 39610388SAndreas.Sandberg@ARM.com} 39710388SAndreas.Sandberg@ARM.com 39810388SAndreas.Sandberg@ARM.comvoid 39910388SAndreas.Sandberg@ARM.comDRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 40010388SAndreas.Sandberg@ARM.com{ 40110388SAndreas.Sandberg@ARM.com // only add to the write queue here. whenever the request is 40210388SAndreas.Sandberg@ARM.com // eventually done, set the readyTime, and call schedule() 40310388SAndreas.Sandberg@ARM.com assert(pkt->isWrite()); 40410388SAndreas.Sandberg@ARM.com 40510388SAndreas.Sandberg@ARM.com // if the request size is larger than burst size, the pkt is split into 40610388SAndreas.Sandberg@ARM.com // multiple DRAM packets 40710388SAndreas.Sandberg@ARM.com Addr addr = pkt->getAddr(); 40810388SAndreas.Sandberg@ARM.com for (int cnt = 0; cnt < pktCount; ++cnt) { 40910388SAndreas.Sandberg@ARM.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 41010388SAndreas.Sandberg@ARM.com pkt->getAddr() + pkt->getSize()) - addr; 41110388SAndreas.Sandberg@ARM.com writePktSize[ceilLog2(size)]++; 41210388SAndreas.Sandberg@ARM.com writeBursts++; 41310388SAndreas.Sandberg@ARM.com 41410388SAndreas.Sandberg@ARM.com // see if we can merge with an existing item in the write 41510388SAndreas.Sandberg@ARM.com // queue and keep track of whether we have merged or not so we 41610388SAndreas.Sandberg@ARM.com // can stop at that point and also avoid enqueueing a new 41710388SAndreas.Sandberg@ARM.com // request 41810388SAndreas.Sandberg@ARM.com bool merged = false; 41910388SAndreas.Sandberg@ARM.com auto w = writeQueue.begin(); 42010388SAndreas.Sandberg@ARM.com 42110388SAndreas.Sandberg@ARM.com while(!merged && w != writeQueue.end()) { 42210388SAndreas.Sandberg@ARM.com // either of the two could be first, if they are the same 42310388SAndreas.Sandberg@ARM.com // it does not matter which way we go 42410388SAndreas.Sandberg@ARM.com if ((*w)->addr >= addr) { 42510602SAndreas.Sandberg@ARM.com // the existing one starts after the new one, figure 42610388SAndreas.Sandberg@ARM.com // out where the new one ends with respect to the 42710388SAndreas.Sandberg@ARM.com // existing one 42810388SAndreas.Sandberg@ARM.com if ((addr + size) >= ((*w)->addr + (*w)->size)) { 42910388SAndreas.Sandberg@ARM.com // check if the existing one is completely 43010388SAndreas.Sandberg@ARM.com // subsumed in the new one 43110388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Merging write covering existing burst\n"); 43210388SAndreas.Sandberg@ARM.com merged = true; 43310388SAndreas.Sandberg@ARM.com // update both the address and the size 43410388SAndreas.Sandberg@ARM.com (*w)->addr = addr; 43510388SAndreas.Sandberg@ARM.com (*w)->size = size; 43610388SAndreas.Sandberg@ARM.com } else if ((addr + size) >= (*w)->addr && 43710602SAndreas.Sandberg@ARM.com ((*w)->addr + (*w)->size - addr) <= burstSize) { 43810388SAndreas.Sandberg@ARM.com // the new one is just before or partially 43910388SAndreas.Sandberg@ARM.com // overlapping with the existing one, and together 44010388SAndreas.Sandberg@ARM.com // they fit within a burst 44110388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Merging write before existing burst\n"); 44210388SAndreas.Sandberg@ARM.com merged = true; 44310388SAndreas.Sandberg@ARM.com // the existing queue item needs to be adjusted with 44410388SAndreas.Sandberg@ARM.com // respect to both address and size 44510388SAndreas.Sandberg@ARM.com (*w)->size = (*w)->addr + (*w)->size - addr; 44610388SAndreas.Sandberg@ARM.com (*w)->addr = addr; 44710388SAndreas.Sandberg@ARM.com } 44810388SAndreas.Sandberg@ARM.com } else { 44910388SAndreas.Sandberg@ARM.com // the new one starts after the current one, figure 45010388SAndreas.Sandberg@ARM.com // out where the existing one ends with respect to the 45110388SAndreas.Sandberg@ARM.com // new one 45210388SAndreas.Sandberg@ARM.com if (((*w)->addr + (*w)->size) >= (addr + size)) { 45310388SAndreas.Sandberg@ARM.com // check if the new one is completely subsumed in the 45410388SAndreas.Sandberg@ARM.com // existing one 45510388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Merging write into existing burst\n"); 45610388SAndreas.Sandberg@ARM.com merged = true; 45710388SAndreas.Sandberg@ARM.com // no adjustments necessary 45810388SAndreas.Sandberg@ARM.com } else if (((*w)->addr + (*w)->size) >= addr && 45910388SAndreas.Sandberg@ARM.com (addr + size - (*w)->addr) <= burstSize) { 46010388SAndreas.Sandberg@ARM.com // the existing one is just before or partially 46110388SAndreas.Sandberg@ARM.com // overlapping with the new one, and together 46210388SAndreas.Sandberg@ARM.com // they fit within a burst 46310388SAndreas.Sandberg@ARM.com DPRINTF(DRAM, "Merging write after existing burst\n"); 46410388SAndreas.Sandberg@ARM.com merged = true; 46510388SAndreas.Sandberg@ARM.com // the address is right, and only the size has 46610388SAndreas.Sandberg@ARM.com // to be adjusted 46710388SAndreas.Sandberg@ARM.com (*w)->size = addr + size - (*w)->addr; 46810388SAndreas.Sandberg@ARM.com } 46910388SAndreas.Sandberg@ARM.com } 47010388SAndreas.Sandberg@ARM.com ++w; 47110388SAndreas.Sandberg@ARM.com } 47210388SAndreas.Sandberg@ARM.com 47310388SAndreas.Sandberg@ARM.com // if the item was not merged we need to create a new write 47410388SAndreas.Sandberg@ARM.com // and enqueue it 47510388SAndreas.Sandberg@ARM.com if (!merged) { 47610388SAndreas.Sandberg@ARM.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 47710388SAndreas.Sandberg@ARM.com 47810388SAndreas.Sandberg@ARM.com assert(writeQueue.size() < writeBufferSize); 479 wrQLenPdf[writeQueue.size()]++; 480 481 DPRINTF(DRAM, "Adding to write queue\n"); 482 483 writeQueue.push_back(dram_pkt); 484 485 // Update stats 486 avgWrQLen = writeQueue.size(); 487 } else { 488 // keep track of the fact that this burst effectively 489 // disappeared as it was merged with an existing one 490 mergedWrBursts++; 491 } 492 493 // Starting address of next dram pkt (aligend to burstSize boundary) 494 addr = (addr | (burstSize - 1)) + 1; 495 } 496 497 // we do not wait for the writes to be send to the actual memory, 498 // but instead take responsibility for the consistency here and 499 // snoop the write queue for any upcoming reads 500 // @todo, if a pkt size is larger than burst size, we might need a 501 // different front end latency 502 accessAndRespond(pkt, frontendLatency); 503 504 // If we are not already scheduled to get a request out of the 505 // queue, do so now 506 if (!nextReqEvent.scheduled()) { 507 DPRINTF(DRAM, "Request scheduled immediately\n"); 508 schedule(nextReqEvent, curTick()); 509 } 510} 511 512void 513DRAMCtrl::printQs() const { 514 DPRINTF(DRAM, "===READ QUEUE===\n\n"); 515 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 516 DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 517 } 518 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 519 for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 520 DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 521 } 522 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 523 for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 524 DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 525 } 526} 527 528bool 529DRAMCtrl::recvTimingReq(PacketPtr pkt) 530{ 531 /// @todo temporary hack to deal with memory corruption issues until 532 /// 4-phase transactions are complete 533 for (int x = 0; x < pendingDelete.size(); x++) 534 delete pendingDelete[x]; 535 pendingDelete.clear(); 536 537 // This is where we enter from the outside world 538 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 539 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 540 541 // simply drop inhibited packets for now 542 if (pkt->memInhibitAsserted()) { 543 DPRINTF(DRAM, "Inhibited packet -- Dropping it now\n"); 544 pendingDelete.push_back(pkt); 545 return true; 546 } 547 548 // Calc avg gap between requests 549 if (prevArrival != 0) { 550 totGap += curTick() - prevArrival; 551 } 552 prevArrival = curTick(); 553 554 555 // Find out how many dram packets a pkt translates to 556 // If the burst size is equal or larger than the pkt size, then a pkt 557 // translates to only one dram packet. Otherwise, a pkt translates to 558 // multiple dram packets 559 unsigned size = pkt->getSize(); 560 unsigned offset = pkt->getAddr() & (burstSize - 1); 561 unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 562 563 // check local buffers and do not accept if full 564 if (pkt->isRead()) { 565 assert(size != 0); 566 if (readQueueFull(dram_pkt_count)) { 567 DPRINTF(DRAM, "Read queue full, not accepting\n"); 568 // remember that we have to retry this port 569 retryRdReq = true; 570 numRdRetry++; 571 return false; 572 } else { 573 addToReadQueue(pkt, dram_pkt_count); 574 readReqs++; 575 bytesReadSys += size; 576 } 577 } else if (pkt->isWrite()) { 578 assert(size != 0); 579 if (writeQueueFull(dram_pkt_count)) { 580 DPRINTF(DRAM, "Write queue full, not accepting\n"); 581 // remember that we have to retry this port 582 retryWrReq = true; 583 numWrRetry++; 584 return false; 585 } else { 586 addToWriteQueue(pkt, dram_pkt_count); 587 writeReqs++; 588 bytesWrittenSys += size; 589 } 590 } else { 591 DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 592 neitherReadNorWrite++; 593 accessAndRespond(pkt, 1); 594 } 595 596 return true; 597} 598 599void 600DRAMCtrl::processRespondEvent() 601{ 602 DPRINTF(DRAM, 603 "processRespondEvent(): Some req has reached its readyTime\n"); 604 605 DRAMPacket* dram_pkt = respQueue.front(); 606 607 if (dram_pkt->burstHelper) { 608 // it is a split packet 609 dram_pkt->burstHelper->burstsServiced++; 610 if (dram_pkt->burstHelper->burstsServiced == 611 dram_pkt->burstHelper->burstCount) { 612 // we have now serviced all children packets of a system packet 613 // so we can now respond to the requester 614 // @todo we probably want to have a different front end and back 615 // end latency for split packets 616 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 617 delete dram_pkt->burstHelper; 618 dram_pkt->burstHelper = NULL; 619 } 620 } else { 621 // it is not a split packet 622 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 623 } 624 625 delete respQueue.front(); 626 respQueue.pop_front(); 627 628 if (!respQueue.empty()) { 629 assert(respQueue.front()->readyTime >= curTick()); 630 assert(!respondEvent.scheduled()); 631 schedule(respondEvent, respQueue.front()->readyTime); 632 } else { 633 // if there is nothing left in any queue, signal a drain 634 if (writeQueue.empty() && readQueue.empty() && 635 drainManager) { 636 drainManager->signalDrainDone(); 637 drainManager = NULL; 638 } 639 } 640 641 // We have made a location in the queue available at this point, 642 // so if there is a read that was forced to wait, retry now 643 if (retryRdReq) { 644 retryRdReq = false; 645 port.sendRetry(); 646 } 647} 648 649void 650DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue) 651{ 652 // This method does the arbitration between requests. The chosen 653 // packet is simply moved to the head of the queue. The other 654 // methods know that this is the place to look. For example, with 655 // FCFS, this method does nothing 656 assert(!queue.empty()); 657 658 if (queue.size() == 1) { 659 DPRINTF(DRAM, "Single request, nothing to do\n"); 660 return; 661 } 662 663 if (memSchedPolicy == Enums::fcfs) { 664 // Do nothing, since the correct request is already head 665 } else if (memSchedPolicy == Enums::frfcfs) { 666 reorderQueue(queue); 667 } else 668 panic("No scheduling policy chosen\n"); 669} 670 671void 672DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue) 673{ 674 // Only determine this when needed 675 uint64_t earliest_banks = 0; 676 677 // Search for row hits first, if no row hit is found then schedule the 678 // packet to one of the earliest banks available 679 bool found_earliest_pkt = false; 680 auto selected_pkt_it = queue.begin(); 681 682 for (auto i = queue.begin(); i != queue.end() ; ++i) { 683 DRAMPacket* dram_pkt = *i; 684 const Bank& bank = dram_pkt->bankRef; 685 // Check if it is a row hit 686 if (bank.openRow == dram_pkt->row) { 687 // FCFS within the hits 688 DPRINTF(DRAM, "Row buffer hit\n"); 689 selected_pkt_it = i; 690 break; 691 } else if (!found_earliest_pkt) { 692 // No row hit, go for first ready 693 if (earliest_banks == 0) 694 earliest_banks = minBankActAt(queue); 695 696 // simplistic approximation of when the bank can issue an 697 // activate, this is calculated in minBankActAt and could 698 // be cached 699 Tick act_at = bank.openRow == Bank::NO_ROW ? 700 bank.actAllowedAt : 701 std::max(bank.preAllowedAt, curTick()) + tRP; 702 703 // Bank is ready or is the first available bank 704 if (act_at <= curTick() || 705 bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 706 // Remember the packet to be scheduled to one of the earliest 707 // banks available, FCFS amongst the earliest banks 708 selected_pkt_it = i; 709 found_earliest_pkt = true; 710 } 711 } 712 } 713 714 DRAMPacket* selected_pkt = *selected_pkt_it; 715 queue.erase(selected_pkt_it); 716 queue.push_front(selected_pkt); 717} 718 719void 720DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 721{ 722 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 723 724 bool needsResponse = pkt->needsResponse(); 725 // do the actual memory access which also turns the packet into a 726 // response 727 access(pkt); 728 729 // turn packet around to go back to requester if response expected 730 if (needsResponse) { 731 // access already turned the packet into a response 732 assert(pkt->isResponse()); 733 734 // @todo someone should pay for this 735 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 736 737 // queue the packet in the response queue to be sent out after 738 // the static latency has passed 739 port.schedTimingResp(pkt, curTick() + static_latency); 740 } else { 741 // @todo the packet is going to be deleted, and the DRAMPacket 742 // is still having a pointer to it 743 pendingDelete.push_back(pkt); 744 } 745 746 DPRINTF(DRAM, "Done\n"); 747 748 return; 749} 750 751void 752DRAMCtrl::activateBank(Tick act_tick, uint8_t rank, uint8_t bank, 753 uint16_t row, Bank& bank_ref) 754{ 755 assert(0 <= rank && rank < ranksPerChannel); 756 assert(actTicks[rank].size() == activationLimit); 757 758 DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 759 760 // update the open row 761 assert(bank_ref.openRow == Bank::NO_ROW); 762 bank_ref.openRow = row; 763 764 // start counting anew, this covers both the case when we 765 // auto-precharged, and when this access is forced to 766 // precharge 767 bank_ref.bytesAccessed = 0; 768 bank_ref.rowAccesses = 0; 769 770 ++numBanksActive; 771 assert(numBanksActive <= banksPerRank * ranksPerChannel); 772 773 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n", 774 act_tick, numBanksActive); 775 776 // The next access has to respect tRAS for this bank 777 bank_ref.preAllowedAt = act_tick + tRAS; 778 779 // Respect the row-to-column command delay 780 bank_ref.colAllowedAt = act_tick + tRCD; 781 782 // start by enforcing tRRD 783 for(int i = 0; i < banksPerRank; i++) { 784 // next activate to any bank in this rank must not happen 785 // before tRRD 786 banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 787 banks[rank][i].actAllowedAt); 788 } 789 790 // next, we deal with tXAW, if the activation limit is disabled 791 // then we are done 792 if (actTicks[rank].empty()) 793 return; 794 795 // sanity check 796 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 797 panic("Got %d activates in window %d (%llu - %llu) which is smaller " 798 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 799 act_tick, actTicks[rank].back(), tXAW); 800 } 801 802 // shift the times used for the book keeping, the last element 803 // (highest index) is the oldest one and hence the lowest value 804 actTicks[rank].pop_back(); 805 806 // record an new activation (in the future) 807 actTicks[rank].push_front(act_tick); 808 809 // cannot activate more than X times in time window tXAW, push the 810 // next one (the X + 1'st activate) to be tXAW away from the 811 // oldest in our window of X 812 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 813 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " 814 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); 815 for(int j = 0; j < banksPerRank; j++) 816 // next activate must not happen before end of window 817 banks[rank][j].actAllowedAt = 818 std::max(actTicks[rank].back() + tXAW, 819 banks[rank][j].actAllowedAt); 820 } 821 822 // at the point when this activate takes place, make sure we 823 // transition to the active power state 824 if (!activateEvent.scheduled()) 825 schedule(activateEvent, act_tick); 826 else if (activateEvent.when() > act_tick) 827 // move it sooner in time 828 reschedule(activateEvent, act_tick); 829} 830 831void 832DRAMCtrl::processActivateEvent() 833{ 834 // we should transition to the active state as soon as any bank is active 835 if (pwrState != PWR_ACT) 836 // note that at this point numBanksActive could be back at 837 // zero again due to a precharge scheduled in the future 838 schedulePowerEvent(PWR_ACT, curTick()); 839} 840 841void 842DRAMCtrl::prechargeBank(Bank& bank, Tick pre_at) 843{ 844 // make sure the bank has an open row 845 assert(bank.openRow != Bank::NO_ROW); 846 847 // sample the bytes per activate here since we are closing 848 // the page 849 bytesPerActivate.sample(bank.bytesAccessed); 850 851 bank.openRow = Bank::NO_ROW; 852 853 // no precharge allowed before this one 854 bank.preAllowedAt = pre_at; 855 856 Tick pre_done_at = pre_at + tRP; 857 858 bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 859 860 assert(numBanksActive != 0); 861 --numBanksActive; 862 863 DPRINTF(DRAM, "Precharging bank at tick %lld, now got %d active\n", 864 pre_at, numBanksActive); 865 866 // if we look at the current number of active banks we might be 867 // tempted to think the DRAM is now idle, however this can be 868 // undone by an activate that is scheduled to happen before we 869 // would have reached the idle state, so schedule an event and 870 // rather check once we actually make it to the point in time when 871 // the (last) precharge takes place 872 if (!prechargeEvent.scheduled()) 873 schedule(prechargeEvent, pre_done_at); 874 else if (prechargeEvent.when() < pre_done_at) 875 reschedule(prechargeEvent, pre_done_at); 876} 877 878void 879DRAMCtrl::processPrechargeEvent() 880{ 881 // if we reached zero, then special conditions apply as we track 882 // if all banks are precharged for the power models 883 if (numBanksActive == 0) { 884 // we should transition to the idle state when the last bank 885 // is precharged 886 schedulePowerEvent(PWR_IDLE, curTick()); 887 } 888} 889 890void 891DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 892{ 893 DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 894 dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 895 896 // get the bank 897 Bank& bank = dram_pkt->bankRef; 898 899 // for the state we need to track if it is a row hit or not 900 bool row_hit = true; 901 902 // respect any constraints on the command (e.g. tRCD or tCCD) 903 Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 904 905 // Determine the access latency and update the bank state 906 if (bank.openRow == dram_pkt->row) { 907 // nothing to do 908 } else { 909 row_hit = false; 910 911 // If there is a page open, precharge it. 912 if (bank.openRow != Bank::NO_ROW) { 913 prechargeBank(bank, std::max(bank.preAllowedAt, curTick())); 914 } 915 916 // next we need to account for the delay in activating the 917 // page 918 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 919 920 // Record the activation and deal with all the global timing 921 // constraints caused be a new activation (tRRD and tXAW) 922 activateBank(act_tick, dram_pkt->rank, dram_pkt->bank, 923 dram_pkt->row, bank); 924 925 // issue the command as early as possible 926 cmd_at = bank.colAllowedAt; 927 } 928 929 // we need to wait until the bus is available before we can issue 930 // the command 931 cmd_at = std::max(cmd_at, busBusyUntil - tCL); 932 933 // update the packet ready time 934 dram_pkt->readyTime = cmd_at + tCL + tBURST; 935 936 // only one burst can use the bus at any one point in time 937 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 938 939 // not strictly necessary, but update the time for the next 940 // read/write (add a max with tCCD here) 941 bank.colAllowedAt = cmd_at + tBURST; 942 943 // If this is a write, we also need to respect the write recovery 944 // time before a precharge, in the case of a read, respect the 945 // read to precharge constraint 946 bank.preAllowedAt = std::max(bank.preAllowedAt, 947 dram_pkt->isRead ? cmd_at + tRTP : 948 dram_pkt->readyTime + tWR); 949 950 // increment the bytes accessed and the accesses per row 951 bank.bytesAccessed += burstSize; 952 ++bank.rowAccesses; 953 954 // if we reached the max, then issue with an auto-precharge 955 bool auto_precharge = pageMgmt == Enums::close || 956 bank.rowAccesses == maxAccessesPerRow; 957 958 // if we did not hit the limit, we might still want to 959 // auto-precharge 960 if (!auto_precharge && 961 (pageMgmt == Enums::open_adaptive || 962 pageMgmt == Enums::close_adaptive)) { 963 // a twist on the open and close page policies: 964 // 1) open_adaptive page policy does not blindly keep the 965 // page open, but close it if there are no row hits, and there 966 // are bank conflicts in the queue 967 // 2) close_adaptive page policy does not blindly close the 968 // page, but closes it only if there are no row hits in the queue. 969 // In this case, only force an auto precharge when there 970 // are no same page hits in the queue 971 bool got_more_hits = false; 972 bool got_bank_conflict = false; 973 974 // either look at the read queue or write queue 975 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 976 writeQueue; 977 auto p = queue.begin(); 978 // make sure we are not considering the packet that we are 979 // currently dealing with (which is the head of the queue) 980 ++p; 981 982 // keep on looking until we have found required condition or 983 // reached the end 984 while (!(got_more_hits && 985 (got_bank_conflict || pageMgmt == Enums::close_adaptive)) && 986 p != queue.end()) { 987 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 988 (dram_pkt->bank == (*p)->bank); 989 bool same_row = dram_pkt->row == (*p)->row; 990 got_more_hits |= same_rank_bank && same_row; 991 got_bank_conflict |= same_rank_bank && !same_row; 992 ++p; 993 } 994 995 // auto pre-charge when either 996 // 1) open_adaptive policy, we have not got any more hits, and 997 // have a bank conflict 998 // 2) close_adaptive policy and we have not got any more hits 999 auto_precharge = !got_more_hits && 1000 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1001 } 1002 1003 // if this access should use auto-precharge, then we are 1004 // closing the row 1005 if (auto_precharge) { 1006 prechargeBank(bank, std::max(curTick(), bank.preAllowedAt)); 1007 1008 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1009 } 1010 1011 // Update bus state 1012 busBusyUntil = dram_pkt->readyTime; 1013 1014 DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 1015 dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 1016 1017 // Update the minimum timing between the requests, this is a 1018 // conservative estimate of when we have to schedule the next 1019 // request to not introduce any unecessary bubbles. In most cases 1020 // we will wake up sooner than we have to. 1021 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1022 1023 // Update the stats and schedule the next request 1024 if (dram_pkt->isRead) { 1025 ++readsThisTime; 1026 if (row_hit) 1027 readRowHits++; 1028 bytesReadDRAM += burstSize; 1029 perBankRdBursts[dram_pkt->bankId]++; 1030 1031 // Update latency stats 1032 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 1033 totBusLat += tBURST; 1034 totQLat += cmd_at - dram_pkt->entryTime; 1035 } else { 1036 ++writesThisTime; 1037 if (row_hit) 1038 writeRowHits++; 1039 bytesWritten += burstSize; 1040 perBankWrBursts[dram_pkt->bankId]++; 1041 } 1042} 1043 1044void 1045DRAMCtrl::processNextReqEvent() 1046{ 1047 if (busState == READ_TO_WRITE) { 1048 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1049 "waiting\n", readsThisTime, readQueue.size()); 1050 1051 // sample and reset the read-related stats as we are now 1052 // transitioning to writes, and all reads are done 1053 rdPerTurnAround.sample(readsThisTime); 1054 readsThisTime = 0; 1055 1056 // now proceed to do the actual writes 1057 busState = WRITE; 1058 } else if (busState == WRITE_TO_READ) { 1059 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1060 "waiting\n", writesThisTime, writeQueue.size()); 1061 1062 wrPerTurnAround.sample(writesThisTime); 1063 writesThisTime = 0; 1064 1065 busState = READ; 1066 } 1067 1068 if (refreshState != REF_IDLE) { 1069 // if a refresh waiting for this event loop to finish, then hand 1070 // over now, and do not schedule a new nextReqEvent 1071 if (refreshState == REF_DRAIN) { 1072 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1073 1074 refreshState = REF_PRE; 1075 1076 // hand control back to the refresh event loop 1077 schedule(refreshEvent, curTick()); 1078 } 1079 1080 // let the refresh finish before issuing any further requests 1081 return; 1082 } 1083 1084 // when we get here it is either a read or a write 1085 if (busState == READ) { 1086 1087 // track if we should switch or not 1088 bool switch_to_writes = false; 1089 1090 if (readQueue.empty()) { 1091 // In the case there is no read request to go next, 1092 // trigger writes if we have passed the low threshold (or 1093 // if we are draining) 1094 if (!writeQueue.empty() && 1095 (drainManager || writeQueue.size() > writeLowThreshold)) { 1096 1097 switch_to_writes = true; 1098 } else { 1099 // check if we are drained 1100 if (respQueue.empty () && drainManager) { 1101 drainManager->signalDrainDone(); 1102 drainManager = NULL; 1103 } 1104 1105 // nothing to do, not even any point in scheduling an 1106 // event for the next request 1107 return; 1108 } 1109 } else { 1110 // Figure out which read request goes next, and move it to the 1111 // front of the read queue 1112 chooseNext(readQueue); 1113 1114 DRAMPacket* dram_pkt = readQueue.front(); 1115 1116 doDRAMAccess(dram_pkt); 1117 1118 // At this point we're done dealing with the request 1119 readQueue.pop_front(); 1120 1121 // sanity check 1122 assert(dram_pkt->size <= burstSize); 1123 assert(dram_pkt->readyTime >= curTick()); 1124 1125 // Insert into response queue. It will be sent back to the 1126 // requestor at its readyTime 1127 if (respQueue.empty()) { 1128 assert(!respondEvent.scheduled()); 1129 schedule(respondEvent, dram_pkt->readyTime); 1130 } else { 1131 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1132 assert(respondEvent.scheduled()); 1133 } 1134 1135 respQueue.push_back(dram_pkt); 1136 1137 // we have so many writes that we have to transition 1138 if (writeQueue.size() > writeHighThreshold) { 1139 switch_to_writes = true; 1140 } 1141 } 1142 1143 // switching to writes, either because the read queue is empty 1144 // and the writes have passed the low threshold (or we are 1145 // draining), or because the writes hit the hight threshold 1146 if (switch_to_writes) { 1147 // transition to writing 1148 busState = READ_TO_WRITE; 1149 1150 // add a bubble to the data bus, as defined by the 1151 // tRTW parameter 1152 busBusyUntil += tRTW; 1153 1154 // update the minimum timing between the requests, 1155 // this shifts us back in time far enough to do any 1156 // bank preparation 1157 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1158 } 1159 } else { 1160 chooseNext(writeQueue); 1161 DRAMPacket* dram_pkt = writeQueue.front(); 1162 // sanity check 1163 assert(dram_pkt->size <= burstSize); 1164 doDRAMAccess(dram_pkt); 1165 1166 writeQueue.pop_front(); 1167 delete dram_pkt; 1168 1169 // If we emptied the write queue, or got sufficiently below the 1170 // threshold (using the minWritesPerSwitch as the hysteresis) and 1171 // are not draining, or we have reads waiting and have done enough 1172 // writes, then switch to reads. 1173 if (writeQueue.empty() || 1174 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1175 !drainManager) || 1176 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1177 // turn the bus back around for reads again 1178 busState = WRITE_TO_READ; 1179 1180 // note that the we switch back to reads also in the idle 1181 // case, which eventually will check for any draining and 1182 // also pause any further scheduling if there is really 1183 // nothing to do 1184 1185 // here we get a bit creative and shift the bus busy time not 1186 // just the tWTR, but also a CAS latency to capture the fact 1187 // that we are allowed to prepare a new bank, but not issue a 1188 // read command until after tWTR, in essence we capture a 1189 // bubble on the data bus that is tWTR + tCL 1190 busBusyUntil += tWTR + tCL; 1191 1192 // update the minimum timing between the requests, this shifts 1193 // us back in time far enough to do any bank preparation 1194 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1195 } 1196 } 1197 1198 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1199 1200 // If there is space available and we have writes waiting then let 1201 // them retry. This is done here to ensure that the retry does not 1202 // cause a nextReqEvent to be scheduled before we do so as part of 1203 // the next request processing 1204 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1205 retryWrReq = false; 1206 port.sendRetry(); 1207 } 1208} 1209 1210uint64_t 1211DRAMCtrl::minBankActAt(const deque<DRAMPacket*>& queue) const 1212{ 1213 uint64_t bank_mask = 0; 1214 Tick min_act_at = MaxTick; 1215 1216 // deterimne if we have queued transactions targetting a 1217 // bank in question 1218 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 1219 for (auto p = queue.begin(); p != queue.end(); ++p) { 1220 got_waiting[(*p)->bankId] = true; 1221 } 1222 1223 for (int i = 0; i < ranksPerChannel; i++) { 1224 for (int j = 0; j < banksPerRank; j++) { 1225 uint8_t bank_id = i * banksPerRank + j; 1226 1227 // if we have waiting requests for the bank, and it is 1228 // amongst the first available, update the mask 1229 if (got_waiting[bank_id]) { 1230 // simplistic approximation of when the bank can issue 1231 // an activate, ignoring any rank-to-rank switching 1232 // cost 1233 Tick act_at = banks[i][j].openRow == Bank::NO_ROW ? 1234 banks[i][j].actAllowedAt : 1235 std::max(banks[i][j].preAllowedAt, curTick()) + tRP; 1236 1237 if (act_at <= min_act_at) { 1238 // reset bank mask if new minimum is found 1239 if (act_at < min_act_at) 1240 bank_mask = 0; 1241 // set the bit corresponding to the available bank 1242 replaceBits(bank_mask, bank_id, bank_id, 1); 1243 min_act_at = act_at; 1244 } 1245 } 1246 } 1247 } 1248 1249 return bank_mask; 1250} 1251 1252void 1253DRAMCtrl::processRefreshEvent() 1254{ 1255 // when first preparing the refresh, remember when it was due 1256 if (refreshState == REF_IDLE) { 1257 // remember when the refresh is due 1258 refreshDueAt = curTick(); 1259 1260 // proceed to drain 1261 refreshState = REF_DRAIN; 1262 1263 DPRINTF(DRAM, "Refresh due\n"); 1264 } 1265 1266 // let any scheduled read or write go ahead, after which it will 1267 // hand control back to this event loop 1268 if (refreshState == REF_DRAIN) { 1269 if (nextReqEvent.scheduled()) { 1270 // hand control over to the request loop until it is 1271 // evaluated next 1272 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1273 1274 return; 1275 } else { 1276 refreshState = REF_PRE; 1277 } 1278 } 1279 1280 // at this point, ensure that all banks are precharged 1281 if (refreshState == REF_PRE) { 1282 // precharge any active bank if we are not already in the idle 1283 // state 1284 if (pwrState != PWR_IDLE) { 1285 // at the moment, we use a precharge all even if there is 1286 // only a single bank open 1287 DPRINTF(DRAM, "Precharging all\n"); 1288 1289 // first determine when we can precharge 1290 Tick pre_at = curTick(); 1291 for (int i = 0; i < ranksPerChannel; i++) { 1292 for (int j = 0; j < banksPerRank; j++) { 1293 // respect both causality and any existing bank 1294 // constraints, some banks could already have a 1295 // (auto) precharge scheduled 1296 pre_at = std::max(banks[i][j].preAllowedAt, pre_at); 1297 } 1298 } 1299 1300 // make sure all banks are precharged, and for those that 1301 // already are, update their availability 1302 Tick act_allowed_at = pre_at + tRP; 1303 1304 for (int i = 0; i < ranksPerChannel; i++) { 1305 for (int j = 0; j < banksPerRank; j++) { 1306 if (banks[i][j].openRow != Bank::NO_ROW) { 1307 prechargeBank(banks[i][j], pre_at); 1308 } else { 1309 banks[i][j].actAllowedAt = 1310 std::max(banks[i][j].actAllowedAt, act_allowed_at); 1311 banks[i][j].preAllowedAt = 1312 std::max(banks[i][j].preAllowedAt, pre_at); 1313 } 1314 } 1315 } 1316 } else { 1317 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1318 1319 // go ahead and kick the power state machine into gear if 1320 // we are already idle 1321 schedulePowerEvent(PWR_REF, curTick()); 1322 } 1323 1324 refreshState = REF_RUN; 1325 assert(numBanksActive == 0); 1326 1327 // wait for all banks to be precharged, at which point the 1328 // power state machine will transition to the idle state, and 1329 // automatically move to a refresh, at that point it will also 1330 // call this method to get the refresh event loop going again 1331 return; 1332 } 1333 1334 // last but not least we perform the actual refresh 1335 if (refreshState == REF_RUN) { 1336 // should never get here with any banks active 1337 assert(numBanksActive == 0); 1338 assert(pwrState == PWR_REF); 1339 1340 Tick ref_done_at = curTick() + tRFC; 1341 1342 for (int i = 0; i < ranksPerChannel; i++) { 1343 for (int j = 0; j < banksPerRank; j++) { 1344 banks[i][j].actAllowedAt = ref_done_at; 1345 } 1346 } 1347 1348 // make sure we did not wait so long that we cannot make up 1349 // for it 1350 if (refreshDueAt + tREFI < ref_done_at) { 1351 fatal("Refresh was delayed so long we cannot catch up\n"); 1352 } 1353 1354 // compensate for the delay in actually performing the refresh 1355 // when scheduling the next one 1356 schedule(refreshEvent, refreshDueAt + tREFI - tRP); 1357 1358 assert(!powerEvent.scheduled()); 1359 1360 // move to the idle power state once the refresh is done, this 1361 // will also move the refresh state machine to the refresh 1362 // idle state 1363 schedulePowerEvent(PWR_IDLE, ref_done_at); 1364 1365 DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 1366 ref_done_at, refreshDueAt + tREFI); 1367 } 1368} 1369 1370void 1371DRAMCtrl::schedulePowerEvent(PowerState pwr_state, Tick tick) 1372{ 1373 // respect causality 1374 assert(tick >= curTick()); 1375 1376 if (!powerEvent.scheduled()) { 1377 DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 1378 tick, pwr_state); 1379 1380 // insert the new transition 1381 pwrStateTrans = pwr_state; 1382 1383 schedule(powerEvent, tick); 1384 } else { 1385 panic("Scheduled power event at %llu to state %d, " 1386 "with scheduled event at %llu to %d\n", tick, pwr_state, 1387 powerEvent.when(), pwrStateTrans); 1388 } 1389} 1390 1391void 1392DRAMCtrl::processPowerEvent() 1393{ 1394 // remember where we were, and for how long 1395 Tick duration = curTick() - pwrStateTick; 1396 PowerState prev_state = pwrState; 1397 1398 // update the accounting 1399 pwrStateTime[prev_state] += duration; 1400 1401 pwrState = pwrStateTrans; 1402 pwrStateTick = curTick(); 1403 1404 if (pwrState == PWR_IDLE) { 1405 DPRINTF(DRAMState, "All banks precharged\n"); 1406 1407 // if we were refreshing, make sure we start scheduling requests again 1408 if (prev_state == PWR_REF) { 1409 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 1410 assert(pwrState == PWR_IDLE); 1411 1412 // kick things into action again 1413 refreshState = REF_IDLE; 1414 assert(!nextReqEvent.scheduled()); 1415 schedule(nextReqEvent, curTick()); 1416 } else { 1417 assert(prev_state == PWR_ACT); 1418 1419 // if we have a pending refresh, and are now moving to 1420 // the idle state, direclty transition to a refresh 1421 if (refreshState == REF_RUN) { 1422 // there should be nothing waiting at this point 1423 assert(!powerEvent.scheduled()); 1424 1425 // update the state in zero time and proceed below 1426 pwrState = PWR_REF; 1427 } 1428 } 1429 } 1430 1431 // we transition to the refresh state, let the refresh state 1432 // machine know of this state update and let it deal with the 1433 // scheduling of the next power state transition as well as the 1434 // following refresh 1435 if (pwrState == PWR_REF) { 1436 DPRINTF(DRAMState, "Refreshing\n"); 1437 // kick the refresh event loop into action again, and that 1438 // in turn will schedule a transition to the idle power 1439 // state once the refresh is done 1440 assert(refreshState == REF_RUN); 1441 processRefreshEvent(); 1442 } 1443} 1444 1445void 1446DRAMCtrl::regStats() 1447{ 1448 using namespace Stats; 1449 1450 AbstractMemory::regStats(); 1451 1452 readReqs 1453 .name(name() + ".readReqs") 1454 .desc("Number of read requests accepted"); 1455 1456 writeReqs 1457 .name(name() + ".writeReqs") 1458 .desc("Number of write requests accepted"); 1459 1460 readBursts 1461 .name(name() + ".readBursts") 1462 .desc("Number of DRAM read bursts, " 1463 "including those serviced by the write queue"); 1464 1465 writeBursts 1466 .name(name() + ".writeBursts") 1467 .desc("Number of DRAM write bursts, " 1468 "including those merged in the write queue"); 1469 1470 servicedByWrQ 1471 .name(name() + ".servicedByWrQ") 1472 .desc("Number of DRAM read bursts serviced by the write queue"); 1473 1474 mergedWrBursts 1475 .name(name() + ".mergedWrBursts") 1476 .desc("Number of DRAM write bursts merged with an existing one"); 1477 1478 neitherReadNorWrite 1479 .name(name() + ".neitherReadNorWriteReqs") 1480 .desc("Number of requests that are neither read nor write"); 1481 1482 perBankRdBursts 1483 .init(banksPerRank * ranksPerChannel) 1484 .name(name() + ".perBankRdBursts") 1485 .desc("Per bank write bursts"); 1486 1487 perBankWrBursts 1488 .init(banksPerRank * ranksPerChannel) 1489 .name(name() + ".perBankWrBursts") 1490 .desc("Per bank write bursts"); 1491 1492 avgRdQLen 1493 .name(name() + ".avgRdQLen") 1494 .desc("Average read queue length when enqueuing") 1495 .precision(2); 1496 1497 avgWrQLen 1498 .name(name() + ".avgWrQLen") 1499 .desc("Average write queue length when enqueuing") 1500 .precision(2); 1501 1502 totQLat 1503 .name(name() + ".totQLat") 1504 .desc("Total ticks spent queuing"); 1505 1506 totBusLat 1507 .name(name() + ".totBusLat") 1508 .desc("Total ticks spent in databus transfers"); 1509 1510 totMemAccLat 1511 .name(name() + ".totMemAccLat") 1512 .desc("Total ticks spent from burst creation until serviced " 1513 "by the DRAM"); 1514 1515 avgQLat 1516 .name(name() + ".avgQLat") 1517 .desc("Average queueing delay per DRAM burst") 1518 .precision(2); 1519 1520 avgQLat = totQLat / (readBursts - servicedByWrQ); 1521 1522 avgBusLat 1523 .name(name() + ".avgBusLat") 1524 .desc("Average bus latency per DRAM burst") 1525 .precision(2); 1526 1527 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1528 1529 avgMemAccLat 1530 .name(name() + ".avgMemAccLat") 1531 .desc("Average memory access latency per DRAM burst") 1532 .precision(2); 1533 1534 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1535 1536 numRdRetry 1537 .name(name() + ".numRdRetry") 1538 .desc("Number of times read queue was full causing retry"); 1539 1540 numWrRetry 1541 .name(name() + ".numWrRetry") 1542 .desc("Number of times write queue was full causing retry"); 1543 1544 readRowHits 1545 .name(name() + ".readRowHits") 1546 .desc("Number of row buffer hits during reads"); 1547 1548 writeRowHits 1549 .name(name() + ".writeRowHits") 1550 .desc("Number of row buffer hits during writes"); 1551 1552 readRowHitRate 1553 .name(name() + ".readRowHitRate") 1554 .desc("Row buffer hit rate for reads") 1555 .precision(2); 1556 1557 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 1558 1559 writeRowHitRate 1560 .name(name() + ".writeRowHitRate") 1561 .desc("Row buffer hit rate for writes") 1562 .precision(2); 1563 1564 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 1565 1566 readPktSize 1567 .init(ceilLog2(burstSize) + 1) 1568 .name(name() + ".readPktSize") 1569 .desc("Read request sizes (log2)"); 1570 1571 writePktSize 1572 .init(ceilLog2(burstSize) + 1) 1573 .name(name() + ".writePktSize") 1574 .desc("Write request sizes (log2)"); 1575 1576 rdQLenPdf 1577 .init(readBufferSize) 1578 .name(name() + ".rdQLenPdf") 1579 .desc("What read queue length does an incoming req see"); 1580 1581 wrQLenPdf 1582 .init(writeBufferSize) 1583 .name(name() + ".wrQLenPdf") 1584 .desc("What write queue length does an incoming req see"); 1585 1586 bytesPerActivate 1587 .init(maxAccessesPerRow) 1588 .name(name() + ".bytesPerActivate") 1589 .desc("Bytes accessed per row activation") 1590 .flags(nozero); 1591 1592 rdPerTurnAround 1593 .init(readBufferSize) 1594 .name(name() + ".rdPerTurnAround") 1595 .desc("Reads before turning the bus around for writes") 1596 .flags(nozero); 1597 1598 wrPerTurnAround 1599 .init(writeBufferSize) 1600 .name(name() + ".wrPerTurnAround") 1601 .desc("Writes before turning the bus around for reads") 1602 .flags(nozero); 1603 1604 bytesReadDRAM 1605 .name(name() + ".bytesReadDRAM") 1606 .desc("Total number of bytes read from DRAM"); 1607 1608 bytesReadWrQ 1609 .name(name() + ".bytesReadWrQ") 1610 .desc("Total number of bytes read from write queue"); 1611 1612 bytesWritten 1613 .name(name() + ".bytesWritten") 1614 .desc("Total number of bytes written to DRAM"); 1615 1616 bytesReadSys 1617 .name(name() + ".bytesReadSys") 1618 .desc("Total read bytes from the system interface side"); 1619 1620 bytesWrittenSys 1621 .name(name() + ".bytesWrittenSys") 1622 .desc("Total written bytes from the system interface side"); 1623 1624 avgRdBW 1625 .name(name() + ".avgRdBW") 1626 .desc("Average DRAM read bandwidth in MiByte/s") 1627 .precision(2); 1628 1629 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 1630 1631 avgWrBW 1632 .name(name() + ".avgWrBW") 1633 .desc("Average achieved write bandwidth in MiByte/s") 1634 .precision(2); 1635 1636 avgWrBW = (bytesWritten / 1000000) / simSeconds; 1637 1638 avgRdBWSys 1639 .name(name() + ".avgRdBWSys") 1640 .desc("Average system read bandwidth in MiByte/s") 1641 .precision(2); 1642 1643 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 1644 1645 avgWrBWSys 1646 .name(name() + ".avgWrBWSys") 1647 .desc("Average system write bandwidth in MiByte/s") 1648 .precision(2); 1649 1650 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 1651 1652 peakBW 1653 .name(name() + ".peakBW") 1654 .desc("Theoretical peak bandwidth in MiByte/s") 1655 .precision(2); 1656 1657 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 1658 1659 busUtil 1660 .name(name() + ".busUtil") 1661 .desc("Data bus utilization in percentage") 1662 .precision(2); 1663 1664 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 1665 1666 totGap 1667 .name(name() + ".totGap") 1668 .desc("Total gap between requests"); 1669 1670 avgGap 1671 .name(name() + ".avgGap") 1672 .desc("Average gap between requests") 1673 .precision(2); 1674 1675 avgGap = totGap / (readReqs + writeReqs); 1676 1677 // Stats for DRAM Power calculation based on Micron datasheet 1678 busUtilRead 1679 .name(name() + ".busUtilRead") 1680 .desc("Data bus utilization in percentage for reads") 1681 .precision(2); 1682 1683 busUtilRead = avgRdBW / peakBW * 100; 1684 1685 busUtilWrite 1686 .name(name() + ".busUtilWrite") 1687 .desc("Data bus utilization in percentage for writes") 1688 .precision(2); 1689 1690 busUtilWrite = avgWrBW / peakBW * 100; 1691 1692 pageHitRate 1693 .name(name() + ".pageHitRate") 1694 .desc("Row buffer hit rate, read and write combined") 1695 .precision(2); 1696 1697 pageHitRate = (writeRowHits + readRowHits) / 1698 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 1699 1700 pwrStateTime 1701 .init(5) 1702 .name(name() + ".memoryStateTime") 1703 .desc("Time in different power states"); 1704 pwrStateTime.subname(0, "IDLE"); 1705 pwrStateTime.subname(1, "REF"); 1706 pwrStateTime.subname(2, "PRE_PDN"); 1707 pwrStateTime.subname(3, "ACT"); 1708 pwrStateTime.subname(4, "ACT_PDN"); 1709} 1710 1711void 1712DRAMCtrl::recvFunctional(PacketPtr pkt) 1713{ 1714 // rely on the abstract memory 1715 functionalAccess(pkt); 1716} 1717 1718BaseSlavePort& 1719DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 1720{ 1721 if (if_name != "port") { 1722 return MemObject::getSlavePort(if_name, idx); 1723 } else { 1724 return port; 1725 } 1726} 1727 1728unsigned int 1729DRAMCtrl::drain(DrainManager *dm) 1730{ 1731 unsigned int count = port.drain(dm); 1732 1733 // if there is anything in any of our internal queues, keep track 1734 // of that as well 1735 if (!(writeQueue.empty() && readQueue.empty() && 1736 respQueue.empty())) { 1737 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 1738 " resp: %d\n", writeQueue.size(), readQueue.size(), 1739 respQueue.size()); 1740 ++count; 1741 drainManager = dm; 1742 1743 // the only part that is not drained automatically over time 1744 // is the write queue, thus kick things into action if needed 1745 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 1746 schedule(nextReqEvent, curTick()); 1747 } 1748 } 1749 1750 if (count) 1751 setDrainState(Drainable::Draining); 1752 else 1753 setDrainState(Drainable::Drained); 1754 return count; 1755} 1756 1757DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 1758 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 1759 memory(_memory) 1760{ } 1761 1762AddrRangeList 1763DRAMCtrl::MemoryPort::getAddrRanges() const 1764{ 1765 AddrRangeList ranges; 1766 ranges.push_back(memory.getAddrRange()); 1767 return ranges; 1768} 1769 1770void 1771DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 1772{ 1773 pkt->pushLabel(memory.name()); 1774 1775 if (!queue.checkFunctional(pkt)) { 1776 // Default implementation of SimpleTimingPort::recvFunctional() 1777 // calls recvAtomic() and throws away the latency; we can save a 1778 // little here by just not calculating the latency. 1779 memory.recvFunctional(pkt); 1780 } 1781 1782 pkt->popLabel(); 1783} 1784 1785Tick 1786DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 1787{ 1788 return memory.recvAtomic(pkt); 1789} 1790 1791bool 1792DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 1793{ 1794 // pass it to the memory controller 1795 return memory.recvTimingReq(pkt); 1796} 1797 1798DRAMCtrl* 1799DRAMCtrlParams::create() 1800{ 1801 return new DRAMCtrl(this); 1802} 1803