dram_ctrl.cc revision 10394
110259SAndrew.Bardsley@arm.com/* 210259SAndrew.Bardsley@arm.com * Copyright (c) 2010-2014 ARM Limited 310259SAndrew.Bardsley@arm.com * All rights reserved 410259SAndrew.Bardsley@arm.com * 510259SAndrew.Bardsley@arm.com * The license below extends only to copyright in the software and shall 610259SAndrew.Bardsley@arm.com * not be construed as granting a license to any other intellectual 710259SAndrew.Bardsley@arm.com * property including but not limited to intellectual property relating 810259SAndrew.Bardsley@arm.com * to a hardware implementation of the functionality of the software 910259SAndrew.Bardsley@arm.com * licensed hereunder. You may use the software subject to the license 1010259SAndrew.Bardsley@arm.com * terms below provided that you ensure that this notice is replicated 1110259SAndrew.Bardsley@arm.com * unmodified and in its entirety in all distributions of the software, 1210259SAndrew.Bardsley@arm.com * modified or unmodified, in source code or in binary form. 1310259SAndrew.Bardsley@arm.com * 1410259SAndrew.Bardsley@arm.com * Copyright (c) 2013 Amin Farmahini-Farahani 1510259SAndrew.Bardsley@arm.com * All rights reserved. 1610259SAndrew.Bardsley@arm.com * 1710259SAndrew.Bardsley@arm.com * Redistribution and use in source and binary forms, with or without 1810259SAndrew.Bardsley@arm.com * modification, are permitted provided that the following conditions are 1910259SAndrew.Bardsley@arm.com * met: redistributions of source code must retain the above copyright 2010259SAndrew.Bardsley@arm.com * notice, this list of conditions and the following disclaimer; 2110259SAndrew.Bardsley@arm.com * redistributions in binary form must reproduce the above copyright 2210259SAndrew.Bardsley@arm.com * notice, this list of conditions and the following disclaimer in the 2310259SAndrew.Bardsley@arm.com * documentation and/or other materials provided with the distribution; 2410259SAndrew.Bardsley@arm.com * neither the name of the copyright holders nor the names of its 2510259SAndrew.Bardsley@arm.com * contributors may be used to endorse or promote products derived from 2610259SAndrew.Bardsley@arm.com * this software without specific prior written permission. 2710259SAndrew.Bardsley@arm.com * 2810259SAndrew.Bardsley@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2910259SAndrew.Bardsley@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3010259SAndrew.Bardsley@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3110259SAndrew.Bardsley@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3210259SAndrew.Bardsley@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3310259SAndrew.Bardsley@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3410259SAndrew.Bardsley@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3510259SAndrew.Bardsley@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3610259SAndrew.Bardsley@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3710259SAndrew.Bardsley@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3810259SAndrew.Bardsley@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3910259SAndrew.Bardsley@arm.com * 4010259SAndrew.Bardsley@arm.com * Authors: Andreas Hansson 4110259SAndrew.Bardsley@arm.com * Ani Udipi 4210259SAndrew.Bardsley@arm.com * Neha Agarwal 4310259SAndrew.Bardsley@arm.com */ 4410259SAndrew.Bardsley@arm.com 4510259SAndrew.Bardsley@arm.com#include "base/bitfield.hh" 4610259SAndrew.Bardsley@arm.com#include "base/trace.hh" 4710259SAndrew.Bardsley@arm.com#include "debug/DRAM.hh" 4810259SAndrew.Bardsley@arm.com#include "debug/DRAMPower.hh" 4910259SAndrew.Bardsley@arm.com#include "debug/DRAMState.hh" 5010259SAndrew.Bardsley@arm.com#include "debug/Drain.hh" 5110259SAndrew.Bardsley@arm.com#include "mem/dram_ctrl.hh" 5210259SAndrew.Bardsley@arm.com#include "sim/system.hh" 5310259SAndrew.Bardsley@arm.com 5410259SAndrew.Bardsley@arm.comusing namespace std; 5510259SAndrew.Bardsley@arm.com 5610259SAndrew.Bardsley@arm.comDRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 5710259SAndrew.Bardsley@arm.com AbstractMemory(p), 5810259SAndrew.Bardsley@arm.com port(name() + ".port", *this), 5910259SAndrew.Bardsley@arm.com retryRdReq(false), retryWrReq(false), 6010259SAndrew.Bardsley@arm.com busState(READ), 6110259SAndrew.Bardsley@arm.com nextReqEvent(this), respondEvent(this), activateEvent(this), 6210259SAndrew.Bardsley@arm.com prechargeEvent(this), refreshEvent(this), powerEvent(this), 6310259SAndrew.Bardsley@arm.com drainManager(NULL), 6410259SAndrew.Bardsley@arm.com deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 6510259SAndrew.Bardsley@arm.com deviceRowBufferSize(p->device_rowbuffer_size), 6610259SAndrew.Bardsley@arm.com devicesPerRank(p->devices_per_rank), 6710259SAndrew.Bardsley@arm.com burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 6810259SAndrew.Bardsley@arm.com rowBufferSize(devicesPerRank * deviceRowBufferSize), 6910259SAndrew.Bardsley@arm.com columnsPerRowBuffer(rowBufferSize / burstSize), 7010259SAndrew.Bardsley@arm.com columnsPerStripe(range.granularity() / burstSize), 7110259SAndrew.Bardsley@arm.com ranksPerChannel(p->ranks_per_channel), 7210259SAndrew.Bardsley@arm.com bankGroupsPerRank(p->bank_groups_per_rank), 7310259SAndrew.Bardsley@arm.com bankGroupArch(p->bank_groups_per_rank > 0), 7410259SAndrew.Bardsley@arm.com banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 7510259SAndrew.Bardsley@arm.com readBufferSize(p->read_buffer_size), 7610259SAndrew.Bardsley@arm.com writeBufferSize(p->write_buffer_size), 7710259SAndrew.Bardsley@arm.com writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 7810259SAndrew.Bardsley@arm.com writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 7910259SAndrew.Bardsley@arm.com minWritesPerSwitch(p->min_writes_per_switch), 8010259SAndrew.Bardsley@arm.com writesThisTime(0), readsThisTime(0), 8110259SAndrew.Bardsley@arm.com tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST), 8210259SAndrew.Bardsley@arm.com tCCD_L(p->tCCD_L), tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), 8310259SAndrew.Bardsley@arm.com tWR(p->tWR), tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 8410259SAndrew.Bardsley@arm.com tRRD_L(p->tRRD_L), tXAW(p->tXAW), activationLimit(p->activation_limit), 8510259SAndrew.Bardsley@arm.com memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 8610259SAndrew.Bardsley@arm.com pageMgmt(p->page_policy), 8710259SAndrew.Bardsley@arm.com maxAccessesPerRow(p->max_accesses_per_row), 8810259SAndrew.Bardsley@arm.com frontendLatency(p->static_frontend_latency), 8910259SAndrew.Bardsley@arm.com backendLatency(p->static_backend_latency), 9010259SAndrew.Bardsley@arm.com busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), 9110259SAndrew.Bardsley@arm.com pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), prevArrival(0), 9210259SAndrew.Bardsley@arm.com nextReqTime(0), pwrStateTick(0), numBanksActive(0), 9310259SAndrew.Bardsley@arm.com activeRank(0) 9410259SAndrew.Bardsley@arm.com{ 9510259SAndrew.Bardsley@arm.com // create the bank states based on the dimensions of the ranks and 9610259SAndrew.Bardsley@arm.com // banks 9710259SAndrew.Bardsley@arm.com banks.resize(ranksPerChannel); 9810259SAndrew.Bardsley@arm.com actTicks.resize(ranksPerChannel); 9910259SAndrew.Bardsley@arm.com for (size_t c = 0; c < ranksPerChannel; ++c) { 10010259SAndrew.Bardsley@arm.com banks[c].resize(banksPerRank); 10110259SAndrew.Bardsley@arm.com actTicks[c].resize(activationLimit, 0); 10210259SAndrew.Bardsley@arm.com } 10310259SAndrew.Bardsley@arm.com 10410259SAndrew.Bardsley@arm.com // set the bank indices 10510259SAndrew.Bardsley@arm.com for (int r = 0; r < ranksPerChannel; r++) { 10610259SAndrew.Bardsley@arm.com for (int b = 0; b < banksPerRank; b++) { 10710259SAndrew.Bardsley@arm.com banks[r][b].rank = r; 10810259SAndrew.Bardsley@arm.com banks[r][b].bank = b; 10910259SAndrew.Bardsley@arm.com if (bankGroupArch) { 11010259SAndrew.Bardsley@arm.com // Simply assign lower bits to bank group in order to 11110259SAndrew.Bardsley@arm.com // rotate across bank groups as banks are incremented 11210259SAndrew.Bardsley@arm.com // e.g. with 4 banks per bank group and 16 banks total: 11310259SAndrew.Bardsley@arm.com // banks 0,4,8,12 are in bank group 0 11410259SAndrew.Bardsley@arm.com // banks 1,5,9,13 are in bank group 1 11510259SAndrew.Bardsley@arm.com // banks 2,6,10,14 are in bank group 2 11610259SAndrew.Bardsley@arm.com // banks 3,7,11,15 are in bank group 3 11710259SAndrew.Bardsley@arm.com banks[r][b].bankgr = b % bankGroupsPerRank; 11810259SAndrew.Bardsley@arm.com } else { 11910259SAndrew.Bardsley@arm.com // No bank groups; simply assign to bank number 12010259SAndrew.Bardsley@arm.com banks[r][b].bankgr = b; 12110259SAndrew.Bardsley@arm.com } 12210259SAndrew.Bardsley@arm.com } 12310259SAndrew.Bardsley@arm.com } 12410259SAndrew.Bardsley@arm.com 12510259SAndrew.Bardsley@arm.com // perform a basic check of the write thresholds 12610259SAndrew.Bardsley@arm.com if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 12710259SAndrew.Bardsley@arm.com fatal("Write buffer low threshold %d must be smaller than the " 12810259SAndrew.Bardsley@arm.com "high threshold %d\n", p->write_low_thresh_perc, 12910259SAndrew.Bardsley@arm.com p->write_high_thresh_perc); 13010259SAndrew.Bardsley@arm.com 13110259SAndrew.Bardsley@arm.com // determine the rows per bank by looking at the total capacity 13210259SAndrew.Bardsley@arm.com uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 13310259SAndrew.Bardsley@arm.com 13410259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 13510259SAndrew.Bardsley@arm.com AbstractMemory::size()); 13610259SAndrew.Bardsley@arm.com 13710259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 13810259SAndrew.Bardsley@arm.com rowBufferSize, columnsPerRowBuffer); 13910259SAndrew.Bardsley@arm.com 14010259SAndrew.Bardsley@arm.com rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 14110259SAndrew.Bardsley@arm.com 14210259SAndrew.Bardsley@arm.com // a bit of sanity checks on the interleaving 14310259SAndrew.Bardsley@arm.com if (range.interleaved()) { 14410259SAndrew.Bardsley@arm.com if (channels != range.stripes()) 14510259SAndrew.Bardsley@arm.com fatal("%s has %d interleaved address stripes but %d channel(s)\n", 14610259SAndrew.Bardsley@arm.com name(), range.stripes(), channels); 14710259SAndrew.Bardsley@arm.com 14810259SAndrew.Bardsley@arm.com if (addrMapping == Enums::RoRaBaChCo) { 14910259SAndrew.Bardsley@arm.com if (rowBufferSize != range.granularity()) { 15010259SAndrew.Bardsley@arm.com fatal("Channel interleaving of %s doesn't match RoRaBaChCo " 15110259SAndrew.Bardsley@arm.com "address map\n", name()); 15210259SAndrew.Bardsley@arm.com } 15310259SAndrew.Bardsley@arm.com } else if (addrMapping == Enums::RoRaBaCoCh || 15410259SAndrew.Bardsley@arm.com addrMapping == Enums::RoCoRaBaCh) { 15510259SAndrew.Bardsley@arm.com // for the interleavings with channel bits in the bottom, 15610259SAndrew.Bardsley@arm.com // if the system uses a channel striping granularity that 15710259SAndrew.Bardsley@arm.com // is larger than the DRAM burst size, then map the 15810259SAndrew.Bardsley@arm.com // sequential accesses within a stripe to a number of 15910259SAndrew.Bardsley@arm.com // columns in the DRAM, effectively placing some of the 16010259SAndrew.Bardsley@arm.com // lower-order column bits as the least-significant bits 16110259SAndrew.Bardsley@arm.com // of the address (above the ones denoting the burst size) 16210259SAndrew.Bardsley@arm.com assert(columnsPerStripe >= 1); 16310259SAndrew.Bardsley@arm.com 16410259SAndrew.Bardsley@arm.com // channel striping has to be done at a granularity that 16510259SAndrew.Bardsley@arm.com // is equal or larger to a cache line 16610259SAndrew.Bardsley@arm.com if (system()->cacheLineSize() > range.granularity()) { 16710259SAndrew.Bardsley@arm.com fatal("Channel interleaving of %s must be at least as large " 16810259SAndrew.Bardsley@arm.com "as the cache line size\n", name()); 16910259SAndrew.Bardsley@arm.com } 17010259SAndrew.Bardsley@arm.com 17110259SAndrew.Bardsley@arm.com // ...and equal or smaller than the row-buffer size 17210259SAndrew.Bardsley@arm.com if (rowBufferSize < range.granularity()) { 17310259SAndrew.Bardsley@arm.com fatal("Channel interleaving of %s must be at most as large " 17410259SAndrew.Bardsley@arm.com "as the row-buffer size\n", name()); 17510259SAndrew.Bardsley@arm.com } 17610259SAndrew.Bardsley@arm.com // this is essentially the check above, so just to be sure 17710259SAndrew.Bardsley@arm.com assert(columnsPerStripe <= columnsPerRowBuffer); 17810259SAndrew.Bardsley@arm.com } 17910259SAndrew.Bardsley@arm.com } 18010259SAndrew.Bardsley@arm.com 18110259SAndrew.Bardsley@arm.com // some basic sanity checks 18210259SAndrew.Bardsley@arm.com if (tREFI <= tRP || tREFI <= tRFC) { 18310259SAndrew.Bardsley@arm.com fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 18410259SAndrew.Bardsley@arm.com tREFI, tRP, tRFC); 18510259SAndrew.Bardsley@arm.com } 18610259SAndrew.Bardsley@arm.com 18710259SAndrew.Bardsley@arm.com // basic bank group architecture checks -> 18810259SAndrew.Bardsley@arm.com if (bankGroupArch) { 18910259SAndrew.Bardsley@arm.com // must have at least one bank per bank group 19010259SAndrew.Bardsley@arm.com if (bankGroupsPerRank > banksPerRank) { 19110259SAndrew.Bardsley@arm.com fatal("banks per rank (%d) must be equal to or larger than " 19210259SAndrew.Bardsley@arm.com "banks groups per rank (%d)\n", 19310259SAndrew.Bardsley@arm.com banksPerRank, bankGroupsPerRank); 19410259SAndrew.Bardsley@arm.com } 19510259SAndrew.Bardsley@arm.com // must have same number of banks in each bank group 19610259SAndrew.Bardsley@arm.com if ((banksPerRank % bankGroupsPerRank) != 0) { 19710259SAndrew.Bardsley@arm.com fatal("Banks per rank (%d) must be evenly divisible by bank groups " 19810259SAndrew.Bardsley@arm.com "per rank (%d) for equal banks per bank group\n", 19910259SAndrew.Bardsley@arm.com banksPerRank, bankGroupsPerRank); 20010259SAndrew.Bardsley@arm.com } 20110259SAndrew.Bardsley@arm.com // tCCD_L should be greater than minimal, back-to-back burst delay 20210259SAndrew.Bardsley@arm.com if (tCCD_L <= tBURST) { 20310259SAndrew.Bardsley@arm.com fatal("tCCD_L (%d) should be larger than tBURST (%d) when " 20410259SAndrew.Bardsley@arm.com "bank groups per rank (%d) is greater than 1\n", 20510259SAndrew.Bardsley@arm.com tCCD_L, tBURST, bankGroupsPerRank); 20610259SAndrew.Bardsley@arm.com } 20710259SAndrew.Bardsley@arm.com // tRRD_L is greater than minimal, same bank group ACT-to-ACT delay 20810259SAndrew.Bardsley@arm.com if (tRRD_L <= tRRD) { 20910259SAndrew.Bardsley@arm.com fatal("tRRD_L (%d) should be larger than tRRD (%d) when " 21010259SAndrew.Bardsley@arm.com "bank groups per rank (%d) is greater than 1\n", 21110259SAndrew.Bardsley@arm.com tRRD_L, tRRD, bankGroupsPerRank); 21210259SAndrew.Bardsley@arm.com } 21310259SAndrew.Bardsley@arm.com } 21410259SAndrew.Bardsley@arm.com 21510259SAndrew.Bardsley@arm.com} 21610259SAndrew.Bardsley@arm.com 21710259SAndrew.Bardsley@arm.comvoid 21810259SAndrew.Bardsley@arm.comDRAMCtrl::init() 21910259SAndrew.Bardsley@arm.com{ 22010259SAndrew.Bardsley@arm.com if (!port.isConnected()) { 22110259SAndrew.Bardsley@arm.com fatal("DRAMCtrl %s is unconnected!\n", name()); 22210259SAndrew.Bardsley@arm.com } else { 22310259SAndrew.Bardsley@arm.com port.sendRangeChange(); 22410259SAndrew.Bardsley@arm.com } 22510259SAndrew.Bardsley@arm.com} 22610259SAndrew.Bardsley@arm.com 22710259SAndrew.Bardsley@arm.comvoid 22810259SAndrew.Bardsley@arm.comDRAMCtrl::startup() 22910379Sandreas.hansson@arm.com{ 23010379Sandreas.hansson@arm.com // update the start tick for the precharge accounting to the 23110259SAndrew.Bardsley@arm.com // current tick 23210259SAndrew.Bardsley@arm.com pwrStateTick = curTick(); 23310259SAndrew.Bardsley@arm.com 23410259SAndrew.Bardsley@arm.com // shift the bus busy time sufficiently far ahead that we never 23510259SAndrew.Bardsley@arm.com // have to worry about negative values when computing the time for 23610259SAndrew.Bardsley@arm.com // the next request, this will add an insignificant bubble at the 23710259SAndrew.Bardsley@arm.com // start of simulation 23810259SAndrew.Bardsley@arm.com busBusyUntil = curTick() + tRP + tRCD + tCL; 23910259SAndrew.Bardsley@arm.com 24010259SAndrew.Bardsley@arm.com // kick off the refresh, and give ourselves enough time to 24110259SAndrew.Bardsley@arm.com // precharge 24210259SAndrew.Bardsley@arm.com schedule(refreshEvent, curTick() + tREFI - tRP); 24310259SAndrew.Bardsley@arm.com} 24410259SAndrew.Bardsley@arm.com 24510259SAndrew.Bardsley@arm.comTick 24610259SAndrew.Bardsley@arm.comDRAMCtrl::recvAtomic(PacketPtr pkt) 24710259SAndrew.Bardsley@arm.com{ 24810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 24910259SAndrew.Bardsley@arm.com 25010259SAndrew.Bardsley@arm.com // do the actual memory access and turn the packet into a response 25110259SAndrew.Bardsley@arm.com access(pkt); 25210259SAndrew.Bardsley@arm.com 25310259SAndrew.Bardsley@arm.com Tick latency = 0; 25410259SAndrew.Bardsley@arm.com if (!pkt->memInhibitAsserted() && pkt->hasData()) { 25510259SAndrew.Bardsley@arm.com // this value is not supposed to be accurate, just enough to 25610259SAndrew.Bardsley@arm.com // keep things going, mimic a closed page 25710259SAndrew.Bardsley@arm.com latency = tRP + tRCD + tCL; 25810259SAndrew.Bardsley@arm.com } 25910259SAndrew.Bardsley@arm.com return latency; 26010259SAndrew.Bardsley@arm.com} 26110259SAndrew.Bardsley@arm.com 26210259SAndrew.Bardsley@arm.combool 26310259SAndrew.Bardsley@arm.comDRAMCtrl::readQueueFull(unsigned int neededEntries) const 26410259SAndrew.Bardsley@arm.com{ 26510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 26610259SAndrew.Bardsley@arm.com readBufferSize, readQueue.size() + respQueue.size(), 26710259SAndrew.Bardsley@arm.com neededEntries); 26810259SAndrew.Bardsley@arm.com 26910259SAndrew.Bardsley@arm.com return 27010259SAndrew.Bardsley@arm.com (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 27110259SAndrew.Bardsley@arm.com} 27210259SAndrew.Bardsley@arm.com 27310259SAndrew.Bardsley@arm.combool 27410259SAndrew.Bardsley@arm.comDRAMCtrl::writeQueueFull(unsigned int neededEntries) const 27510259SAndrew.Bardsley@arm.com{ 27610379Sandreas.hansson@arm.com DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 27710379Sandreas.hansson@arm.com writeBufferSize, writeQueue.size(), neededEntries); 27810259SAndrew.Bardsley@arm.com return (writeQueue.size() + neededEntries) > writeBufferSize; 27910259SAndrew.Bardsley@arm.com} 28010259SAndrew.Bardsley@arm.com 28110259SAndrew.Bardsley@arm.comDRAMCtrl::DRAMPacket* 28210259SAndrew.Bardsley@arm.comDRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 28310259SAndrew.Bardsley@arm.com bool isRead) 28410259SAndrew.Bardsley@arm.com{ 28510259SAndrew.Bardsley@arm.com // decode the address based on the address mapping scheme, with 28610259SAndrew.Bardsley@arm.com // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 28710259SAndrew.Bardsley@arm.com // channel, respectively 28810259SAndrew.Bardsley@arm.com uint8_t rank; 28910259SAndrew.Bardsley@arm.com uint8_t bank; 29010259SAndrew.Bardsley@arm.com // use a 64-bit unsigned during the computations as the row is 29110259SAndrew.Bardsley@arm.com // always the top bits, and check before creating the DRAMPacket 29210259SAndrew.Bardsley@arm.com uint64_t row; 29310259SAndrew.Bardsley@arm.com 29410259SAndrew.Bardsley@arm.com // truncate the address to a DRAM burst, which makes it unique to 29510259SAndrew.Bardsley@arm.com // a specific column, row, bank, rank and channel 29610259SAndrew.Bardsley@arm.com Addr addr = dramPktAddr / burstSize; 29710259SAndrew.Bardsley@arm.com 29810259SAndrew.Bardsley@arm.com // we have removed the lowest order address bits that denote the 29910259SAndrew.Bardsley@arm.com // position within the column 30010259SAndrew.Bardsley@arm.com if (addrMapping == Enums::RoRaBaChCo) { 30110259SAndrew.Bardsley@arm.com // the lowest order bits denote the column to ensure that 30210259SAndrew.Bardsley@arm.com // sequential cache lines occupy the same row 30310259SAndrew.Bardsley@arm.com addr = addr / columnsPerRowBuffer; 30410259SAndrew.Bardsley@arm.com 30510259SAndrew.Bardsley@arm.com // take out the channel part of the address 30610259SAndrew.Bardsley@arm.com addr = addr / channels; 30710259SAndrew.Bardsley@arm.com 30810259SAndrew.Bardsley@arm.com // after the channel bits, get the bank bits to interleave 30910259SAndrew.Bardsley@arm.com // over the banks 31010259SAndrew.Bardsley@arm.com bank = addr % banksPerRank; 31110259SAndrew.Bardsley@arm.com addr = addr / banksPerRank; 31210259SAndrew.Bardsley@arm.com 31310259SAndrew.Bardsley@arm.com // after the bank, we get the rank bits which thus interleaves 31410259SAndrew.Bardsley@arm.com // over the ranks 31510259SAndrew.Bardsley@arm.com rank = addr % ranksPerChannel; 31610259SAndrew.Bardsley@arm.com addr = addr / ranksPerChannel; 31710259SAndrew.Bardsley@arm.com 31810259SAndrew.Bardsley@arm.com // lastly, get the row bits 31910259SAndrew.Bardsley@arm.com row = addr % rowsPerBank; 32010259SAndrew.Bardsley@arm.com addr = addr / rowsPerBank; 32110259SAndrew.Bardsley@arm.com } else if (addrMapping == Enums::RoRaBaCoCh) { 32210259SAndrew.Bardsley@arm.com // take out the lower-order column bits 32310259SAndrew.Bardsley@arm.com addr = addr / columnsPerStripe; 32410259SAndrew.Bardsley@arm.com 32510259SAndrew.Bardsley@arm.com // take out the channel part of the address 32610259SAndrew.Bardsley@arm.com addr = addr / channels; 32710259SAndrew.Bardsley@arm.com 32810259SAndrew.Bardsley@arm.com // next, the higher-order column bites 32910259SAndrew.Bardsley@arm.com addr = addr / (columnsPerRowBuffer / columnsPerStripe); 33010259SAndrew.Bardsley@arm.com 33110259SAndrew.Bardsley@arm.com // after the column bits, we get the bank bits to interleave 33210259SAndrew.Bardsley@arm.com // over the banks 33310259SAndrew.Bardsley@arm.com bank = addr % banksPerRank; 33410259SAndrew.Bardsley@arm.com addr = addr / banksPerRank; 33510259SAndrew.Bardsley@arm.com 33610259SAndrew.Bardsley@arm.com // after the bank, we get the rank bits which thus interleaves 33710259SAndrew.Bardsley@arm.com // over the ranks 33810259SAndrew.Bardsley@arm.com rank = addr % ranksPerChannel; 33910259SAndrew.Bardsley@arm.com addr = addr / ranksPerChannel; 34010259SAndrew.Bardsley@arm.com 34110259SAndrew.Bardsley@arm.com // lastly, get the row bits 34210259SAndrew.Bardsley@arm.com row = addr % rowsPerBank; 34310259SAndrew.Bardsley@arm.com addr = addr / rowsPerBank; 34410259SAndrew.Bardsley@arm.com } else if (addrMapping == Enums::RoCoRaBaCh) { 34510259SAndrew.Bardsley@arm.com // optimise for closed page mode and utilise maximum 34610259SAndrew.Bardsley@arm.com // parallelism of the DRAM (at the cost of power) 34710259SAndrew.Bardsley@arm.com 34810259SAndrew.Bardsley@arm.com // take out the lower-order column bits 34910259SAndrew.Bardsley@arm.com addr = addr / columnsPerStripe; 35010259SAndrew.Bardsley@arm.com 35110259SAndrew.Bardsley@arm.com // take out the channel part of the address, not that this has 35210259SAndrew.Bardsley@arm.com // to match with how accesses are interleaved between the 35310259SAndrew.Bardsley@arm.com // controllers in the address mapping 35410259SAndrew.Bardsley@arm.com addr = addr / channels; 35510259SAndrew.Bardsley@arm.com 35610259SAndrew.Bardsley@arm.com // start with the bank bits, as this provides the maximum 35710259SAndrew.Bardsley@arm.com // opportunity for parallelism between requests 35810259SAndrew.Bardsley@arm.com bank = addr % banksPerRank; 35910259SAndrew.Bardsley@arm.com addr = addr / banksPerRank; 36010259SAndrew.Bardsley@arm.com 36110259SAndrew.Bardsley@arm.com // next get the rank bits 36210259SAndrew.Bardsley@arm.com rank = addr % ranksPerChannel; 36310259SAndrew.Bardsley@arm.com addr = addr / ranksPerChannel; 36410259SAndrew.Bardsley@arm.com 36510259SAndrew.Bardsley@arm.com // next, the higher-order column bites 36610259SAndrew.Bardsley@arm.com addr = addr / (columnsPerRowBuffer / columnsPerStripe); 36710259SAndrew.Bardsley@arm.com 36810259SAndrew.Bardsley@arm.com // lastly, get the row bits 36910259SAndrew.Bardsley@arm.com row = addr % rowsPerBank; 37010259SAndrew.Bardsley@arm.com addr = addr / rowsPerBank; 37110259SAndrew.Bardsley@arm.com } else 37210259SAndrew.Bardsley@arm.com panic("Unknown address mapping policy chosen!"); 37310259SAndrew.Bardsley@arm.com 37410259SAndrew.Bardsley@arm.com assert(rank < ranksPerChannel); 37510259SAndrew.Bardsley@arm.com assert(bank < banksPerRank); 37610259SAndrew.Bardsley@arm.com assert(row < rowsPerBank); 37710259SAndrew.Bardsley@arm.com assert(row < Bank::NO_ROW); 37810259SAndrew.Bardsley@arm.com 37910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 38010259SAndrew.Bardsley@arm.com dramPktAddr, rank, bank, row); 38110259SAndrew.Bardsley@arm.com 38210259SAndrew.Bardsley@arm.com // create the corresponding DRAM packet with the entry time and 38310259SAndrew.Bardsley@arm.com // ready time set to the current tick, the latter will be updated 38410259SAndrew.Bardsley@arm.com // later 38510259SAndrew.Bardsley@arm.com uint16_t bank_id = banksPerRank * rank + bank; 38610259SAndrew.Bardsley@arm.com return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 38710259SAndrew.Bardsley@arm.com size, banks[rank][bank]); 38810259SAndrew.Bardsley@arm.com} 38910259SAndrew.Bardsley@arm.com 39010259SAndrew.Bardsley@arm.comvoid 39110259SAndrew.Bardsley@arm.comDRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 39210259SAndrew.Bardsley@arm.com{ 39310259SAndrew.Bardsley@arm.com // only add to the read queue here. whenever the request is 39410259SAndrew.Bardsley@arm.com // eventually done, set the readyTime, and call schedule() 39510259SAndrew.Bardsley@arm.com assert(!pkt->isWrite()); 39610259SAndrew.Bardsley@arm.com 39710259SAndrew.Bardsley@arm.com assert(pktCount != 0); 39810259SAndrew.Bardsley@arm.com 39910259SAndrew.Bardsley@arm.com // if the request size is larger than burst size, the pkt is split into 40010259SAndrew.Bardsley@arm.com // multiple DRAM packets 40110259SAndrew.Bardsley@arm.com // Note if the pkt starting address is not aligened to burst size, the 40210259SAndrew.Bardsley@arm.com // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 40310259SAndrew.Bardsley@arm.com // are aligned to burst size boundaries. This is to ensure we accurately 40410259SAndrew.Bardsley@arm.com // check read packets against packets in write queue. 40510259SAndrew.Bardsley@arm.com Addr addr = pkt->getAddr(); 40610259SAndrew.Bardsley@arm.com unsigned pktsServicedByWrQ = 0; 40710259SAndrew.Bardsley@arm.com BurstHelper* burst_helper = NULL; 40810259SAndrew.Bardsley@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 40910259SAndrew.Bardsley@arm.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 41010259SAndrew.Bardsley@arm.com pkt->getAddr() + pkt->getSize()) - addr; 41110259SAndrew.Bardsley@arm.com readPktSize[ceilLog2(size)]++; 41210259SAndrew.Bardsley@arm.com readBursts++; 41310259SAndrew.Bardsley@arm.com 41410259SAndrew.Bardsley@arm.com // First check write buffer to see if the data is already at 41510259SAndrew.Bardsley@arm.com // the controller 41610259SAndrew.Bardsley@arm.com bool foundInWrQ = false; 41710259SAndrew.Bardsley@arm.com for (auto i = writeQueue.begin(); i != writeQueue.end(); ++i) { 41810259SAndrew.Bardsley@arm.com // check if the read is subsumed in the write entry we are 41910259SAndrew.Bardsley@arm.com // looking at 42010259SAndrew.Bardsley@arm.com if ((*i)->addr <= addr && 42110259SAndrew.Bardsley@arm.com (addr + size) <= ((*i)->addr + (*i)->size)) { 42210259SAndrew.Bardsley@arm.com foundInWrQ = true; 42310259SAndrew.Bardsley@arm.com servicedByWrQ++; 42410259SAndrew.Bardsley@arm.com pktsServicedByWrQ++; 42510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 42610259SAndrew.Bardsley@arm.com "write queue\n", addr, size); 42710259SAndrew.Bardsley@arm.com bytesReadWrQ += burstSize; 42810259SAndrew.Bardsley@arm.com break; 42910259SAndrew.Bardsley@arm.com } 43010259SAndrew.Bardsley@arm.com } 43110259SAndrew.Bardsley@arm.com 43210259SAndrew.Bardsley@arm.com // If not found in the write q, make a DRAM packet and 43310259SAndrew.Bardsley@arm.com // push it onto the read queue 43410259SAndrew.Bardsley@arm.com if (!foundInWrQ) { 43510259SAndrew.Bardsley@arm.com 43610259SAndrew.Bardsley@arm.com // Make the burst helper for split packets 43710259SAndrew.Bardsley@arm.com if (pktCount > 1 && burst_helper == NULL) { 43810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Read to addr %lld translates to %d " 43910259SAndrew.Bardsley@arm.com "dram requests\n", pkt->getAddr(), pktCount); 44010259SAndrew.Bardsley@arm.com burst_helper = new BurstHelper(pktCount); 44110259SAndrew.Bardsley@arm.com } 44210259SAndrew.Bardsley@arm.com 44310259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 44410259SAndrew.Bardsley@arm.com dram_pkt->burstHelper = burst_helper; 44510259SAndrew.Bardsley@arm.com 44610259SAndrew.Bardsley@arm.com assert(!readQueueFull(1)); 44710259SAndrew.Bardsley@arm.com rdQLenPdf[readQueue.size() + respQueue.size()]++; 44810259SAndrew.Bardsley@arm.com 44910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Adding to read queue\n"); 45010259SAndrew.Bardsley@arm.com 45110259SAndrew.Bardsley@arm.com readQueue.push_back(dram_pkt); 45210259SAndrew.Bardsley@arm.com 45310259SAndrew.Bardsley@arm.com // Update stats 45410259SAndrew.Bardsley@arm.com avgRdQLen = readQueue.size() + respQueue.size(); 45510259SAndrew.Bardsley@arm.com } 45610259SAndrew.Bardsley@arm.com 45710259SAndrew.Bardsley@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 45810259SAndrew.Bardsley@arm.com addr = (addr | (burstSize - 1)) + 1; 45910259SAndrew.Bardsley@arm.com } 46010259SAndrew.Bardsley@arm.com 46110259SAndrew.Bardsley@arm.com // If all packets are serviced by write queue, we send the repsonse back 46210259SAndrew.Bardsley@arm.com if (pktsServicedByWrQ == pktCount) { 46310259SAndrew.Bardsley@arm.com accessAndRespond(pkt, frontendLatency); 46410259SAndrew.Bardsley@arm.com return; 46510259SAndrew.Bardsley@arm.com } 46610259SAndrew.Bardsley@arm.com 46710259SAndrew.Bardsley@arm.com // Update how many split packets are serviced by write queue 46810259SAndrew.Bardsley@arm.com if (burst_helper != NULL) 46910259SAndrew.Bardsley@arm.com burst_helper->burstsServiced = pktsServicedByWrQ; 47010259SAndrew.Bardsley@arm.com 47110259SAndrew.Bardsley@arm.com // If we are not already scheduled to get a request out of the 47210259SAndrew.Bardsley@arm.com // queue, do so now 47310259SAndrew.Bardsley@arm.com if (!nextReqEvent.scheduled()) { 47410259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 47510259SAndrew.Bardsley@arm.com schedule(nextReqEvent, curTick()); 47610259SAndrew.Bardsley@arm.com } 47710259SAndrew.Bardsley@arm.com} 47810259SAndrew.Bardsley@arm.com 47910368SAndrew.Bardsley@arm.comvoid 48010368SAndrew.Bardsley@arm.comDRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 48110259SAndrew.Bardsley@arm.com{ 48210259SAndrew.Bardsley@arm.com // only add to the write queue here. whenever the request is 48310259SAndrew.Bardsley@arm.com // eventually done, set the readyTime, and call schedule() 48410259SAndrew.Bardsley@arm.com assert(pkt->isWrite()); 48510259SAndrew.Bardsley@arm.com 48610259SAndrew.Bardsley@arm.com // if the request size is larger than burst size, the pkt is split into 48710259SAndrew.Bardsley@arm.com // multiple DRAM packets 48810259SAndrew.Bardsley@arm.com Addr addr = pkt->getAddr(); 48910259SAndrew.Bardsley@arm.com for (int cnt = 0; cnt < pktCount; ++cnt) { 49010259SAndrew.Bardsley@arm.com unsigned size = std::min((addr | (burstSize - 1)) + 1, 49110259SAndrew.Bardsley@arm.com pkt->getAddr() + pkt->getSize()) - addr; 49210259SAndrew.Bardsley@arm.com writePktSize[ceilLog2(size)]++; 49310259SAndrew.Bardsley@arm.com writeBursts++; 49410259SAndrew.Bardsley@arm.com 49510259SAndrew.Bardsley@arm.com // see if we can merge with an existing item in the write 49610259SAndrew.Bardsley@arm.com // queue and keep track of whether we have merged or not so we 49710259SAndrew.Bardsley@arm.com // can stop at that point and also avoid enqueueing a new 49810259SAndrew.Bardsley@arm.com // request 49910259SAndrew.Bardsley@arm.com bool merged = false; 50010259SAndrew.Bardsley@arm.com auto w = writeQueue.begin(); 50110259SAndrew.Bardsley@arm.com 50210259SAndrew.Bardsley@arm.com while(!merged && w != writeQueue.end()) { 50310259SAndrew.Bardsley@arm.com // either of the two could be first, if they are the same 50410259SAndrew.Bardsley@arm.com // it does not matter which way we go 50510259SAndrew.Bardsley@arm.com if ((*w)->addr >= addr) { 50610259SAndrew.Bardsley@arm.com // the existing one starts after the new one, figure 50710259SAndrew.Bardsley@arm.com // out where the new one ends with respect to the 50810259SAndrew.Bardsley@arm.com // existing one 50910259SAndrew.Bardsley@arm.com if ((addr + size) >= ((*w)->addr + (*w)->size)) { 51010259SAndrew.Bardsley@arm.com // check if the existing one is completely 51110259SAndrew.Bardsley@arm.com // subsumed in the new one 51210259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Merging write covering existing burst\n"); 51310259SAndrew.Bardsley@arm.com merged = true; 51410259SAndrew.Bardsley@arm.com // update both the address and the size 51510259SAndrew.Bardsley@arm.com (*w)->addr = addr; 51610259SAndrew.Bardsley@arm.com (*w)->size = size; 51710259SAndrew.Bardsley@arm.com } else if ((addr + size) >= (*w)->addr && 51810259SAndrew.Bardsley@arm.com ((*w)->addr + (*w)->size - addr) <= burstSize) { 51910259SAndrew.Bardsley@arm.com // the new one is just before or partially 52010259SAndrew.Bardsley@arm.com // overlapping with the existing one, and together 52110259SAndrew.Bardsley@arm.com // they fit within a burst 52210259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Merging write before existing burst\n"); 52310259SAndrew.Bardsley@arm.com merged = true; 52410259SAndrew.Bardsley@arm.com // the existing queue item needs to be adjusted with 52510259SAndrew.Bardsley@arm.com // respect to both address and size 52610259SAndrew.Bardsley@arm.com (*w)->size = (*w)->addr + (*w)->size - addr; 52710259SAndrew.Bardsley@arm.com (*w)->addr = addr; 52810259SAndrew.Bardsley@arm.com } 52910259SAndrew.Bardsley@arm.com } else { 53010259SAndrew.Bardsley@arm.com // the new one starts after the current one, figure 53110259SAndrew.Bardsley@arm.com // out where the existing one ends with respect to the 53210259SAndrew.Bardsley@arm.com // new one 53310259SAndrew.Bardsley@arm.com if (((*w)->addr + (*w)->size) >= (addr + size)) { 53410259SAndrew.Bardsley@arm.com // check if the new one is completely subsumed in the 53510259SAndrew.Bardsley@arm.com // existing one 53610259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Merging write into existing burst\n"); 53710259SAndrew.Bardsley@arm.com merged = true; 53810259SAndrew.Bardsley@arm.com // no adjustments necessary 53910259SAndrew.Bardsley@arm.com } else if (((*w)->addr + (*w)->size) >= addr && 54010259SAndrew.Bardsley@arm.com (addr + size - (*w)->addr) <= burstSize) { 54110259SAndrew.Bardsley@arm.com // the existing one is just before or partially 54210259SAndrew.Bardsley@arm.com // overlapping with the new one, and together 54310259SAndrew.Bardsley@arm.com // they fit within a burst 54410259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Merging write after existing burst\n"); 54510259SAndrew.Bardsley@arm.com merged = true; 54610259SAndrew.Bardsley@arm.com // the address is right, and only the size has 54710259SAndrew.Bardsley@arm.com // to be adjusted 54810259SAndrew.Bardsley@arm.com (*w)->size = addr + size - (*w)->addr; 54910259SAndrew.Bardsley@arm.com } 55010259SAndrew.Bardsley@arm.com } 55110259SAndrew.Bardsley@arm.com ++w; 55210259SAndrew.Bardsley@arm.com } 55310259SAndrew.Bardsley@arm.com 55410259SAndrew.Bardsley@arm.com // if the item was not merged we need to create a new write 55510259SAndrew.Bardsley@arm.com // and enqueue it 55610259SAndrew.Bardsley@arm.com if (!merged) { 55710259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 55810259SAndrew.Bardsley@arm.com 55910259SAndrew.Bardsley@arm.com assert(writeQueue.size() < writeBufferSize); 56010259SAndrew.Bardsley@arm.com wrQLenPdf[writeQueue.size()]++; 56110259SAndrew.Bardsley@arm.com 56210259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Adding to write queue\n"); 56310563Sandreas.hansson@arm.com 56410259SAndrew.Bardsley@arm.com writeQueue.push_back(dram_pkt); 56510259SAndrew.Bardsley@arm.com 56610259SAndrew.Bardsley@arm.com // Update stats 56710259SAndrew.Bardsley@arm.com avgWrQLen = writeQueue.size(); 56810259SAndrew.Bardsley@arm.com } else { 56910259SAndrew.Bardsley@arm.com // keep track of the fact that this burst effectively 57010259SAndrew.Bardsley@arm.com // disappeared as it was merged with an existing one 57110259SAndrew.Bardsley@arm.com mergedWrBursts++; 57210259SAndrew.Bardsley@arm.com } 57310259SAndrew.Bardsley@arm.com 57410259SAndrew.Bardsley@arm.com // Starting address of next dram pkt (aligend to burstSize boundary) 57510259SAndrew.Bardsley@arm.com addr = (addr | (burstSize - 1)) + 1; 57610259SAndrew.Bardsley@arm.com } 57710259SAndrew.Bardsley@arm.com 57810259SAndrew.Bardsley@arm.com // we do not wait for the writes to be send to the actual memory, 57910259SAndrew.Bardsley@arm.com // but instead take responsibility for the consistency here and 58010259SAndrew.Bardsley@arm.com // snoop the write queue for any upcoming reads 58110259SAndrew.Bardsley@arm.com // @todo, if a pkt size is larger than burst size, we might need a 58210259SAndrew.Bardsley@arm.com // different front end latency 58310259SAndrew.Bardsley@arm.com accessAndRespond(pkt, frontendLatency); 58410259SAndrew.Bardsley@arm.com 58510259SAndrew.Bardsley@arm.com // If we are not already scheduled to get a request out of the 58610259SAndrew.Bardsley@arm.com // queue, do so now 58710259SAndrew.Bardsley@arm.com if (!nextReqEvent.scheduled()) { 58810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Request scheduled immediately\n"); 58910259SAndrew.Bardsley@arm.com schedule(nextReqEvent, curTick()); 59010259SAndrew.Bardsley@arm.com } 59110259SAndrew.Bardsley@arm.com} 59210259SAndrew.Bardsley@arm.com 59310259SAndrew.Bardsley@arm.comvoid 59410259SAndrew.Bardsley@arm.comDRAMCtrl::printQs() const { 59510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "===READ QUEUE===\n\n"); 59610259SAndrew.Bardsley@arm.com for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 59710259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 59810259SAndrew.Bardsley@arm.com } 59910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 60010259SAndrew.Bardsley@arm.com for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 60110259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 60210259SAndrew.Bardsley@arm.com } 60310259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 60410259SAndrew.Bardsley@arm.com for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 60510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 60610259SAndrew.Bardsley@arm.com } 60710259SAndrew.Bardsley@arm.com} 60810259SAndrew.Bardsley@arm.com 60910259SAndrew.Bardsley@arm.combool 61010259SAndrew.Bardsley@arm.comDRAMCtrl::recvTimingReq(PacketPtr pkt) 61110259SAndrew.Bardsley@arm.com{ 61210259SAndrew.Bardsley@arm.com /// @todo temporary hack to deal with memory corruption issues until 61310259SAndrew.Bardsley@arm.com /// 4-phase transactions are complete 61410259SAndrew.Bardsley@arm.com for (int x = 0; x < pendingDelete.size(); x++) 61510259SAndrew.Bardsley@arm.com delete pendingDelete[x]; 61610259SAndrew.Bardsley@arm.com pendingDelete.clear(); 61710259SAndrew.Bardsley@arm.com 61810259SAndrew.Bardsley@arm.com // This is where we enter from the outside world 61910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 62010259SAndrew.Bardsley@arm.com pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 62110259SAndrew.Bardsley@arm.com 62210259SAndrew.Bardsley@arm.com // simply drop inhibited packets for now 62310259SAndrew.Bardsley@arm.com if (pkt->memInhibitAsserted()) { 62410259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Inhibited packet -- Dropping it now\n"); 62510259SAndrew.Bardsley@arm.com pendingDelete.push_back(pkt); 62610259SAndrew.Bardsley@arm.com return true; 62710259SAndrew.Bardsley@arm.com } 62810259SAndrew.Bardsley@arm.com 62910259SAndrew.Bardsley@arm.com // Calc avg gap between requests 63010259SAndrew.Bardsley@arm.com if (prevArrival != 0) { 63110259SAndrew.Bardsley@arm.com totGap += curTick() - prevArrival; 63210259SAndrew.Bardsley@arm.com } 63310259SAndrew.Bardsley@arm.com prevArrival = curTick(); 63410259SAndrew.Bardsley@arm.com 63510259SAndrew.Bardsley@arm.com 63610259SAndrew.Bardsley@arm.com // Find out how many dram packets a pkt translates to 63710259SAndrew.Bardsley@arm.com // If the burst size is equal or larger than the pkt size, then a pkt 63810259SAndrew.Bardsley@arm.com // translates to only one dram packet. Otherwise, a pkt translates to 63910259SAndrew.Bardsley@arm.com // multiple dram packets 64010259SAndrew.Bardsley@arm.com unsigned size = pkt->getSize(); 64110259SAndrew.Bardsley@arm.com unsigned offset = pkt->getAddr() & (burstSize - 1); 64210259SAndrew.Bardsley@arm.com unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 64310259SAndrew.Bardsley@arm.com 64410259SAndrew.Bardsley@arm.com // check local buffers and do not accept if full 64510259SAndrew.Bardsley@arm.com if (pkt->isRead()) { 64610259SAndrew.Bardsley@arm.com assert(size != 0); 64710259SAndrew.Bardsley@arm.com if (readQueueFull(dram_pkt_count)) { 64810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Read queue full, not accepting\n"); 64910259SAndrew.Bardsley@arm.com // remember that we have to retry this port 65010259SAndrew.Bardsley@arm.com retryRdReq = true; 65110259SAndrew.Bardsley@arm.com numRdRetry++; 65210259SAndrew.Bardsley@arm.com return false; 65310259SAndrew.Bardsley@arm.com } else { 65410259SAndrew.Bardsley@arm.com addToReadQueue(pkt, dram_pkt_count); 65510259SAndrew.Bardsley@arm.com readReqs++; 65610259SAndrew.Bardsley@arm.com bytesReadSys += size; 65710259SAndrew.Bardsley@arm.com } 65810259SAndrew.Bardsley@arm.com } else if (pkt->isWrite()) { 65910259SAndrew.Bardsley@arm.com assert(size != 0); 66010259SAndrew.Bardsley@arm.com if (writeQueueFull(dram_pkt_count)) { 66110259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Write queue full, not accepting\n"); 66210259SAndrew.Bardsley@arm.com // remember that we have to retry this port 66310259SAndrew.Bardsley@arm.com retryWrReq = true; 66410259SAndrew.Bardsley@arm.com numWrRetry++; 66510259SAndrew.Bardsley@arm.com return false; 66610259SAndrew.Bardsley@arm.com } else { 66710259SAndrew.Bardsley@arm.com addToWriteQueue(pkt, dram_pkt_count); 66810259SAndrew.Bardsley@arm.com writeReqs++; 66910259SAndrew.Bardsley@arm.com bytesWrittenSys += size; 67010259SAndrew.Bardsley@arm.com } 67110259SAndrew.Bardsley@arm.com } else { 67210259SAndrew.Bardsley@arm.com DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 67310259SAndrew.Bardsley@arm.com neitherReadNorWrite++; 67410259SAndrew.Bardsley@arm.com accessAndRespond(pkt, 1); 67510259SAndrew.Bardsley@arm.com } 67610259SAndrew.Bardsley@arm.com 67710259SAndrew.Bardsley@arm.com return true; 67810259SAndrew.Bardsley@arm.com} 67910259SAndrew.Bardsley@arm.com 68010259SAndrew.Bardsley@arm.comvoid 68110259SAndrew.Bardsley@arm.comDRAMCtrl::processRespondEvent() 68210259SAndrew.Bardsley@arm.com{ 68310259SAndrew.Bardsley@arm.com DPRINTF(DRAM, 68410259SAndrew.Bardsley@arm.com "processRespondEvent(): Some req has reached its readyTime\n"); 68510259SAndrew.Bardsley@arm.com 68610259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = respQueue.front(); 68710259SAndrew.Bardsley@arm.com 68810259SAndrew.Bardsley@arm.com if (dram_pkt->burstHelper) { 68910259SAndrew.Bardsley@arm.com // it is a split packet 69010259SAndrew.Bardsley@arm.com dram_pkt->burstHelper->burstsServiced++; 69110259SAndrew.Bardsley@arm.com if (dram_pkt->burstHelper->burstsServiced == 69210259SAndrew.Bardsley@arm.com dram_pkt->burstHelper->burstCount) { 69310259SAndrew.Bardsley@arm.com // we have now serviced all children packets of a system packet 69410259SAndrew.Bardsley@arm.com // so we can now respond to the requester 69510259SAndrew.Bardsley@arm.com // @todo we probably want to have a different front end and back 69610259SAndrew.Bardsley@arm.com // end latency for split packets 69710259SAndrew.Bardsley@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 69810259SAndrew.Bardsley@arm.com delete dram_pkt->burstHelper; 69910259SAndrew.Bardsley@arm.com dram_pkt->burstHelper = NULL; 70010259SAndrew.Bardsley@arm.com } 70110259SAndrew.Bardsley@arm.com } else { 70210259SAndrew.Bardsley@arm.com // it is not a split packet 70310259SAndrew.Bardsley@arm.com accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 70410259SAndrew.Bardsley@arm.com } 70510259SAndrew.Bardsley@arm.com 70610259SAndrew.Bardsley@arm.com delete respQueue.front(); 70710259SAndrew.Bardsley@arm.com respQueue.pop_front(); 70810259SAndrew.Bardsley@arm.com 70910259SAndrew.Bardsley@arm.com if (!respQueue.empty()) { 71010259SAndrew.Bardsley@arm.com assert(respQueue.front()->readyTime >= curTick()); 71110259SAndrew.Bardsley@arm.com assert(!respondEvent.scheduled()); 71210259SAndrew.Bardsley@arm.com schedule(respondEvent, respQueue.front()->readyTime); 71310259SAndrew.Bardsley@arm.com } else { 71410259SAndrew.Bardsley@arm.com // if there is nothing left in any queue, signal a drain 71510259SAndrew.Bardsley@arm.com if (writeQueue.empty() && readQueue.empty() && 71610259SAndrew.Bardsley@arm.com drainManager) { 71710259SAndrew.Bardsley@arm.com drainManager->signalDrainDone(); 71810259SAndrew.Bardsley@arm.com drainManager = NULL; 71910259SAndrew.Bardsley@arm.com } 72010259SAndrew.Bardsley@arm.com } 72110259SAndrew.Bardsley@arm.com 72210259SAndrew.Bardsley@arm.com // We have made a location in the queue available at this point, 72310259SAndrew.Bardsley@arm.com // so if there is a read that was forced to wait, retry now 72410259SAndrew.Bardsley@arm.com if (retryRdReq) { 72510259SAndrew.Bardsley@arm.com retryRdReq = false; 72610259SAndrew.Bardsley@arm.com port.sendRetry(); 72710259SAndrew.Bardsley@arm.com } 72810259SAndrew.Bardsley@arm.com} 72910259SAndrew.Bardsley@arm.com 73010259SAndrew.Bardsley@arm.comvoid 73110259SAndrew.Bardsley@arm.comDRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, bool switched_cmd_type) 73210259SAndrew.Bardsley@arm.com{ 73310259SAndrew.Bardsley@arm.com // This method does the arbitration between requests. The chosen 73410259SAndrew.Bardsley@arm.com // packet is simply moved to the head of the queue. The other 73510259SAndrew.Bardsley@arm.com // methods know that this is the place to look. For example, with 73610259SAndrew.Bardsley@arm.com // FCFS, this method does nothing 73710259SAndrew.Bardsley@arm.com assert(!queue.empty()); 73810259SAndrew.Bardsley@arm.com 73910259SAndrew.Bardsley@arm.com if (queue.size() == 1) { 74010259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Single request, nothing to do\n"); 74110259SAndrew.Bardsley@arm.com return; 74210259SAndrew.Bardsley@arm.com } 74310259SAndrew.Bardsley@arm.com 74410259SAndrew.Bardsley@arm.com if (memSchedPolicy == Enums::fcfs) { 74510259SAndrew.Bardsley@arm.com // Do nothing, since the correct request is already head 74610259SAndrew.Bardsley@arm.com } else if (memSchedPolicy == Enums::frfcfs) { 74710259SAndrew.Bardsley@arm.com reorderQueue(queue, switched_cmd_type); 74810259SAndrew.Bardsley@arm.com } else 74910259SAndrew.Bardsley@arm.com panic("No scheduling policy chosen\n"); 75010259SAndrew.Bardsley@arm.com} 75110259SAndrew.Bardsley@arm.com 75210259SAndrew.Bardsley@arm.comvoid 75310259SAndrew.Bardsley@arm.comDRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, bool switched_cmd_type) 75410259SAndrew.Bardsley@arm.com{ 75510259SAndrew.Bardsley@arm.com // Only determine this when needed 75610259SAndrew.Bardsley@arm.com uint64_t earliest_banks = 0; 75710259SAndrew.Bardsley@arm.com 75810259SAndrew.Bardsley@arm.com // Search for row hits first, if no row hit is found then schedule the 75910259SAndrew.Bardsley@arm.com // packet to one of the earliest banks available 76010259SAndrew.Bardsley@arm.com bool found_earliest_pkt = false; 76110259SAndrew.Bardsley@arm.com bool found_prepped_diff_rank_pkt = false; 76210259SAndrew.Bardsley@arm.com auto selected_pkt_it = queue.begin(); 76310259SAndrew.Bardsley@arm.com 76410259SAndrew.Bardsley@arm.com for (auto i = queue.begin(); i != queue.end() ; ++i) { 76510259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = *i; 76610259SAndrew.Bardsley@arm.com const Bank& bank = dram_pkt->bankRef; 76710259SAndrew.Bardsley@arm.com // Check if it is a row hit 76810259SAndrew.Bardsley@arm.com if (bank.openRow == dram_pkt->row) { 76910259SAndrew.Bardsley@arm.com if (dram_pkt->rank == activeRank || switched_cmd_type) { 77010259SAndrew.Bardsley@arm.com // FCFS within the hits, giving priority to commands 77110259SAndrew.Bardsley@arm.com // that access the same rank as the previous burst 77210259SAndrew.Bardsley@arm.com // to minimize bus turnaround delays 77310259SAndrew.Bardsley@arm.com // Only give rank prioity when command type is not changing 77410259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Row buffer hit\n"); 77510259SAndrew.Bardsley@arm.com selected_pkt_it = i; 77610259SAndrew.Bardsley@arm.com break; 77710259SAndrew.Bardsley@arm.com } else if (!found_prepped_diff_rank_pkt) { 77810259SAndrew.Bardsley@arm.com // found row hit for command on different rank than prev burst 77910259SAndrew.Bardsley@arm.com selected_pkt_it = i; 78010259SAndrew.Bardsley@arm.com found_prepped_diff_rank_pkt = true; 78110259SAndrew.Bardsley@arm.com } 78210259SAndrew.Bardsley@arm.com } else if (!found_earliest_pkt & !found_prepped_diff_rank_pkt) { 78310259SAndrew.Bardsley@arm.com // No row hit and 78410259SAndrew.Bardsley@arm.com // haven't found an entry with a row hit to a new rank 78510259SAndrew.Bardsley@arm.com if (earliest_banks == 0) 78610259SAndrew.Bardsley@arm.com // Determine entries with earliest bank prep delay 78710259SAndrew.Bardsley@arm.com // Function will give priority to commands that access the 78810259SAndrew.Bardsley@arm.com // same rank as previous burst and can prep the bank seamlessly 78910259SAndrew.Bardsley@arm.com earliest_banks = minBankPrep(queue, switched_cmd_type); 79010259SAndrew.Bardsley@arm.com 79110259SAndrew.Bardsley@arm.com // FCFS - Bank is first available bank 79210259SAndrew.Bardsley@arm.com if (bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 79310259SAndrew.Bardsley@arm.com // Remember the packet to be scheduled to one of the earliest 79410259SAndrew.Bardsley@arm.com // banks available, FCFS amongst the earliest banks 79510259SAndrew.Bardsley@arm.com selected_pkt_it = i; 79610259SAndrew.Bardsley@arm.com found_earliest_pkt = true; 79710259SAndrew.Bardsley@arm.com } 79810259SAndrew.Bardsley@arm.com } 79910259SAndrew.Bardsley@arm.com } 80010259SAndrew.Bardsley@arm.com 80110259SAndrew.Bardsley@arm.com DRAMPacket* selected_pkt = *selected_pkt_it; 80210259SAndrew.Bardsley@arm.com queue.erase(selected_pkt_it); 80310259SAndrew.Bardsley@arm.com queue.push_front(selected_pkt); 80410259SAndrew.Bardsley@arm.com} 80510259SAndrew.Bardsley@arm.com 80610259SAndrew.Bardsley@arm.comvoid 80710259SAndrew.Bardsley@arm.comDRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 80810259SAndrew.Bardsley@arm.com{ 80910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 81010259SAndrew.Bardsley@arm.com 81110259SAndrew.Bardsley@arm.com bool needsResponse = pkt->needsResponse(); 81210259SAndrew.Bardsley@arm.com // do the actual memory access which also turns the packet into a 81310259SAndrew.Bardsley@arm.com // response 81410259SAndrew.Bardsley@arm.com access(pkt); 81510259SAndrew.Bardsley@arm.com 81610259SAndrew.Bardsley@arm.com // turn packet around to go back to requester if response expected 81710259SAndrew.Bardsley@arm.com if (needsResponse) { 81810259SAndrew.Bardsley@arm.com // access already turned the packet into a response 81910259SAndrew.Bardsley@arm.com assert(pkt->isResponse()); 82010259SAndrew.Bardsley@arm.com 82110259SAndrew.Bardsley@arm.com // @todo someone should pay for this 82210259SAndrew.Bardsley@arm.com pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 82310259SAndrew.Bardsley@arm.com 82410259SAndrew.Bardsley@arm.com // queue the packet in the response queue to be sent out after 82510259SAndrew.Bardsley@arm.com // the static latency has passed 82610259SAndrew.Bardsley@arm.com port.schedTimingResp(pkt, curTick() + static_latency); 82710259SAndrew.Bardsley@arm.com } else { 82810259SAndrew.Bardsley@arm.com // @todo the packet is going to be deleted, and the DRAMPacket 82910259SAndrew.Bardsley@arm.com // is still having a pointer to it 83010259SAndrew.Bardsley@arm.com pendingDelete.push_back(pkt); 83110259SAndrew.Bardsley@arm.com } 83210259SAndrew.Bardsley@arm.com 83310259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Done\n"); 83410259SAndrew.Bardsley@arm.com 83510259SAndrew.Bardsley@arm.com return; 83610259SAndrew.Bardsley@arm.com} 83710259SAndrew.Bardsley@arm.com 83810259SAndrew.Bardsley@arm.comvoid 83910259SAndrew.Bardsley@arm.comDRAMCtrl::activateBank(Bank& bank, Tick act_tick, uint32_t row) 84010259SAndrew.Bardsley@arm.com{ 84110259SAndrew.Bardsley@arm.com // get the rank index from the bank 84210259SAndrew.Bardsley@arm.com uint8_t rank = bank.rank; 84310259SAndrew.Bardsley@arm.com 84410259SAndrew.Bardsley@arm.com assert(actTicks[rank].size() == activationLimit); 84510259SAndrew.Bardsley@arm.com 84610259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 84710259SAndrew.Bardsley@arm.com 84810259SAndrew.Bardsley@arm.com // update the open row 84910259SAndrew.Bardsley@arm.com assert(bank.openRow == Bank::NO_ROW); 85010259SAndrew.Bardsley@arm.com bank.openRow = row; 85110259SAndrew.Bardsley@arm.com 85210259SAndrew.Bardsley@arm.com // start counting anew, this covers both the case when we 85310259SAndrew.Bardsley@arm.com // auto-precharged, and when this access is forced to 85410259SAndrew.Bardsley@arm.com // precharge 85510259SAndrew.Bardsley@arm.com bank.bytesAccessed = 0; 85610259SAndrew.Bardsley@arm.com bank.rowAccesses = 0; 85710259SAndrew.Bardsley@arm.com 85810259SAndrew.Bardsley@arm.com ++numBanksActive; 85910259SAndrew.Bardsley@arm.com assert(numBanksActive <= banksPerRank * ranksPerChannel); 86010259SAndrew.Bardsley@arm.com 86110259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Activate bank %d, rank %d at tick %lld, now got %d active\n", 86210259SAndrew.Bardsley@arm.com bank.bank, bank.rank, act_tick, numBanksActive); 86310259SAndrew.Bardsley@arm.com 86410259SAndrew.Bardsley@arm.com DPRINTF(DRAMPower, "%llu,ACT,%d,%d\n", divCeil(act_tick, tCK), bank.bank, 86510259SAndrew.Bardsley@arm.com bank.rank); 86610259SAndrew.Bardsley@arm.com 86710259SAndrew.Bardsley@arm.com // The next access has to respect tRAS for this bank 86810259SAndrew.Bardsley@arm.com bank.preAllowedAt = act_tick + tRAS; 86910259SAndrew.Bardsley@arm.com 87010259SAndrew.Bardsley@arm.com // Respect the row-to-column command delay 87110259SAndrew.Bardsley@arm.com bank.colAllowedAt = std::max(act_tick + tRCD, bank.colAllowedAt); 87210259SAndrew.Bardsley@arm.com 87310259SAndrew.Bardsley@arm.com // start by enforcing tRRD 87410259SAndrew.Bardsley@arm.com for(int i = 0; i < banksPerRank; i++) { 87510259SAndrew.Bardsley@arm.com // next activate to any bank in this rank must not happen 87610259SAndrew.Bardsley@arm.com // before tRRD 87710259SAndrew.Bardsley@arm.com if (bankGroupArch && (bank.bankgr == banks[rank][i].bankgr)) { 87810259SAndrew.Bardsley@arm.com // bank group architecture requires longer delays between 87910259SAndrew.Bardsley@arm.com // ACT commands within the same bank group. Use tRRD_L 88010259SAndrew.Bardsley@arm.com // in this case 88110259SAndrew.Bardsley@arm.com banks[rank][i].actAllowedAt = std::max(act_tick + tRRD_L, 88210259SAndrew.Bardsley@arm.com banks[rank][i].actAllowedAt); 88310259SAndrew.Bardsley@arm.com } else { 88410259SAndrew.Bardsley@arm.com // use shorter tRRD value when either 88510259SAndrew.Bardsley@arm.com // 1) bank group architecture is not supportted 88610259SAndrew.Bardsley@arm.com // 2) bank is in a different bank group 88710259SAndrew.Bardsley@arm.com banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 88810259SAndrew.Bardsley@arm.com banks[rank][i].actAllowedAt); 88910259SAndrew.Bardsley@arm.com } 89010259SAndrew.Bardsley@arm.com } 89110259SAndrew.Bardsley@arm.com 89210259SAndrew.Bardsley@arm.com // next, we deal with tXAW, if the activation limit is disabled 89310259SAndrew.Bardsley@arm.com // then we are done 89410259SAndrew.Bardsley@arm.com if (actTicks[rank].empty()) 89510259SAndrew.Bardsley@arm.com return; 89610259SAndrew.Bardsley@arm.com 89710259SAndrew.Bardsley@arm.com // sanity check 89810259SAndrew.Bardsley@arm.com if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 89910259SAndrew.Bardsley@arm.com panic("Got %d activates in window %d (%llu - %llu) which is smaller " 90010259SAndrew.Bardsley@arm.com "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 90110259SAndrew.Bardsley@arm.com act_tick, actTicks[rank].back(), tXAW); 90210259SAndrew.Bardsley@arm.com } 90310259SAndrew.Bardsley@arm.com 90410259SAndrew.Bardsley@arm.com // shift the times used for the book keeping, the last element 90510259SAndrew.Bardsley@arm.com // (highest index) is the oldest one and hence the lowest value 90610259SAndrew.Bardsley@arm.com actTicks[rank].pop_back(); 90710259SAndrew.Bardsley@arm.com 90810259SAndrew.Bardsley@arm.com // record an new activation (in the future) 90910259SAndrew.Bardsley@arm.com actTicks[rank].push_front(act_tick); 91010259SAndrew.Bardsley@arm.com 91110259SAndrew.Bardsley@arm.com // cannot activate more than X times in time window tXAW, push the 91210259SAndrew.Bardsley@arm.com // next one (the X + 1'st activate) to be tXAW away from the 91310259SAndrew.Bardsley@arm.com // oldest in our window of X 91410259SAndrew.Bardsley@arm.com if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 91510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " 91610259SAndrew.Bardsley@arm.com "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); 91710259SAndrew.Bardsley@arm.com for(int j = 0; j < banksPerRank; j++) 91810259SAndrew.Bardsley@arm.com // next activate must not happen before end of window 91910259SAndrew.Bardsley@arm.com banks[rank][j].actAllowedAt = 92010259SAndrew.Bardsley@arm.com std::max(actTicks[rank].back() + tXAW, 92110259SAndrew.Bardsley@arm.com banks[rank][j].actAllowedAt); 92210259SAndrew.Bardsley@arm.com } 92310259SAndrew.Bardsley@arm.com 92410259SAndrew.Bardsley@arm.com // at the point when this activate takes place, make sure we 92510259SAndrew.Bardsley@arm.com // transition to the active power state 92610259SAndrew.Bardsley@arm.com if (!activateEvent.scheduled()) 92710259SAndrew.Bardsley@arm.com schedule(activateEvent, act_tick); 92810259SAndrew.Bardsley@arm.com else if (activateEvent.when() > act_tick) 92910259SAndrew.Bardsley@arm.com // move it sooner in time 93010259SAndrew.Bardsley@arm.com reschedule(activateEvent, act_tick); 93110259SAndrew.Bardsley@arm.com} 93210259SAndrew.Bardsley@arm.com 93310259SAndrew.Bardsley@arm.comvoid 93410259SAndrew.Bardsley@arm.comDRAMCtrl::processActivateEvent() 93510259SAndrew.Bardsley@arm.com{ 93610259SAndrew.Bardsley@arm.com // we should transition to the active state as soon as any bank is active 93710259SAndrew.Bardsley@arm.com if (pwrState != PWR_ACT) 93810259SAndrew.Bardsley@arm.com // note that at this point numBanksActive could be back at 93910259SAndrew.Bardsley@arm.com // zero again due to a precharge scheduled in the future 94010259SAndrew.Bardsley@arm.com schedulePowerEvent(PWR_ACT, curTick()); 94110259SAndrew.Bardsley@arm.com} 94210259SAndrew.Bardsley@arm.com 94310259SAndrew.Bardsley@arm.comvoid 94410259SAndrew.Bardsley@arm.comDRAMCtrl::prechargeBank(Bank& bank, Tick pre_at, bool trace) 94510259SAndrew.Bardsley@arm.com{ 94610259SAndrew.Bardsley@arm.com // make sure the bank has an open row 94710259SAndrew.Bardsley@arm.com assert(bank.openRow != Bank::NO_ROW); 94810259SAndrew.Bardsley@arm.com 94910259SAndrew.Bardsley@arm.com // sample the bytes per activate here since we are closing 95010259SAndrew.Bardsley@arm.com // the page 95110259SAndrew.Bardsley@arm.com bytesPerActivate.sample(bank.bytesAccessed); 95210259SAndrew.Bardsley@arm.com 95310259SAndrew.Bardsley@arm.com bank.openRow = Bank::NO_ROW; 95410259SAndrew.Bardsley@arm.com 95510259SAndrew.Bardsley@arm.com // no precharge allowed before this one 95610259SAndrew.Bardsley@arm.com bank.preAllowedAt = pre_at; 95710259SAndrew.Bardsley@arm.com 95810259SAndrew.Bardsley@arm.com Tick pre_done_at = pre_at + tRP; 95910259SAndrew.Bardsley@arm.com 96010259SAndrew.Bardsley@arm.com bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 96110259SAndrew.Bardsley@arm.com 96210259SAndrew.Bardsley@arm.com assert(numBanksActive != 0); 96310259SAndrew.Bardsley@arm.com --numBanksActive; 96410259SAndrew.Bardsley@arm.com 96510259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Precharging bank %d, rank %d at tick %lld, now got " 96610259SAndrew.Bardsley@arm.com "%d active\n", bank.bank, bank.rank, pre_at, numBanksActive); 96710259SAndrew.Bardsley@arm.com 96810259SAndrew.Bardsley@arm.com if (trace) 96910259SAndrew.Bardsley@arm.com DPRINTF(DRAMPower, "%llu,PRE,%d,%d\n", divCeil(pre_at, tCK), 97010259SAndrew.Bardsley@arm.com bank.bank, bank.rank); 97110259SAndrew.Bardsley@arm.com 97210259SAndrew.Bardsley@arm.com // if we look at the current number of active banks we might be 97310259SAndrew.Bardsley@arm.com // tempted to think the DRAM is now idle, however this can be 97410259SAndrew.Bardsley@arm.com // undone by an activate that is scheduled to happen before we 97510259SAndrew.Bardsley@arm.com // would have reached the idle state, so schedule an event and 97610259SAndrew.Bardsley@arm.com // rather check once we actually make it to the point in time when 97710259SAndrew.Bardsley@arm.com // the (last) precharge takes place 97810259SAndrew.Bardsley@arm.com if (!prechargeEvent.scheduled()) 97910259SAndrew.Bardsley@arm.com schedule(prechargeEvent, pre_done_at); 98010259SAndrew.Bardsley@arm.com else if (prechargeEvent.when() < pre_done_at) 98110259SAndrew.Bardsley@arm.com reschedule(prechargeEvent, pre_done_at); 98210259SAndrew.Bardsley@arm.com} 98310259SAndrew.Bardsley@arm.com 98410259SAndrew.Bardsley@arm.comvoid 98510259SAndrew.Bardsley@arm.comDRAMCtrl::processPrechargeEvent() 98610259SAndrew.Bardsley@arm.com{ 98710259SAndrew.Bardsley@arm.com // if we reached zero, then special conditions apply as we track 98810259SAndrew.Bardsley@arm.com // if all banks are precharged for the power models 98910259SAndrew.Bardsley@arm.com if (numBanksActive == 0) { 99010259SAndrew.Bardsley@arm.com // we should transition to the idle state when the last bank 99110259SAndrew.Bardsley@arm.com // is precharged 99210259SAndrew.Bardsley@arm.com schedulePowerEvent(PWR_IDLE, curTick()); 99310259SAndrew.Bardsley@arm.com } 99410259SAndrew.Bardsley@arm.com} 99510259SAndrew.Bardsley@arm.com 99610259SAndrew.Bardsley@arm.comvoid 99710259SAndrew.Bardsley@arm.comDRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 99810259SAndrew.Bardsley@arm.com{ 99910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 100010259SAndrew.Bardsley@arm.com dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 100110259SAndrew.Bardsley@arm.com 100210259SAndrew.Bardsley@arm.com // get the bank 100310259SAndrew.Bardsley@arm.com Bank& bank = dram_pkt->bankRef; 100410259SAndrew.Bardsley@arm.com 100510259SAndrew.Bardsley@arm.com // for the state we need to track if it is a row hit or not 100610259SAndrew.Bardsley@arm.com bool row_hit = true; 100710259SAndrew.Bardsley@arm.com 100810259SAndrew.Bardsley@arm.com // respect any constraints on the command (e.g. tRCD or tCCD) 100910259SAndrew.Bardsley@arm.com Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 101010259SAndrew.Bardsley@arm.com 101110259SAndrew.Bardsley@arm.com // Determine the access latency and update the bank state 101210259SAndrew.Bardsley@arm.com if (bank.openRow == dram_pkt->row) { 101310259SAndrew.Bardsley@arm.com // nothing to do 101410259SAndrew.Bardsley@arm.com } else { 101510259SAndrew.Bardsley@arm.com row_hit = false; 101610259SAndrew.Bardsley@arm.com 101710259SAndrew.Bardsley@arm.com // If there is a page open, precharge it. 101810259SAndrew.Bardsley@arm.com if (bank.openRow != Bank::NO_ROW) { 101910259SAndrew.Bardsley@arm.com prechargeBank(bank, std::max(bank.preAllowedAt, curTick())); 102010259SAndrew.Bardsley@arm.com } 102110259SAndrew.Bardsley@arm.com 102210259SAndrew.Bardsley@arm.com // next we need to account for the delay in activating the 102310259SAndrew.Bardsley@arm.com // page 102410259SAndrew.Bardsley@arm.com Tick act_tick = std::max(bank.actAllowedAt, curTick()); 102510259SAndrew.Bardsley@arm.com 102610259SAndrew.Bardsley@arm.com // Record the activation and deal with all the global timing 102710259SAndrew.Bardsley@arm.com // constraints caused be a new activation (tRRD and tXAW) 102810259SAndrew.Bardsley@arm.com activateBank(bank, act_tick, dram_pkt->row); 102910259SAndrew.Bardsley@arm.com 103010259SAndrew.Bardsley@arm.com // issue the command as early as possible 103110259SAndrew.Bardsley@arm.com cmd_at = bank.colAllowedAt; 103210259SAndrew.Bardsley@arm.com } 103310259SAndrew.Bardsley@arm.com 103410368SAndrew.Bardsley@arm.com // we need to wait until the bus is available before we can issue 103510259SAndrew.Bardsley@arm.com // the command 103610259SAndrew.Bardsley@arm.com cmd_at = std::max(cmd_at, busBusyUntil - tCL); 103710259SAndrew.Bardsley@arm.com 103810259SAndrew.Bardsley@arm.com // update the packet ready time 103910259SAndrew.Bardsley@arm.com dram_pkt->readyTime = cmd_at + tCL + tBURST; 104010259SAndrew.Bardsley@arm.com 104110259SAndrew.Bardsley@arm.com // only one burst can use the bus at any one point in time 104210259SAndrew.Bardsley@arm.com assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 104310259SAndrew.Bardsley@arm.com 104410259SAndrew.Bardsley@arm.com // update the time for the next read/write burst for each 104510259SAndrew.Bardsley@arm.com // bank (add a max with tCCD/tCCD_L here) 104610259SAndrew.Bardsley@arm.com Tick cmd_dly; 104710259SAndrew.Bardsley@arm.com for(int j = 0; j < ranksPerChannel; j++) { 104810259SAndrew.Bardsley@arm.com for(int i = 0; i < banksPerRank; i++) { 104910259SAndrew.Bardsley@arm.com // next burst to same bank group in this rank must not happen 105010259SAndrew.Bardsley@arm.com // before tCCD_L. Different bank group timing requirement is 105110259SAndrew.Bardsley@arm.com // tBURST; Add tCS for different ranks 105210259SAndrew.Bardsley@arm.com if (dram_pkt->rank == j) { 105310259SAndrew.Bardsley@arm.com if (bankGroupArch && (bank.bankgr == banks[j][i].bankgr)) { 105410259SAndrew.Bardsley@arm.com // bank group architecture requires longer delays between 105510259SAndrew.Bardsley@arm.com // RD/WR burst commands to the same bank group. 105610259SAndrew.Bardsley@arm.com // Use tCCD_L in this case 105710259SAndrew.Bardsley@arm.com cmd_dly = tCCD_L; 105810259SAndrew.Bardsley@arm.com } else { 105910259SAndrew.Bardsley@arm.com // use tBURST (equivalent to tCCD_S), the shorter 106010259SAndrew.Bardsley@arm.com // cas-to-cas delay value, when either: 106110259SAndrew.Bardsley@arm.com // 1) bank group architecture is not supportted 106210259SAndrew.Bardsley@arm.com // 2) bank is in a different bank group 106310259SAndrew.Bardsley@arm.com cmd_dly = tBURST; 106410259SAndrew.Bardsley@arm.com } 106510259SAndrew.Bardsley@arm.com } else { 106610259SAndrew.Bardsley@arm.com // different rank is by default in a different bank group 106710259SAndrew.Bardsley@arm.com // use tBURST (equivalent to tCCD_S), which is the shorter 106810259SAndrew.Bardsley@arm.com // cas-to-cas delay in this case 106910259SAndrew.Bardsley@arm.com // Add tCS to account for rank-to-rank bus delay requirements 107010259SAndrew.Bardsley@arm.com cmd_dly = tBURST + tCS; 107110259SAndrew.Bardsley@arm.com } 107210259SAndrew.Bardsley@arm.com banks[j][i].colAllowedAt = std::max(cmd_at + cmd_dly, 107310259SAndrew.Bardsley@arm.com banks[j][i].colAllowedAt); 107410259SAndrew.Bardsley@arm.com } 107510259SAndrew.Bardsley@arm.com } 107610259SAndrew.Bardsley@arm.com 107710259SAndrew.Bardsley@arm.com // Save rank of current access 107810259SAndrew.Bardsley@arm.com activeRank = dram_pkt->rank; 107910259SAndrew.Bardsley@arm.com 108010259SAndrew.Bardsley@arm.com // If this is a write, we also need to respect the write recovery 108110259SAndrew.Bardsley@arm.com // time before a precharge, in the case of a read, respect the 108210259SAndrew.Bardsley@arm.com // read to precharge constraint 108310259SAndrew.Bardsley@arm.com bank.preAllowedAt = std::max(bank.preAllowedAt, 108410259SAndrew.Bardsley@arm.com dram_pkt->isRead ? cmd_at + tRTP : 108510259SAndrew.Bardsley@arm.com dram_pkt->readyTime + tWR); 108610259SAndrew.Bardsley@arm.com 108710259SAndrew.Bardsley@arm.com // increment the bytes accessed and the accesses per row 108810259SAndrew.Bardsley@arm.com bank.bytesAccessed += burstSize; 108910259SAndrew.Bardsley@arm.com ++bank.rowAccesses; 109010259SAndrew.Bardsley@arm.com 109110259SAndrew.Bardsley@arm.com // if we reached the max, then issue with an auto-precharge 109210259SAndrew.Bardsley@arm.com bool auto_precharge = pageMgmt == Enums::close || 109310259SAndrew.Bardsley@arm.com bank.rowAccesses == maxAccessesPerRow; 109410259SAndrew.Bardsley@arm.com 109510259SAndrew.Bardsley@arm.com // if we did not hit the limit, we might still want to 109610259SAndrew.Bardsley@arm.com // auto-precharge 109710259SAndrew.Bardsley@arm.com if (!auto_precharge && 109810259SAndrew.Bardsley@arm.com (pageMgmt == Enums::open_adaptive || 109910259SAndrew.Bardsley@arm.com pageMgmt == Enums::close_adaptive)) { 110010259SAndrew.Bardsley@arm.com // a twist on the open and close page policies: 110110259SAndrew.Bardsley@arm.com // 1) open_adaptive page policy does not blindly keep the 110210259SAndrew.Bardsley@arm.com // page open, but close it if there are no row hits, and there 110310259SAndrew.Bardsley@arm.com // are bank conflicts in the queue 110410259SAndrew.Bardsley@arm.com // 2) close_adaptive page policy does not blindly close the 110510259SAndrew.Bardsley@arm.com // page, but closes it only if there are no row hits in the queue. 110610259SAndrew.Bardsley@arm.com // In this case, only force an auto precharge when there 110710259SAndrew.Bardsley@arm.com // are no same page hits in the queue 110810259SAndrew.Bardsley@arm.com bool got_more_hits = false; 110910259SAndrew.Bardsley@arm.com bool got_bank_conflict = false; 111010259SAndrew.Bardsley@arm.com 111110259SAndrew.Bardsley@arm.com // either look at the read queue or write queue 111210259SAndrew.Bardsley@arm.com const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 111310259SAndrew.Bardsley@arm.com writeQueue; 111410259SAndrew.Bardsley@arm.com auto p = queue.begin(); 111510259SAndrew.Bardsley@arm.com // make sure we are not considering the packet that we are 111610259SAndrew.Bardsley@arm.com // currently dealing with (which is the head of the queue) 111710259SAndrew.Bardsley@arm.com ++p; 111810259SAndrew.Bardsley@arm.com 111910259SAndrew.Bardsley@arm.com // keep on looking until we have found required condition or 112010259SAndrew.Bardsley@arm.com // reached the end 112110259SAndrew.Bardsley@arm.com while (!(got_more_hits && 112210259SAndrew.Bardsley@arm.com (got_bank_conflict || pageMgmt == Enums::close_adaptive)) && 112310259SAndrew.Bardsley@arm.com p != queue.end()) { 112410259SAndrew.Bardsley@arm.com bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 112510259SAndrew.Bardsley@arm.com (dram_pkt->bank == (*p)->bank); 112610259SAndrew.Bardsley@arm.com bool same_row = dram_pkt->row == (*p)->row; 112710259SAndrew.Bardsley@arm.com got_more_hits |= same_rank_bank && same_row; 112810259SAndrew.Bardsley@arm.com got_bank_conflict |= same_rank_bank && !same_row; 112910259SAndrew.Bardsley@arm.com ++p; 113010259SAndrew.Bardsley@arm.com } 113110259SAndrew.Bardsley@arm.com 113210259SAndrew.Bardsley@arm.com // auto pre-charge when either 113310259SAndrew.Bardsley@arm.com // 1) open_adaptive policy, we have not got any more hits, and 113410259SAndrew.Bardsley@arm.com // have a bank conflict 113510259SAndrew.Bardsley@arm.com // 2) close_adaptive policy and we have not got any more hits 113610259SAndrew.Bardsley@arm.com auto_precharge = !got_more_hits && 113710259SAndrew.Bardsley@arm.com (got_bank_conflict || pageMgmt == Enums::close_adaptive); 113810259SAndrew.Bardsley@arm.com } 113910259SAndrew.Bardsley@arm.com 114010259SAndrew.Bardsley@arm.com // DRAMPower trace command to be written 114110259SAndrew.Bardsley@arm.com std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR"; 114210259SAndrew.Bardsley@arm.com 114310259SAndrew.Bardsley@arm.com // if this access should use auto-precharge, then we are 114410259SAndrew.Bardsley@arm.com // closing the row 114510259SAndrew.Bardsley@arm.com if (auto_precharge) { 114610259SAndrew.Bardsley@arm.com prechargeBank(bank, std::max(curTick(), bank.preAllowedAt), false); 114710259SAndrew.Bardsley@arm.com 114810259SAndrew.Bardsley@arm.com mem_cmd.append("A"); 114910259SAndrew.Bardsley@arm.com 115010259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 115110259SAndrew.Bardsley@arm.com } 115210259SAndrew.Bardsley@arm.com 115310259SAndrew.Bardsley@arm.com // Update bus state 115410259SAndrew.Bardsley@arm.com busBusyUntil = dram_pkt->readyTime; 115510259SAndrew.Bardsley@arm.com 115610259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 115710259SAndrew.Bardsley@arm.com dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 115810259SAndrew.Bardsley@arm.com 115910259SAndrew.Bardsley@arm.com DPRINTF(DRAMPower, "%llu,%s,%d,%d\n", divCeil(cmd_at, tCK), mem_cmd, 116010259SAndrew.Bardsley@arm.com dram_pkt->bank, dram_pkt->rank); 116110259SAndrew.Bardsley@arm.com 116210259SAndrew.Bardsley@arm.com // Update the minimum timing between the requests, this is a 116310259SAndrew.Bardsley@arm.com // conservative estimate of when we have to schedule the next 116410259SAndrew.Bardsley@arm.com // request to not introduce any unecessary bubbles. In most cases 116510259SAndrew.Bardsley@arm.com // we will wake up sooner than we have to. 116610259SAndrew.Bardsley@arm.com nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 116710259SAndrew.Bardsley@arm.com 116810259SAndrew.Bardsley@arm.com // Update the stats and schedule the next request 116910259SAndrew.Bardsley@arm.com if (dram_pkt->isRead) { 117010259SAndrew.Bardsley@arm.com ++readsThisTime; 117110259SAndrew.Bardsley@arm.com if (row_hit) 117210259SAndrew.Bardsley@arm.com readRowHits++; 117310259SAndrew.Bardsley@arm.com bytesReadDRAM += burstSize; 117410259SAndrew.Bardsley@arm.com perBankRdBursts[dram_pkt->bankId]++; 117510259SAndrew.Bardsley@arm.com 117610259SAndrew.Bardsley@arm.com // Update latency stats 117710259SAndrew.Bardsley@arm.com totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 117810259SAndrew.Bardsley@arm.com totBusLat += tBURST; 117910259SAndrew.Bardsley@arm.com totQLat += cmd_at - dram_pkt->entryTime; 118010259SAndrew.Bardsley@arm.com } else { 118110259SAndrew.Bardsley@arm.com ++writesThisTime; 118210259SAndrew.Bardsley@arm.com if (row_hit) 118310259SAndrew.Bardsley@arm.com writeRowHits++; 118410259SAndrew.Bardsley@arm.com bytesWritten += burstSize; 118510259SAndrew.Bardsley@arm.com perBankWrBursts[dram_pkt->bankId]++; 118610259SAndrew.Bardsley@arm.com } 118710259SAndrew.Bardsley@arm.com} 118810259SAndrew.Bardsley@arm.com 118910259SAndrew.Bardsley@arm.comvoid 119010259SAndrew.Bardsley@arm.comDRAMCtrl::processNextReqEvent() 119110259SAndrew.Bardsley@arm.com{ 119210259SAndrew.Bardsley@arm.com // pre-emptively set to false. Overwrite if in READ_TO_WRITE 119310259SAndrew.Bardsley@arm.com // or WRITE_TO_READ state 119410259SAndrew.Bardsley@arm.com bool switched_cmd_type = false; 119510259SAndrew.Bardsley@arm.com if (busState == READ_TO_WRITE) { 119610259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 119710259SAndrew.Bardsley@arm.com "waiting\n", readsThisTime, readQueue.size()); 119810259SAndrew.Bardsley@arm.com 119910259SAndrew.Bardsley@arm.com // sample and reset the read-related stats as we are now 120010259SAndrew.Bardsley@arm.com // transitioning to writes, and all reads are done 120110259SAndrew.Bardsley@arm.com rdPerTurnAround.sample(readsThisTime); 120210259SAndrew.Bardsley@arm.com readsThisTime = 0; 120310259SAndrew.Bardsley@arm.com 120410259SAndrew.Bardsley@arm.com // now proceed to do the actual writes 120510259SAndrew.Bardsley@arm.com busState = WRITE; 120610259SAndrew.Bardsley@arm.com switched_cmd_type = true; 120710259SAndrew.Bardsley@arm.com } else if (busState == WRITE_TO_READ) { 120810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 120910259SAndrew.Bardsley@arm.com "waiting\n", writesThisTime, writeQueue.size()); 121010259SAndrew.Bardsley@arm.com 121110259SAndrew.Bardsley@arm.com wrPerTurnAround.sample(writesThisTime); 121210259SAndrew.Bardsley@arm.com writesThisTime = 0; 121310259SAndrew.Bardsley@arm.com 121410259SAndrew.Bardsley@arm.com busState = READ; 121510259SAndrew.Bardsley@arm.com switched_cmd_type = true; 121610259SAndrew.Bardsley@arm.com } 121710259SAndrew.Bardsley@arm.com 121810259SAndrew.Bardsley@arm.com if (refreshState != REF_IDLE) { 121910259SAndrew.Bardsley@arm.com // if a refresh waiting for this event loop to finish, then hand 122010259SAndrew.Bardsley@arm.com // over now, and do not schedule a new nextReqEvent 122110259SAndrew.Bardsley@arm.com if (refreshState == REF_DRAIN) { 122210259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 122310259SAndrew.Bardsley@arm.com 122410259SAndrew.Bardsley@arm.com refreshState = REF_PRE; 122510259SAndrew.Bardsley@arm.com 122610259SAndrew.Bardsley@arm.com // hand control back to the refresh event loop 122710259SAndrew.Bardsley@arm.com schedule(refreshEvent, curTick()); 122810259SAndrew.Bardsley@arm.com } 122910259SAndrew.Bardsley@arm.com 123010259SAndrew.Bardsley@arm.com // let the refresh finish before issuing any further requests 123110259SAndrew.Bardsley@arm.com return; 123210259SAndrew.Bardsley@arm.com } 123310259SAndrew.Bardsley@arm.com 123410259SAndrew.Bardsley@arm.com // when we get here it is either a read or a write 123510259SAndrew.Bardsley@arm.com if (busState == READ) { 123610259SAndrew.Bardsley@arm.com 123710259SAndrew.Bardsley@arm.com // track if we should switch or not 123810259SAndrew.Bardsley@arm.com bool switch_to_writes = false; 123910259SAndrew.Bardsley@arm.com 124010259SAndrew.Bardsley@arm.com if (readQueue.empty()) { 124110259SAndrew.Bardsley@arm.com // In the case there is no read request to go next, 124210259SAndrew.Bardsley@arm.com // trigger writes if we have passed the low threshold (or 124310259SAndrew.Bardsley@arm.com // if we are draining) 124410259SAndrew.Bardsley@arm.com if (!writeQueue.empty() && 124510259SAndrew.Bardsley@arm.com (drainManager || writeQueue.size() > writeLowThreshold)) { 124610259SAndrew.Bardsley@arm.com 124710259SAndrew.Bardsley@arm.com switch_to_writes = true; 124810259SAndrew.Bardsley@arm.com } else { 124910259SAndrew.Bardsley@arm.com // check if we are drained 125010259SAndrew.Bardsley@arm.com if (respQueue.empty () && drainManager) { 125110259SAndrew.Bardsley@arm.com drainManager->signalDrainDone(); 125210259SAndrew.Bardsley@arm.com drainManager = NULL; 125310259SAndrew.Bardsley@arm.com } 125410259SAndrew.Bardsley@arm.com 125510259SAndrew.Bardsley@arm.com // nothing to do, not even any point in scheduling an 125610259SAndrew.Bardsley@arm.com // event for the next request 125710259SAndrew.Bardsley@arm.com return; 125810259SAndrew.Bardsley@arm.com } 125910259SAndrew.Bardsley@arm.com } else { 126010259SAndrew.Bardsley@arm.com // Figure out which read request goes next, and move it to the 126110259SAndrew.Bardsley@arm.com // front of the read queue 126210259SAndrew.Bardsley@arm.com chooseNext(readQueue, switched_cmd_type); 126310259SAndrew.Bardsley@arm.com 126410259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = readQueue.front(); 126510259SAndrew.Bardsley@arm.com 126610259SAndrew.Bardsley@arm.com // here we get a bit creative and shift the bus busy time not 126710259SAndrew.Bardsley@arm.com // just the tWTR, but also a CAS latency to capture the fact 126810259SAndrew.Bardsley@arm.com // that we are allowed to prepare a new bank, but not issue a 126910259SAndrew.Bardsley@arm.com // read command until after tWTR, in essence we capture a 127010259SAndrew.Bardsley@arm.com // bubble on the data bus that is tWTR + tCL 127110259SAndrew.Bardsley@arm.com if (switched_cmd_type && dram_pkt->rank == activeRank) { 127210259SAndrew.Bardsley@arm.com busBusyUntil += tWTR + tCL; 127310259SAndrew.Bardsley@arm.com } 127410259SAndrew.Bardsley@arm.com 127510259SAndrew.Bardsley@arm.com doDRAMAccess(dram_pkt); 127610259SAndrew.Bardsley@arm.com 127710259SAndrew.Bardsley@arm.com // At this point we're done dealing with the request 127810259SAndrew.Bardsley@arm.com readQueue.pop_front(); 127910259SAndrew.Bardsley@arm.com 128010259SAndrew.Bardsley@arm.com // sanity check 128110259SAndrew.Bardsley@arm.com assert(dram_pkt->size <= burstSize); 128210259SAndrew.Bardsley@arm.com assert(dram_pkt->readyTime >= curTick()); 128310259SAndrew.Bardsley@arm.com 128410259SAndrew.Bardsley@arm.com // Insert into response queue. It will be sent back to the 128510259SAndrew.Bardsley@arm.com // requestor at its readyTime 128610259SAndrew.Bardsley@arm.com if (respQueue.empty()) { 128710259SAndrew.Bardsley@arm.com assert(!respondEvent.scheduled()); 128810259SAndrew.Bardsley@arm.com schedule(respondEvent, dram_pkt->readyTime); 128910259SAndrew.Bardsley@arm.com } else { 129010259SAndrew.Bardsley@arm.com assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 129110259SAndrew.Bardsley@arm.com assert(respondEvent.scheduled()); 129210259SAndrew.Bardsley@arm.com } 129310259SAndrew.Bardsley@arm.com 129410259SAndrew.Bardsley@arm.com respQueue.push_back(dram_pkt); 129510259SAndrew.Bardsley@arm.com 129610259SAndrew.Bardsley@arm.com // we have so many writes that we have to transition 129710259SAndrew.Bardsley@arm.com if (writeQueue.size() > writeHighThreshold) { 129810259SAndrew.Bardsley@arm.com switch_to_writes = true; 129910259SAndrew.Bardsley@arm.com } 130010259SAndrew.Bardsley@arm.com } 130110259SAndrew.Bardsley@arm.com 130210259SAndrew.Bardsley@arm.com // switching to writes, either because the read queue is empty 130310259SAndrew.Bardsley@arm.com // and the writes have passed the low threshold (or we are 130410259SAndrew.Bardsley@arm.com // draining), or because the writes hit the hight threshold 130510259SAndrew.Bardsley@arm.com if (switch_to_writes) { 130610259SAndrew.Bardsley@arm.com // transition to writing 130710259SAndrew.Bardsley@arm.com busState = READ_TO_WRITE; 130810259SAndrew.Bardsley@arm.com } 130910259SAndrew.Bardsley@arm.com } else { 131010259SAndrew.Bardsley@arm.com chooseNext(writeQueue, switched_cmd_type); 131110259SAndrew.Bardsley@arm.com DRAMPacket* dram_pkt = writeQueue.front(); 131210259SAndrew.Bardsley@arm.com // sanity check 131310259SAndrew.Bardsley@arm.com assert(dram_pkt->size <= burstSize); 131410259SAndrew.Bardsley@arm.com 131510259SAndrew.Bardsley@arm.com // add a bubble to the data bus, as defined by the 131610259SAndrew.Bardsley@arm.com // tRTW when access is to the same rank as previous burst 131710259SAndrew.Bardsley@arm.com // Different rank timing is handled with tCS, which is 131810259SAndrew.Bardsley@arm.com // applied to colAllowedAt 131910259SAndrew.Bardsley@arm.com if (switched_cmd_type && dram_pkt->rank == activeRank) { 132010259SAndrew.Bardsley@arm.com busBusyUntil += tRTW; 132110259SAndrew.Bardsley@arm.com } 132210259SAndrew.Bardsley@arm.com 132310259SAndrew.Bardsley@arm.com doDRAMAccess(dram_pkt); 132410259SAndrew.Bardsley@arm.com 132510259SAndrew.Bardsley@arm.com writeQueue.pop_front(); 132610259SAndrew.Bardsley@arm.com delete dram_pkt; 132710259SAndrew.Bardsley@arm.com 132810259SAndrew.Bardsley@arm.com // If we emptied the write queue, or got sufficiently below the 132910259SAndrew.Bardsley@arm.com // threshold (using the minWritesPerSwitch as the hysteresis) and 133010259SAndrew.Bardsley@arm.com // are not draining, or we have reads waiting and have done enough 133110259SAndrew.Bardsley@arm.com // writes, then switch to reads. 133210259SAndrew.Bardsley@arm.com if (writeQueue.empty() || 133310259SAndrew.Bardsley@arm.com (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 133410259SAndrew.Bardsley@arm.com !drainManager) || 133510259SAndrew.Bardsley@arm.com (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 133610259SAndrew.Bardsley@arm.com // turn the bus back around for reads again 133710259SAndrew.Bardsley@arm.com busState = WRITE_TO_READ; 133810259SAndrew.Bardsley@arm.com 133910259SAndrew.Bardsley@arm.com // note that the we switch back to reads also in the idle 134010259SAndrew.Bardsley@arm.com // case, which eventually will check for any draining and 134110259SAndrew.Bardsley@arm.com // also pause any further scheduling if there is really 134210259SAndrew.Bardsley@arm.com // nothing to do 134310259SAndrew.Bardsley@arm.com } 134410259SAndrew.Bardsley@arm.com } 134510259SAndrew.Bardsley@arm.com 134610259SAndrew.Bardsley@arm.com schedule(nextReqEvent, std::max(nextReqTime, curTick())); 134710259SAndrew.Bardsley@arm.com 134810259SAndrew.Bardsley@arm.com // If there is space available and we have writes waiting then let 134910259SAndrew.Bardsley@arm.com // them retry. This is done here to ensure that the retry does not 135010259SAndrew.Bardsley@arm.com // cause a nextReqEvent to be scheduled before we do so as part of 135110259SAndrew.Bardsley@arm.com // the next request processing 135210259SAndrew.Bardsley@arm.com if (retryWrReq && writeQueue.size() < writeBufferSize) { 135310259SAndrew.Bardsley@arm.com retryWrReq = false; 135410259SAndrew.Bardsley@arm.com port.sendRetry(); 135510259SAndrew.Bardsley@arm.com } 135610259SAndrew.Bardsley@arm.com} 135710259SAndrew.Bardsley@arm.com 135810259SAndrew.Bardsley@arm.comuint64_t 135910259SAndrew.Bardsley@arm.comDRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue, 136010259SAndrew.Bardsley@arm.com bool switched_cmd_type) const 136110259SAndrew.Bardsley@arm.com{ 136210259SAndrew.Bardsley@arm.com uint64_t bank_mask = 0; 136310259SAndrew.Bardsley@arm.com Tick min_act_at = MaxTick; 136410259SAndrew.Bardsley@arm.com 136510259SAndrew.Bardsley@arm.com uint64_t bank_mask_same_rank = 0; 136610259SAndrew.Bardsley@arm.com Tick min_act_at_same_rank = MaxTick; 136710259SAndrew.Bardsley@arm.com 136810259SAndrew.Bardsley@arm.com // Give precedence to commands that access same rank as previous command 136910259SAndrew.Bardsley@arm.com bool same_rank_match = false; 137010259SAndrew.Bardsley@arm.com 137110259SAndrew.Bardsley@arm.com // determine if we have queued transactions targetting the 137210259SAndrew.Bardsley@arm.com // bank in question 137310504SAndrew.Bardsley@arm.com vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 137410504SAndrew.Bardsley@arm.com for (auto p = queue.begin(); p != queue.end(); ++p) { 137510504SAndrew.Bardsley@arm.com got_waiting[(*p)->bankId] = true; 137610504SAndrew.Bardsley@arm.com } 137710504SAndrew.Bardsley@arm.com 137810504SAndrew.Bardsley@arm.com for (int i = 0; i < ranksPerChannel; i++) { 137910504SAndrew.Bardsley@arm.com for (int j = 0; j < banksPerRank; j++) { 138010259SAndrew.Bardsley@arm.com uint8_t bank_id = i * banksPerRank + j; 138110259SAndrew.Bardsley@arm.com 138210259SAndrew.Bardsley@arm.com // if we have waiting requests for the bank, and it is 138310259SAndrew.Bardsley@arm.com // amongst the first available, update the mask 138410259SAndrew.Bardsley@arm.com if (got_waiting[bank_id]) { 138510259SAndrew.Bardsley@arm.com // simplistic approximation of when the bank can issue 138610259SAndrew.Bardsley@arm.com // an activate, ignoring any rank-to-rank switching 138710259SAndrew.Bardsley@arm.com // cost in this calculation 138810259SAndrew.Bardsley@arm.com Tick act_at = banks[i][j].openRow == Bank::NO_ROW ? 138910259SAndrew.Bardsley@arm.com banks[i][j].actAllowedAt : 139010259SAndrew.Bardsley@arm.com std::max(banks[i][j].preAllowedAt, curTick()) + tRP; 139110259SAndrew.Bardsley@arm.com 139210259SAndrew.Bardsley@arm.com // prioritize commands that access the 139310259SAndrew.Bardsley@arm.com // same rank as previous burst 139410259SAndrew.Bardsley@arm.com // Calculate bank mask separately for the case and 139510259SAndrew.Bardsley@arm.com // evaluate after loop iterations complete 139610259SAndrew.Bardsley@arm.com if (i == activeRank && ranksPerChannel > 1) { 139710259SAndrew.Bardsley@arm.com if (act_at <= min_act_at_same_rank) { 139810259SAndrew.Bardsley@arm.com // reset same rank bank mask if new minimum is found 139910259SAndrew.Bardsley@arm.com // and previous minimum could not immediately send ACT 140010259SAndrew.Bardsley@arm.com if (act_at < min_act_at_same_rank && 140110259SAndrew.Bardsley@arm.com min_act_at_same_rank > curTick()) 140210259SAndrew.Bardsley@arm.com bank_mask_same_rank = 0; 140310259SAndrew.Bardsley@arm.com 140410259SAndrew.Bardsley@arm.com // Set flag indicating that a same rank 140510259SAndrew.Bardsley@arm.com // opportunity was found 140610259SAndrew.Bardsley@arm.com same_rank_match = true; 140710259SAndrew.Bardsley@arm.com 140810259SAndrew.Bardsley@arm.com // set the bit corresponding to the available bank 140910259SAndrew.Bardsley@arm.com replaceBits(bank_mask_same_rank, bank_id, bank_id, 1); 141010259SAndrew.Bardsley@arm.com min_act_at_same_rank = act_at; 141110259SAndrew.Bardsley@arm.com } 141210259SAndrew.Bardsley@arm.com } else { 141310259SAndrew.Bardsley@arm.com if (act_at <= min_act_at) { 141410259SAndrew.Bardsley@arm.com // reset bank mask if new minimum is found 141510259SAndrew.Bardsley@arm.com // and either previous minimum could not immediately send ACT 141610259SAndrew.Bardsley@arm.com if (act_at < min_act_at && min_act_at > curTick()) 141710259SAndrew.Bardsley@arm.com bank_mask = 0; 141810259SAndrew.Bardsley@arm.com // set the bit corresponding to the available bank 141910259SAndrew.Bardsley@arm.com replaceBits(bank_mask, bank_id, bank_id, 1); 142010259SAndrew.Bardsley@arm.com min_act_at = act_at; 142110259SAndrew.Bardsley@arm.com } 142210259SAndrew.Bardsley@arm.com } 142310259SAndrew.Bardsley@arm.com } 142410259SAndrew.Bardsley@arm.com } 142510259SAndrew.Bardsley@arm.com } 142610259SAndrew.Bardsley@arm.com 142710259SAndrew.Bardsley@arm.com // Determine the earliest time when the next burst can issue based 142810259SAndrew.Bardsley@arm.com // on the current busBusyUntil delay. 142910259SAndrew.Bardsley@arm.com // Offset by tRCD to correlate with ACT timing variables 143010259SAndrew.Bardsley@arm.com Tick min_cmd_at = busBusyUntil - tCL - tRCD; 143110259SAndrew.Bardsley@arm.com 143210259SAndrew.Bardsley@arm.com // Prioritize same rank accesses that can issue B2B 143310259SAndrew.Bardsley@arm.com // Only optimize for same ranks when the command type 143410259SAndrew.Bardsley@arm.com // does not change; do not want to unnecessarily incur tWTR 143510259SAndrew.Bardsley@arm.com // 143610259SAndrew.Bardsley@arm.com // Resulting FCFS prioritization Order is: 143710259SAndrew.Bardsley@arm.com // 1) Commands that access the same rank as previous burst 143810259SAndrew.Bardsley@arm.com // and can prep the bank seamlessly. 143910259SAndrew.Bardsley@arm.com // 2) Commands (any rank) with earliest bank prep 144010259SAndrew.Bardsley@arm.com if (!switched_cmd_type && same_rank_match && 144110259SAndrew.Bardsley@arm.com min_act_at_same_rank <= min_cmd_at) { 144210259SAndrew.Bardsley@arm.com bank_mask = bank_mask_same_rank; 144310259SAndrew.Bardsley@arm.com } 144410259SAndrew.Bardsley@arm.com 144510259SAndrew.Bardsley@arm.com return bank_mask; 144610259SAndrew.Bardsley@arm.com} 144710259SAndrew.Bardsley@arm.com 144810259SAndrew.Bardsley@arm.comvoid 144910259SAndrew.Bardsley@arm.comDRAMCtrl::processRefreshEvent() 145010259SAndrew.Bardsley@arm.com{ 145110259SAndrew.Bardsley@arm.com // when first preparing the refresh, remember when it was due 145210259SAndrew.Bardsley@arm.com if (refreshState == REF_IDLE) { 145310259SAndrew.Bardsley@arm.com // remember when the refresh is due 145410259SAndrew.Bardsley@arm.com refreshDueAt = curTick(); 145510259SAndrew.Bardsley@arm.com 145610259SAndrew.Bardsley@arm.com // proceed to drain 145710259SAndrew.Bardsley@arm.com refreshState = REF_DRAIN; 145810259SAndrew.Bardsley@arm.com 145910259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Refresh due\n"); 146010259SAndrew.Bardsley@arm.com } 146110259SAndrew.Bardsley@arm.com 146210259SAndrew.Bardsley@arm.com // let any scheduled read or write go ahead, after which it will 146310259SAndrew.Bardsley@arm.com // hand control back to this event loop 146410259SAndrew.Bardsley@arm.com if (refreshState == REF_DRAIN) { 146510259SAndrew.Bardsley@arm.com if (nextReqEvent.scheduled()) { 146610259SAndrew.Bardsley@arm.com // hand control over to the request loop until it is 146710259SAndrew.Bardsley@arm.com // evaluated next 146810259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Refresh awaiting draining\n"); 146910259SAndrew.Bardsley@arm.com 147010259SAndrew.Bardsley@arm.com return; 147110259SAndrew.Bardsley@arm.com } else { 147210259SAndrew.Bardsley@arm.com refreshState = REF_PRE; 147310259SAndrew.Bardsley@arm.com } 147410259SAndrew.Bardsley@arm.com } 147510259SAndrew.Bardsley@arm.com 147610259SAndrew.Bardsley@arm.com // at this point, ensure that all banks are precharged 147710259SAndrew.Bardsley@arm.com if (refreshState == REF_PRE) { 147810259SAndrew.Bardsley@arm.com // precharge any active bank if we are not already in the idle 147910259SAndrew.Bardsley@arm.com // state 148010259SAndrew.Bardsley@arm.com if (pwrState != PWR_IDLE) { 148110259SAndrew.Bardsley@arm.com // at the moment, we use a precharge all even if there is 148210259SAndrew.Bardsley@arm.com // only a single bank open 148310259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "Precharging all\n"); 148410259SAndrew.Bardsley@arm.com 148510259SAndrew.Bardsley@arm.com // first determine when we can precharge 148610259SAndrew.Bardsley@arm.com Tick pre_at = curTick(); 148710259SAndrew.Bardsley@arm.com for (int i = 0; i < ranksPerChannel; i++) { 148810259SAndrew.Bardsley@arm.com for (int j = 0; j < banksPerRank; j++) { 148910259SAndrew.Bardsley@arm.com // respect both causality and any existing bank 149010259SAndrew.Bardsley@arm.com // constraints, some banks could already have a 149110259SAndrew.Bardsley@arm.com // (auto) precharge scheduled 149210259SAndrew.Bardsley@arm.com pre_at = std::max(banks[i][j].preAllowedAt, pre_at); 149310259SAndrew.Bardsley@arm.com } 149410259SAndrew.Bardsley@arm.com } 149510259SAndrew.Bardsley@arm.com 149610259SAndrew.Bardsley@arm.com // make sure all banks are precharged, and for those that 149710259SAndrew.Bardsley@arm.com // already are, update their availability 149810259SAndrew.Bardsley@arm.com Tick act_allowed_at = pre_at + tRP; 149910259SAndrew.Bardsley@arm.com 150010259SAndrew.Bardsley@arm.com for (int i = 0; i < ranksPerChannel; i++) { 150110259SAndrew.Bardsley@arm.com for (int j = 0; j < banksPerRank; j++) { 150210259SAndrew.Bardsley@arm.com if (banks[i][j].openRow != Bank::NO_ROW) { 150310259SAndrew.Bardsley@arm.com prechargeBank(banks[i][j], pre_at, false); 150410259SAndrew.Bardsley@arm.com } else { 150510259SAndrew.Bardsley@arm.com banks[i][j].actAllowedAt = 150610259SAndrew.Bardsley@arm.com std::max(banks[i][j].actAllowedAt, act_allowed_at); 150710259SAndrew.Bardsley@arm.com banks[i][j].preAllowedAt = 150810259SAndrew.Bardsley@arm.com std::max(banks[i][j].preAllowedAt, pre_at); 150910259SAndrew.Bardsley@arm.com } 151010259SAndrew.Bardsley@arm.com } 151110259SAndrew.Bardsley@arm.com 151210259SAndrew.Bardsley@arm.com // at the moment this affects all ranks 151310259SAndrew.Bardsley@arm.com DPRINTF(DRAMPower, "%llu,PREA,0,%d\n", divCeil(pre_at, tCK), 151410259SAndrew.Bardsley@arm.com i); 151510259SAndrew.Bardsley@arm.com } 151610259SAndrew.Bardsley@arm.com } else { 151710259SAndrew.Bardsley@arm.com DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 151810259SAndrew.Bardsley@arm.com 151910259SAndrew.Bardsley@arm.com // go ahead and kick the power state machine into gear if 152010259SAndrew.Bardsley@arm.com // we are already idle 152110259SAndrew.Bardsley@arm.com schedulePowerEvent(PWR_REF, curTick()); 152210259SAndrew.Bardsley@arm.com } 152310259SAndrew.Bardsley@arm.com 152410259SAndrew.Bardsley@arm.com refreshState = REF_RUN; 152510259SAndrew.Bardsley@arm.com assert(numBanksActive == 0); 152610259SAndrew.Bardsley@arm.com 152710259SAndrew.Bardsley@arm.com // wait for all banks to be precharged, at which point the 152810259SAndrew.Bardsley@arm.com // power state machine will transition to the idle state, and 152910259SAndrew.Bardsley@arm.com // automatically move to a refresh, at that point it will also 153010259SAndrew.Bardsley@arm.com // call this method to get the refresh event loop going again 153110259SAndrew.Bardsley@arm.com return; 153210259SAndrew.Bardsley@arm.com } 153310259SAndrew.Bardsley@arm.com 153410259SAndrew.Bardsley@arm.com // last but not least we perform the actual refresh 153510259SAndrew.Bardsley@arm.com if (refreshState == REF_RUN) { 153610259SAndrew.Bardsley@arm.com // should never get here with any banks active 153710259SAndrew.Bardsley@arm.com assert(numBanksActive == 0); 153810259SAndrew.Bardsley@arm.com assert(pwrState == PWR_REF); 153910259SAndrew.Bardsley@arm.com 154010259SAndrew.Bardsley@arm.com Tick ref_done_at = curTick() + tRFC; 154110259SAndrew.Bardsley@arm.com 154210259SAndrew.Bardsley@arm.com for (int i = 0; i < ranksPerChannel; i++) { 154310259SAndrew.Bardsley@arm.com for (int j = 0; j < banksPerRank; j++) { 154410259SAndrew.Bardsley@arm.com banks[i][j].actAllowedAt = ref_done_at; 154510259SAndrew.Bardsley@arm.com } 154610259SAndrew.Bardsley@arm.com 154710259SAndrew.Bardsley@arm.com // at the moment this affects all ranks 154810259SAndrew.Bardsley@arm.com DPRINTF(DRAMPower, "%llu,REF,0,%d\n", divCeil(curTick(), tCK), i); 154910259SAndrew.Bardsley@arm.com } 155010259SAndrew.Bardsley@arm.com 155110259SAndrew.Bardsley@arm.com // make sure we did not wait so long that we cannot make up 155210259SAndrew.Bardsley@arm.com // for it 155310259SAndrew.Bardsley@arm.com if (refreshDueAt + tREFI < ref_done_at) { 155410259SAndrew.Bardsley@arm.com fatal("Refresh was delayed so long we cannot catch up\n"); 155510259SAndrew.Bardsley@arm.com } 155610259SAndrew.Bardsley@arm.com 155710259SAndrew.Bardsley@arm.com // compensate for the delay in actually performing the refresh 155810259SAndrew.Bardsley@arm.com // when scheduling the next one 155910259SAndrew.Bardsley@arm.com schedule(refreshEvent, refreshDueAt + tREFI - tRP); 156010259SAndrew.Bardsley@arm.com 156110259SAndrew.Bardsley@arm.com assert(!powerEvent.scheduled()); 156210259SAndrew.Bardsley@arm.com 156310259SAndrew.Bardsley@arm.com // move to the idle power state once the refresh is done, this 156410259SAndrew.Bardsley@arm.com // will also move the refresh state machine to the refresh 156510259SAndrew.Bardsley@arm.com // idle state 156610259SAndrew.Bardsley@arm.com schedulePowerEvent(PWR_IDLE, ref_done_at); 156710259SAndrew.Bardsley@arm.com 156810259SAndrew.Bardsley@arm.com DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 156910259SAndrew.Bardsley@arm.com ref_done_at, refreshDueAt + tREFI); 157010259SAndrew.Bardsley@arm.com } 157110259SAndrew.Bardsley@arm.com} 157210259SAndrew.Bardsley@arm.com 157310259SAndrew.Bardsley@arm.comvoid 157410259SAndrew.Bardsley@arm.comDRAMCtrl::schedulePowerEvent(PowerState pwr_state, Tick tick) 157510259SAndrew.Bardsley@arm.com{ 157610259SAndrew.Bardsley@arm.com // respect causality 157710259SAndrew.Bardsley@arm.com assert(tick >= curTick()); 157810259SAndrew.Bardsley@arm.com 157910259SAndrew.Bardsley@arm.com if (!powerEvent.scheduled()) { 158010259SAndrew.Bardsley@arm.com DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 158110259SAndrew.Bardsley@arm.com tick, pwr_state); 158210259SAndrew.Bardsley@arm.com 158310259SAndrew.Bardsley@arm.com // insert the new transition 158410259SAndrew.Bardsley@arm.com pwrStateTrans = pwr_state; 158510259SAndrew.Bardsley@arm.com 158610259SAndrew.Bardsley@arm.com schedule(powerEvent, tick); 158710259SAndrew.Bardsley@arm.com } else { 158810259SAndrew.Bardsley@arm.com panic("Scheduled power event at %llu to state %d, " 158910259SAndrew.Bardsley@arm.com "with scheduled event at %llu to %d\n", tick, pwr_state, 159010259SAndrew.Bardsley@arm.com powerEvent.when(), pwrStateTrans); 159110259SAndrew.Bardsley@arm.com } 159210259SAndrew.Bardsley@arm.com} 159310259SAndrew.Bardsley@arm.com 159410259SAndrew.Bardsley@arm.comvoid 159510259SAndrew.Bardsley@arm.comDRAMCtrl::processPowerEvent() 159610259SAndrew.Bardsley@arm.com{ 159710259SAndrew.Bardsley@arm.com // remember where we were, and for how long 159810259SAndrew.Bardsley@arm.com Tick duration = curTick() - pwrStateTick; 159910259SAndrew.Bardsley@arm.com PowerState prev_state = pwrState; 160010259SAndrew.Bardsley@arm.com 160110259SAndrew.Bardsley@arm.com // update the accounting 160210259SAndrew.Bardsley@arm.com pwrStateTime[prev_state] += duration; 160310259SAndrew.Bardsley@arm.com 160410259SAndrew.Bardsley@arm.com pwrState = pwrStateTrans; 160510259SAndrew.Bardsley@arm.com pwrStateTick = curTick(); 160610259SAndrew.Bardsley@arm.com 160710259SAndrew.Bardsley@arm.com if (pwrState == PWR_IDLE) { 160810259SAndrew.Bardsley@arm.com DPRINTF(DRAMState, "All banks precharged\n"); 160910259SAndrew.Bardsley@arm.com 161010259SAndrew.Bardsley@arm.com // if we were refreshing, make sure we start scheduling requests again 161110259SAndrew.Bardsley@arm.com if (prev_state == PWR_REF) { 161210259SAndrew.Bardsley@arm.com DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 161310259SAndrew.Bardsley@arm.com assert(pwrState == PWR_IDLE); 161410259SAndrew.Bardsley@arm.com 161510259SAndrew.Bardsley@arm.com // kick things into action again 161610259SAndrew.Bardsley@arm.com refreshState = REF_IDLE; 161710259SAndrew.Bardsley@arm.com assert(!nextReqEvent.scheduled()); 161810259SAndrew.Bardsley@arm.com schedule(nextReqEvent, curTick()); 161910259SAndrew.Bardsley@arm.com } else { 162010259SAndrew.Bardsley@arm.com assert(prev_state == PWR_ACT); 1621 1622 // if we have a pending refresh, and are now moving to 1623 // the idle state, direclty transition to a refresh 1624 if (refreshState == REF_RUN) { 1625 // there should be nothing waiting at this point 1626 assert(!powerEvent.scheduled()); 1627 1628 // update the state in zero time and proceed below 1629 pwrState = PWR_REF; 1630 } 1631 } 1632 } 1633 1634 // we transition to the refresh state, let the refresh state 1635 // machine know of this state update and let it deal with the 1636 // scheduling of the next power state transition as well as the 1637 // following refresh 1638 if (pwrState == PWR_REF) { 1639 DPRINTF(DRAMState, "Refreshing\n"); 1640 // kick the refresh event loop into action again, and that 1641 // in turn will schedule a transition to the idle power 1642 // state once the refresh is done 1643 assert(refreshState == REF_RUN); 1644 processRefreshEvent(); 1645 } 1646} 1647 1648void 1649DRAMCtrl::regStats() 1650{ 1651 using namespace Stats; 1652 1653 AbstractMemory::regStats(); 1654 1655 readReqs 1656 .name(name() + ".readReqs") 1657 .desc("Number of read requests accepted"); 1658 1659 writeReqs 1660 .name(name() + ".writeReqs") 1661 .desc("Number of write requests accepted"); 1662 1663 readBursts 1664 .name(name() + ".readBursts") 1665 .desc("Number of DRAM read bursts, " 1666 "including those serviced by the write queue"); 1667 1668 writeBursts 1669 .name(name() + ".writeBursts") 1670 .desc("Number of DRAM write bursts, " 1671 "including those merged in the write queue"); 1672 1673 servicedByWrQ 1674 .name(name() + ".servicedByWrQ") 1675 .desc("Number of DRAM read bursts serviced by the write queue"); 1676 1677 mergedWrBursts 1678 .name(name() + ".mergedWrBursts") 1679 .desc("Number of DRAM write bursts merged with an existing one"); 1680 1681 neitherReadNorWrite 1682 .name(name() + ".neitherReadNorWriteReqs") 1683 .desc("Number of requests that are neither read nor write"); 1684 1685 perBankRdBursts 1686 .init(banksPerRank * ranksPerChannel) 1687 .name(name() + ".perBankRdBursts") 1688 .desc("Per bank write bursts"); 1689 1690 perBankWrBursts 1691 .init(banksPerRank * ranksPerChannel) 1692 .name(name() + ".perBankWrBursts") 1693 .desc("Per bank write bursts"); 1694 1695 avgRdQLen 1696 .name(name() + ".avgRdQLen") 1697 .desc("Average read queue length when enqueuing") 1698 .precision(2); 1699 1700 avgWrQLen 1701 .name(name() + ".avgWrQLen") 1702 .desc("Average write queue length when enqueuing") 1703 .precision(2); 1704 1705 totQLat 1706 .name(name() + ".totQLat") 1707 .desc("Total ticks spent queuing"); 1708 1709 totBusLat 1710 .name(name() + ".totBusLat") 1711 .desc("Total ticks spent in databus transfers"); 1712 1713 totMemAccLat 1714 .name(name() + ".totMemAccLat") 1715 .desc("Total ticks spent from burst creation until serviced " 1716 "by the DRAM"); 1717 1718 avgQLat 1719 .name(name() + ".avgQLat") 1720 .desc("Average queueing delay per DRAM burst") 1721 .precision(2); 1722 1723 avgQLat = totQLat / (readBursts - servicedByWrQ); 1724 1725 avgBusLat 1726 .name(name() + ".avgBusLat") 1727 .desc("Average bus latency per DRAM burst") 1728 .precision(2); 1729 1730 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1731 1732 avgMemAccLat 1733 .name(name() + ".avgMemAccLat") 1734 .desc("Average memory access latency per DRAM burst") 1735 .precision(2); 1736 1737 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1738 1739 numRdRetry 1740 .name(name() + ".numRdRetry") 1741 .desc("Number of times read queue was full causing retry"); 1742 1743 numWrRetry 1744 .name(name() + ".numWrRetry") 1745 .desc("Number of times write queue was full causing retry"); 1746 1747 readRowHits 1748 .name(name() + ".readRowHits") 1749 .desc("Number of row buffer hits during reads"); 1750 1751 writeRowHits 1752 .name(name() + ".writeRowHits") 1753 .desc("Number of row buffer hits during writes"); 1754 1755 readRowHitRate 1756 .name(name() + ".readRowHitRate") 1757 .desc("Row buffer hit rate for reads") 1758 .precision(2); 1759 1760 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 1761 1762 writeRowHitRate 1763 .name(name() + ".writeRowHitRate") 1764 .desc("Row buffer hit rate for writes") 1765 .precision(2); 1766 1767 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 1768 1769 readPktSize 1770 .init(ceilLog2(burstSize) + 1) 1771 .name(name() + ".readPktSize") 1772 .desc("Read request sizes (log2)"); 1773 1774 writePktSize 1775 .init(ceilLog2(burstSize) + 1) 1776 .name(name() + ".writePktSize") 1777 .desc("Write request sizes (log2)"); 1778 1779 rdQLenPdf 1780 .init(readBufferSize) 1781 .name(name() + ".rdQLenPdf") 1782 .desc("What read queue length does an incoming req see"); 1783 1784 wrQLenPdf 1785 .init(writeBufferSize) 1786 .name(name() + ".wrQLenPdf") 1787 .desc("What write queue length does an incoming req see"); 1788 1789 bytesPerActivate 1790 .init(maxAccessesPerRow) 1791 .name(name() + ".bytesPerActivate") 1792 .desc("Bytes accessed per row activation") 1793 .flags(nozero); 1794 1795 rdPerTurnAround 1796 .init(readBufferSize) 1797 .name(name() + ".rdPerTurnAround") 1798 .desc("Reads before turning the bus around for writes") 1799 .flags(nozero); 1800 1801 wrPerTurnAround 1802 .init(writeBufferSize) 1803 .name(name() + ".wrPerTurnAround") 1804 .desc("Writes before turning the bus around for reads") 1805 .flags(nozero); 1806 1807 bytesReadDRAM 1808 .name(name() + ".bytesReadDRAM") 1809 .desc("Total number of bytes read from DRAM"); 1810 1811 bytesReadWrQ 1812 .name(name() + ".bytesReadWrQ") 1813 .desc("Total number of bytes read from write queue"); 1814 1815 bytesWritten 1816 .name(name() + ".bytesWritten") 1817 .desc("Total number of bytes written to DRAM"); 1818 1819 bytesReadSys 1820 .name(name() + ".bytesReadSys") 1821 .desc("Total read bytes from the system interface side"); 1822 1823 bytesWrittenSys 1824 .name(name() + ".bytesWrittenSys") 1825 .desc("Total written bytes from the system interface side"); 1826 1827 avgRdBW 1828 .name(name() + ".avgRdBW") 1829 .desc("Average DRAM read bandwidth in MiByte/s") 1830 .precision(2); 1831 1832 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 1833 1834 avgWrBW 1835 .name(name() + ".avgWrBW") 1836 .desc("Average achieved write bandwidth in MiByte/s") 1837 .precision(2); 1838 1839 avgWrBW = (bytesWritten / 1000000) / simSeconds; 1840 1841 avgRdBWSys 1842 .name(name() + ".avgRdBWSys") 1843 .desc("Average system read bandwidth in MiByte/s") 1844 .precision(2); 1845 1846 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 1847 1848 avgWrBWSys 1849 .name(name() + ".avgWrBWSys") 1850 .desc("Average system write bandwidth in MiByte/s") 1851 .precision(2); 1852 1853 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 1854 1855 peakBW 1856 .name(name() + ".peakBW") 1857 .desc("Theoretical peak bandwidth in MiByte/s") 1858 .precision(2); 1859 1860 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 1861 1862 busUtil 1863 .name(name() + ".busUtil") 1864 .desc("Data bus utilization in percentage") 1865 .precision(2); 1866 1867 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 1868 1869 totGap 1870 .name(name() + ".totGap") 1871 .desc("Total gap between requests"); 1872 1873 avgGap 1874 .name(name() + ".avgGap") 1875 .desc("Average gap between requests") 1876 .precision(2); 1877 1878 avgGap = totGap / (readReqs + writeReqs); 1879 1880 // Stats for DRAM Power calculation based on Micron datasheet 1881 busUtilRead 1882 .name(name() + ".busUtilRead") 1883 .desc("Data bus utilization in percentage for reads") 1884 .precision(2); 1885 1886 busUtilRead = avgRdBW / peakBW * 100; 1887 1888 busUtilWrite 1889 .name(name() + ".busUtilWrite") 1890 .desc("Data bus utilization in percentage for writes") 1891 .precision(2); 1892 1893 busUtilWrite = avgWrBW / peakBW * 100; 1894 1895 pageHitRate 1896 .name(name() + ".pageHitRate") 1897 .desc("Row buffer hit rate, read and write combined") 1898 .precision(2); 1899 1900 pageHitRate = (writeRowHits + readRowHits) / 1901 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 1902 1903 pwrStateTime 1904 .init(5) 1905 .name(name() + ".memoryStateTime") 1906 .desc("Time in different power states"); 1907 pwrStateTime.subname(0, "IDLE"); 1908 pwrStateTime.subname(1, "REF"); 1909 pwrStateTime.subname(2, "PRE_PDN"); 1910 pwrStateTime.subname(3, "ACT"); 1911 pwrStateTime.subname(4, "ACT_PDN"); 1912} 1913 1914void 1915DRAMCtrl::recvFunctional(PacketPtr pkt) 1916{ 1917 // rely on the abstract memory 1918 functionalAccess(pkt); 1919} 1920 1921BaseSlavePort& 1922DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 1923{ 1924 if (if_name != "port") { 1925 return MemObject::getSlavePort(if_name, idx); 1926 } else { 1927 return port; 1928 } 1929} 1930 1931unsigned int 1932DRAMCtrl::drain(DrainManager *dm) 1933{ 1934 unsigned int count = port.drain(dm); 1935 1936 // if there is anything in any of our internal queues, keep track 1937 // of that as well 1938 if (!(writeQueue.empty() && readQueue.empty() && 1939 respQueue.empty())) { 1940 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 1941 " resp: %d\n", writeQueue.size(), readQueue.size(), 1942 respQueue.size()); 1943 ++count; 1944 drainManager = dm; 1945 1946 // the only part that is not drained automatically over time 1947 // is the write queue, thus kick things into action if needed 1948 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 1949 schedule(nextReqEvent, curTick()); 1950 } 1951 } 1952 1953 if (count) 1954 setDrainState(Drainable::Draining); 1955 else 1956 setDrainState(Drainable::Drained); 1957 return count; 1958} 1959 1960DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 1961 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 1962 memory(_memory) 1963{ } 1964 1965AddrRangeList 1966DRAMCtrl::MemoryPort::getAddrRanges() const 1967{ 1968 AddrRangeList ranges; 1969 ranges.push_back(memory.getAddrRange()); 1970 return ranges; 1971} 1972 1973void 1974DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 1975{ 1976 pkt->pushLabel(memory.name()); 1977 1978 if (!queue.checkFunctional(pkt)) { 1979 // Default implementation of SimpleTimingPort::recvFunctional() 1980 // calls recvAtomic() and throws away the latency; we can save a 1981 // little here by just not calculating the latency. 1982 memory.recvFunctional(pkt); 1983 } 1984 1985 pkt->popLabel(); 1986} 1987 1988Tick 1989DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 1990{ 1991 return memory.recvAtomic(pkt); 1992} 1993 1994bool 1995DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 1996{ 1997 // pass it to the memory controller 1998 return memory.recvTimingReq(pkt); 1999} 2000 2001DRAMCtrl* 2002DRAMCtrlParams::create() 2003{ 2004 return new DRAMCtrl(this); 2005} 2006