1/* 2 * Copyright (c) 2010-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2013 Amin Farmahini-Farahani 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Andreas Hansson 41 * Ani Udipi 42 * Neha Agarwal 43 */ 44 45#include "base/bitfield.hh" 46#include "base/trace.hh" 47#include "debug/DRAM.hh"
| 1/* 2 * Copyright (c) 2010-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2013 Amin Farmahini-Farahani 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Andreas Hansson 41 * Ani Udipi 42 * Neha Agarwal 43 */ 44 45#include "base/bitfield.hh" 46#include "base/trace.hh" 47#include "debug/DRAM.hh"
|
| 48#include "debug/DRAMPower.hh"
|
48#include "debug/DRAMState.hh" 49#include "debug/Drain.hh" 50#include "mem/dram_ctrl.hh" 51#include "sim/system.hh" 52 53using namespace std; 54 55DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 56 AbstractMemory(p), 57 port(name() + ".port", *this), 58 retryRdReq(false), retryWrReq(false), 59 busState(READ), 60 nextReqEvent(this), respondEvent(this), activateEvent(this), 61 prechargeEvent(this), refreshEvent(this), powerEvent(this), 62 drainManager(NULL), 63 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 64 deviceRowBufferSize(p->device_rowbuffer_size), 65 devicesPerRank(p->devices_per_rank), 66 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 67 rowBufferSize(devicesPerRank * deviceRowBufferSize), 68 columnsPerRowBuffer(rowBufferSize / burstSize), 69 ranksPerChannel(p->ranks_per_channel), 70 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 71 readBufferSize(p->read_buffer_size), 72 writeBufferSize(p->write_buffer_size), 73 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 74 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 75 minWritesPerSwitch(p->min_writes_per_switch), 76 writesThisTime(0), readsThisTime(0), 77 tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), 78 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR), 79 tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 80 tXAW(p->tXAW), activationLimit(p->activation_limit), 81 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 82 pageMgmt(p->page_policy), 83 maxAccessesPerRow(p->max_accesses_per_row), 84 frontendLatency(p->static_frontend_latency), 85 backendLatency(p->static_backend_latency), 86 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), 87 pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), prevArrival(0), 88 nextReqTime(0), pwrStateTick(0), numBanksActive(0) 89{ 90 // create the bank states based on the dimensions of the ranks and 91 // banks 92 banks.resize(ranksPerChannel); 93 actTicks.resize(ranksPerChannel); 94 for (size_t c = 0; c < ranksPerChannel; ++c) { 95 banks[c].resize(banksPerRank); 96 actTicks[c].resize(activationLimit, 0); 97 } 98 99 // set the bank indices 100 for (int r = 0; r < ranksPerChannel; r++) { 101 for (int b = 0; b < banksPerRank; b++) { 102 banks[r][b].rank = r; 103 banks[r][b].bank = b; 104 } 105 } 106 107 // perform a basic check of the write thresholds 108 if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 109 fatal("Write buffer low threshold %d must be smaller than the " 110 "high threshold %d\n", p->write_low_thresh_perc, 111 p->write_high_thresh_perc); 112 113 // determine the rows per bank by looking at the total capacity 114 uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 115 116 DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 117 AbstractMemory::size()); 118 119 DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 120 rowBufferSize, columnsPerRowBuffer); 121 122 rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 123 124 if (range.interleaved()) { 125 if (channels != range.stripes()) 126 fatal("%s has %d interleaved address stripes but %d channel(s)\n", 127 name(), range.stripes(), channels); 128 129 if (addrMapping == Enums::RoRaBaChCo) { 130 if (rowBufferSize != range.granularity()) { 131 fatal("Interleaving of %s doesn't match RoRaBaChCo " 132 "address map\n", name()); 133 } 134 } else if (addrMapping == Enums::RoRaBaCoCh) { 135 if (system()->cacheLineSize() != range.granularity()) { 136 fatal("Interleaving of %s doesn't match RoRaBaCoCh " 137 "address map\n", name()); 138 } 139 } else if (addrMapping == Enums::RoCoRaBaCh) { 140 if (system()->cacheLineSize() != range.granularity()) 141 fatal("Interleaving of %s doesn't match RoCoRaBaCh " 142 "address map\n", name()); 143 } 144 } 145 146 // some basic sanity checks 147 if (tREFI <= tRP || tREFI <= tRFC) { 148 fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 149 tREFI, tRP, tRFC); 150 } 151} 152 153void 154DRAMCtrl::init() 155{ 156 if (!port.isConnected()) { 157 fatal("DRAMCtrl %s is unconnected!\n", name()); 158 } else { 159 port.sendRangeChange(); 160 } 161} 162 163void 164DRAMCtrl::startup() 165{ 166 // update the start tick for the precharge accounting to the 167 // current tick 168 pwrStateTick = curTick(); 169 170 // shift the bus busy time sufficiently far ahead that we never 171 // have to worry about negative values when computing the time for 172 // the next request, this will add an insignificant bubble at the 173 // start of simulation 174 busBusyUntil = curTick() + tRP + tRCD + tCL; 175 176 // kick off the refresh, and give ourselves enough time to 177 // precharge 178 schedule(refreshEvent, curTick() + tREFI - tRP); 179} 180 181Tick 182DRAMCtrl::recvAtomic(PacketPtr pkt) 183{ 184 DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 185 186 // do the actual memory access and turn the packet into a response 187 access(pkt); 188 189 Tick latency = 0; 190 if (!pkt->memInhibitAsserted() && pkt->hasData()) { 191 // this value is not supposed to be accurate, just enough to 192 // keep things going, mimic a closed page 193 latency = tRP + tRCD + tCL; 194 } 195 return latency; 196} 197 198bool 199DRAMCtrl::readQueueFull(unsigned int neededEntries) const 200{ 201 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 202 readBufferSize, readQueue.size() + respQueue.size(), 203 neededEntries); 204 205 return 206 (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 207} 208 209bool 210DRAMCtrl::writeQueueFull(unsigned int neededEntries) const 211{ 212 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 213 writeBufferSize, writeQueue.size(), neededEntries); 214 return (writeQueue.size() + neededEntries) > writeBufferSize; 215} 216 217DRAMCtrl::DRAMPacket* 218DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 219 bool isRead) 220{ 221 // decode the address based on the address mapping scheme, with 222 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 223 // channel, respectively 224 uint8_t rank; 225 uint8_t bank; 226 // use a 64-bit unsigned during the computations as the row is 227 // always the top bits, and check before creating the DRAMPacket 228 uint64_t row; 229 230 // truncate the address to the access granularity 231 Addr addr = dramPktAddr / burstSize; 232 233 // we have removed the lowest order address bits that denote the 234 // position within the column 235 if (addrMapping == Enums::RoRaBaChCo) { 236 // the lowest order bits denote the column to ensure that 237 // sequential cache lines occupy the same row 238 addr = addr / columnsPerRowBuffer; 239 240 // take out the channel part of the address 241 addr = addr / channels; 242 243 // after the channel bits, get the bank bits to interleave 244 // over the banks 245 bank = addr % banksPerRank; 246 addr = addr / banksPerRank; 247 248 // after the bank, we get the rank bits which thus interleaves 249 // over the ranks 250 rank = addr % ranksPerChannel; 251 addr = addr / ranksPerChannel; 252 253 // lastly, get the row bits 254 row = addr % rowsPerBank; 255 addr = addr / rowsPerBank; 256 } else if (addrMapping == Enums::RoRaBaCoCh) { 257 // take out the channel part of the address 258 addr = addr / channels; 259 260 // next, the column 261 addr = addr / columnsPerRowBuffer; 262 263 // after the column bits, we get the bank bits to interleave 264 // over the banks 265 bank = addr % banksPerRank; 266 addr = addr / banksPerRank; 267 268 // after the bank, we get the rank bits which thus interleaves 269 // over the ranks 270 rank = addr % ranksPerChannel; 271 addr = addr / ranksPerChannel; 272 273 // lastly, get the row bits 274 row = addr % rowsPerBank; 275 addr = addr / rowsPerBank; 276 } else if (addrMapping == Enums::RoCoRaBaCh) { 277 // optimise for closed page mode and utilise maximum 278 // parallelism of the DRAM (at the cost of power) 279 280 // take out the channel part of the address, not that this has 281 // to match with how accesses are interleaved between the 282 // controllers in the address mapping 283 addr = addr / channels; 284 285 // start with the bank bits, as this provides the maximum 286 // opportunity for parallelism between requests 287 bank = addr % banksPerRank; 288 addr = addr / banksPerRank; 289 290 // next get the rank bits 291 rank = addr % ranksPerChannel; 292 addr = addr / ranksPerChannel; 293 294 // next the column bits which we do not need to keep track of 295 // and simply skip past 296 addr = addr / columnsPerRowBuffer; 297 298 // lastly, get the row bits 299 row = addr % rowsPerBank; 300 addr = addr / rowsPerBank; 301 } else 302 panic("Unknown address mapping policy chosen!"); 303 304 assert(rank < ranksPerChannel); 305 assert(bank < banksPerRank); 306 assert(row < rowsPerBank); 307 assert(row < Bank::NO_ROW); 308 309 DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 310 dramPktAddr, rank, bank, row); 311 312 // create the corresponding DRAM packet with the entry time and 313 // ready time set to the current tick, the latter will be updated 314 // later 315 uint16_t bank_id = banksPerRank * rank + bank; 316 return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 317 size, banks[rank][bank]); 318} 319 320void 321DRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 322{ 323 // only add to the read queue here. whenever the request is 324 // eventually done, set the readyTime, and call schedule() 325 assert(!pkt->isWrite()); 326 327 assert(pktCount != 0); 328 329 // if the request size is larger than burst size, the pkt is split into 330 // multiple DRAM packets 331 // Note if the pkt starting address is not aligened to burst size, the 332 // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 333 // are aligned to burst size boundaries. This is to ensure we accurately 334 // check read packets against packets in write queue. 335 Addr addr = pkt->getAddr(); 336 unsigned pktsServicedByWrQ = 0; 337 BurstHelper* burst_helper = NULL; 338 for (int cnt = 0; cnt < pktCount; ++cnt) { 339 unsigned size = std::min((addr | (burstSize - 1)) + 1, 340 pkt->getAddr() + pkt->getSize()) - addr; 341 readPktSize[ceilLog2(size)]++; 342 readBursts++; 343 344 // First check write buffer to see if the data is already at 345 // the controller 346 bool foundInWrQ = false; 347 for (auto i = writeQueue.begin(); i != writeQueue.end(); ++i) { 348 // check if the read is subsumed in the write entry we are 349 // looking at 350 if ((*i)->addr <= addr && 351 (addr + size) <= ((*i)->addr + (*i)->size)) { 352 foundInWrQ = true; 353 servicedByWrQ++; 354 pktsServicedByWrQ++; 355 DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 356 "write queue\n", addr, size); 357 bytesReadWrQ += burstSize; 358 break; 359 } 360 } 361 362 // If not found in the write q, make a DRAM packet and 363 // push it onto the read queue 364 if (!foundInWrQ) { 365 366 // Make the burst helper for split packets 367 if (pktCount > 1 && burst_helper == NULL) { 368 DPRINTF(DRAM, "Read to addr %lld translates to %d " 369 "dram requests\n", pkt->getAddr(), pktCount); 370 burst_helper = new BurstHelper(pktCount); 371 } 372 373 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 374 dram_pkt->burstHelper = burst_helper; 375 376 assert(!readQueueFull(1)); 377 rdQLenPdf[readQueue.size() + respQueue.size()]++; 378 379 DPRINTF(DRAM, "Adding to read queue\n"); 380 381 readQueue.push_back(dram_pkt); 382 383 // Update stats 384 avgRdQLen = readQueue.size() + respQueue.size(); 385 } 386 387 // Starting address of next dram pkt (aligend to burstSize boundary) 388 addr = (addr | (burstSize - 1)) + 1; 389 } 390 391 // If all packets are serviced by write queue, we send the repsonse back 392 if (pktsServicedByWrQ == pktCount) { 393 accessAndRespond(pkt, frontendLatency); 394 return; 395 } 396 397 // Update how many split packets are serviced by write queue 398 if (burst_helper != NULL) 399 burst_helper->burstsServiced = pktsServicedByWrQ; 400 401 // If we are not already scheduled to get a request out of the 402 // queue, do so now 403 if (!nextReqEvent.scheduled()) { 404 DPRINTF(DRAM, "Request scheduled immediately\n"); 405 schedule(nextReqEvent, curTick()); 406 } 407} 408 409void 410DRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 411{ 412 // only add to the write queue here. whenever the request is 413 // eventually done, set the readyTime, and call schedule() 414 assert(pkt->isWrite()); 415 416 // if the request size is larger than burst size, the pkt is split into 417 // multiple DRAM packets 418 Addr addr = pkt->getAddr(); 419 for (int cnt = 0; cnt < pktCount; ++cnt) { 420 unsigned size = std::min((addr | (burstSize - 1)) + 1, 421 pkt->getAddr() + pkt->getSize()) - addr; 422 writePktSize[ceilLog2(size)]++; 423 writeBursts++; 424 425 // see if we can merge with an existing item in the write 426 // queue and keep track of whether we have merged or not so we 427 // can stop at that point and also avoid enqueueing a new 428 // request 429 bool merged = false; 430 auto w = writeQueue.begin(); 431 432 while(!merged && w != writeQueue.end()) { 433 // either of the two could be first, if they are the same 434 // it does not matter which way we go 435 if ((*w)->addr >= addr) { 436 // the existing one starts after the new one, figure 437 // out where the new one ends with respect to the 438 // existing one 439 if ((addr + size) >= ((*w)->addr + (*w)->size)) { 440 // check if the existing one is completely 441 // subsumed in the new one 442 DPRINTF(DRAM, "Merging write covering existing burst\n"); 443 merged = true; 444 // update both the address and the size 445 (*w)->addr = addr; 446 (*w)->size = size; 447 } else if ((addr + size) >= (*w)->addr && 448 ((*w)->addr + (*w)->size - addr) <= burstSize) { 449 // the new one is just before or partially 450 // overlapping with the existing one, and together 451 // they fit within a burst 452 DPRINTF(DRAM, "Merging write before existing burst\n"); 453 merged = true; 454 // the existing queue item needs to be adjusted with 455 // respect to both address and size 456 (*w)->size = (*w)->addr + (*w)->size - addr; 457 (*w)->addr = addr; 458 } 459 } else { 460 // the new one starts after the current one, figure 461 // out where the existing one ends with respect to the 462 // new one 463 if (((*w)->addr + (*w)->size) >= (addr + size)) { 464 // check if the new one is completely subsumed in the 465 // existing one 466 DPRINTF(DRAM, "Merging write into existing burst\n"); 467 merged = true; 468 // no adjustments necessary 469 } else if (((*w)->addr + (*w)->size) >= addr && 470 (addr + size - (*w)->addr) <= burstSize) { 471 // the existing one is just before or partially 472 // overlapping with the new one, and together 473 // they fit within a burst 474 DPRINTF(DRAM, "Merging write after existing burst\n"); 475 merged = true; 476 // the address is right, and only the size has 477 // to be adjusted 478 (*w)->size = addr + size - (*w)->addr; 479 } 480 } 481 ++w; 482 } 483 484 // if the item was not merged we need to create a new write 485 // and enqueue it 486 if (!merged) { 487 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 488 489 assert(writeQueue.size() < writeBufferSize); 490 wrQLenPdf[writeQueue.size()]++; 491 492 DPRINTF(DRAM, "Adding to write queue\n"); 493 494 writeQueue.push_back(dram_pkt); 495 496 // Update stats 497 avgWrQLen = writeQueue.size(); 498 } else { 499 // keep track of the fact that this burst effectively 500 // disappeared as it was merged with an existing one 501 mergedWrBursts++; 502 } 503 504 // Starting address of next dram pkt (aligend to burstSize boundary) 505 addr = (addr | (burstSize - 1)) + 1; 506 } 507 508 // we do not wait for the writes to be send to the actual memory, 509 // but instead take responsibility for the consistency here and 510 // snoop the write queue for any upcoming reads 511 // @todo, if a pkt size is larger than burst size, we might need a 512 // different front end latency 513 accessAndRespond(pkt, frontendLatency); 514 515 // If we are not already scheduled to get a request out of the 516 // queue, do so now 517 if (!nextReqEvent.scheduled()) { 518 DPRINTF(DRAM, "Request scheduled immediately\n"); 519 schedule(nextReqEvent, curTick()); 520 } 521} 522 523void 524DRAMCtrl::printQs() const { 525 DPRINTF(DRAM, "===READ QUEUE===\n\n"); 526 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 527 DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 528 } 529 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 530 for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 531 DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 532 } 533 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 534 for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 535 DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 536 } 537} 538 539bool 540DRAMCtrl::recvTimingReq(PacketPtr pkt) 541{ 542 /// @todo temporary hack to deal with memory corruption issues until 543 /// 4-phase transactions are complete 544 for (int x = 0; x < pendingDelete.size(); x++) 545 delete pendingDelete[x]; 546 pendingDelete.clear(); 547 548 // This is where we enter from the outside world 549 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 550 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 551 552 // simply drop inhibited packets for now 553 if (pkt->memInhibitAsserted()) { 554 DPRINTF(DRAM, "Inhibited packet -- Dropping it now\n"); 555 pendingDelete.push_back(pkt); 556 return true; 557 } 558 559 // Calc avg gap between requests 560 if (prevArrival != 0) { 561 totGap += curTick() - prevArrival; 562 } 563 prevArrival = curTick(); 564 565 566 // Find out how many dram packets a pkt translates to 567 // If the burst size is equal or larger than the pkt size, then a pkt 568 // translates to only one dram packet. Otherwise, a pkt translates to 569 // multiple dram packets 570 unsigned size = pkt->getSize(); 571 unsigned offset = pkt->getAddr() & (burstSize - 1); 572 unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 573 574 // check local buffers and do not accept if full 575 if (pkt->isRead()) { 576 assert(size != 0); 577 if (readQueueFull(dram_pkt_count)) { 578 DPRINTF(DRAM, "Read queue full, not accepting\n"); 579 // remember that we have to retry this port 580 retryRdReq = true; 581 numRdRetry++; 582 return false; 583 } else { 584 addToReadQueue(pkt, dram_pkt_count); 585 readReqs++; 586 bytesReadSys += size; 587 } 588 } else if (pkt->isWrite()) { 589 assert(size != 0); 590 if (writeQueueFull(dram_pkt_count)) { 591 DPRINTF(DRAM, "Write queue full, not accepting\n"); 592 // remember that we have to retry this port 593 retryWrReq = true; 594 numWrRetry++; 595 return false; 596 } else { 597 addToWriteQueue(pkt, dram_pkt_count); 598 writeReqs++; 599 bytesWrittenSys += size; 600 } 601 } else { 602 DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 603 neitherReadNorWrite++; 604 accessAndRespond(pkt, 1); 605 } 606 607 return true; 608} 609 610void 611DRAMCtrl::processRespondEvent() 612{ 613 DPRINTF(DRAM, 614 "processRespondEvent(): Some req has reached its readyTime\n"); 615 616 DRAMPacket* dram_pkt = respQueue.front(); 617 618 if (dram_pkt->burstHelper) { 619 // it is a split packet 620 dram_pkt->burstHelper->burstsServiced++; 621 if (dram_pkt->burstHelper->burstsServiced == 622 dram_pkt->burstHelper->burstCount) { 623 // we have now serviced all children packets of a system packet 624 // so we can now respond to the requester 625 // @todo we probably want to have a different front end and back 626 // end latency for split packets 627 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 628 delete dram_pkt->burstHelper; 629 dram_pkt->burstHelper = NULL; 630 } 631 } else { 632 // it is not a split packet 633 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 634 } 635 636 delete respQueue.front(); 637 respQueue.pop_front(); 638 639 if (!respQueue.empty()) { 640 assert(respQueue.front()->readyTime >= curTick()); 641 assert(!respondEvent.scheduled()); 642 schedule(respondEvent, respQueue.front()->readyTime); 643 } else { 644 // if there is nothing left in any queue, signal a drain 645 if (writeQueue.empty() && readQueue.empty() && 646 drainManager) { 647 drainManager->signalDrainDone(); 648 drainManager = NULL; 649 } 650 } 651 652 // We have made a location in the queue available at this point, 653 // so if there is a read that was forced to wait, retry now 654 if (retryRdReq) { 655 retryRdReq = false; 656 port.sendRetry(); 657 } 658} 659 660void 661DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue) 662{ 663 // This method does the arbitration between requests. The chosen 664 // packet is simply moved to the head of the queue. The other 665 // methods know that this is the place to look. For example, with 666 // FCFS, this method does nothing 667 assert(!queue.empty()); 668 669 if (queue.size() == 1) { 670 DPRINTF(DRAM, "Single request, nothing to do\n"); 671 return; 672 } 673 674 if (memSchedPolicy == Enums::fcfs) { 675 // Do nothing, since the correct request is already head 676 } else if (memSchedPolicy == Enums::frfcfs) { 677 reorderQueue(queue); 678 } else 679 panic("No scheduling policy chosen\n"); 680} 681 682void 683DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue) 684{ 685 // Only determine this when needed 686 uint64_t earliest_banks = 0; 687 688 // Search for row hits first, if no row hit is found then schedule the 689 // packet to one of the earliest banks available 690 bool found_earliest_pkt = false; 691 auto selected_pkt_it = queue.begin(); 692 693 for (auto i = queue.begin(); i != queue.end() ; ++i) { 694 DRAMPacket* dram_pkt = *i; 695 const Bank& bank = dram_pkt->bankRef; 696 // Check if it is a row hit 697 if (bank.openRow == dram_pkt->row) { 698 // FCFS within the hits 699 DPRINTF(DRAM, "Row buffer hit\n"); 700 selected_pkt_it = i; 701 break; 702 } else if (!found_earliest_pkt) { 703 // No row hit, go for first ready 704 if (earliest_banks == 0) 705 earliest_banks = minBankActAt(queue); 706 707 // simplistic approximation of when the bank can issue an 708 // activate, this is calculated in minBankActAt and could 709 // be cached 710 Tick act_at = bank.openRow == Bank::NO_ROW ? 711 bank.actAllowedAt : 712 std::max(bank.preAllowedAt, curTick()) + tRP; 713 714 // Bank is ready or is the first available bank 715 if (act_at <= curTick() || 716 bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 717 // Remember the packet to be scheduled to one of the earliest 718 // banks available, FCFS amongst the earliest banks 719 selected_pkt_it = i; 720 found_earliest_pkt = true; 721 } 722 } 723 } 724 725 DRAMPacket* selected_pkt = *selected_pkt_it; 726 queue.erase(selected_pkt_it); 727 queue.push_front(selected_pkt); 728} 729 730void 731DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 732{ 733 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 734 735 bool needsResponse = pkt->needsResponse(); 736 // do the actual memory access which also turns the packet into a 737 // response 738 access(pkt); 739 740 // turn packet around to go back to requester if response expected 741 if (needsResponse) { 742 // access already turned the packet into a response 743 assert(pkt->isResponse()); 744 745 // @todo someone should pay for this 746 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 747 748 // queue the packet in the response queue to be sent out after 749 // the static latency has passed 750 port.schedTimingResp(pkt, curTick() + static_latency); 751 } else { 752 // @todo the packet is going to be deleted, and the DRAMPacket 753 // is still having a pointer to it 754 pendingDelete.push_back(pkt); 755 } 756 757 DPRINTF(DRAM, "Done\n"); 758 759 return; 760} 761 762void 763DRAMCtrl::activateBank(Bank& bank, Tick act_tick, uint32_t row) 764{ 765 // get the rank index from the bank 766 uint8_t rank = bank.rank; 767 768 assert(actTicks[rank].size() == activationLimit); 769 770 DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 771 772 // update the open row 773 assert(bank.openRow == Bank::NO_ROW); 774 bank.openRow = row; 775 776 // start counting anew, this covers both the case when we 777 // auto-precharged, and when this access is forced to 778 // precharge 779 bank.bytesAccessed = 0; 780 bank.rowAccesses = 0; 781 782 ++numBanksActive; 783 assert(numBanksActive <= banksPerRank * ranksPerChannel); 784
| 49#include "debug/DRAMState.hh" 50#include "debug/Drain.hh" 51#include "mem/dram_ctrl.hh" 52#include "sim/system.hh" 53 54using namespace std; 55 56DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 57 AbstractMemory(p), 58 port(name() + ".port", *this), 59 retryRdReq(false), retryWrReq(false), 60 busState(READ), 61 nextReqEvent(this), respondEvent(this), activateEvent(this), 62 prechargeEvent(this), refreshEvent(this), powerEvent(this), 63 drainManager(NULL), 64 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 65 deviceRowBufferSize(p->device_rowbuffer_size), 66 devicesPerRank(p->devices_per_rank), 67 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 68 rowBufferSize(devicesPerRank * deviceRowBufferSize), 69 columnsPerRowBuffer(rowBufferSize / burstSize), 70 ranksPerChannel(p->ranks_per_channel), 71 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0), 72 readBufferSize(p->read_buffer_size), 73 writeBufferSize(p->write_buffer_size), 74 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0), 75 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0), 76 minWritesPerSwitch(p->min_writes_per_switch), 77 writesThisTime(0), readsThisTime(0), 78 tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST), 79 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR), 80 tRTP(p->tRTP), tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD), 81 tXAW(p->tXAW), activationLimit(p->activation_limit), 82 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping), 83 pageMgmt(p->page_policy), 84 maxAccessesPerRow(p->max_accesses_per_row), 85 frontendLatency(p->static_frontend_latency), 86 backendLatency(p->static_backend_latency), 87 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), 88 pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), prevArrival(0), 89 nextReqTime(0), pwrStateTick(0), numBanksActive(0) 90{ 91 // create the bank states based on the dimensions of the ranks and 92 // banks 93 banks.resize(ranksPerChannel); 94 actTicks.resize(ranksPerChannel); 95 for (size_t c = 0; c < ranksPerChannel; ++c) { 96 banks[c].resize(banksPerRank); 97 actTicks[c].resize(activationLimit, 0); 98 } 99 100 // set the bank indices 101 for (int r = 0; r < ranksPerChannel; r++) { 102 for (int b = 0; b < banksPerRank; b++) { 103 banks[r][b].rank = r; 104 banks[r][b].bank = b; 105 } 106 } 107 108 // perform a basic check of the write thresholds 109 if (p->write_low_thresh_perc >= p->write_high_thresh_perc) 110 fatal("Write buffer low threshold %d must be smaller than the " 111 "high threshold %d\n", p->write_low_thresh_perc, 112 p->write_high_thresh_perc); 113 114 // determine the rows per bank by looking at the total capacity 115 uint64_t capacity = ULL(1) << ceilLog2(AbstractMemory::size()); 116 117 DPRINTF(DRAM, "Memory capacity %lld (%lld) bytes\n", capacity, 118 AbstractMemory::size()); 119 120 DPRINTF(DRAM, "Row buffer size %d bytes with %d columns per row buffer\n", 121 rowBufferSize, columnsPerRowBuffer); 122 123 rowsPerBank = capacity / (rowBufferSize * banksPerRank * ranksPerChannel); 124 125 if (range.interleaved()) { 126 if (channels != range.stripes()) 127 fatal("%s has %d interleaved address stripes but %d channel(s)\n", 128 name(), range.stripes(), channels); 129 130 if (addrMapping == Enums::RoRaBaChCo) { 131 if (rowBufferSize != range.granularity()) { 132 fatal("Interleaving of %s doesn't match RoRaBaChCo " 133 "address map\n", name()); 134 } 135 } else if (addrMapping == Enums::RoRaBaCoCh) { 136 if (system()->cacheLineSize() != range.granularity()) { 137 fatal("Interleaving of %s doesn't match RoRaBaCoCh " 138 "address map\n", name()); 139 } 140 } else if (addrMapping == Enums::RoCoRaBaCh) { 141 if (system()->cacheLineSize() != range.granularity()) 142 fatal("Interleaving of %s doesn't match RoCoRaBaCh " 143 "address map\n", name()); 144 } 145 } 146 147 // some basic sanity checks 148 if (tREFI <= tRP || tREFI <= tRFC) { 149 fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", 150 tREFI, tRP, tRFC); 151 } 152} 153 154void 155DRAMCtrl::init() 156{ 157 if (!port.isConnected()) { 158 fatal("DRAMCtrl %s is unconnected!\n", name()); 159 } else { 160 port.sendRangeChange(); 161 } 162} 163 164void 165DRAMCtrl::startup() 166{ 167 // update the start tick for the precharge accounting to the 168 // current tick 169 pwrStateTick = curTick(); 170 171 // shift the bus busy time sufficiently far ahead that we never 172 // have to worry about negative values when computing the time for 173 // the next request, this will add an insignificant bubble at the 174 // start of simulation 175 busBusyUntil = curTick() + tRP + tRCD + tCL; 176 177 // kick off the refresh, and give ourselves enough time to 178 // precharge 179 schedule(refreshEvent, curTick() + tREFI - tRP); 180} 181 182Tick 183DRAMCtrl::recvAtomic(PacketPtr pkt) 184{ 185 DPRINTF(DRAM, "recvAtomic: %s 0x%x\n", pkt->cmdString(), pkt->getAddr()); 186 187 // do the actual memory access and turn the packet into a response 188 access(pkt); 189 190 Tick latency = 0; 191 if (!pkt->memInhibitAsserted() && pkt->hasData()) { 192 // this value is not supposed to be accurate, just enough to 193 // keep things going, mimic a closed page 194 latency = tRP + tRCD + tCL; 195 } 196 return latency; 197} 198 199bool 200DRAMCtrl::readQueueFull(unsigned int neededEntries) const 201{ 202 DPRINTF(DRAM, "Read queue limit %d, current size %d, entries needed %d\n", 203 readBufferSize, readQueue.size() + respQueue.size(), 204 neededEntries); 205 206 return 207 (readQueue.size() + respQueue.size() + neededEntries) > readBufferSize; 208} 209 210bool 211DRAMCtrl::writeQueueFull(unsigned int neededEntries) const 212{ 213 DPRINTF(DRAM, "Write queue limit %d, current size %d, entries needed %d\n", 214 writeBufferSize, writeQueue.size(), neededEntries); 215 return (writeQueue.size() + neededEntries) > writeBufferSize; 216} 217 218DRAMCtrl::DRAMPacket* 219DRAMCtrl::decodeAddr(PacketPtr pkt, Addr dramPktAddr, unsigned size, 220 bool isRead) 221{ 222 // decode the address based on the address mapping scheme, with 223 // Ro, Ra, Co, Ba and Ch denoting row, rank, column, bank and 224 // channel, respectively 225 uint8_t rank; 226 uint8_t bank; 227 // use a 64-bit unsigned during the computations as the row is 228 // always the top bits, and check before creating the DRAMPacket 229 uint64_t row; 230 231 // truncate the address to the access granularity 232 Addr addr = dramPktAddr / burstSize; 233 234 // we have removed the lowest order address bits that denote the 235 // position within the column 236 if (addrMapping == Enums::RoRaBaChCo) { 237 // the lowest order bits denote the column to ensure that 238 // sequential cache lines occupy the same row 239 addr = addr / columnsPerRowBuffer; 240 241 // take out the channel part of the address 242 addr = addr / channels; 243 244 // after the channel bits, get the bank bits to interleave 245 // over the banks 246 bank = addr % banksPerRank; 247 addr = addr / banksPerRank; 248 249 // after the bank, we get the rank bits which thus interleaves 250 // over the ranks 251 rank = addr % ranksPerChannel; 252 addr = addr / ranksPerChannel; 253 254 // lastly, get the row bits 255 row = addr % rowsPerBank; 256 addr = addr / rowsPerBank; 257 } else if (addrMapping == Enums::RoRaBaCoCh) { 258 // take out the channel part of the address 259 addr = addr / channels; 260 261 // next, the column 262 addr = addr / columnsPerRowBuffer; 263 264 // after the column bits, we get the bank bits to interleave 265 // over the banks 266 bank = addr % banksPerRank; 267 addr = addr / banksPerRank; 268 269 // after the bank, we get the rank bits which thus interleaves 270 // over the ranks 271 rank = addr % ranksPerChannel; 272 addr = addr / ranksPerChannel; 273 274 // lastly, get the row bits 275 row = addr % rowsPerBank; 276 addr = addr / rowsPerBank; 277 } else if (addrMapping == Enums::RoCoRaBaCh) { 278 // optimise for closed page mode and utilise maximum 279 // parallelism of the DRAM (at the cost of power) 280 281 // take out the channel part of the address, not that this has 282 // to match with how accesses are interleaved between the 283 // controllers in the address mapping 284 addr = addr / channels; 285 286 // start with the bank bits, as this provides the maximum 287 // opportunity for parallelism between requests 288 bank = addr % banksPerRank; 289 addr = addr / banksPerRank; 290 291 // next get the rank bits 292 rank = addr % ranksPerChannel; 293 addr = addr / ranksPerChannel; 294 295 // next the column bits which we do not need to keep track of 296 // and simply skip past 297 addr = addr / columnsPerRowBuffer; 298 299 // lastly, get the row bits 300 row = addr % rowsPerBank; 301 addr = addr / rowsPerBank; 302 } else 303 panic("Unknown address mapping policy chosen!"); 304 305 assert(rank < ranksPerChannel); 306 assert(bank < banksPerRank); 307 assert(row < rowsPerBank); 308 assert(row < Bank::NO_ROW); 309 310 DPRINTF(DRAM, "Address: %lld Rank %d Bank %d Row %d\n", 311 dramPktAddr, rank, bank, row); 312 313 // create the corresponding DRAM packet with the entry time and 314 // ready time set to the current tick, the latter will be updated 315 // later 316 uint16_t bank_id = banksPerRank * rank + bank; 317 return new DRAMPacket(pkt, isRead, rank, bank, row, bank_id, dramPktAddr, 318 size, banks[rank][bank]); 319} 320 321void 322DRAMCtrl::addToReadQueue(PacketPtr pkt, unsigned int pktCount) 323{ 324 // only add to the read queue here. whenever the request is 325 // eventually done, set the readyTime, and call schedule() 326 assert(!pkt->isWrite()); 327 328 assert(pktCount != 0); 329 330 // if the request size is larger than burst size, the pkt is split into 331 // multiple DRAM packets 332 // Note if the pkt starting address is not aligened to burst size, the 333 // address of first DRAM packet is kept unaliged. Subsequent DRAM packets 334 // are aligned to burst size boundaries. This is to ensure we accurately 335 // check read packets against packets in write queue. 336 Addr addr = pkt->getAddr(); 337 unsigned pktsServicedByWrQ = 0; 338 BurstHelper* burst_helper = NULL; 339 for (int cnt = 0; cnt < pktCount; ++cnt) { 340 unsigned size = std::min((addr | (burstSize - 1)) + 1, 341 pkt->getAddr() + pkt->getSize()) - addr; 342 readPktSize[ceilLog2(size)]++; 343 readBursts++; 344 345 // First check write buffer to see if the data is already at 346 // the controller 347 bool foundInWrQ = false; 348 for (auto i = writeQueue.begin(); i != writeQueue.end(); ++i) { 349 // check if the read is subsumed in the write entry we are 350 // looking at 351 if ((*i)->addr <= addr && 352 (addr + size) <= ((*i)->addr + (*i)->size)) { 353 foundInWrQ = true; 354 servicedByWrQ++; 355 pktsServicedByWrQ++; 356 DPRINTF(DRAM, "Read to addr %lld with size %d serviced by " 357 "write queue\n", addr, size); 358 bytesReadWrQ += burstSize; 359 break; 360 } 361 } 362 363 // If not found in the write q, make a DRAM packet and 364 // push it onto the read queue 365 if (!foundInWrQ) { 366 367 // Make the burst helper for split packets 368 if (pktCount > 1 && burst_helper == NULL) { 369 DPRINTF(DRAM, "Read to addr %lld translates to %d " 370 "dram requests\n", pkt->getAddr(), pktCount); 371 burst_helper = new BurstHelper(pktCount); 372 } 373 374 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, true); 375 dram_pkt->burstHelper = burst_helper; 376 377 assert(!readQueueFull(1)); 378 rdQLenPdf[readQueue.size() + respQueue.size()]++; 379 380 DPRINTF(DRAM, "Adding to read queue\n"); 381 382 readQueue.push_back(dram_pkt); 383 384 // Update stats 385 avgRdQLen = readQueue.size() + respQueue.size(); 386 } 387 388 // Starting address of next dram pkt (aligend to burstSize boundary) 389 addr = (addr | (burstSize - 1)) + 1; 390 } 391 392 // If all packets are serviced by write queue, we send the repsonse back 393 if (pktsServicedByWrQ == pktCount) { 394 accessAndRespond(pkt, frontendLatency); 395 return; 396 } 397 398 // Update how many split packets are serviced by write queue 399 if (burst_helper != NULL) 400 burst_helper->burstsServiced = pktsServicedByWrQ; 401 402 // If we are not already scheduled to get a request out of the 403 // queue, do so now 404 if (!nextReqEvent.scheduled()) { 405 DPRINTF(DRAM, "Request scheduled immediately\n"); 406 schedule(nextReqEvent, curTick()); 407 } 408} 409 410void 411DRAMCtrl::addToWriteQueue(PacketPtr pkt, unsigned int pktCount) 412{ 413 // only add to the write queue here. whenever the request is 414 // eventually done, set the readyTime, and call schedule() 415 assert(pkt->isWrite()); 416 417 // if the request size is larger than burst size, the pkt is split into 418 // multiple DRAM packets 419 Addr addr = pkt->getAddr(); 420 for (int cnt = 0; cnt < pktCount; ++cnt) { 421 unsigned size = std::min((addr | (burstSize - 1)) + 1, 422 pkt->getAddr() + pkt->getSize()) - addr; 423 writePktSize[ceilLog2(size)]++; 424 writeBursts++; 425 426 // see if we can merge with an existing item in the write 427 // queue and keep track of whether we have merged or not so we 428 // can stop at that point and also avoid enqueueing a new 429 // request 430 bool merged = false; 431 auto w = writeQueue.begin(); 432 433 while(!merged && w != writeQueue.end()) { 434 // either of the two could be first, if they are the same 435 // it does not matter which way we go 436 if ((*w)->addr >= addr) { 437 // the existing one starts after the new one, figure 438 // out where the new one ends with respect to the 439 // existing one 440 if ((addr + size) >= ((*w)->addr + (*w)->size)) { 441 // check if the existing one is completely 442 // subsumed in the new one 443 DPRINTF(DRAM, "Merging write covering existing burst\n"); 444 merged = true; 445 // update both the address and the size 446 (*w)->addr = addr; 447 (*w)->size = size; 448 } else if ((addr + size) >= (*w)->addr && 449 ((*w)->addr + (*w)->size - addr) <= burstSize) { 450 // the new one is just before or partially 451 // overlapping with the existing one, and together 452 // they fit within a burst 453 DPRINTF(DRAM, "Merging write before existing burst\n"); 454 merged = true; 455 // the existing queue item needs to be adjusted with 456 // respect to both address and size 457 (*w)->size = (*w)->addr + (*w)->size - addr; 458 (*w)->addr = addr; 459 } 460 } else { 461 // the new one starts after the current one, figure 462 // out where the existing one ends with respect to the 463 // new one 464 if (((*w)->addr + (*w)->size) >= (addr + size)) { 465 // check if the new one is completely subsumed in the 466 // existing one 467 DPRINTF(DRAM, "Merging write into existing burst\n"); 468 merged = true; 469 // no adjustments necessary 470 } else if (((*w)->addr + (*w)->size) >= addr && 471 (addr + size - (*w)->addr) <= burstSize) { 472 // the existing one is just before or partially 473 // overlapping with the new one, and together 474 // they fit within a burst 475 DPRINTF(DRAM, "Merging write after existing burst\n"); 476 merged = true; 477 // the address is right, and only the size has 478 // to be adjusted 479 (*w)->size = addr + size - (*w)->addr; 480 } 481 } 482 ++w; 483 } 484 485 // if the item was not merged we need to create a new write 486 // and enqueue it 487 if (!merged) { 488 DRAMPacket* dram_pkt = decodeAddr(pkt, addr, size, false); 489 490 assert(writeQueue.size() < writeBufferSize); 491 wrQLenPdf[writeQueue.size()]++; 492 493 DPRINTF(DRAM, "Adding to write queue\n"); 494 495 writeQueue.push_back(dram_pkt); 496 497 // Update stats 498 avgWrQLen = writeQueue.size(); 499 } else { 500 // keep track of the fact that this burst effectively 501 // disappeared as it was merged with an existing one 502 mergedWrBursts++; 503 } 504 505 // Starting address of next dram pkt (aligend to burstSize boundary) 506 addr = (addr | (burstSize - 1)) + 1; 507 } 508 509 // we do not wait for the writes to be send to the actual memory, 510 // but instead take responsibility for the consistency here and 511 // snoop the write queue for any upcoming reads 512 // @todo, if a pkt size is larger than burst size, we might need a 513 // different front end latency 514 accessAndRespond(pkt, frontendLatency); 515 516 // If we are not already scheduled to get a request out of the 517 // queue, do so now 518 if (!nextReqEvent.scheduled()) { 519 DPRINTF(DRAM, "Request scheduled immediately\n"); 520 schedule(nextReqEvent, curTick()); 521 } 522} 523 524void 525DRAMCtrl::printQs() const { 526 DPRINTF(DRAM, "===READ QUEUE===\n\n"); 527 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) { 528 DPRINTF(DRAM, "Read %lu\n", (*i)->addr); 529 } 530 DPRINTF(DRAM, "\n===RESP QUEUE===\n\n"); 531 for (auto i = respQueue.begin() ; i != respQueue.end() ; ++i) { 532 DPRINTF(DRAM, "Response %lu\n", (*i)->addr); 533 } 534 DPRINTF(DRAM, "\n===WRITE QUEUE===\n\n"); 535 for (auto i = writeQueue.begin() ; i != writeQueue.end() ; ++i) { 536 DPRINTF(DRAM, "Write %lu\n", (*i)->addr); 537 } 538} 539 540bool 541DRAMCtrl::recvTimingReq(PacketPtr pkt) 542{ 543 /// @todo temporary hack to deal with memory corruption issues until 544 /// 4-phase transactions are complete 545 for (int x = 0; x < pendingDelete.size(); x++) 546 delete pendingDelete[x]; 547 pendingDelete.clear(); 548 549 // This is where we enter from the outside world 550 DPRINTF(DRAM, "recvTimingReq: request %s addr %lld size %d\n", 551 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 552 553 // simply drop inhibited packets for now 554 if (pkt->memInhibitAsserted()) { 555 DPRINTF(DRAM, "Inhibited packet -- Dropping it now\n"); 556 pendingDelete.push_back(pkt); 557 return true; 558 } 559 560 // Calc avg gap between requests 561 if (prevArrival != 0) { 562 totGap += curTick() - prevArrival; 563 } 564 prevArrival = curTick(); 565 566 567 // Find out how many dram packets a pkt translates to 568 // If the burst size is equal or larger than the pkt size, then a pkt 569 // translates to only one dram packet. Otherwise, a pkt translates to 570 // multiple dram packets 571 unsigned size = pkt->getSize(); 572 unsigned offset = pkt->getAddr() & (burstSize - 1); 573 unsigned int dram_pkt_count = divCeil(offset + size, burstSize); 574 575 // check local buffers and do not accept if full 576 if (pkt->isRead()) { 577 assert(size != 0); 578 if (readQueueFull(dram_pkt_count)) { 579 DPRINTF(DRAM, "Read queue full, not accepting\n"); 580 // remember that we have to retry this port 581 retryRdReq = true; 582 numRdRetry++; 583 return false; 584 } else { 585 addToReadQueue(pkt, dram_pkt_count); 586 readReqs++; 587 bytesReadSys += size; 588 } 589 } else if (pkt->isWrite()) { 590 assert(size != 0); 591 if (writeQueueFull(dram_pkt_count)) { 592 DPRINTF(DRAM, "Write queue full, not accepting\n"); 593 // remember that we have to retry this port 594 retryWrReq = true; 595 numWrRetry++; 596 return false; 597 } else { 598 addToWriteQueue(pkt, dram_pkt_count); 599 writeReqs++; 600 bytesWrittenSys += size; 601 } 602 } else { 603 DPRINTF(DRAM,"Neither read nor write, ignore timing\n"); 604 neitherReadNorWrite++; 605 accessAndRespond(pkt, 1); 606 } 607 608 return true; 609} 610 611void 612DRAMCtrl::processRespondEvent() 613{ 614 DPRINTF(DRAM, 615 "processRespondEvent(): Some req has reached its readyTime\n"); 616 617 DRAMPacket* dram_pkt = respQueue.front(); 618 619 if (dram_pkt->burstHelper) { 620 // it is a split packet 621 dram_pkt->burstHelper->burstsServiced++; 622 if (dram_pkt->burstHelper->burstsServiced == 623 dram_pkt->burstHelper->burstCount) { 624 // we have now serviced all children packets of a system packet 625 // so we can now respond to the requester 626 // @todo we probably want to have a different front end and back 627 // end latency for split packets 628 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 629 delete dram_pkt->burstHelper; 630 dram_pkt->burstHelper = NULL; 631 } 632 } else { 633 // it is not a split packet 634 accessAndRespond(dram_pkt->pkt, frontendLatency + backendLatency); 635 } 636 637 delete respQueue.front(); 638 respQueue.pop_front(); 639 640 if (!respQueue.empty()) { 641 assert(respQueue.front()->readyTime >= curTick()); 642 assert(!respondEvent.scheduled()); 643 schedule(respondEvent, respQueue.front()->readyTime); 644 } else { 645 // if there is nothing left in any queue, signal a drain 646 if (writeQueue.empty() && readQueue.empty() && 647 drainManager) { 648 drainManager->signalDrainDone(); 649 drainManager = NULL; 650 } 651 } 652 653 // We have made a location in the queue available at this point, 654 // so if there is a read that was forced to wait, retry now 655 if (retryRdReq) { 656 retryRdReq = false; 657 port.sendRetry(); 658 } 659} 660 661void 662DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue) 663{ 664 // This method does the arbitration between requests. The chosen 665 // packet is simply moved to the head of the queue. The other 666 // methods know that this is the place to look. For example, with 667 // FCFS, this method does nothing 668 assert(!queue.empty()); 669 670 if (queue.size() == 1) { 671 DPRINTF(DRAM, "Single request, nothing to do\n"); 672 return; 673 } 674 675 if (memSchedPolicy == Enums::fcfs) { 676 // Do nothing, since the correct request is already head 677 } else if (memSchedPolicy == Enums::frfcfs) { 678 reorderQueue(queue); 679 } else 680 panic("No scheduling policy chosen\n"); 681} 682 683void 684DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue) 685{ 686 // Only determine this when needed 687 uint64_t earliest_banks = 0; 688 689 // Search for row hits first, if no row hit is found then schedule the 690 // packet to one of the earliest banks available 691 bool found_earliest_pkt = false; 692 auto selected_pkt_it = queue.begin(); 693 694 for (auto i = queue.begin(); i != queue.end() ; ++i) { 695 DRAMPacket* dram_pkt = *i; 696 const Bank& bank = dram_pkt->bankRef; 697 // Check if it is a row hit 698 if (bank.openRow == dram_pkt->row) { 699 // FCFS within the hits 700 DPRINTF(DRAM, "Row buffer hit\n"); 701 selected_pkt_it = i; 702 break; 703 } else if (!found_earliest_pkt) { 704 // No row hit, go for first ready 705 if (earliest_banks == 0) 706 earliest_banks = minBankActAt(queue); 707 708 // simplistic approximation of when the bank can issue an 709 // activate, this is calculated in minBankActAt and could 710 // be cached 711 Tick act_at = bank.openRow == Bank::NO_ROW ? 712 bank.actAllowedAt : 713 std::max(bank.preAllowedAt, curTick()) + tRP; 714 715 // Bank is ready or is the first available bank 716 if (act_at <= curTick() || 717 bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) { 718 // Remember the packet to be scheduled to one of the earliest 719 // banks available, FCFS amongst the earliest banks 720 selected_pkt_it = i; 721 found_earliest_pkt = true; 722 } 723 } 724 } 725 726 DRAMPacket* selected_pkt = *selected_pkt_it; 727 queue.erase(selected_pkt_it); 728 queue.push_front(selected_pkt); 729} 730 731void 732DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency) 733{ 734 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr()); 735 736 bool needsResponse = pkt->needsResponse(); 737 // do the actual memory access which also turns the packet into a 738 // response 739 access(pkt); 740 741 // turn packet around to go back to requester if response expected 742 if (needsResponse) { 743 // access already turned the packet into a response 744 assert(pkt->isResponse()); 745 746 // @todo someone should pay for this 747 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 748 749 // queue the packet in the response queue to be sent out after 750 // the static latency has passed 751 port.schedTimingResp(pkt, curTick() + static_latency); 752 } else { 753 // @todo the packet is going to be deleted, and the DRAMPacket 754 // is still having a pointer to it 755 pendingDelete.push_back(pkt); 756 } 757 758 DPRINTF(DRAM, "Done\n"); 759 760 return; 761} 762 763void 764DRAMCtrl::activateBank(Bank& bank, Tick act_tick, uint32_t row) 765{ 766 // get the rank index from the bank 767 uint8_t rank = bank.rank; 768 769 assert(actTicks[rank].size() == activationLimit); 770 771 DPRINTF(DRAM, "Activate at tick %d\n", act_tick); 772 773 // update the open row 774 assert(bank.openRow == Bank::NO_ROW); 775 bank.openRow = row; 776 777 // start counting anew, this covers both the case when we 778 // auto-precharged, and when this access is forced to 779 // precharge 780 bank.bytesAccessed = 0; 781 bank.rowAccesses = 0; 782 783 ++numBanksActive; 784 assert(numBanksActive <= banksPerRank * ranksPerChannel); 785
|
785 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n", 786 act_tick, numBanksActive);
| 786 DPRINTF(DRAM, "Activate bank %d, rank %d at tick %lld, now got %d active\n", 787 bank.bank, bank.rank, act_tick, numBanksActive);
|
787
| 788
|
| 789 DPRINTF(DRAMPower, "%llu,ACT,%d,%d\n", divCeil(act_tick, tCK), bank.bank, 790 bank.rank); 791
|
788 // The next access has to respect tRAS for this bank 789 bank.preAllowedAt = act_tick + tRAS; 790 791 // Respect the row-to-column command delay 792 bank.colAllowedAt = act_tick + tRCD; 793 794 // start by enforcing tRRD 795 for(int i = 0; i < banksPerRank; i++) { 796 // next activate to any bank in this rank must not happen 797 // before tRRD 798 banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 799 banks[rank][i].actAllowedAt); 800 } 801 802 // next, we deal with tXAW, if the activation limit is disabled 803 // then we are done 804 if (actTicks[rank].empty()) 805 return; 806 807 // sanity check 808 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 809 panic("Got %d activates in window %d (%llu - %llu) which is smaller " 810 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 811 act_tick, actTicks[rank].back(), tXAW); 812 } 813 814 // shift the times used for the book keeping, the last element 815 // (highest index) is the oldest one and hence the lowest value 816 actTicks[rank].pop_back(); 817 818 // record an new activation (in the future) 819 actTicks[rank].push_front(act_tick); 820 821 // cannot activate more than X times in time window tXAW, push the 822 // next one (the X + 1'st activate) to be tXAW away from the 823 // oldest in our window of X 824 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 825 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " 826 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); 827 for(int j = 0; j < banksPerRank; j++) 828 // next activate must not happen before end of window 829 banks[rank][j].actAllowedAt = 830 std::max(actTicks[rank].back() + tXAW, 831 banks[rank][j].actAllowedAt); 832 } 833 834 // at the point when this activate takes place, make sure we 835 // transition to the active power state 836 if (!activateEvent.scheduled()) 837 schedule(activateEvent, act_tick); 838 else if (activateEvent.when() > act_tick) 839 // move it sooner in time 840 reschedule(activateEvent, act_tick); 841} 842 843void 844DRAMCtrl::processActivateEvent() 845{ 846 // we should transition to the active state as soon as any bank is active 847 if (pwrState != PWR_ACT) 848 // note that at this point numBanksActive could be back at 849 // zero again due to a precharge scheduled in the future 850 schedulePowerEvent(PWR_ACT, curTick()); 851} 852 853void
| 792 // The next access has to respect tRAS for this bank 793 bank.preAllowedAt = act_tick + tRAS; 794 795 // Respect the row-to-column command delay 796 bank.colAllowedAt = act_tick + tRCD; 797 798 // start by enforcing tRRD 799 for(int i = 0; i < banksPerRank; i++) { 800 // next activate to any bank in this rank must not happen 801 // before tRRD 802 banks[rank][i].actAllowedAt = std::max(act_tick + tRRD, 803 banks[rank][i].actAllowedAt); 804 } 805 806 // next, we deal with tXAW, if the activation limit is disabled 807 // then we are done 808 if (actTicks[rank].empty()) 809 return; 810 811 // sanity check 812 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 813 panic("Got %d activates in window %d (%llu - %llu) which is smaller " 814 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(), 815 act_tick, actTicks[rank].back(), tXAW); 816 } 817 818 // shift the times used for the book keeping, the last element 819 // (highest index) is the oldest one and hence the lowest value 820 actTicks[rank].pop_back(); 821 822 // record an new activation (in the future) 823 actTicks[rank].push_front(act_tick); 824 825 // cannot activate more than X times in time window tXAW, push the 826 // next one (the X + 1'st activate) to be tXAW away from the 827 // oldest in our window of X 828 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) { 829 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier " 830 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW); 831 for(int j = 0; j < banksPerRank; j++) 832 // next activate must not happen before end of window 833 banks[rank][j].actAllowedAt = 834 std::max(actTicks[rank].back() + tXAW, 835 banks[rank][j].actAllowedAt); 836 } 837 838 // at the point when this activate takes place, make sure we 839 // transition to the active power state 840 if (!activateEvent.scheduled()) 841 schedule(activateEvent, act_tick); 842 else if (activateEvent.when() > act_tick) 843 // move it sooner in time 844 reschedule(activateEvent, act_tick); 845} 846 847void 848DRAMCtrl::processActivateEvent() 849{ 850 // we should transition to the active state as soon as any bank is active 851 if (pwrState != PWR_ACT) 852 // note that at this point numBanksActive could be back at 853 // zero again due to a precharge scheduled in the future 854 schedulePowerEvent(PWR_ACT, curTick()); 855} 856 857void
|
854DRAMCtrl::prechargeBank(Bank& bank, Tick pre_at)
| 858DRAMCtrl::prechargeBank(Bank& bank, Tick pre_at, bool trace)
|
855{ 856 // make sure the bank has an open row 857 assert(bank.openRow != Bank::NO_ROW); 858 859 // sample the bytes per activate here since we are closing 860 // the page 861 bytesPerActivate.sample(bank.bytesAccessed); 862 863 bank.openRow = Bank::NO_ROW; 864 865 // no precharge allowed before this one 866 bank.preAllowedAt = pre_at; 867 868 Tick pre_done_at = pre_at + tRP; 869 870 bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 871 872 assert(numBanksActive != 0); 873 --numBanksActive; 874
| 859{ 860 // make sure the bank has an open row 861 assert(bank.openRow != Bank::NO_ROW); 862 863 // sample the bytes per activate here since we are closing 864 // the page 865 bytesPerActivate.sample(bank.bytesAccessed); 866 867 bank.openRow = Bank::NO_ROW; 868 869 // no precharge allowed before this one 870 bank.preAllowedAt = pre_at; 871 872 Tick pre_done_at = pre_at + tRP; 873 874 bank.actAllowedAt = std::max(bank.actAllowedAt, pre_done_at); 875 876 assert(numBanksActive != 0); 877 --numBanksActive; 878
|
875 DPRINTF(DRAM, "Precharging bank at tick %lld, now got %d active\n", 876 pre_at, numBanksActive);
| 879 DPRINTF(DRAM, "Precharging bank %d, rank %d at tick %lld, now got " 880 "%d active\n", bank.bank, bank.rank, pre_at, numBanksActive);
|
877
| 881
|
| 882 if (trace) 883 DPRINTF(DRAMPower, "%llu,PRE,%d,%d\n", divCeil(pre_at, tCK), 884 bank.bank, bank.rank); 885
|
878 // if we look at the current number of active banks we might be 879 // tempted to think the DRAM is now idle, however this can be 880 // undone by an activate that is scheduled to happen before we 881 // would have reached the idle state, so schedule an event and 882 // rather check once we actually make it to the point in time when 883 // the (last) precharge takes place 884 if (!prechargeEvent.scheduled()) 885 schedule(prechargeEvent, pre_done_at); 886 else if (prechargeEvent.when() < pre_done_at) 887 reschedule(prechargeEvent, pre_done_at); 888} 889 890void 891DRAMCtrl::processPrechargeEvent() 892{ 893 // if we reached zero, then special conditions apply as we track 894 // if all banks are precharged for the power models 895 if (numBanksActive == 0) { 896 // we should transition to the idle state when the last bank 897 // is precharged 898 schedulePowerEvent(PWR_IDLE, curTick()); 899 } 900} 901 902void 903DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 904{ 905 DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 906 dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 907 908 // get the bank 909 Bank& bank = dram_pkt->bankRef; 910 911 // for the state we need to track if it is a row hit or not 912 bool row_hit = true; 913 914 // respect any constraints on the command (e.g. tRCD or tCCD) 915 Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 916 917 // Determine the access latency and update the bank state 918 if (bank.openRow == dram_pkt->row) { 919 // nothing to do 920 } else { 921 row_hit = false; 922 923 // If there is a page open, precharge it. 924 if (bank.openRow != Bank::NO_ROW) { 925 prechargeBank(bank, std::max(bank.preAllowedAt, curTick())); 926 } 927 928 // next we need to account for the delay in activating the 929 // page 930 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 931 932 // Record the activation and deal with all the global timing 933 // constraints caused be a new activation (tRRD and tXAW) 934 activateBank(bank, act_tick, dram_pkt->row); 935 936 // issue the command as early as possible 937 cmd_at = bank.colAllowedAt; 938 } 939 940 // we need to wait until the bus is available before we can issue 941 // the command 942 cmd_at = std::max(cmd_at, busBusyUntil - tCL); 943 944 // update the packet ready time 945 dram_pkt->readyTime = cmd_at + tCL + tBURST; 946 947 // only one burst can use the bus at any one point in time 948 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 949 950 // not strictly necessary, but update the time for the next 951 // read/write (add a max with tCCD here) 952 bank.colAllowedAt = cmd_at + tBURST; 953 954 // If this is a write, we also need to respect the write recovery 955 // time before a precharge, in the case of a read, respect the 956 // read to precharge constraint 957 bank.preAllowedAt = std::max(bank.preAllowedAt, 958 dram_pkt->isRead ? cmd_at + tRTP : 959 dram_pkt->readyTime + tWR); 960 961 // increment the bytes accessed and the accesses per row 962 bank.bytesAccessed += burstSize; 963 ++bank.rowAccesses; 964 965 // if we reached the max, then issue with an auto-precharge 966 bool auto_precharge = pageMgmt == Enums::close || 967 bank.rowAccesses == maxAccessesPerRow; 968 969 // if we did not hit the limit, we might still want to 970 // auto-precharge 971 if (!auto_precharge && 972 (pageMgmt == Enums::open_adaptive || 973 pageMgmt == Enums::close_adaptive)) { 974 // a twist on the open and close page policies: 975 // 1) open_adaptive page policy does not blindly keep the 976 // page open, but close it if there are no row hits, and there 977 // are bank conflicts in the queue 978 // 2) close_adaptive page policy does not blindly close the 979 // page, but closes it only if there are no row hits in the queue. 980 // In this case, only force an auto precharge when there 981 // are no same page hits in the queue 982 bool got_more_hits = false; 983 bool got_bank_conflict = false; 984 985 // either look at the read queue or write queue 986 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 987 writeQueue; 988 auto p = queue.begin(); 989 // make sure we are not considering the packet that we are 990 // currently dealing with (which is the head of the queue) 991 ++p; 992 993 // keep on looking until we have found required condition or 994 // reached the end 995 while (!(got_more_hits && 996 (got_bank_conflict || pageMgmt == Enums::close_adaptive)) && 997 p != queue.end()) { 998 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 999 (dram_pkt->bank == (*p)->bank); 1000 bool same_row = dram_pkt->row == (*p)->row; 1001 got_more_hits |= same_rank_bank && same_row; 1002 got_bank_conflict |= same_rank_bank && !same_row; 1003 ++p; 1004 } 1005 1006 // auto pre-charge when either 1007 // 1) open_adaptive policy, we have not got any more hits, and 1008 // have a bank conflict 1009 // 2) close_adaptive policy and we have not got any more hits 1010 auto_precharge = !got_more_hits && 1011 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1012 } 1013
| 886 // if we look at the current number of active banks we might be 887 // tempted to think the DRAM is now idle, however this can be 888 // undone by an activate that is scheduled to happen before we 889 // would have reached the idle state, so schedule an event and 890 // rather check once we actually make it to the point in time when 891 // the (last) precharge takes place 892 if (!prechargeEvent.scheduled()) 893 schedule(prechargeEvent, pre_done_at); 894 else if (prechargeEvent.when() < pre_done_at) 895 reschedule(prechargeEvent, pre_done_at); 896} 897 898void 899DRAMCtrl::processPrechargeEvent() 900{ 901 // if we reached zero, then special conditions apply as we track 902 // if all banks are precharged for the power models 903 if (numBanksActive == 0) { 904 // we should transition to the idle state when the last bank 905 // is precharged 906 schedulePowerEvent(PWR_IDLE, curTick()); 907 } 908} 909 910void 911DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 912{ 913 DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 914 dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 915 916 // get the bank 917 Bank& bank = dram_pkt->bankRef; 918 919 // for the state we need to track if it is a row hit or not 920 bool row_hit = true; 921 922 // respect any constraints on the command (e.g. tRCD or tCCD) 923 Tick cmd_at = std::max(bank.colAllowedAt, curTick()); 924 925 // Determine the access latency and update the bank state 926 if (bank.openRow == dram_pkt->row) { 927 // nothing to do 928 } else { 929 row_hit = false; 930 931 // If there is a page open, precharge it. 932 if (bank.openRow != Bank::NO_ROW) { 933 prechargeBank(bank, std::max(bank.preAllowedAt, curTick())); 934 } 935 936 // next we need to account for the delay in activating the 937 // page 938 Tick act_tick = std::max(bank.actAllowedAt, curTick()); 939 940 // Record the activation and deal with all the global timing 941 // constraints caused be a new activation (tRRD and tXAW) 942 activateBank(bank, act_tick, dram_pkt->row); 943 944 // issue the command as early as possible 945 cmd_at = bank.colAllowedAt; 946 } 947 948 // we need to wait until the bus is available before we can issue 949 // the command 950 cmd_at = std::max(cmd_at, busBusyUntil - tCL); 951 952 // update the packet ready time 953 dram_pkt->readyTime = cmd_at + tCL + tBURST; 954 955 // only one burst can use the bus at any one point in time 956 assert(dram_pkt->readyTime - busBusyUntil >= tBURST); 957 958 // not strictly necessary, but update the time for the next 959 // read/write (add a max with tCCD here) 960 bank.colAllowedAt = cmd_at + tBURST; 961 962 // If this is a write, we also need to respect the write recovery 963 // time before a precharge, in the case of a read, respect the 964 // read to precharge constraint 965 bank.preAllowedAt = std::max(bank.preAllowedAt, 966 dram_pkt->isRead ? cmd_at + tRTP : 967 dram_pkt->readyTime + tWR); 968 969 // increment the bytes accessed and the accesses per row 970 bank.bytesAccessed += burstSize; 971 ++bank.rowAccesses; 972 973 // if we reached the max, then issue with an auto-precharge 974 bool auto_precharge = pageMgmt == Enums::close || 975 bank.rowAccesses == maxAccessesPerRow; 976 977 // if we did not hit the limit, we might still want to 978 // auto-precharge 979 if (!auto_precharge && 980 (pageMgmt == Enums::open_adaptive || 981 pageMgmt == Enums::close_adaptive)) { 982 // a twist on the open and close page policies: 983 // 1) open_adaptive page policy does not blindly keep the 984 // page open, but close it if there are no row hits, and there 985 // are bank conflicts in the queue 986 // 2) close_adaptive page policy does not blindly close the 987 // page, but closes it only if there are no row hits in the queue. 988 // In this case, only force an auto precharge when there 989 // are no same page hits in the queue 990 bool got_more_hits = false; 991 bool got_bank_conflict = false; 992 993 // either look at the read queue or write queue 994 const deque<DRAMPacket*>& queue = dram_pkt->isRead ? readQueue : 995 writeQueue; 996 auto p = queue.begin(); 997 // make sure we are not considering the packet that we are 998 // currently dealing with (which is the head of the queue) 999 ++p; 1000 1001 // keep on looking until we have found required condition or 1002 // reached the end 1003 while (!(got_more_hits && 1004 (got_bank_conflict || pageMgmt == Enums::close_adaptive)) && 1005 p != queue.end()) { 1006 bool same_rank_bank = (dram_pkt->rank == (*p)->rank) && 1007 (dram_pkt->bank == (*p)->bank); 1008 bool same_row = dram_pkt->row == (*p)->row; 1009 got_more_hits |= same_rank_bank && same_row; 1010 got_bank_conflict |= same_rank_bank && !same_row; 1011 ++p; 1012 } 1013 1014 // auto pre-charge when either 1015 // 1) open_adaptive policy, we have not got any more hits, and 1016 // have a bank conflict 1017 // 2) close_adaptive policy and we have not got any more hits 1018 auto_precharge = !got_more_hits && 1019 (got_bank_conflict || pageMgmt == Enums::close_adaptive); 1020 } 1021
|
| 1022 // DRAMPower trace command to be written 1023 std::string mem_cmd = dram_pkt->isRead ? "RD" : "WR"; 1024
|
1014 // if this access should use auto-precharge, then we are 1015 // closing the row 1016 if (auto_precharge) {
| 1025 // if this access should use auto-precharge, then we are 1026 // closing the row 1027 if (auto_precharge) {
|
1017 prechargeBank(bank, std::max(curTick(), bank.preAllowedAt));
| 1028 prechargeBank(bank, std::max(curTick(), bank.preAllowedAt), false);
|
1018
| 1029
|
| 1030 mem_cmd.append("A"); 1031
|
1019 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1020 } 1021 1022 // Update bus state 1023 busBusyUntil = dram_pkt->readyTime; 1024 1025 DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 1026 dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 1027
| 1032 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); 1033 } 1034 1035 // Update bus state 1036 busBusyUntil = dram_pkt->readyTime; 1037 1038 DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n", 1039 dram_pkt->addr, dram_pkt->readyTime, busBusyUntil); 1040
|
| 1041 DPRINTF(DRAMPower, "%llu,%s,%d,%d\n", divCeil(cmd_at, tCK), mem_cmd, 1042 dram_pkt->bank, dram_pkt->rank); 1043
|
1028 // Update the minimum timing between the requests, this is a 1029 // conservative estimate of when we have to schedule the next 1030 // request to not introduce any unecessary bubbles. In most cases 1031 // we will wake up sooner than we have to. 1032 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1033 1034 // Update the stats and schedule the next request 1035 if (dram_pkt->isRead) { 1036 ++readsThisTime; 1037 if (row_hit) 1038 readRowHits++; 1039 bytesReadDRAM += burstSize; 1040 perBankRdBursts[dram_pkt->bankId]++; 1041 1042 // Update latency stats 1043 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 1044 totBusLat += tBURST; 1045 totQLat += cmd_at - dram_pkt->entryTime; 1046 } else { 1047 ++writesThisTime; 1048 if (row_hit) 1049 writeRowHits++; 1050 bytesWritten += burstSize; 1051 perBankWrBursts[dram_pkt->bankId]++; 1052 } 1053} 1054 1055void 1056DRAMCtrl::processNextReqEvent() 1057{ 1058 if (busState == READ_TO_WRITE) { 1059 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1060 "waiting\n", readsThisTime, readQueue.size()); 1061 1062 // sample and reset the read-related stats as we are now 1063 // transitioning to writes, and all reads are done 1064 rdPerTurnAround.sample(readsThisTime); 1065 readsThisTime = 0; 1066 1067 // now proceed to do the actual writes 1068 busState = WRITE; 1069 } else if (busState == WRITE_TO_READ) { 1070 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1071 "waiting\n", writesThisTime, writeQueue.size()); 1072 1073 wrPerTurnAround.sample(writesThisTime); 1074 writesThisTime = 0; 1075 1076 busState = READ; 1077 } 1078 1079 if (refreshState != REF_IDLE) { 1080 // if a refresh waiting for this event loop to finish, then hand 1081 // over now, and do not schedule a new nextReqEvent 1082 if (refreshState == REF_DRAIN) { 1083 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1084 1085 refreshState = REF_PRE; 1086 1087 // hand control back to the refresh event loop 1088 schedule(refreshEvent, curTick()); 1089 } 1090 1091 // let the refresh finish before issuing any further requests 1092 return; 1093 } 1094 1095 // when we get here it is either a read or a write 1096 if (busState == READ) { 1097 1098 // track if we should switch or not 1099 bool switch_to_writes = false; 1100 1101 if (readQueue.empty()) { 1102 // In the case there is no read request to go next, 1103 // trigger writes if we have passed the low threshold (or 1104 // if we are draining) 1105 if (!writeQueue.empty() && 1106 (drainManager || writeQueue.size() > writeLowThreshold)) { 1107 1108 switch_to_writes = true; 1109 } else { 1110 // check if we are drained 1111 if (respQueue.empty () && drainManager) { 1112 drainManager->signalDrainDone(); 1113 drainManager = NULL; 1114 } 1115 1116 // nothing to do, not even any point in scheduling an 1117 // event for the next request 1118 return; 1119 } 1120 } else { 1121 // Figure out which read request goes next, and move it to the 1122 // front of the read queue 1123 chooseNext(readQueue); 1124 1125 DRAMPacket* dram_pkt = readQueue.front(); 1126 1127 doDRAMAccess(dram_pkt); 1128 1129 // At this point we're done dealing with the request 1130 readQueue.pop_front(); 1131 1132 // sanity check 1133 assert(dram_pkt->size <= burstSize); 1134 assert(dram_pkt->readyTime >= curTick()); 1135 1136 // Insert into response queue. It will be sent back to the 1137 // requestor at its readyTime 1138 if (respQueue.empty()) { 1139 assert(!respondEvent.scheduled()); 1140 schedule(respondEvent, dram_pkt->readyTime); 1141 } else { 1142 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1143 assert(respondEvent.scheduled()); 1144 } 1145 1146 respQueue.push_back(dram_pkt); 1147 1148 // we have so many writes that we have to transition 1149 if (writeQueue.size() > writeHighThreshold) { 1150 switch_to_writes = true; 1151 } 1152 } 1153 1154 // switching to writes, either because the read queue is empty 1155 // and the writes have passed the low threshold (or we are 1156 // draining), or because the writes hit the hight threshold 1157 if (switch_to_writes) { 1158 // transition to writing 1159 busState = READ_TO_WRITE; 1160 1161 // add a bubble to the data bus, as defined by the 1162 // tRTW parameter 1163 busBusyUntil += tRTW; 1164 1165 // update the minimum timing between the requests, 1166 // this shifts us back in time far enough to do any 1167 // bank preparation 1168 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1169 } 1170 } else { 1171 chooseNext(writeQueue); 1172 DRAMPacket* dram_pkt = writeQueue.front(); 1173 // sanity check 1174 assert(dram_pkt->size <= burstSize); 1175 doDRAMAccess(dram_pkt); 1176 1177 writeQueue.pop_front(); 1178 delete dram_pkt; 1179 1180 // If we emptied the write queue, or got sufficiently below the 1181 // threshold (using the minWritesPerSwitch as the hysteresis) and 1182 // are not draining, or we have reads waiting and have done enough 1183 // writes, then switch to reads. 1184 if (writeQueue.empty() || 1185 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1186 !drainManager) || 1187 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1188 // turn the bus back around for reads again 1189 busState = WRITE_TO_READ; 1190 1191 // note that the we switch back to reads also in the idle 1192 // case, which eventually will check for any draining and 1193 // also pause any further scheduling if there is really 1194 // nothing to do 1195 1196 // here we get a bit creative and shift the bus busy time not 1197 // just the tWTR, but also a CAS latency to capture the fact 1198 // that we are allowed to prepare a new bank, but not issue a 1199 // read command until after tWTR, in essence we capture a 1200 // bubble on the data bus that is tWTR + tCL 1201 busBusyUntil += tWTR + tCL; 1202 1203 // update the minimum timing between the requests, this shifts 1204 // us back in time far enough to do any bank preparation 1205 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1206 } 1207 } 1208 1209 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1210 1211 // If there is space available and we have writes waiting then let 1212 // them retry. This is done here to ensure that the retry does not 1213 // cause a nextReqEvent to be scheduled before we do so as part of 1214 // the next request processing 1215 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1216 retryWrReq = false; 1217 port.sendRetry(); 1218 } 1219} 1220 1221uint64_t 1222DRAMCtrl::minBankActAt(const deque<DRAMPacket*>& queue) const 1223{ 1224 uint64_t bank_mask = 0; 1225 Tick min_act_at = MaxTick; 1226 1227 // deterimne if we have queued transactions targetting a 1228 // bank in question 1229 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 1230 for (auto p = queue.begin(); p != queue.end(); ++p) { 1231 got_waiting[(*p)->bankId] = true; 1232 } 1233 1234 for (int i = 0; i < ranksPerChannel; i++) { 1235 for (int j = 0; j < banksPerRank; j++) { 1236 uint8_t bank_id = i * banksPerRank + j; 1237 1238 // if we have waiting requests for the bank, and it is 1239 // amongst the first available, update the mask 1240 if (got_waiting[bank_id]) { 1241 // simplistic approximation of when the bank can issue 1242 // an activate, ignoring any rank-to-rank switching 1243 // cost 1244 Tick act_at = banks[i][j].openRow == Bank::NO_ROW ? 1245 banks[i][j].actAllowedAt : 1246 std::max(banks[i][j].preAllowedAt, curTick()) + tRP; 1247 1248 if (act_at <= min_act_at) { 1249 // reset bank mask if new minimum is found 1250 if (act_at < min_act_at) 1251 bank_mask = 0; 1252 // set the bit corresponding to the available bank 1253 replaceBits(bank_mask, bank_id, bank_id, 1); 1254 min_act_at = act_at; 1255 } 1256 } 1257 } 1258 } 1259 1260 return bank_mask; 1261} 1262 1263void 1264DRAMCtrl::processRefreshEvent() 1265{ 1266 // when first preparing the refresh, remember when it was due 1267 if (refreshState == REF_IDLE) { 1268 // remember when the refresh is due 1269 refreshDueAt = curTick(); 1270 1271 // proceed to drain 1272 refreshState = REF_DRAIN; 1273 1274 DPRINTF(DRAM, "Refresh due\n"); 1275 } 1276 1277 // let any scheduled read or write go ahead, after which it will 1278 // hand control back to this event loop 1279 if (refreshState == REF_DRAIN) { 1280 if (nextReqEvent.scheduled()) { 1281 // hand control over to the request loop until it is 1282 // evaluated next 1283 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1284 1285 return; 1286 } else { 1287 refreshState = REF_PRE; 1288 } 1289 } 1290 1291 // at this point, ensure that all banks are precharged 1292 if (refreshState == REF_PRE) { 1293 // precharge any active bank if we are not already in the idle 1294 // state 1295 if (pwrState != PWR_IDLE) { 1296 // at the moment, we use a precharge all even if there is 1297 // only a single bank open 1298 DPRINTF(DRAM, "Precharging all\n"); 1299 1300 // first determine when we can precharge 1301 Tick pre_at = curTick(); 1302 for (int i = 0; i < ranksPerChannel; i++) { 1303 for (int j = 0; j < banksPerRank; j++) { 1304 // respect both causality and any existing bank 1305 // constraints, some banks could already have a 1306 // (auto) precharge scheduled 1307 pre_at = std::max(banks[i][j].preAllowedAt, pre_at); 1308 } 1309 } 1310 1311 // make sure all banks are precharged, and for those that 1312 // already are, update their availability 1313 Tick act_allowed_at = pre_at + tRP; 1314 1315 for (int i = 0; i < ranksPerChannel; i++) { 1316 for (int j = 0; j < banksPerRank; j++) { 1317 if (banks[i][j].openRow != Bank::NO_ROW) {
| 1044 // Update the minimum timing between the requests, this is a 1045 // conservative estimate of when we have to schedule the next 1046 // request to not introduce any unecessary bubbles. In most cases 1047 // we will wake up sooner than we have to. 1048 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1049 1050 // Update the stats and schedule the next request 1051 if (dram_pkt->isRead) { 1052 ++readsThisTime; 1053 if (row_hit) 1054 readRowHits++; 1055 bytesReadDRAM += burstSize; 1056 perBankRdBursts[dram_pkt->bankId]++; 1057 1058 // Update latency stats 1059 totMemAccLat += dram_pkt->readyTime - dram_pkt->entryTime; 1060 totBusLat += tBURST; 1061 totQLat += cmd_at - dram_pkt->entryTime; 1062 } else { 1063 ++writesThisTime; 1064 if (row_hit) 1065 writeRowHits++; 1066 bytesWritten += burstSize; 1067 perBankWrBursts[dram_pkt->bankId]++; 1068 } 1069} 1070 1071void 1072DRAMCtrl::processNextReqEvent() 1073{ 1074 if (busState == READ_TO_WRITE) { 1075 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1076 "waiting\n", readsThisTime, readQueue.size()); 1077 1078 // sample and reset the read-related stats as we are now 1079 // transitioning to writes, and all reads are done 1080 rdPerTurnAround.sample(readsThisTime); 1081 readsThisTime = 0; 1082 1083 // now proceed to do the actual writes 1084 busState = WRITE; 1085 } else if (busState == WRITE_TO_READ) { 1086 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1087 "waiting\n", writesThisTime, writeQueue.size()); 1088 1089 wrPerTurnAround.sample(writesThisTime); 1090 writesThisTime = 0; 1091 1092 busState = READ; 1093 } 1094 1095 if (refreshState != REF_IDLE) { 1096 // if a refresh waiting for this event loop to finish, then hand 1097 // over now, and do not schedule a new nextReqEvent 1098 if (refreshState == REF_DRAIN) { 1099 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1100 1101 refreshState = REF_PRE; 1102 1103 // hand control back to the refresh event loop 1104 schedule(refreshEvent, curTick()); 1105 } 1106 1107 // let the refresh finish before issuing any further requests 1108 return; 1109 } 1110 1111 // when we get here it is either a read or a write 1112 if (busState == READ) { 1113 1114 // track if we should switch or not 1115 bool switch_to_writes = false; 1116 1117 if (readQueue.empty()) { 1118 // In the case there is no read request to go next, 1119 // trigger writes if we have passed the low threshold (or 1120 // if we are draining) 1121 if (!writeQueue.empty() && 1122 (drainManager || writeQueue.size() > writeLowThreshold)) { 1123 1124 switch_to_writes = true; 1125 } else { 1126 // check if we are drained 1127 if (respQueue.empty () && drainManager) { 1128 drainManager->signalDrainDone(); 1129 drainManager = NULL; 1130 } 1131 1132 // nothing to do, not even any point in scheduling an 1133 // event for the next request 1134 return; 1135 } 1136 } else { 1137 // Figure out which read request goes next, and move it to the 1138 // front of the read queue 1139 chooseNext(readQueue); 1140 1141 DRAMPacket* dram_pkt = readQueue.front(); 1142 1143 doDRAMAccess(dram_pkt); 1144 1145 // At this point we're done dealing with the request 1146 readQueue.pop_front(); 1147 1148 // sanity check 1149 assert(dram_pkt->size <= burstSize); 1150 assert(dram_pkt->readyTime >= curTick()); 1151 1152 // Insert into response queue. It will be sent back to the 1153 // requestor at its readyTime 1154 if (respQueue.empty()) { 1155 assert(!respondEvent.scheduled()); 1156 schedule(respondEvent, dram_pkt->readyTime); 1157 } else { 1158 assert(respQueue.back()->readyTime <= dram_pkt->readyTime); 1159 assert(respondEvent.scheduled()); 1160 } 1161 1162 respQueue.push_back(dram_pkt); 1163 1164 // we have so many writes that we have to transition 1165 if (writeQueue.size() > writeHighThreshold) { 1166 switch_to_writes = true; 1167 } 1168 } 1169 1170 // switching to writes, either because the read queue is empty 1171 // and the writes have passed the low threshold (or we are 1172 // draining), or because the writes hit the hight threshold 1173 if (switch_to_writes) { 1174 // transition to writing 1175 busState = READ_TO_WRITE; 1176 1177 // add a bubble to the data bus, as defined by the 1178 // tRTW parameter 1179 busBusyUntil += tRTW; 1180 1181 // update the minimum timing between the requests, 1182 // this shifts us back in time far enough to do any 1183 // bank preparation 1184 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1185 } 1186 } else { 1187 chooseNext(writeQueue); 1188 DRAMPacket* dram_pkt = writeQueue.front(); 1189 // sanity check 1190 assert(dram_pkt->size <= burstSize); 1191 doDRAMAccess(dram_pkt); 1192 1193 writeQueue.pop_front(); 1194 delete dram_pkt; 1195 1196 // If we emptied the write queue, or got sufficiently below the 1197 // threshold (using the minWritesPerSwitch as the hysteresis) and 1198 // are not draining, or we have reads waiting and have done enough 1199 // writes, then switch to reads. 1200 if (writeQueue.empty() || 1201 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1202 !drainManager) || 1203 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1204 // turn the bus back around for reads again 1205 busState = WRITE_TO_READ; 1206 1207 // note that the we switch back to reads also in the idle 1208 // case, which eventually will check for any draining and 1209 // also pause any further scheduling if there is really 1210 // nothing to do 1211 1212 // here we get a bit creative and shift the bus busy time not 1213 // just the tWTR, but also a CAS latency to capture the fact 1214 // that we are allowed to prepare a new bank, but not issue a 1215 // read command until after tWTR, in essence we capture a 1216 // bubble on the data bus that is tWTR + tCL 1217 busBusyUntil += tWTR + tCL; 1218 1219 // update the minimum timing between the requests, this shifts 1220 // us back in time far enough to do any bank preparation 1221 nextReqTime = busBusyUntil - (tRP + tRCD + tCL); 1222 } 1223 } 1224 1225 schedule(nextReqEvent, std::max(nextReqTime, curTick())); 1226 1227 // If there is space available and we have writes waiting then let 1228 // them retry. This is done here to ensure that the retry does not 1229 // cause a nextReqEvent to be scheduled before we do so as part of 1230 // the next request processing 1231 if (retryWrReq && writeQueue.size() < writeBufferSize) { 1232 retryWrReq = false; 1233 port.sendRetry(); 1234 } 1235} 1236 1237uint64_t 1238DRAMCtrl::minBankActAt(const deque<DRAMPacket*>& queue) const 1239{ 1240 uint64_t bank_mask = 0; 1241 Tick min_act_at = MaxTick; 1242 1243 // deterimne if we have queued transactions targetting a 1244 // bank in question 1245 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false); 1246 for (auto p = queue.begin(); p != queue.end(); ++p) { 1247 got_waiting[(*p)->bankId] = true; 1248 } 1249 1250 for (int i = 0; i < ranksPerChannel; i++) { 1251 for (int j = 0; j < banksPerRank; j++) { 1252 uint8_t bank_id = i * banksPerRank + j; 1253 1254 // if we have waiting requests for the bank, and it is 1255 // amongst the first available, update the mask 1256 if (got_waiting[bank_id]) { 1257 // simplistic approximation of when the bank can issue 1258 // an activate, ignoring any rank-to-rank switching 1259 // cost 1260 Tick act_at = banks[i][j].openRow == Bank::NO_ROW ? 1261 banks[i][j].actAllowedAt : 1262 std::max(banks[i][j].preAllowedAt, curTick()) + tRP; 1263 1264 if (act_at <= min_act_at) { 1265 // reset bank mask if new minimum is found 1266 if (act_at < min_act_at) 1267 bank_mask = 0; 1268 // set the bit corresponding to the available bank 1269 replaceBits(bank_mask, bank_id, bank_id, 1); 1270 min_act_at = act_at; 1271 } 1272 } 1273 } 1274 } 1275 1276 return bank_mask; 1277} 1278 1279void 1280DRAMCtrl::processRefreshEvent() 1281{ 1282 // when first preparing the refresh, remember when it was due 1283 if (refreshState == REF_IDLE) { 1284 // remember when the refresh is due 1285 refreshDueAt = curTick(); 1286 1287 // proceed to drain 1288 refreshState = REF_DRAIN; 1289 1290 DPRINTF(DRAM, "Refresh due\n"); 1291 } 1292 1293 // let any scheduled read or write go ahead, after which it will 1294 // hand control back to this event loop 1295 if (refreshState == REF_DRAIN) { 1296 if (nextReqEvent.scheduled()) { 1297 // hand control over to the request loop until it is 1298 // evaluated next 1299 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1300 1301 return; 1302 } else { 1303 refreshState = REF_PRE; 1304 } 1305 } 1306 1307 // at this point, ensure that all banks are precharged 1308 if (refreshState == REF_PRE) { 1309 // precharge any active bank if we are not already in the idle 1310 // state 1311 if (pwrState != PWR_IDLE) { 1312 // at the moment, we use a precharge all even if there is 1313 // only a single bank open 1314 DPRINTF(DRAM, "Precharging all\n"); 1315 1316 // first determine when we can precharge 1317 Tick pre_at = curTick(); 1318 for (int i = 0; i < ranksPerChannel; i++) { 1319 for (int j = 0; j < banksPerRank; j++) { 1320 // respect both causality and any existing bank 1321 // constraints, some banks could already have a 1322 // (auto) precharge scheduled 1323 pre_at = std::max(banks[i][j].preAllowedAt, pre_at); 1324 } 1325 } 1326 1327 // make sure all banks are precharged, and for those that 1328 // already are, update their availability 1329 Tick act_allowed_at = pre_at + tRP; 1330 1331 for (int i = 0; i < ranksPerChannel; i++) { 1332 for (int j = 0; j < banksPerRank; j++) { 1333 if (banks[i][j].openRow != Bank::NO_ROW) {
|
1318 prechargeBank(banks[i][j], pre_at);
| 1334 prechargeBank(banks[i][j], pre_at, false);
|
1319 } else { 1320 banks[i][j].actAllowedAt = 1321 std::max(banks[i][j].actAllowedAt, act_allowed_at); 1322 banks[i][j].preAllowedAt = 1323 std::max(banks[i][j].preAllowedAt, pre_at); 1324 } 1325 }
| 1335 } else { 1336 banks[i][j].actAllowedAt = 1337 std::max(banks[i][j].actAllowedAt, act_allowed_at); 1338 banks[i][j].preAllowedAt = 1339 std::max(banks[i][j].preAllowedAt, pre_at); 1340 } 1341 }
|
| 1342 1343 // at the moment this affects all ranks 1344 DPRINTF(DRAMPower, "%llu,PREA,0,%d\n", divCeil(pre_at, tCK), 1345 i);
|
1326 } 1327 } else { 1328 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1329 1330 // go ahead and kick the power state machine into gear if 1331 // we are already idle 1332 schedulePowerEvent(PWR_REF, curTick()); 1333 } 1334 1335 refreshState = REF_RUN; 1336 assert(numBanksActive == 0); 1337 1338 // wait for all banks to be precharged, at which point the 1339 // power state machine will transition to the idle state, and 1340 // automatically move to a refresh, at that point it will also 1341 // call this method to get the refresh event loop going again 1342 return; 1343 } 1344 1345 // last but not least we perform the actual refresh 1346 if (refreshState == REF_RUN) { 1347 // should never get here with any banks active 1348 assert(numBanksActive == 0); 1349 assert(pwrState == PWR_REF); 1350 1351 Tick ref_done_at = curTick() + tRFC; 1352 1353 for (int i = 0; i < ranksPerChannel; i++) { 1354 for (int j = 0; j < banksPerRank; j++) { 1355 banks[i][j].actAllowedAt = ref_done_at; 1356 }
| 1346 } 1347 } else { 1348 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1349 1350 // go ahead and kick the power state machine into gear if 1351 // we are already idle 1352 schedulePowerEvent(PWR_REF, curTick()); 1353 } 1354 1355 refreshState = REF_RUN; 1356 assert(numBanksActive == 0); 1357 1358 // wait for all banks to be precharged, at which point the 1359 // power state machine will transition to the idle state, and 1360 // automatically move to a refresh, at that point it will also 1361 // call this method to get the refresh event loop going again 1362 return; 1363 } 1364 1365 // last but not least we perform the actual refresh 1366 if (refreshState == REF_RUN) { 1367 // should never get here with any banks active 1368 assert(numBanksActive == 0); 1369 assert(pwrState == PWR_REF); 1370 1371 Tick ref_done_at = curTick() + tRFC; 1372 1373 for (int i = 0; i < ranksPerChannel; i++) { 1374 for (int j = 0; j < banksPerRank; j++) { 1375 banks[i][j].actAllowedAt = ref_done_at; 1376 }
|
| 1377 1378 // at the moment this affects all ranks 1379 DPRINTF(DRAMPower, "%llu,REF,0,%d\n", divCeil(curTick(), tCK), i);
|
1357 } 1358 1359 // make sure we did not wait so long that we cannot make up 1360 // for it 1361 if (refreshDueAt + tREFI < ref_done_at) { 1362 fatal("Refresh was delayed so long we cannot catch up\n"); 1363 } 1364 1365 // compensate for the delay in actually performing the refresh 1366 // when scheduling the next one 1367 schedule(refreshEvent, refreshDueAt + tREFI - tRP); 1368 1369 assert(!powerEvent.scheduled()); 1370 1371 // move to the idle power state once the refresh is done, this 1372 // will also move the refresh state machine to the refresh 1373 // idle state 1374 schedulePowerEvent(PWR_IDLE, ref_done_at); 1375 1376 DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 1377 ref_done_at, refreshDueAt + tREFI); 1378 } 1379} 1380 1381void 1382DRAMCtrl::schedulePowerEvent(PowerState pwr_state, Tick tick) 1383{ 1384 // respect causality 1385 assert(tick >= curTick()); 1386 1387 if (!powerEvent.scheduled()) { 1388 DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 1389 tick, pwr_state); 1390 1391 // insert the new transition 1392 pwrStateTrans = pwr_state; 1393 1394 schedule(powerEvent, tick); 1395 } else { 1396 panic("Scheduled power event at %llu to state %d, " 1397 "with scheduled event at %llu to %d\n", tick, pwr_state, 1398 powerEvent.when(), pwrStateTrans); 1399 } 1400} 1401 1402void 1403DRAMCtrl::processPowerEvent() 1404{ 1405 // remember where we were, and for how long 1406 Tick duration = curTick() - pwrStateTick; 1407 PowerState prev_state = pwrState; 1408 1409 // update the accounting 1410 pwrStateTime[prev_state] += duration; 1411 1412 pwrState = pwrStateTrans; 1413 pwrStateTick = curTick(); 1414 1415 if (pwrState == PWR_IDLE) { 1416 DPRINTF(DRAMState, "All banks precharged\n"); 1417 1418 // if we were refreshing, make sure we start scheduling requests again 1419 if (prev_state == PWR_REF) { 1420 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 1421 assert(pwrState == PWR_IDLE); 1422 1423 // kick things into action again 1424 refreshState = REF_IDLE; 1425 assert(!nextReqEvent.scheduled()); 1426 schedule(nextReqEvent, curTick()); 1427 } else { 1428 assert(prev_state == PWR_ACT); 1429 1430 // if we have a pending refresh, and are now moving to 1431 // the idle state, direclty transition to a refresh 1432 if (refreshState == REF_RUN) { 1433 // there should be nothing waiting at this point 1434 assert(!powerEvent.scheduled()); 1435 1436 // update the state in zero time and proceed below 1437 pwrState = PWR_REF; 1438 } 1439 } 1440 } 1441 1442 // we transition to the refresh state, let the refresh state 1443 // machine know of this state update and let it deal with the 1444 // scheduling of the next power state transition as well as the 1445 // following refresh 1446 if (pwrState == PWR_REF) { 1447 DPRINTF(DRAMState, "Refreshing\n"); 1448 // kick the refresh event loop into action again, and that 1449 // in turn will schedule a transition to the idle power 1450 // state once the refresh is done 1451 assert(refreshState == REF_RUN); 1452 processRefreshEvent(); 1453 } 1454} 1455 1456void 1457DRAMCtrl::regStats() 1458{ 1459 using namespace Stats; 1460 1461 AbstractMemory::regStats(); 1462 1463 readReqs 1464 .name(name() + ".readReqs") 1465 .desc("Number of read requests accepted"); 1466 1467 writeReqs 1468 .name(name() + ".writeReqs") 1469 .desc("Number of write requests accepted"); 1470 1471 readBursts 1472 .name(name() + ".readBursts") 1473 .desc("Number of DRAM read bursts, " 1474 "including those serviced by the write queue"); 1475 1476 writeBursts 1477 .name(name() + ".writeBursts") 1478 .desc("Number of DRAM write bursts, " 1479 "including those merged in the write queue"); 1480 1481 servicedByWrQ 1482 .name(name() + ".servicedByWrQ") 1483 .desc("Number of DRAM read bursts serviced by the write queue"); 1484 1485 mergedWrBursts 1486 .name(name() + ".mergedWrBursts") 1487 .desc("Number of DRAM write bursts merged with an existing one"); 1488 1489 neitherReadNorWrite 1490 .name(name() + ".neitherReadNorWriteReqs") 1491 .desc("Number of requests that are neither read nor write"); 1492 1493 perBankRdBursts 1494 .init(banksPerRank * ranksPerChannel) 1495 .name(name() + ".perBankRdBursts") 1496 .desc("Per bank write bursts"); 1497 1498 perBankWrBursts 1499 .init(banksPerRank * ranksPerChannel) 1500 .name(name() + ".perBankWrBursts") 1501 .desc("Per bank write bursts"); 1502 1503 avgRdQLen 1504 .name(name() + ".avgRdQLen") 1505 .desc("Average read queue length when enqueuing") 1506 .precision(2); 1507 1508 avgWrQLen 1509 .name(name() + ".avgWrQLen") 1510 .desc("Average write queue length when enqueuing") 1511 .precision(2); 1512 1513 totQLat 1514 .name(name() + ".totQLat") 1515 .desc("Total ticks spent queuing"); 1516 1517 totBusLat 1518 .name(name() + ".totBusLat") 1519 .desc("Total ticks spent in databus transfers"); 1520 1521 totMemAccLat 1522 .name(name() + ".totMemAccLat") 1523 .desc("Total ticks spent from burst creation until serviced " 1524 "by the DRAM"); 1525 1526 avgQLat 1527 .name(name() + ".avgQLat") 1528 .desc("Average queueing delay per DRAM burst") 1529 .precision(2); 1530 1531 avgQLat = totQLat / (readBursts - servicedByWrQ); 1532 1533 avgBusLat 1534 .name(name() + ".avgBusLat") 1535 .desc("Average bus latency per DRAM burst") 1536 .precision(2); 1537 1538 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1539 1540 avgMemAccLat 1541 .name(name() + ".avgMemAccLat") 1542 .desc("Average memory access latency per DRAM burst") 1543 .precision(2); 1544 1545 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1546 1547 numRdRetry 1548 .name(name() + ".numRdRetry") 1549 .desc("Number of times read queue was full causing retry"); 1550 1551 numWrRetry 1552 .name(name() + ".numWrRetry") 1553 .desc("Number of times write queue was full causing retry"); 1554 1555 readRowHits 1556 .name(name() + ".readRowHits") 1557 .desc("Number of row buffer hits during reads"); 1558 1559 writeRowHits 1560 .name(name() + ".writeRowHits") 1561 .desc("Number of row buffer hits during writes"); 1562 1563 readRowHitRate 1564 .name(name() + ".readRowHitRate") 1565 .desc("Row buffer hit rate for reads") 1566 .precision(2); 1567 1568 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 1569 1570 writeRowHitRate 1571 .name(name() + ".writeRowHitRate") 1572 .desc("Row buffer hit rate for writes") 1573 .precision(2); 1574 1575 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 1576 1577 readPktSize 1578 .init(ceilLog2(burstSize) + 1) 1579 .name(name() + ".readPktSize") 1580 .desc("Read request sizes (log2)"); 1581 1582 writePktSize 1583 .init(ceilLog2(burstSize) + 1) 1584 .name(name() + ".writePktSize") 1585 .desc("Write request sizes (log2)"); 1586 1587 rdQLenPdf 1588 .init(readBufferSize) 1589 .name(name() + ".rdQLenPdf") 1590 .desc("What read queue length does an incoming req see"); 1591 1592 wrQLenPdf 1593 .init(writeBufferSize) 1594 .name(name() + ".wrQLenPdf") 1595 .desc("What write queue length does an incoming req see"); 1596 1597 bytesPerActivate 1598 .init(maxAccessesPerRow) 1599 .name(name() + ".bytesPerActivate") 1600 .desc("Bytes accessed per row activation") 1601 .flags(nozero); 1602 1603 rdPerTurnAround 1604 .init(readBufferSize) 1605 .name(name() + ".rdPerTurnAround") 1606 .desc("Reads before turning the bus around for writes") 1607 .flags(nozero); 1608 1609 wrPerTurnAround 1610 .init(writeBufferSize) 1611 .name(name() + ".wrPerTurnAround") 1612 .desc("Writes before turning the bus around for reads") 1613 .flags(nozero); 1614 1615 bytesReadDRAM 1616 .name(name() + ".bytesReadDRAM") 1617 .desc("Total number of bytes read from DRAM"); 1618 1619 bytesReadWrQ 1620 .name(name() + ".bytesReadWrQ") 1621 .desc("Total number of bytes read from write queue"); 1622 1623 bytesWritten 1624 .name(name() + ".bytesWritten") 1625 .desc("Total number of bytes written to DRAM"); 1626 1627 bytesReadSys 1628 .name(name() + ".bytesReadSys") 1629 .desc("Total read bytes from the system interface side"); 1630 1631 bytesWrittenSys 1632 .name(name() + ".bytesWrittenSys") 1633 .desc("Total written bytes from the system interface side"); 1634 1635 avgRdBW 1636 .name(name() + ".avgRdBW") 1637 .desc("Average DRAM read bandwidth in MiByte/s") 1638 .precision(2); 1639 1640 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 1641 1642 avgWrBW 1643 .name(name() + ".avgWrBW") 1644 .desc("Average achieved write bandwidth in MiByte/s") 1645 .precision(2); 1646 1647 avgWrBW = (bytesWritten / 1000000) / simSeconds; 1648 1649 avgRdBWSys 1650 .name(name() + ".avgRdBWSys") 1651 .desc("Average system read bandwidth in MiByte/s") 1652 .precision(2); 1653 1654 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 1655 1656 avgWrBWSys 1657 .name(name() + ".avgWrBWSys") 1658 .desc("Average system write bandwidth in MiByte/s") 1659 .precision(2); 1660 1661 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 1662 1663 peakBW 1664 .name(name() + ".peakBW") 1665 .desc("Theoretical peak bandwidth in MiByte/s") 1666 .precision(2); 1667 1668 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 1669 1670 busUtil 1671 .name(name() + ".busUtil") 1672 .desc("Data bus utilization in percentage") 1673 .precision(2); 1674 1675 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 1676 1677 totGap 1678 .name(name() + ".totGap") 1679 .desc("Total gap between requests"); 1680 1681 avgGap 1682 .name(name() + ".avgGap") 1683 .desc("Average gap between requests") 1684 .precision(2); 1685 1686 avgGap = totGap / (readReqs + writeReqs); 1687 1688 // Stats for DRAM Power calculation based on Micron datasheet 1689 busUtilRead 1690 .name(name() + ".busUtilRead") 1691 .desc("Data bus utilization in percentage for reads") 1692 .precision(2); 1693 1694 busUtilRead = avgRdBW / peakBW * 100; 1695 1696 busUtilWrite 1697 .name(name() + ".busUtilWrite") 1698 .desc("Data bus utilization in percentage for writes") 1699 .precision(2); 1700 1701 busUtilWrite = avgWrBW / peakBW * 100; 1702 1703 pageHitRate 1704 .name(name() + ".pageHitRate") 1705 .desc("Row buffer hit rate, read and write combined") 1706 .precision(2); 1707 1708 pageHitRate = (writeRowHits + readRowHits) / 1709 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 1710 1711 pwrStateTime 1712 .init(5) 1713 .name(name() + ".memoryStateTime") 1714 .desc("Time in different power states"); 1715 pwrStateTime.subname(0, "IDLE"); 1716 pwrStateTime.subname(1, "REF"); 1717 pwrStateTime.subname(2, "PRE_PDN"); 1718 pwrStateTime.subname(3, "ACT"); 1719 pwrStateTime.subname(4, "ACT_PDN"); 1720} 1721 1722void 1723DRAMCtrl::recvFunctional(PacketPtr pkt) 1724{ 1725 // rely on the abstract memory 1726 functionalAccess(pkt); 1727} 1728 1729BaseSlavePort& 1730DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 1731{ 1732 if (if_name != "port") { 1733 return MemObject::getSlavePort(if_name, idx); 1734 } else { 1735 return port; 1736 } 1737} 1738 1739unsigned int 1740DRAMCtrl::drain(DrainManager *dm) 1741{ 1742 unsigned int count = port.drain(dm); 1743 1744 // if there is anything in any of our internal queues, keep track 1745 // of that as well 1746 if (!(writeQueue.empty() && readQueue.empty() && 1747 respQueue.empty())) { 1748 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 1749 " resp: %d\n", writeQueue.size(), readQueue.size(), 1750 respQueue.size()); 1751 ++count; 1752 drainManager = dm; 1753 1754 // the only part that is not drained automatically over time 1755 // is the write queue, thus kick things into action if needed 1756 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 1757 schedule(nextReqEvent, curTick()); 1758 } 1759 } 1760 1761 if (count) 1762 setDrainState(Drainable::Draining); 1763 else 1764 setDrainState(Drainable::Drained); 1765 return count; 1766} 1767 1768DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 1769 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 1770 memory(_memory) 1771{ } 1772 1773AddrRangeList 1774DRAMCtrl::MemoryPort::getAddrRanges() const 1775{ 1776 AddrRangeList ranges; 1777 ranges.push_back(memory.getAddrRange()); 1778 return ranges; 1779} 1780 1781void 1782DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 1783{ 1784 pkt->pushLabel(memory.name()); 1785 1786 if (!queue.checkFunctional(pkt)) { 1787 // Default implementation of SimpleTimingPort::recvFunctional() 1788 // calls recvAtomic() and throws away the latency; we can save a 1789 // little here by just not calculating the latency. 1790 memory.recvFunctional(pkt); 1791 } 1792 1793 pkt->popLabel(); 1794} 1795 1796Tick 1797DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 1798{ 1799 return memory.recvAtomic(pkt); 1800} 1801 1802bool 1803DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 1804{ 1805 // pass it to the memory controller 1806 return memory.recvTimingReq(pkt); 1807} 1808 1809DRAMCtrl* 1810DRAMCtrlParams::create() 1811{ 1812 return new DRAMCtrl(this); 1813}
| 1380 } 1381 1382 // make sure we did not wait so long that we cannot make up 1383 // for it 1384 if (refreshDueAt + tREFI < ref_done_at) { 1385 fatal("Refresh was delayed so long we cannot catch up\n"); 1386 } 1387 1388 // compensate for the delay in actually performing the refresh 1389 // when scheduling the next one 1390 schedule(refreshEvent, refreshDueAt + tREFI - tRP); 1391 1392 assert(!powerEvent.scheduled()); 1393 1394 // move to the idle power state once the refresh is done, this 1395 // will also move the refresh state machine to the refresh 1396 // idle state 1397 schedulePowerEvent(PWR_IDLE, ref_done_at); 1398 1399 DPRINTF(DRAMState, "Refresh done at %llu and next refresh at %llu\n", 1400 ref_done_at, refreshDueAt + tREFI); 1401 } 1402} 1403 1404void 1405DRAMCtrl::schedulePowerEvent(PowerState pwr_state, Tick tick) 1406{ 1407 // respect causality 1408 assert(tick >= curTick()); 1409 1410 if (!powerEvent.scheduled()) { 1411 DPRINTF(DRAMState, "Scheduling power event at %llu to state %d\n", 1412 tick, pwr_state); 1413 1414 // insert the new transition 1415 pwrStateTrans = pwr_state; 1416 1417 schedule(powerEvent, tick); 1418 } else { 1419 panic("Scheduled power event at %llu to state %d, " 1420 "with scheduled event at %llu to %d\n", tick, pwr_state, 1421 powerEvent.when(), pwrStateTrans); 1422 } 1423} 1424 1425void 1426DRAMCtrl::processPowerEvent() 1427{ 1428 // remember where we were, and for how long 1429 Tick duration = curTick() - pwrStateTick; 1430 PowerState prev_state = pwrState; 1431 1432 // update the accounting 1433 pwrStateTime[prev_state] += duration; 1434 1435 pwrState = pwrStateTrans; 1436 pwrStateTick = curTick(); 1437 1438 if (pwrState == PWR_IDLE) { 1439 DPRINTF(DRAMState, "All banks precharged\n"); 1440 1441 // if we were refreshing, make sure we start scheduling requests again 1442 if (prev_state == PWR_REF) { 1443 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 1444 assert(pwrState == PWR_IDLE); 1445 1446 // kick things into action again 1447 refreshState = REF_IDLE; 1448 assert(!nextReqEvent.scheduled()); 1449 schedule(nextReqEvent, curTick()); 1450 } else { 1451 assert(prev_state == PWR_ACT); 1452 1453 // if we have a pending refresh, and are now moving to 1454 // the idle state, direclty transition to a refresh 1455 if (refreshState == REF_RUN) { 1456 // there should be nothing waiting at this point 1457 assert(!powerEvent.scheduled()); 1458 1459 // update the state in zero time and proceed below 1460 pwrState = PWR_REF; 1461 } 1462 } 1463 } 1464 1465 // we transition to the refresh state, let the refresh state 1466 // machine know of this state update and let it deal with the 1467 // scheduling of the next power state transition as well as the 1468 // following refresh 1469 if (pwrState == PWR_REF) { 1470 DPRINTF(DRAMState, "Refreshing\n"); 1471 // kick the refresh event loop into action again, and that 1472 // in turn will schedule a transition to the idle power 1473 // state once the refresh is done 1474 assert(refreshState == REF_RUN); 1475 processRefreshEvent(); 1476 } 1477} 1478 1479void 1480DRAMCtrl::regStats() 1481{ 1482 using namespace Stats; 1483 1484 AbstractMemory::regStats(); 1485 1486 readReqs 1487 .name(name() + ".readReqs") 1488 .desc("Number of read requests accepted"); 1489 1490 writeReqs 1491 .name(name() + ".writeReqs") 1492 .desc("Number of write requests accepted"); 1493 1494 readBursts 1495 .name(name() + ".readBursts") 1496 .desc("Number of DRAM read bursts, " 1497 "including those serviced by the write queue"); 1498 1499 writeBursts 1500 .name(name() + ".writeBursts") 1501 .desc("Number of DRAM write bursts, " 1502 "including those merged in the write queue"); 1503 1504 servicedByWrQ 1505 .name(name() + ".servicedByWrQ") 1506 .desc("Number of DRAM read bursts serviced by the write queue"); 1507 1508 mergedWrBursts 1509 .name(name() + ".mergedWrBursts") 1510 .desc("Number of DRAM write bursts merged with an existing one"); 1511 1512 neitherReadNorWrite 1513 .name(name() + ".neitherReadNorWriteReqs") 1514 .desc("Number of requests that are neither read nor write"); 1515 1516 perBankRdBursts 1517 .init(banksPerRank * ranksPerChannel) 1518 .name(name() + ".perBankRdBursts") 1519 .desc("Per bank write bursts"); 1520 1521 perBankWrBursts 1522 .init(banksPerRank * ranksPerChannel) 1523 .name(name() + ".perBankWrBursts") 1524 .desc("Per bank write bursts"); 1525 1526 avgRdQLen 1527 .name(name() + ".avgRdQLen") 1528 .desc("Average read queue length when enqueuing") 1529 .precision(2); 1530 1531 avgWrQLen 1532 .name(name() + ".avgWrQLen") 1533 .desc("Average write queue length when enqueuing") 1534 .precision(2); 1535 1536 totQLat 1537 .name(name() + ".totQLat") 1538 .desc("Total ticks spent queuing"); 1539 1540 totBusLat 1541 .name(name() + ".totBusLat") 1542 .desc("Total ticks spent in databus transfers"); 1543 1544 totMemAccLat 1545 .name(name() + ".totMemAccLat") 1546 .desc("Total ticks spent from burst creation until serviced " 1547 "by the DRAM"); 1548 1549 avgQLat 1550 .name(name() + ".avgQLat") 1551 .desc("Average queueing delay per DRAM burst") 1552 .precision(2); 1553 1554 avgQLat = totQLat / (readBursts - servicedByWrQ); 1555 1556 avgBusLat 1557 .name(name() + ".avgBusLat") 1558 .desc("Average bus latency per DRAM burst") 1559 .precision(2); 1560 1561 avgBusLat = totBusLat / (readBursts - servicedByWrQ); 1562 1563 avgMemAccLat 1564 .name(name() + ".avgMemAccLat") 1565 .desc("Average memory access latency per DRAM burst") 1566 .precision(2); 1567 1568 avgMemAccLat = totMemAccLat / (readBursts - servicedByWrQ); 1569 1570 numRdRetry 1571 .name(name() + ".numRdRetry") 1572 .desc("Number of times read queue was full causing retry"); 1573 1574 numWrRetry 1575 .name(name() + ".numWrRetry") 1576 .desc("Number of times write queue was full causing retry"); 1577 1578 readRowHits 1579 .name(name() + ".readRowHits") 1580 .desc("Number of row buffer hits during reads"); 1581 1582 writeRowHits 1583 .name(name() + ".writeRowHits") 1584 .desc("Number of row buffer hits during writes"); 1585 1586 readRowHitRate 1587 .name(name() + ".readRowHitRate") 1588 .desc("Row buffer hit rate for reads") 1589 .precision(2); 1590 1591 readRowHitRate = (readRowHits / (readBursts - servicedByWrQ)) * 100; 1592 1593 writeRowHitRate 1594 .name(name() + ".writeRowHitRate") 1595 .desc("Row buffer hit rate for writes") 1596 .precision(2); 1597 1598 writeRowHitRate = (writeRowHits / (writeBursts - mergedWrBursts)) * 100; 1599 1600 readPktSize 1601 .init(ceilLog2(burstSize) + 1) 1602 .name(name() + ".readPktSize") 1603 .desc("Read request sizes (log2)"); 1604 1605 writePktSize 1606 .init(ceilLog2(burstSize) + 1) 1607 .name(name() + ".writePktSize") 1608 .desc("Write request sizes (log2)"); 1609 1610 rdQLenPdf 1611 .init(readBufferSize) 1612 .name(name() + ".rdQLenPdf") 1613 .desc("What read queue length does an incoming req see"); 1614 1615 wrQLenPdf 1616 .init(writeBufferSize) 1617 .name(name() + ".wrQLenPdf") 1618 .desc("What write queue length does an incoming req see"); 1619 1620 bytesPerActivate 1621 .init(maxAccessesPerRow) 1622 .name(name() + ".bytesPerActivate") 1623 .desc("Bytes accessed per row activation") 1624 .flags(nozero); 1625 1626 rdPerTurnAround 1627 .init(readBufferSize) 1628 .name(name() + ".rdPerTurnAround") 1629 .desc("Reads before turning the bus around for writes") 1630 .flags(nozero); 1631 1632 wrPerTurnAround 1633 .init(writeBufferSize) 1634 .name(name() + ".wrPerTurnAround") 1635 .desc("Writes before turning the bus around for reads") 1636 .flags(nozero); 1637 1638 bytesReadDRAM 1639 .name(name() + ".bytesReadDRAM") 1640 .desc("Total number of bytes read from DRAM"); 1641 1642 bytesReadWrQ 1643 .name(name() + ".bytesReadWrQ") 1644 .desc("Total number of bytes read from write queue"); 1645 1646 bytesWritten 1647 .name(name() + ".bytesWritten") 1648 .desc("Total number of bytes written to DRAM"); 1649 1650 bytesReadSys 1651 .name(name() + ".bytesReadSys") 1652 .desc("Total read bytes from the system interface side"); 1653 1654 bytesWrittenSys 1655 .name(name() + ".bytesWrittenSys") 1656 .desc("Total written bytes from the system interface side"); 1657 1658 avgRdBW 1659 .name(name() + ".avgRdBW") 1660 .desc("Average DRAM read bandwidth in MiByte/s") 1661 .precision(2); 1662 1663 avgRdBW = (bytesReadDRAM / 1000000) / simSeconds; 1664 1665 avgWrBW 1666 .name(name() + ".avgWrBW") 1667 .desc("Average achieved write bandwidth in MiByte/s") 1668 .precision(2); 1669 1670 avgWrBW = (bytesWritten / 1000000) / simSeconds; 1671 1672 avgRdBWSys 1673 .name(name() + ".avgRdBWSys") 1674 .desc("Average system read bandwidth in MiByte/s") 1675 .precision(2); 1676 1677 avgRdBWSys = (bytesReadSys / 1000000) / simSeconds; 1678 1679 avgWrBWSys 1680 .name(name() + ".avgWrBWSys") 1681 .desc("Average system write bandwidth in MiByte/s") 1682 .precision(2); 1683 1684 avgWrBWSys = (bytesWrittenSys / 1000000) / simSeconds; 1685 1686 peakBW 1687 .name(name() + ".peakBW") 1688 .desc("Theoretical peak bandwidth in MiByte/s") 1689 .precision(2); 1690 1691 peakBW = (SimClock::Frequency / tBURST) * burstSize / 1000000; 1692 1693 busUtil 1694 .name(name() + ".busUtil") 1695 .desc("Data bus utilization in percentage") 1696 .precision(2); 1697 1698 busUtil = (avgRdBW + avgWrBW) / peakBW * 100; 1699 1700 totGap 1701 .name(name() + ".totGap") 1702 .desc("Total gap between requests"); 1703 1704 avgGap 1705 .name(name() + ".avgGap") 1706 .desc("Average gap between requests") 1707 .precision(2); 1708 1709 avgGap = totGap / (readReqs + writeReqs); 1710 1711 // Stats for DRAM Power calculation based on Micron datasheet 1712 busUtilRead 1713 .name(name() + ".busUtilRead") 1714 .desc("Data bus utilization in percentage for reads") 1715 .precision(2); 1716 1717 busUtilRead = avgRdBW / peakBW * 100; 1718 1719 busUtilWrite 1720 .name(name() + ".busUtilWrite") 1721 .desc("Data bus utilization in percentage for writes") 1722 .precision(2); 1723 1724 busUtilWrite = avgWrBW / peakBW * 100; 1725 1726 pageHitRate 1727 .name(name() + ".pageHitRate") 1728 .desc("Row buffer hit rate, read and write combined") 1729 .precision(2); 1730 1731 pageHitRate = (writeRowHits + readRowHits) / 1732 (writeBursts - mergedWrBursts + readBursts - servicedByWrQ) * 100; 1733 1734 pwrStateTime 1735 .init(5) 1736 .name(name() + ".memoryStateTime") 1737 .desc("Time in different power states"); 1738 pwrStateTime.subname(0, "IDLE"); 1739 pwrStateTime.subname(1, "REF"); 1740 pwrStateTime.subname(2, "PRE_PDN"); 1741 pwrStateTime.subname(3, "ACT"); 1742 pwrStateTime.subname(4, "ACT_PDN"); 1743} 1744 1745void 1746DRAMCtrl::recvFunctional(PacketPtr pkt) 1747{ 1748 // rely on the abstract memory 1749 functionalAccess(pkt); 1750} 1751 1752BaseSlavePort& 1753DRAMCtrl::getSlavePort(const string &if_name, PortID idx) 1754{ 1755 if (if_name != "port") { 1756 return MemObject::getSlavePort(if_name, idx); 1757 } else { 1758 return port; 1759 } 1760} 1761 1762unsigned int 1763DRAMCtrl::drain(DrainManager *dm) 1764{ 1765 unsigned int count = port.drain(dm); 1766 1767 // if there is anything in any of our internal queues, keep track 1768 // of that as well 1769 if (!(writeQueue.empty() && readQueue.empty() && 1770 respQueue.empty())) { 1771 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 1772 " resp: %d\n", writeQueue.size(), readQueue.size(), 1773 respQueue.size()); 1774 ++count; 1775 drainManager = dm; 1776 1777 // the only part that is not drained automatically over time 1778 // is the write queue, thus kick things into action if needed 1779 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 1780 schedule(nextReqEvent, curTick()); 1781 } 1782 } 1783 1784 if (count) 1785 setDrainState(Drainable::Draining); 1786 else 1787 setDrainState(Drainable::Drained); 1788 return count; 1789} 1790 1791DRAMCtrl::MemoryPort::MemoryPort(const std::string& name, DRAMCtrl& _memory) 1792 : QueuedSlavePort(name, &_memory, queue), queue(_memory, *this), 1793 memory(_memory) 1794{ } 1795 1796AddrRangeList 1797DRAMCtrl::MemoryPort::getAddrRanges() const 1798{ 1799 AddrRangeList ranges; 1800 ranges.push_back(memory.getAddrRange()); 1801 return ranges; 1802} 1803 1804void 1805DRAMCtrl::MemoryPort::recvFunctional(PacketPtr pkt) 1806{ 1807 pkt->pushLabel(memory.name()); 1808 1809 if (!queue.checkFunctional(pkt)) { 1810 // Default implementation of SimpleTimingPort::recvFunctional() 1811 // calls recvAtomic() and throws away the latency; we can save a 1812 // little here by just not calculating the latency. 1813 memory.recvFunctional(pkt); 1814 } 1815 1816 pkt->popLabel(); 1817} 1818 1819Tick 1820DRAMCtrl::MemoryPort::recvAtomic(PacketPtr pkt) 1821{ 1822 return memory.recvAtomic(pkt); 1823} 1824 1825bool 1826DRAMCtrl::MemoryPort::recvTimingReq(PacketPtr pkt) 1827{ 1828 // pass it to the memory controller 1829 return memory.recvTimingReq(pkt); 1830} 1831 1832DRAMCtrl* 1833DRAMCtrlParams::create() 1834{ 1835 return new DRAMCtrl(this); 1836}
|