1/* 2 * Copyright (c) 2012-2013, 2018-2019 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2003-2005 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Erik Hallnor 41 * Nikos Nikoleris 42 */ 43 44/** 45 * @file 46 * Definition of BaseCache functions. 47 */ 48 49#include "mem/cache/base.hh" 50 51#include "base/compiler.hh" 52#include "base/logging.hh" 53#include "debug/Cache.hh"
| 1/* 2 * Copyright (c) 2012-2013, 2018-2019 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2003-2005 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Erik Hallnor 41 * Nikos Nikoleris 42 */ 43 44/** 45 * @file 46 * Definition of BaseCache functions. 47 */ 48 49#include "mem/cache/base.hh" 50 51#include "base/compiler.hh" 52#include "base/logging.hh" 53#include "debug/Cache.hh"
|
| 54#include "debug/CacheComp.hh"
|
54#include "debug/CachePort.hh" 55#include "debug/CacheRepl.hh" 56#include "debug/CacheVerbose.hh" 57#include "mem/cache/compressors/base.hh" 58#include "mem/cache/mshr.hh" 59#include "mem/cache/prefetch/base.hh" 60#include "mem/cache/queue_entry.hh"
| 55#include "debug/CachePort.hh" 56#include "debug/CacheRepl.hh" 57#include "debug/CacheVerbose.hh" 58#include "mem/cache/compressors/base.hh" 59#include "mem/cache/mshr.hh" 60#include "mem/cache/prefetch/base.hh" 61#include "mem/cache/queue_entry.hh"
|
| 62#include "mem/cache/tags/super_blk.hh"
|
61#include "params/BaseCache.hh" 62#include "params/WriteAllocator.hh" 63#include "sim/core.hh" 64 65class BaseMasterPort; 66class BaseSlavePort; 67 68using namespace std; 69 70BaseCache::CacheSlavePort::CacheSlavePort(const std::string &_name, 71 BaseCache *_cache, 72 const std::string &_label) 73 : QueuedSlavePort(_name, _cache, queue), 74 queue(*_cache, *this, true, _label), 75 blocked(false), mustSendRetry(false), 76 sendRetryEvent([this]{ processSendRetry(); }, _name) 77{ 78} 79 80BaseCache::BaseCache(const BaseCacheParams *p, unsigned blk_size) 81 : ClockedObject(p), 82 cpuSidePort (p->name + ".cpu_side", this, "CpuSidePort"), 83 memSidePort(p->name + ".mem_side", this, "MemSidePort"), 84 mshrQueue("MSHRs", p->mshrs, 0, p->demand_mshr_reserve), // see below 85 writeBuffer("write buffer", p->write_buffers, p->mshrs), // see below 86 tags(p->tags), 87 compressor(p->compressor), 88 prefetcher(p->prefetcher), 89 writeAllocator(p->write_allocator), 90 writebackClean(p->writeback_clean), 91 tempBlockWriteback(nullptr), 92 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); }, 93 name(), false, 94 EventBase::Delayed_Writeback_Pri), 95 blkSize(blk_size), 96 lookupLatency(p->tag_latency), 97 dataLatency(p->data_latency), 98 forwardLatency(p->tag_latency), 99 fillLatency(p->data_latency), 100 responseLatency(p->response_latency), 101 sequentialAccess(p->sequential_access), 102 numTarget(p->tgts_per_mshr), 103 forwardSnoops(true), 104 clusivity(p->clusivity), 105 isReadOnly(p->is_read_only), 106 blocked(0), 107 order(0), 108 noTargetMSHR(nullptr), 109 missCount(p->max_miss_count), 110 addrRanges(p->addr_ranges.begin(), p->addr_ranges.end()), 111 system(p->system) 112{ 113 // the MSHR queue has no reserve entries as we check the MSHR 114 // queue on every single allocation, whereas the write queue has 115 // as many reserve entries as we have MSHRs, since every MSHR may 116 // eventually require a writeback, and we do not check the write 117 // buffer before committing to an MSHR 118 119 // forward snoops is overridden in init() once we can query 120 // whether the connected master is actually snooping or not 121 122 tempBlock = new TempCacheBlk(blkSize); 123 124 tags->tagsInit(); 125 if (prefetcher) 126 prefetcher->setCache(this); 127} 128 129BaseCache::~BaseCache() 130{ 131 delete tempBlock; 132} 133 134void 135BaseCache::CacheSlavePort::setBlocked() 136{ 137 assert(!blocked); 138 DPRINTF(CachePort, "Port is blocking new requests\n"); 139 blocked = true; 140 // if we already scheduled a retry in this cycle, but it has not yet 141 // happened, cancel it 142 if (sendRetryEvent.scheduled()) { 143 owner.deschedule(sendRetryEvent); 144 DPRINTF(CachePort, "Port descheduled retry\n"); 145 mustSendRetry = true; 146 } 147} 148 149void 150BaseCache::CacheSlavePort::clearBlocked() 151{ 152 assert(blocked); 153 DPRINTF(CachePort, "Port is accepting new requests\n"); 154 blocked = false; 155 if (mustSendRetry) { 156 // @TODO: need to find a better time (next cycle?) 157 owner.schedule(sendRetryEvent, curTick() + 1); 158 } 159} 160 161void 162BaseCache::CacheSlavePort::processSendRetry() 163{ 164 DPRINTF(CachePort, "Port is sending retry\n"); 165 166 // reset the flag and call retry 167 mustSendRetry = false; 168 sendRetryReq(); 169} 170 171Addr 172BaseCache::regenerateBlkAddr(CacheBlk* blk) 173{ 174 if (blk != tempBlock) { 175 return tags->regenerateBlkAddr(blk); 176 } else { 177 return tempBlock->getAddr(); 178 } 179} 180 181void 182BaseCache::init() 183{ 184 if (!cpuSidePort.isConnected() || !memSidePort.isConnected()) 185 fatal("Cache ports on %s are not connected\n", name()); 186 cpuSidePort.sendRangeChange(); 187 forwardSnoops = cpuSidePort.isSnooping(); 188} 189 190Port & 191BaseCache::getPort(const std::string &if_name, PortID idx) 192{ 193 if (if_name == "mem_side") { 194 return memSidePort; 195 } else if (if_name == "cpu_side") { 196 return cpuSidePort; 197 } else { 198 return ClockedObject::getPort(if_name, idx); 199 } 200} 201 202bool 203BaseCache::inRange(Addr addr) const 204{ 205 for (const auto& r : addrRanges) { 206 if (r.contains(addr)) { 207 return true; 208 } 209 } 210 return false; 211} 212 213void 214BaseCache::handleTimingReqHit(PacketPtr pkt, CacheBlk *blk, Tick request_time) 215{ 216 if (pkt->needsResponse()) { 217 // These delays should have been consumed by now 218 assert(pkt->headerDelay == 0); 219 assert(pkt->payloadDelay == 0); 220 221 pkt->makeTimingResponse(); 222 223 // In this case we are considering request_time that takes 224 // into account the delay of the xbar, if any, and just 225 // lat, neglecting responseLatency, modelling hit latency 226 // just as the value of lat overriden by access(), which calls 227 // the calculateAccessLatency() function. 228 cpuSidePort.schedTimingResp(pkt, request_time); 229 } else { 230 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__, 231 pkt->print()); 232 233 // queue the packet for deletion, as the sending cache is 234 // still relying on it; if the block is found in access(), 235 // CleanEvict and Writeback messages will be deleted 236 // here as well 237 pendingDelete.reset(pkt); 238 } 239} 240 241void 242BaseCache::handleTimingReqMiss(PacketPtr pkt, MSHR *mshr, CacheBlk *blk, 243 Tick forward_time, Tick request_time) 244{ 245 if (writeAllocator && 246 pkt && pkt->isWrite() && !pkt->req->isUncacheable()) { 247 writeAllocator->updateMode(pkt->getAddr(), pkt->getSize(), 248 pkt->getBlockAddr(blkSize)); 249 } 250 251 if (mshr) { 252 /// MSHR hit 253 /// @note writebacks will be checked in getNextMSHR() 254 /// for any conflicting requests to the same block 255 256 //@todo remove hw_pf here 257 258 // Coalesce unless it was a software prefetch (see above). 259 if (pkt) { 260 assert(!pkt->isWriteback()); 261 // CleanEvicts corresponding to blocks which have 262 // outstanding requests in MSHRs are simply sunk here 263 if (pkt->cmd == MemCmd::CleanEvict) { 264 pendingDelete.reset(pkt); 265 } else if (pkt->cmd == MemCmd::WriteClean) { 266 // A WriteClean should never coalesce with any 267 // outstanding cache maintenance requests. 268 269 // We use forward_time here because there is an 270 // uncached memory write, forwarded to WriteBuffer. 271 allocateWriteBuffer(pkt, forward_time); 272 } else { 273 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__, 274 pkt->print()); 275 276 assert(pkt->req->masterId() < system->maxMasters()); 277 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++; 278 279 // We use forward_time here because it is the same 280 // considering new targets. We have multiple 281 // requests for the same address here. It 282 // specifies the latency to allocate an internal 283 // buffer and to schedule an event to the queued 284 // port and also takes into account the additional 285 // delay of the xbar. 286 mshr->allocateTarget(pkt, forward_time, order++, 287 allocOnFill(pkt->cmd)); 288 if (mshr->getNumTargets() == numTarget) { 289 noTargetMSHR = mshr; 290 setBlocked(Blocked_NoTargets); 291 // need to be careful with this... if this mshr isn't 292 // ready yet (i.e. time > curTick()), we don't want to 293 // move it ahead of mshrs that are ready 294 // mshrQueue.moveToFront(mshr); 295 } 296 } 297 } 298 } else { 299 // no MSHR 300 assert(pkt->req->masterId() < system->maxMasters()); 301 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 302 303 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean) { 304 // We use forward_time here because there is an 305 // writeback or writeclean, forwarded to WriteBuffer. 306 allocateWriteBuffer(pkt, forward_time); 307 } else { 308 if (blk && blk->isValid()) { 309 // If we have a write miss to a valid block, we 310 // need to mark the block non-readable. Otherwise 311 // if we allow reads while there's an outstanding 312 // write miss, the read could return stale data 313 // out of the cache block... a more aggressive 314 // system could detect the overlap (if any) and 315 // forward data out of the MSHRs, but we don't do 316 // that yet. Note that we do need to leave the 317 // block valid so that it stays in the cache, in 318 // case we get an upgrade response (and hence no 319 // new data) when the write miss completes. 320 // As long as CPUs do proper store/load forwarding 321 // internally, and have a sufficiently weak memory 322 // model, this is probably unnecessary, but at some 323 // point it must have seemed like we needed it... 324 assert((pkt->needsWritable() && !blk->isWritable()) || 325 pkt->req->isCacheMaintenance()); 326 blk->status &= ~BlkReadable; 327 } 328 // Here we are using forward_time, modelling the latency of 329 // a miss (outbound) just as forwardLatency, neglecting the 330 // lookupLatency component. 331 allocateMissBuffer(pkt, forward_time); 332 } 333 } 334} 335 336void 337BaseCache::recvTimingReq(PacketPtr pkt) 338{ 339 // anything that is merely forwarded pays for the forward latency and 340 // the delay provided by the crossbar 341 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 342 343 Cycles lat; 344 CacheBlk *blk = nullptr; 345 bool satisfied = false; 346 { 347 PacketList writebacks; 348 // Note that lat is passed by reference here. The function 349 // access() will set the lat value. 350 satisfied = access(pkt, blk, lat, writebacks); 351 352 // After the evicted blocks are selected, they must be forwarded 353 // to the write buffer to ensure they logically precede anything 354 // happening below 355 doWritebacks(writebacks, clockEdge(lat + forwardLatency)); 356 } 357 358 // Here we charge the headerDelay that takes into account the latencies 359 // of the bus, if the packet comes from it. 360 // The latency charged is just the value set by the access() function. 361 // In case of a hit we are neglecting response latency. 362 // In case of a miss we are neglecting forward latency. 363 Tick request_time = clockEdge(lat); 364 // Here we reset the timing of the packet. 365 pkt->headerDelay = pkt->payloadDelay = 0; 366 367 if (satisfied) { 368 // notify before anything else as later handleTimingReqHit might turn 369 // the packet in a response 370 ppHit->notify(pkt); 371 372 if (prefetcher && blk && blk->wasPrefetched()) { 373 blk->status &= ~BlkHWPrefetched; 374 } 375 376 handleTimingReqHit(pkt, blk, request_time); 377 } else { 378 handleTimingReqMiss(pkt, blk, forward_time, request_time); 379 380 ppMiss->notify(pkt); 381 } 382 383 if (prefetcher) { 384 // track time of availability of next prefetch, if any 385 Tick next_pf_time = prefetcher->nextPrefetchReadyTime(); 386 if (next_pf_time != MaxTick) { 387 schedMemSideSendEvent(next_pf_time); 388 } 389 } 390} 391 392void 393BaseCache::handleUncacheableWriteResp(PacketPtr pkt) 394{ 395 Tick completion_time = clockEdge(responseLatency) + 396 pkt->headerDelay + pkt->payloadDelay; 397 398 // Reset the bus additional time as it is now accounted for 399 pkt->headerDelay = pkt->payloadDelay = 0; 400 401 cpuSidePort.schedTimingResp(pkt, completion_time); 402} 403 404void 405BaseCache::recvTimingResp(PacketPtr pkt) 406{ 407 assert(pkt->isResponse()); 408 409 // all header delay should be paid for by the crossbar, unless 410 // this is a prefetch response from above 411 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 412 "%s saw a non-zero packet delay\n", name()); 413 414 const bool is_error = pkt->isError(); 415 416 if (is_error) { 417 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__, 418 pkt->print()); 419 } 420 421 DPRINTF(Cache, "%s: Handling response %s\n", __func__, 422 pkt->print()); 423 424 // if this is a write, we should be looking at an uncacheable 425 // write 426 if (pkt->isWrite()) { 427 assert(pkt->req->isUncacheable()); 428 handleUncacheableWriteResp(pkt); 429 return; 430 } 431 432 // we have dealt with any (uncacheable) writes above, from here on 433 // we know we are dealing with an MSHR due to a miss or a prefetch 434 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState()); 435 assert(mshr); 436 437 if (mshr == noTargetMSHR) { 438 // we always clear at least one target 439 clearBlocked(Blocked_NoTargets); 440 noTargetMSHR = nullptr; 441 } 442 443 // Initial target is used just for stats 444 QueueEntry::Target *initial_tgt = mshr->getTarget(); 445 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 446 Tick miss_latency = curTick() - initial_tgt->recvTime; 447 448 if (pkt->req->isUncacheable()) { 449 assert(pkt->req->masterId() < system->maxMasters()); 450 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 451 miss_latency; 452 } else { 453 assert(pkt->req->masterId() < system->maxMasters()); 454 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 455 miss_latency; 456 } 457 458 PacketList writebacks; 459 460 bool is_fill = !mshr->isForward && 461 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp || 462 mshr->wasWholeLineWrite); 463 464 // make sure that if the mshr was due to a whole line write then 465 // the response is an invalidation 466 assert(!mshr->wasWholeLineWrite || pkt->isInvalidate()); 467 468 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 469 470 if (is_fill && !is_error) { 471 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n", 472 pkt->getAddr()); 473 474 const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ? 475 writeAllocator->allocate() : mshr->allocOnFill(); 476 blk = handleFill(pkt, blk, writebacks, allocate); 477 assert(blk != nullptr); 478 ppFill->notify(pkt); 479 } 480 481 if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) { 482 // The block was marked not readable while there was a pending 483 // cache maintenance operation, restore its flag. 484 blk->status |= BlkReadable; 485 486 // This was a cache clean operation (without invalidate) 487 // and we have a copy of the block already. Since there 488 // is no invalidation, we can promote targets that don't 489 // require a writable copy 490 mshr->promoteReadable(); 491 } 492 493 if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) { 494 // If at this point the referenced block is writable and the 495 // response is not a cache invalidate, we promote targets that 496 // were deferred as we couldn't guarrantee a writable copy 497 mshr->promoteWritable(); 498 } 499 500 serviceMSHRTargets(mshr, pkt, blk); 501 502 if (mshr->promoteDeferredTargets()) { 503 // avoid later read getting stale data while write miss is 504 // outstanding.. see comment in timingAccess() 505 if (blk) { 506 blk->status &= ~BlkReadable; 507 } 508 mshrQueue.markPending(mshr); 509 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 510 } else { 511 // while we deallocate an mshr from the queue we still have to 512 // check the isFull condition before and after as we might 513 // have been using the reserved entries already 514 const bool was_full = mshrQueue.isFull(); 515 mshrQueue.deallocate(mshr); 516 if (was_full && !mshrQueue.isFull()) { 517 clearBlocked(Blocked_NoMSHRs); 518 } 519 520 // Request the bus for a prefetch if this deallocation freed enough 521 // MSHRs for a prefetch to take place 522 if (prefetcher && mshrQueue.canPrefetch()) { 523 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 524 clockEdge()); 525 if (next_pf_time != MaxTick) 526 schedMemSideSendEvent(next_pf_time); 527 } 528 } 529 530 // if we used temp block, check to see if its valid and then clear it out 531 if (blk == tempBlock && tempBlock->isValid()) { 532 evictBlock(blk, writebacks); 533 } 534 535 const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 536 // copy writebacks to write buffer 537 doWritebacks(writebacks, forward_time); 538 539 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print()); 540 delete pkt; 541} 542 543 544Tick 545BaseCache::recvAtomic(PacketPtr pkt) 546{ 547 // should assert here that there are no outstanding MSHRs or 548 // writebacks... that would mean that someone used an atomic 549 // access in timing mode 550 551 // We use lookupLatency here because it is used to specify the latency 552 // to access. 553 Cycles lat = lookupLatency; 554 555 CacheBlk *blk = nullptr; 556 PacketList writebacks; 557 bool satisfied = access(pkt, blk, lat, writebacks); 558 559 if (pkt->isClean() && blk && blk->isDirty()) { 560 // A cache clean opearation is looking for a dirty 561 // block. If a dirty block is encountered a WriteClean 562 // will update any copies to the path to the memory 563 // until the point of reference. 564 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 565 __func__, pkt->print(), blk->print()); 566 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id); 567 writebacks.push_back(wb_pkt); 568 pkt->setSatisfied(); 569 } 570 571 // handle writebacks resulting from the access here to ensure they 572 // logically precede anything happening below 573 doWritebacksAtomic(writebacks); 574 assert(writebacks.empty()); 575 576 if (!satisfied) { 577 lat += handleAtomicReqMiss(pkt, blk, writebacks); 578 } 579 580 // Note that we don't invoke the prefetcher at all in atomic mode. 581 // It's not clear how to do it properly, particularly for 582 // prefetchers that aggressively generate prefetch candidates and 583 // rely on bandwidth contention to throttle them; these will tend 584 // to pollute the cache in atomic mode since there is no bandwidth 585 // contention. If we ever do want to enable prefetching in atomic 586 // mode, though, this is the place to do it... see timingAccess() 587 // for an example (though we'd want to issue the prefetch(es) 588 // immediately rather than calling requestMemSideBus() as we do 589 // there). 590 591 // do any writebacks resulting from the response handling 592 doWritebacksAtomic(writebacks); 593 594 // if we used temp block, check to see if its valid and if so 595 // clear it out, but only do so after the call to recvAtomic is 596 // finished so that any downstream observers (such as a snoop 597 // filter), first see the fill, and only then see the eviction 598 if (blk == tempBlock && tempBlock->isValid()) { 599 // the atomic CPU calls recvAtomic for fetch and load/store 600 // sequentuially, and we may already have a tempBlock 601 // writeback from the fetch that we have not yet sent 602 if (tempBlockWriteback) { 603 // if that is the case, write the prevoius one back, and 604 // do not schedule any new event 605 writebackTempBlockAtomic(); 606 } else { 607 // the writeback/clean eviction happens after the call to 608 // recvAtomic has finished (but before any successive 609 // calls), so that the response handling from the fill is 610 // allowed to happen first 611 schedule(writebackTempBlockAtomicEvent, curTick()); 612 } 613 614 tempBlockWriteback = evictBlock(blk); 615 } 616 617 if (pkt->needsResponse()) { 618 pkt->makeAtomicResponse(); 619 } 620 621 return lat * clockPeriod(); 622} 623 624void 625BaseCache::functionalAccess(PacketPtr pkt, bool from_cpu_side) 626{ 627 Addr blk_addr = pkt->getBlockAddr(blkSize); 628 bool is_secure = pkt->isSecure(); 629 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure); 630 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure); 631 632 pkt->pushLabel(name()); 633 634 CacheBlkPrintWrapper cbpw(blk); 635 636 // Note that just because an L2/L3 has valid data doesn't mean an 637 // L1 doesn't have a more up-to-date modified copy that still 638 // needs to be found. As a result we always update the request if 639 // we have it, but only declare it satisfied if we are the owner. 640 641 // see if we have data at all (owned or otherwise) 642 bool have_data = blk && blk->isValid() 643 && pkt->trySatisfyFunctional(&cbpw, blk_addr, is_secure, blkSize, 644 blk->data); 645 646 // data we have is dirty if marked as such or if we have an 647 // in-service MSHR that is pending a modified line 648 bool have_dirty = 649 have_data && (blk->isDirty() || 650 (mshr && mshr->inService && mshr->isPendingModified())); 651 652 bool done = have_dirty || 653 cpuSidePort.trySatisfyFunctional(pkt) || 654 mshrQueue.trySatisfyFunctional(pkt) || 655 writeBuffer.trySatisfyFunctional(pkt) || 656 memSidePort.trySatisfyFunctional(pkt); 657 658 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(), 659 (blk && blk->isValid()) ? "valid " : "", 660 have_data ? "data " : "", done ? "done " : ""); 661 662 // We're leaving the cache, so pop cache->name() label 663 pkt->popLabel(); 664 665 if (done) { 666 pkt->makeResponse(); 667 } else { 668 // if it came as a request from the CPU side then make sure it 669 // continues towards the memory side 670 if (from_cpu_side) { 671 memSidePort.sendFunctional(pkt); 672 } else if (cpuSidePort.isSnooping()) { 673 // if it came from the memory side, it must be a snoop request 674 // and we should only forward it if we are forwarding snoops 675 cpuSidePort.sendFunctionalSnoop(pkt); 676 } 677 } 678} 679 680 681void 682BaseCache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt) 683{ 684 assert(pkt->isRequest()); 685 686 uint64_t overwrite_val; 687 bool overwrite_mem; 688 uint64_t condition_val64; 689 uint32_t condition_val32; 690 691 int offset = pkt->getOffset(blkSize); 692 uint8_t *blk_data = blk->data + offset; 693 694 assert(sizeof(uint64_t) >= pkt->getSize()); 695 696 overwrite_mem = true; 697 // keep a copy of our possible write value, and copy what is at the 698 // memory address into the packet 699 pkt->writeData((uint8_t *)&overwrite_val); 700 pkt->setData(blk_data); 701 702 if (pkt->req->isCondSwap()) { 703 if (pkt->getSize() == sizeof(uint64_t)) { 704 condition_val64 = pkt->req->getExtraData(); 705 overwrite_mem = !std::memcmp(&condition_val64, blk_data, 706 sizeof(uint64_t)); 707 } else if (pkt->getSize() == sizeof(uint32_t)) { 708 condition_val32 = (uint32_t)pkt->req->getExtraData(); 709 overwrite_mem = !std::memcmp(&condition_val32, blk_data, 710 sizeof(uint32_t)); 711 } else 712 panic("Invalid size for conditional read/write\n"); 713 } 714 715 if (overwrite_mem) { 716 std::memcpy(blk_data, &overwrite_val, pkt->getSize()); 717 blk->status |= BlkDirty; 718 } 719} 720 721QueueEntry* 722BaseCache::getNextQueueEntry() 723{ 724 // Check both MSHR queue and write buffer for potential requests, 725 // note that null does not mean there is no request, it could 726 // simply be that it is not ready 727 MSHR *miss_mshr = mshrQueue.getNext(); 728 WriteQueueEntry *wq_entry = writeBuffer.getNext(); 729 730 // If we got a write buffer request ready, first priority is a 731 // full write buffer, otherwise we favour the miss requests 732 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) { 733 // need to search MSHR queue for conflicting earlier miss. 734 MSHR *conflict_mshr = mshrQueue.findPending(wq_entry); 735 736 if (conflict_mshr && conflict_mshr->order < wq_entry->order) { 737 // Service misses in order until conflict is cleared. 738 return conflict_mshr; 739 740 // @todo Note that we ignore the ready time of the conflict here 741 } 742 743 // No conflicts; issue write 744 return wq_entry; 745 } else if (miss_mshr) { 746 // need to check for conflicting earlier writeback 747 WriteQueueEntry *conflict_mshr = writeBuffer.findPending(miss_mshr); 748 if (conflict_mshr) { 749 // not sure why we don't check order here... it was in the 750 // original code but commented out. 751 752 // The only way this happens is if we are 753 // doing a write and we didn't have permissions 754 // then subsequently saw a writeback (owned got evicted) 755 // We need to make sure to perform the writeback first 756 // To preserve the dirty data, then we can issue the write 757 758 // should we return wq_entry here instead? I.e. do we 759 // have to flush writes in order? I don't think so... not 760 // for Alpha anyway. Maybe for x86? 761 return conflict_mshr; 762 763 // @todo Note that we ignore the ready time of the conflict here 764 } 765 766 // No conflicts; issue read 767 return miss_mshr; 768 } 769 770 // fall through... no pending requests. Try a prefetch. 771 assert(!miss_mshr && !wq_entry); 772 if (prefetcher && mshrQueue.canPrefetch()) { 773 // If we have a miss queue slot, we can try a prefetch 774 PacketPtr pkt = prefetcher->getPacket(); 775 if (pkt) { 776 Addr pf_addr = pkt->getBlockAddr(blkSize); 777 if (!tags->findBlock(pf_addr, pkt->isSecure()) && 778 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 779 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { 780 // Update statistic on number of prefetches issued 781 // (hwpf_mshr_misses) 782 assert(pkt->req->masterId() < system->maxMasters()); 783 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 784 785 // allocate an MSHR and return it, note 786 // that we send the packet straight away, so do not 787 // schedule the send 788 return allocateMissBuffer(pkt, curTick(), false); 789 } else { 790 // free the request and packet 791 delete pkt; 792 } 793 } 794 } 795 796 return nullptr; 797} 798
| 63#include "params/BaseCache.hh" 64#include "params/WriteAllocator.hh" 65#include "sim/core.hh" 66 67class BaseMasterPort; 68class BaseSlavePort; 69 70using namespace std; 71 72BaseCache::CacheSlavePort::CacheSlavePort(const std::string &_name, 73 BaseCache *_cache, 74 const std::string &_label) 75 : QueuedSlavePort(_name, _cache, queue), 76 queue(*_cache, *this, true, _label), 77 blocked(false), mustSendRetry(false), 78 sendRetryEvent([this]{ processSendRetry(); }, _name) 79{ 80} 81 82BaseCache::BaseCache(const BaseCacheParams *p, unsigned blk_size) 83 : ClockedObject(p), 84 cpuSidePort (p->name + ".cpu_side", this, "CpuSidePort"), 85 memSidePort(p->name + ".mem_side", this, "MemSidePort"), 86 mshrQueue("MSHRs", p->mshrs, 0, p->demand_mshr_reserve), // see below 87 writeBuffer("write buffer", p->write_buffers, p->mshrs), // see below 88 tags(p->tags), 89 compressor(p->compressor), 90 prefetcher(p->prefetcher), 91 writeAllocator(p->write_allocator), 92 writebackClean(p->writeback_clean), 93 tempBlockWriteback(nullptr), 94 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); }, 95 name(), false, 96 EventBase::Delayed_Writeback_Pri), 97 blkSize(blk_size), 98 lookupLatency(p->tag_latency), 99 dataLatency(p->data_latency), 100 forwardLatency(p->tag_latency), 101 fillLatency(p->data_latency), 102 responseLatency(p->response_latency), 103 sequentialAccess(p->sequential_access), 104 numTarget(p->tgts_per_mshr), 105 forwardSnoops(true), 106 clusivity(p->clusivity), 107 isReadOnly(p->is_read_only), 108 blocked(0), 109 order(0), 110 noTargetMSHR(nullptr), 111 missCount(p->max_miss_count), 112 addrRanges(p->addr_ranges.begin(), p->addr_ranges.end()), 113 system(p->system) 114{ 115 // the MSHR queue has no reserve entries as we check the MSHR 116 // queue on every single allocation, whereas the write queue has 117 // as many reserve entries as we have MSHRs, since every MSHR may 118 // eventually require a writeback, and we do not check the write 119 // buffer before committing to an MSHR 120 121 // forward snoops is overridden in init() once we can query 122 // whether the connected master is actually snooping or not 123 124 tempBlock = new TempCacheBlk(blkSize); 125 126 tags->tagsInit(); 127 if (prefetcher) 128 prefetcher->setCache(this); 129} 130 131BaseCache::~BaseCache() 132{ 133 delete tempBlock; 134} 135 136void 137BaseCache::CacheSlavePort::setBlocked() 138{ 139 assert(!blocked); 140 DPRINTF(CachePort, "Port is blocking new requests\n"); 141 blocked = true; 142 // if we already scheduled a retry in this cycle, but it has not yet 143 // happened, cancel it 144 if (sendRetryEvent.scheduled()) { 145 owner.deschedule(sendRetryEvent); 146 DPRINTF(CachePort, "Port descheduled retry\n"); 147 mustSendRetry = true; 148 } 149} 150 151void 152BaseCache::CacheSlavePort::clearBlocked() 153{ 154 assert(blocked); 155 DPRINTF(CachePort, "Port is accepting new requests\n"); 156 blocked = false; 157 if (mustSendRetry) { 158 // @TODO: need to find a better time (next cycle?) 159 owner.schedule(sendRetryEvent, curTick() + 1); 160 } 161} 162 163void 164BaseCache::CacheSlavePort::processSendRetry() 165{ 166 DPRINTF(CachePort, "Port is sending retry\n"); 167 168 // reset the flag and call retry 169 mustSendRetry = false; 170 sendRetryReq(); 171} 172 173Addr 174BaseCache::regenerateBlkAddr(CacheBlk* blk) 175{ 176 if (blk != tempBlock) { 177 return tags->regenerateBlkAddr(blk); 178 } else { 179 return tempBlock->getAddr(); 180 } 181} 182 183void 184BaseCache::init() 185{ 186 if (!cpuSidePort.isConnected() || !memSidePort.isConnected()) 187 fatal("Cache ports on %s are not connected\n", name()); 188 cpuSidePort.sendRangeChange(); 189 forwardSnoops = cpuSidePort.isSnooping(); 190} 191 192Port & 193BaseCache::getPort(const std::string &if_name, PortID idx) 194{ 195 if (if_name == "mem_side") { 196 return memSidePort; 197 } else if (if_name == "cpu_side") { 198 return cpuSidePort; 199 } else { 200 return ClockedObject::getPort(if_name, idx); 201 } 202} 203 204bool 205BaseCache::inRange(Addr addr) const 206{ 207 for (const auto& r : addrRanges) { 208 if (r.contains(addr)) { 209 return true; 210 } 211 } 212 return false; 213} 214 215void 216BaseCache::handleTimingReqHit(PacketPtr pkt, CacheBlk *blk, Tick request_time) 217{ 218 if (pkt->needsResponse()) { 219 // These delays should have been consumed by now 220 assert(pkt->headerDelay == 0); 221 assert(pkt->payloadDelay == 0); 222 223 pkt->makeTimingResponse(); 224 225 // In this case we are considering request_time that takes 226 // into account the delay of the xbar, if any, and just 227 // lat, neglecting responseLatency, modelling hit latency 228 // just as the value of lat overriden by access(), which calls 229 // the calculateAccessLatency() function. 230 cpuSidePort.schedTimingResp(pkt, request_time); 231 } else { 232 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__, 233 pkt->print()); 234 235 // queue the packet for deletion, as the sending cache is 236 // still relying on it; if the block is found in access(), 237 // CleanEvict and Writeback messages will be deleted 238 // here as well 239 pendingDelete.reset(pkt); 240 } 241} 242 243void 244BaseCache::handleTimingReqMiss(PacketPtr pkt, MSHR *mshr, CacheBlk *blk, 245 Tick forward_time, Tick request_time) 246{ 247 if (writeAllocator && 248 pkt && pkt->isWrite() && !pkt->req->isUncacheable()) { 249 writeAllocator->updateMode(pkt->getAddr(), pkt->getSize(), 250 pkt->getBlockAddr(blkSize)); 251 } 252 253 if (mshr) { 254 /// MSHR hit 255 /// @note writebacks will be checked in getNextMSHR() 256 /// for any conflicting requests to the same block 257 258 //@todo remove hw_pf here 259 260 // Coalesce unless it was a software prefetch (see above). 261 if (pkt) { 262 assert(!pkt->isWriteback()); 263 // CleanEvicts corresponding to blocks which have 264 // outstanding requests in MSHRs are simply sunk here 265 if (pkt->cmd == MemCmd::CleanEvict) { 266 pendingDelete.reset(pkt); 267 } else if (pkt->cmd == MemCmd::WriteClean) { 268 // A WriteClean should never coalesce with any 269 // outstanding cache maintenance requests. 270 271 // We use forward_time here because there is an 272 // uncached memory write, forwarded to WriteBuffer. 273 allocateWriteBuffer(pkt, forward_time); 274 } else { 275 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__, 276 pkt->print()); 277 278 assert(pkt->req->masterId() < system->maxMasters()); 279 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++; 280 281 // We use forward_time here because it is the same 282 // considering new targets. We have multiple 283 // requests for the same address here. It 284 // specifies the latency to allocate an internal 285 // buffer and to schedule an event to the queued 286 // port and also takes into account the additional 287 // delay of the xbar. 288 mshr->allocateTarget(pkt, forward_time, order++, 289 allocOnFill(pkt->cmd)); 290 if (mshr->getNumTargets() == numTarget) { 291 noTargetMSHR = mshr; 292 setBlocked(Blocked_NoTargets); 293 // need to be careful with this... if this mshr isn't 294 // ready yet (i.e. time > curTick()), we don't want to 295 // move it ahead of mshrs that are ready 296 // mshrQueue.moveToFront(mshr); 297 } 298 } 299 } 300 } else { 301 // no MSHR 302 assert(pkt->req->masterId() < system->maxMasters()); 303 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 304 305 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean) { 306 // We use forward_time here because there is an 307 // writeback or writeclean, forwarded to WriteBuffer. 308 allocateWriteBuffer(pkt, forward_time); 309 } else { 310 if (blk && blk->isValid()) { 311 // If we have a write miss to a valid block, we 312 // need to mark the block non-readable. Otherwise 313 // if we allow reads while there's an outstanding 314 // write miss, the read could return stale data 315 // out of the cache block... a more aggressive 316 // system could detect the overlap (if any) and 317 // forward data out of the MSHRs, but we don't do 318 // that yet. Note that we do need to leave the 319 // block valid so that it stays in the cache, in 320 // case we get an upgrade response (and hence no 321 // new data) when the write miss completes. 322 // As long as CPUs do proper store/load forwarding 323 // internally, and have a sufficiently weak memory 324 // model, this is probably unnecessary, but at some 325 // point it must have seemed like we needed it... 326 assert((pkt->needsWritable() && !blk->isWritable()) || 327 pkt->req->isCacheMaintenance()); 328 blk->status &= ~BlkReadable; 329 } 330 // Here we are using forward_time, modelling the latency of 331 // a miss (outbound) just as forwardLatency, neglecting the 332 // lookupLatency component. 333 allocateMissBuffer(pkt, forward_time); 334 } 335 } 336} 337 338void 339BaseCache::recvTimingReq(PacketPtr pkt) 340{ 341 // anything that is merely forwarded pays for the forward latency and 342 // the delay provided by the crossbar 343 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 344 345 Cycles lat; 346 CacheBlk *blk = nullptr; 347 bool satisfied = false; 348 { 349 PacketList writebacks; 350 // Note that lat is passed by reference here. The function 351 // access() will set the lat value. 352 satisfied = access(pkt, blk, lat, writebacks); 353 354 // After the evicted blocks are selected, they must be forwarded 355 // to the write buffer to ensure they logically precede anything 356 // happening below 357 doWritebacks(writebacks, clockEdge(lat + forwardLatency)); 358 } 359 360 // Here we charge the headerDelay that takes into account the latencies 361 // of the bus, if the packet comes from it. 362 // The latency charged is just the value set by the access() function. 363 // In case of a hit we are neglecting response latency. 364 // In case of a miss we are neglecting forward latency. 365 Tick request_time = clockEdge(lat); 366 // Here we reset the timing of the packet. 367 pkt->headerDelay = pkt->payloadDelay = 0; 368 369 if (satisfied) { 370 // notify before anything else as later handleTimingReqHit might turn 371 // the packet in a response 372 ppHit->notify(pkt); 373 374 if (prefetcher && blk && blk->wasPrefetched()) { 375 blk->status &= ~BlkHWPrefetched; 376 } 377 378 handleTimingReqHit(pkt, blk, request_time); 379 } else { 380 handleTimingReqMiss(pkt, blk, forward_time, request_time); 381 382 ppMiss->notify(pkt); 383 } 384 385 if (prefetcher) { 386 // track time of availability of next prefetch, if any 387 Tick next_pf_time = prefetcher->nextPrefetchReadyTime(); 388 if (next_pf_time != MaxTick) { 389 schedMemSideSendEvent(next_pf_time); 390 } 391 } 392} 393 394void 395BaseCache::handleUncacheableWriteResp(PacketPtr pkt) 396{ 397 Tick completion_time = clockEdge(responseLatency) + 398 pkt->headerDelay + pkt->payloadDelay; 399 400 // Reset the bus additional time as it is now accounted for 401 pkt->headerDelay = pkt->payloadDelay = 0; 402 403 cpuSidePort.schedTimingResp(pkt, completion_time); 404} 405 406void 407BaseCache::recvTimingResp(PacketPtr pkt) 408{ 409 assert(pkt->isResponse()); 410 411 // all header delay should be paid for by the crossbar, unless 412 // this is a prefetch response from above 413 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 414 "%s saw a non-zero packet delay\n", name()); 415 416 const bool is_error = pkt->isError(); 417 418 if (is_error) { 419 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__, 420 pkt->print()); 421 } 422 423 DPRINTF(Cache, "%s: Handling response %s\n", __func__, 424 pkt->print()); 425 426 // if this is a write, we should be looking at an uncacheable 427 // write 428 if (pkt->isWrite()) { 429 assert(pkt->req->isUncacheable()); 430 handleUncacheableWriteResp(pkt); 431 return; 432 } 433 434 // we have dealt with any (uncacheable) writes above, from here on 435 // we know we are dealing with an MSHR due to a miss or a prefetch 436 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState()); 437 assert(mshr); 438 439 if (mshr == noTargetMSHR) { 440 // we always clear at least one target 441 clearBlocked(Blocked_NoTargets); 442 noTargetMSHR = nullptr; 443 } 444 445 // Initial target is used just for stats 446 QueueEntry::Target *initial_tgt = mshr->getTarget(); 447 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 448 Tick miss_latency = curTick() - initial_tgt->recvTime; 449 450 if (pkt->req->isUncacheable()) { 451 assert(pkt->req->masterId() < system->maxMasters()); 452 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 453 miss_latency; 454 } else { 455 assert(pkt->req->masterId() < system->maxMasters()); 456 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 457 miss_latency; 458 } 459 460 PacketList writebacks; 461 462 bool is_fill = !mshr->isForward && 463 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp || 464 mshr->wasWholeLineWrite); 465 466 // make sure that if the mshr was due to a whole line write then 467 // the response is an invalidation 468 assert(!mshr->wasWholeLineWrite || pkt->isInvalidate()); 469 470 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 471 472 if (is_fill && !is_error) { 473 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n", 474 pkt->getAddr()); 475 476 const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ? 477 writeAllocator->allocate() : mshr->allocOnFill(); 478 blk = handleFill(pkt, blk, writebacks, allocate); 479 assert(blk != nullptr); 480 ppFill->notify(pkt); 481 } 482 483 if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) { 484 // The block was marked not readable while there was a pending 485 // cache maintenance operation, restore its flag. 486 blk->status |= BlkReadable; 487 488 // This was a cache clean operation (without invalidate) 489 // and we have a copy of the block already. Since there 490 // is no invalidation, we can promote targets that don't 491 // require a writable copy 492 mshr->promoteReadable(); 493 } 494 495 if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) { 496 // If at this point the referenced block is writable and the 497 // response is not a cache invalidate, we promote targets that 498 // were deferred as we couldn't guarrantee a writable copy 499 mshr->promoteWritable(); 500 } 501 502 serviceMSHRTargets(mshr, pkt, blk); 503 504 if (mshr->promoteDeferredTargets()) { 505 // avoid later read getting stale data while write miss is 506 // outstanding.. see comment in timingAccess() 507 if (blk) { 508 blk->status &= ~BlkReadable; 509 } 510 mshrQueue.markPending(mshr); 511 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 512 } else { 513 // while we deallocate an mshr from the queue we still have to 514 // check the isFull condition before and after as we might 515 // have been using the reserved entries already 516 const bool was_full = mshrQueue.isFull(); 517 mshrQueue.deallocate(mshr); 518 if (was_full && !mshrQueue.isFull()) { 519 clearBlocked(Blocked_NoMSHRs); 520 } 521 522 // Request the bus for a prefetch if this deallocation freed enough 523 // MSHRs for a prefetch to take place 524 if (prefetcher && mshrQueue.canPrefetch()) { 525 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 526 clockEdge()); 527 if (next_pf_time != MaxTick) 528 schedMemSideSendEvent(next_pf_time); 529 } 530 } 531 532 // if we used temp block, check to see if its valid and then clear it out 533 if (blk == tempBlock && tempBlock->isValid()) { 534 evictBlock(blk, writebacks); 535 } 536 537 const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 538 // copy writebacks to write buffer 539 doWritebacks(writebacks, forward_time); 540 541 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print()); 542 delete pkt; 543} 544 545 546Tick 547BaseCache::recvAtomic(PacketPtr pkt) 548{ 549 // should assert here that there are no outstanding MSHRs or 550 // writebacks... that would mean that someone used an atomic 551 // access in timing mode 552 553 // We use lookupLatency here because it is used to specify the latency 554 // to access. 555 Cycles lat = lookupLatency; 556 557 CacheBlk *blk = nullptr; 558 PacketList writebacks; 559 bool satisfied = access(pkt, blk, lat, writebacks); 560 561 if (pkt->isClean() && blk && blk->isDirty()) { 562 // A cache clean opearation is looking for a dirty 563 // block. If a dirty block is encountered a WriteClean 564 // will update any copies to the path to the memory 565 // until the point of reference. 566 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 567 __func__, pkt->print(), blk->print()); 568 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id); 569 writebacks.push_back(wb_pkt); 570 pkt->setSatisfied(); 571 } 572 573 // handle writebacks resulting from the access here to ensure they 574 // logically precede anything happening below 575 doWritebacksAtomic(writebacks); 576 assert(writebacks.empty()); 577 578 if (!satisfied) { 579 lat += handleAtomicReqMiss(pkt, blk, writebacks); 580 } 581 582 // Note that we don't invoke the prefetcher at all in atomic mode. 583 // It's not clear how to do it properly, particularly for 584 // prefetchers that aggressively generate prefetch candidates and 585 // rely on bandwidth contention to throttle them; these will tend 586 // to pollute the cache in atomic mode since there is no bandwidth 587 // contention. If we ever do want to enable prefetching in atomic 588 // mode, though, this is the place to do it... see timingAccess() 589 // for an example (though we'd want to issue the prefetch(es) 590 // immediately rather than calling requestMemSideBus() as we do 591 // there). 592 593 // do any writebacks resulting from the response handling 594 doWritebacksAtomic(writebacks); 595 596 // if we used temp block, check to see if its valid and if so 597 // clear it out, but only do so after the call to recvAtomic is 598 // finished so that any downstream observers (such as a snoop 599 // filter), first see the fill, and only then see the eviction 600 if (blk == tempBlock && tempBlock->isValid()) { 601 // the atomic CPU calls recvAtomic for fetch and load/store 602 // sequentuially, and we may already have a tempBlock 603 // writeback from the fetch that we have not yet sent 604 if (tempBlockWriteback) { 605 // if that is the case, write the prevoius one back, and 606 // do not schedule any new event 607 writebackTempBlockAtomic(); 608 } else { 609 // the writeback/clean eviction happens after the call to 610 // recvAtomic has finished (but before any successive 611 // calls), so that the response handling from the fill is 612 // allowed to happen first 613 schedule(writebackTempBlockAtomicEvent, curTick()); 614 } 615 616 tempBlockWriteback = evictBlock(blk); 617 } 618 619 if (pkt->needsResponse()) { 620 pkt->makeAtomicResponse(); 621 } 622 623 return lat * clockPeriod(); 624} 625 626void 627BaseCache::functionalAccess(PacketPtr pkt, bool from_cpu_side) 628{ 629 Addr blk_addr = pkt->getBlockAddr(blkSize); 630 bool is_secure = pkt->isSecure(); 631 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure); 632 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure); 633 634 pkt->pushLabel(name()); 635 636 CacheBlkPrintWrapper cbpw(blk); 637 638 // Note that just because an L2/L3 has valid data doesn't mean an 639 // L1 doesn't have a more up-to-date modified copy that still 640 // needs to be found. As a result we always update the request if 641 // we have it, but only declare it satisfied if we are the owner. 642 643 // see if we have data at all (owned or otherwise) 644 bool have_data = blk && blk->isValid() 645 && pkt->trySatisfyFunctional(&cbpw, blk_addr, is_secure, blkSize, 646 blk->data); 647 648 // data we have is dirty if marked as such or if we have an 649 // in-service MSHR that is pending a modified line 650 bool have_dirty = 651 have_data && (blk->isDirty() || 652 (mshr && mshr->inService && mshr->isPendingModified())); 653 654 bool done = have_dirty || 655 cpuSidePort.trySatisfyFunctional(pkt) || 656 mshrQueue.trySatisfyFunctional(pkt) || 657 writeBuffer.trySatisfyFunctional(pkt) || 658 memSidePort.trySatisfyFunctional(pkt); 659 660 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(), 661 (blk && blk->isValid()) ? "valid " : "", 662 have_data ? "data " : "", done ? "done " : ""); 663 664 // We're leaving the cache, so pop cache->name() label 665 pkt->popLabel(); 666 667 if (done) { 668 pkt->makeResponse(); 669 } else { 670 // if it came as a request from the CPU side then make sure it 671 // continues towards the memory side 672 if (from_cpu_side) { 673 memSidePort.sendFunctional(pkt); 674 } else if (cpuSidePort.isSnooping()) { 675 // if it came from the memory side, it must be a snoop request 676 // and we should only forward it if we are forwarding snoops 677 cpuSidePort.sendFunctionalSnoop(pkt); 678 } 679 } 680} 681 682 683void 684BaseCache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt) 685{ 686 assert(pkt->isRequest()); 687 688 uint64_t overwrite_val; 689 bool overwrite_mem; 690 uint64_t condition_val64; 691 uint32_t condition_val32; 692 693 int offset = pkt->getOffset(blkSize); 694 uint8_t *blk_data = blk->data + offset; 695 696 assert(sizeof(uint64_t) >= pkt->getSize()); 697 698 overwrite_mem = true; 699 // keep a copy of our possible write value, and copy what is at the 700 // memory address into the packet 701 pkt->writeData((uint8_t *)&overwrite_val); 702 pkt->setData(blk_data); 703 704 if (pkt->req->isCondSwap()) { 705 if (pkt->getSize() == sizeof(uint64_t)) { 706 condition_val64 = pkt->req->getExtraData(); 707 overwrite_mem = !std::memcmp(&condition_val64, blk_data, 708 sizeof(uint64_t)); 709 } else if (pkt->getSize() == sizeof(uint32_t)) { 710 condition_val32 = (uint32_t)pkt->req->getExtraData(); 711 overwrite_mem = !std::memcmp(&condition_val32, blk_data, 712 sizeof(uint32_t)); 713 } else 714 panic("Invalid size for conditional read/write\n"); 715 } 716 717 if (overwrite_mem) { 718 std::memcpy(blk_data, &overwrite_val, pkt->getSize()); 719 blk->status |= BlkDirty; 720 } 721} 722 723QueueEntry* 724BaseCache::getNextQueueEntry() 725{ 726 // Check both MSHR queue and write buffer for potential requests, 727 // note that null does not mean there is no request, it could 728 // simply be that it is not ready 729 MSHR *miss_mshr = mshrQueue.getNext(); 730 WriteQueueEntry *wq_entry = writeBuffer.getNext(); 731 732 // If we got a write buffer request ready, first priority is a 733 // full write buffer, otherwise we favour the miss requests 734 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) { 735 // need to search MSHR queue for conflicting earlier miss. 736 MSHR *conflict_mshr = mshrQueue.findPending(wq_entry); 737 738 if (conflict_mshr && conflict_mshr->order < wq_entry->order) { 739 // Service misses in order until conflict is cleared. 740 return conflict_mshr; 741 742 // @todo Note that we ignore the ready time of the conflict here 743 } 744 745 // No conflicts; issue write 746 return wq_entry; 747 } else if (miss_mshr) { 748 // need to check for conflicting earlier writeback 749 WriteQueueEntry *conflict_mshr = writeBuffer.findPending(miss_mshr); 750 if (conflict_mshr) { 751 // not sure why we don't check order here... it was in the 752 // original code but commented out. 753 754 // The only way this happens is if we are 755 // doing a write and we didn't have permissions 756 // then subsequently saw a writeback (owned got evicted) 757 // We need to make sure to perform the writeback first 758 // To preserve the dirty data, then we can issue the write 759 760 // should we return wq_entry here instead? I.e. do we 761 // have to flush writes in order? I don't think so... not 762 // for Alpha anyway. Maybe for x86? 763 return conflict_mshr; 764 765 // @todo Note that we ignore the ready time of the conflict here 766 } 767 768 // No conflicts; issue read 769 return miss_mshr; 770 } 771 772 // fall through... no pending requests. Try a prefetch. 773 assert(!miss_mshr && !wq_entry); 774 if (prefetcher && mshrQueue.canPrefetch()) { 775 // If we have a miss queue slot, we can try a prefetch 776 PacketPtr pkt = prefetcher->getPacket(); 777 if (pkt) { 778 Addr pf_addr = pkt->getBlockAddr(blkSize); 779 if (!tags->findBlock(pf_addr, pkt->isSecure()) && 780 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 781 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { 782 // Update statistic on number of prefetches issued 783 // (hwpf_mshr_misses) 784 assert(pkt->req->masterId() < system->maxMasters()); 785 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 786 787 // allocate an MSHR and return it, note 788 // that we send the packet straight away, so do not 789 // schedule the send 790 return allocateMissBuffer(pkt, curTick(), false); 791 } else { 792 // free the request and packet 793 delete pkt; 794 } 795 } 796 } 797 798 return nullptr; 799} 800
|
| 801bool 802BaseCache::updateCompressionData(CacheBlk *blk, const uint64_t* data, 803 PacketList &writebacks) 804{ 805 // tempBlock does not exist in the tags, so don't do anything for it. 806 if (blk == tempBlock) { 807 return true; 808 } 809 810 // Get superblock of the given block 811 CompressionBlk* compression_blk = static_cast<CompressionBlk*>(blk); 812 const SuperBlk* superblock = static_cast<const SuperBlk*>( 813 compression_blk->getSectorBlock()); 814 815 // The compressor is called to compress the updated data, so that its 816 // metadata can be updated. 817 std::size_t compression_size = 0; 818 Cycles compression_lat = Cycles(0); 819 Cycles decompression_lat = Cycles(0); 820 compressor->compress(data, compression_lat, decompression_lat, 821 compression_size); 822 823 // If block's compression factor increased, it may not be co-allocatable 824 // anymore. If so, some blocks might need to be evicted to make room for 825 // the bigger block 826 827 // Get previous compressed size 828 const std::size_t M5_VAR_USED prev_size = compression_blk->getSizeBits(); 829 830 // Check if new data is co-allocatable 831 const bool is_co_allocatable = superblock->isCompressed(compression_blk) && 832 superblock->canCoAllocate(compression_size); 833 834 // If block was compressed, possibly co-allocated with other blocks, and 835 // cannot be co-allocated anymore, one or more blocks must be evicted to 836 // make room for the expanded block. As of now we decide to evict the co- 837 // allocated blocks to make room for the expansion, but other approaches 838 // that take the replacement data of the superblock into account may 839 // generate better results 840 std::vector<CacheBlk*> evict_blks; 841 const bool was_compressed = compression_blk->isCompressed(); 842 if (was_compressed && !is_co_allocatable) { 843 // Get all co-allocated blocks 844 for (const auto& sub_blk : superblock->blks) { 845 if (sub_blk->isValid() && (compression_blk != sub_blk)) { 846 // Check for transient state allocations. If any of the 847 // entries listed for eviction has a transient state, the 848 // allocation fails 849 const Addr repl_addr = regenerateBlkAddr(sub_blk); 850 const MSHR *repl_mshr = 851 mshrQueue.findMatch(repl_addr, sub_blk->isSecure()); 852 if (repl_mshr) { 853 DPRINTF(CacheRepl, "Aborting data expansion of %s due " \ 854 "to replacement of block in transient state: %s\n", 855 compression_blk->print(), sub_blk->print()); 856 // Too hard to replace block with transient state, so it 857 // cannot be evicted. Mark the update as failed and expect 858 // the caller to evict this block. Since this is called 859 // only when writebacks arrive, and packets do not contain 860 // compressed data, there is no need to decompress 861 compression_blk->setSizeBits(blkSize * 8); 862 compression_blk->setDecompressionLatency(Cycles(0)); 863 compression_blk->setUncompressed(); 864 return false; 865 } 866 867 evict_blks.push_back(sub_blk); 868 } 869 } 870 871 // Update the number of data expansions 872 dataExpansions++; 873 874 DPRINTF(CacheComp, "Data expansion: expanding [%s] from %d to %d bits" 875 "\n", blk->print(), prev_size, compression_size); 876 } 877 878 // We always store compressed blocks when possible 879 if (is_co_allocatable) { 880 compression_blk->setCompressed(); 881 } else { 882 compression_blk->setUncompressed(); 883 } 884 compression_blk->setSizeBits(compression_size); 885 compression_blk->setDecompressionLatency(decompression_lat); 886 887 // Evict valid blocks 888 for (const auto& evict_blk : evict_blks) { 889 if (evict_blk->isValid()) { 890 if (evict_blk->wasPrefetched()) { 891 unusedPrefetches++; 892 } 893 evictBlock(evict_blk, writebacks); 894 } 895 } 896 897 return true; 898} 899
|
799void 800BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) 801{ 802 assert(pkt->isRequest()); 803 804 assert(blk && blk->isValid()); 805 // Occasionally this is not true... if we are a lower-level cache 806 // satisfying a string of Read and ReadEx requests from 807 // upper-level caches, a Read will mark the block as shared but we 808 // can satisfy a following ReadEx anyway since we can rely on the 809 // Read requester(s) to have buffered the ReadEx snoop and to 810 // invalidate their blocks after receiving them. 811 // assert(!pkt->needsWritable() || blk->isWritable()); 812 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize); 813 814 // Check RMW operations first since both isRead() and 815 // isWrite() will be true for them 816 if (pkt->cmd == MemCmd::SwapReq) { 817 if (pkt->isAtomicOp()) { 818 // extract data from cache and save it into the data field in 819 // the packet as a return value from this atomic op 820 int offset = tags->extractBlkOffset(pkt->getAddr()); 821 uint8_t *blk_data = blk->data + offset; 822 pkt->setData(blk_data); 823 824 // execute AMO operation 825 (*(pkt->getAtomicOp()))(blk_data); 826 827 // set block status to dirty 828 blk->status |= BlkDirty; 829 } else { 830 cmpAndSwap(blk, pkt); 831 } 832 } else if (pkt->isWrite()) { 833 // we have the block in a writable state and can go ahead, 834 // note that the line may be also be considered writable in 835 // downstream caches along the path to memory, but always 836 // Exclusive, and never Modified 837 assert(blk->isWritable()); 838 // Write or WriteLine at the first cache with block in writable state 839 if (blk->checkWrite(pkt)) { 840 pkt->writeDataToBlock(blk->data, blkSize); 841 } 842 // Always mark the line as dirty (and thus transition to the 843 // Modified state) even if we are a failed StoreCond so we 844 // supply data to any snoops that have appended themselves to 845 // this cache before knowing the store will fail. 846 blk->status |= BlkDirty; 847 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print()); 848 } else if (pkt->isRead()) { 849 if (pkt->isLLSC()) { 850 blk->trackLoadLocked(pkt); 851 } 852 853 // all read responses have a data payload 854 assert(pkt->hasRespData()); 855 pkt->setDataFromBlock(blk->data, blkSize); 856 } else if (pkt->isUpgrade()) { 857 // sanity check 858 assert(!pkt->hasSharers()); 859 860 if (blk->isDirty()) { 861 // we were in the Owned state, and a cache above us that 862 // has the line in Shared state needs to be made aware 863 // that the data it already has is in fact dirty 864 pkt->setCacheResponding(); 865 blk->status &= ~BlkDirty; 866 } 867 } else if (pkt->isClean()) { 868 blk->status &= ~BlkDirty; 869 } else { 870 assert(pkt->isInvalidate()); 871 invalidateBlock(blk); 872 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__, 873 pkt->print()); 874 } 875} 876 877///////////////////////////////////////////////////// 878// 879// Access path: requests coming in from the CPU side 880// 881///////////////////////////////////////////////////// 882Cycles 883BaseCache::calculateTagOnlyLatency(const uint32_t delay, 884 const Cycles lookup_lat) const 885{ 886 // A tag-only access has to wait for the packet to arrive in order to 887 // perform the tag lookup. 888 return ticksToCycles(delay) + lookup_lat; 889} 890 891Cycles 892BaseCache::calculateAccessLatency(const CacheBlk* blk, const uint32_t delay, 893 const Cycles lookup_lat) const 894{ 895 Cycles lat(0); 896 897 if (blk != nullptr) { 898 // As soon as the access arrives, for sequential accesses first access 899 // tags, then the data entry. In the case of parallel accesses the 900 // latency is dictated by the slowest of tag and data latencies. 901 if (sequentialAccess) { 902 lat = ticksToCycles(delay) + lookup_lat + dataLatency; 903 } else { 904 lat = ticksToCycles(delay) + std::max(lookup_lat, dataLatency); 905 } 906 907 // Check if the block to be accessed is available. If not, apply the 908 // access latency on top of when the block is ready to be accessed. 909 const Tick tick = curTick() + delay; 910 const Tick when_ready = blk->getWhenReady(); 911 if (when_ready > tick && 912 ticksToCycles(when_ready - tick) > lat) { 913 lat += ticksToCycles(when_ready - tick); 914 } 915 } else { 916 // In case of a miss, we neglect the data access in a parallel 917 // configuration (i.e., the data access will be stopped as soon as 918 // we find out it is a miss), and use the tag-only latency. 919 lat = calculateTagOnlyLatency(delay, lookup_lat); 920 } 921 922 return lat; 923} 924 925bool 926BaseCache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 927 PacketList &writebacks) 928{ 929 // sanity check 930 assert(pkt->isRequest()); 931 932 chatty_assert(!(isReadOnly && pkt->isWrite()), 933 "Should never see a write in a read-only cache %s\n", 934 name()); 935 936 // Access block in the tags 937 Cycles tag_latency(0); 938 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), tag_latency); 939 940 DPRINTF(Cache, "%s for %s %s\n", __func__, pkt->print(), 941 blk ? "hit " + blk->print() : "miss"); 942 943 if (pkt->req->isCacheMaintenance()) { 944 // A cache maintenance operation is always forwarded to the 945 // memory below even if the block is found in dirty state. 946 947 // We defer any changes to the state of the block until we 948 // create and mark as in service the mshr for the downstream 949 // packet. 950 951 // Calculate access latency on top of when the packet arrives. This 952 // takes into account the bus delay. 953 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 954 955 return false; 956 } 957 958 if (pkt->isEviction()) { 959 // We check for presence of block in above caches before issuing 960 // Writeback or CleanEvict to write buffer. Therefore the only 961 // possible cases can be of a CleanEvict packet coming from above 962 // encountering a Writeback generated in this cache peer cache and 963 // waiting in the write buffer. Cases of upper level peer caches 964 // generating CleanEvict and Writeback or simply CleanEvict and 965 // CleanEvict almost simultaneously will be caught by snoops sent out 966 // by crossbar. 967 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(), 968 pkt->isSecure()); 969 if (wb_entry) { 970 assert(wb_entry->getNumTargets() == 1); 971 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 972 assert(wbPkt->isWriteback()); 973 974 if (pkt->isCleanEviction()) { 975 // The CleanEvict and WritebackClean snoops into other 976 // peer caches of the same level while traversing the 977 // crossbar. If a copy of the block is found, the 978 // packet is deleted in the crossbar. Hence, none of 979 // the other upper level caches connected to this 980 // cache have the block, so we can clear the 981 // BLOCK_CACHED flag in the Writeback if set and 982 // discard the CleanEvict by returning true. 983 wbPkt->clearBlockCached(); 984 985 // A clean evict does not need to access the data array 986 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 987 988 return true; 989 } else { 990 assert(pkt->cmd == MemCmd::WritebackDirty); 991 // Dirty writeback from above trumps our clean 992 // writeback... discard here 993 // Note: markInService will remove entry from writeback buffer. 994 markInService(wb_entry); 995 delete wbPkt; 996 } 997 } 998 } 999 1000 // Writeback handling is special case. We can write the block into 1001 // the cache without having a writeable copy (or any copy at all). 1002 if (pkt->isWriteback()) { 1003 assert(blkSize == pkt->getSize()); 1004 1005 // we could get a clean writeback while we are having 1006 // outstanding accesses to a block, do the simple thing for 1007 // now and drop the clean writeback so that we do not upset 1008 // any ordering/decisions about ownership already taken 1009 if (pkt->cmd == MemCmd::WritebackClean && 1010 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) { 1011 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, " 1012 "dropping\n", pkt->getAddr()); 1013 1014 // A writeback searches for the block, then writes the data. 1015 // As the writeback is being dropped, the data is not touched, 1016 // and we just had to wait for the time to find a match in the 1017 // MSHR. As of now assume a mshr queue search takes as long as 1018 // a tag lookup for simplicity. 1019 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1020 1021 return true; 1022 } 1023 1024 if (!blk) { 1025 // need to do a replacement 1026 blk = allocateBlock(pkt, writebacks); 1027 if (!blk) { 1028 // no replaceable block available: give up, fwd to next level. 1029 incMissCount(pkt); 1030 1031 // A writeback searches for the block, then writes the data. 1032 // As the block could not be found, it was a tag-only access. 1033 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1034 1035 return false; 1036 } 1037 1038 blk->status |= BlkReadable;
| 900void 901BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) 902{ 903 assert(pkt->isRequest()); 904 905 assert(blk && blk->isValid()); 906 // Occasionally this is not true... if we are a lower-level cache 907 // satisfying a string of Read and ReadEx requests from 908 // upper-level caches, a Read will mark the block as shared but we 909 // can satisfy a following ReadEx anyway since we can rely on the 910 // Read requester(s) to have buffered the ReadEx snoop and to 911 // invalidate their blocks after receiving them. 912 // assert(!pkt->needsWritable() || blk->isWritable()); 913 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize); 914 915 // Check RMW operations first since both isRead() and 916 // isWrite() will be true for them 917 if (pkt->cmd == MemCmd::SwapReq) { 918 if (pkt->isAtomicOp()) { 919 // extract data from cache and save it into the data field in 920 // the packet as a return value from this atomic op 921 int offset = tags->extractBlkOffset(pkt->getAddr()); 922 uint8_t *blk_data = blk->data + offset; 923 pkt->setData(blk_data); 924 925 // execute AMO operation 926 (*(pkt->getAtomicOp()))(blk_data); 927 928 // set block status to dirty 929 blk->status |= BlkDirty; 930 } else { 931 cmpAndSwap(blk, pkt); 932 } 933 } else if (pkt->isWrite()) { 934 // we have the block in a writable state and can go ahead, 935 // note that the line may be also be considered writable in 936 // downstream caches along the path to memory, but always 937 // Exclusive, and never Modified 938 assert(blk->isWritable()); 939 // Write or WriteLine at the first cache with block in writable state 940 if (blk->checkWrite(pkt)) { 941 pkt->writeDataToBlock(blk->data, blkSize); 942 } 943 // Always mark the line as dirty (and thus transition to the 944 // Modified state) even if we are a failed StoreCond so we 945 // supply data to any snoops that have appended themselves to 946 // this cache before knowing the store will fail. 947 blk->status |= BlkDirty; 948 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print()); 949 } else if (pkt->isRead()) { 950 if (pkt->isLLSC()) { 951 blk->trackLoadLocked(pkt); 952 } 953 954 // all read responses have a data payload 955 assert(pkt->hasRespData()); 956 pkt->setDataFromBlock(blk->data, blkSize); 957 } else if (pkt->isUpgrade()) { 958 // sanity check 959 assert(!pkt->hasSharers()); 960 961 if (blk->isDirty()) { 962 // we were in the Owned state, and a cache above us that 963 // has the line in Shared state needs to be made aware 964 // that the data it already has is in fact dirty 965 pkt->setCacheResponding(); 966 blk->status &= ~BlkDirty; 967 } 968 } else if (pkt->isClean()) { 969 blk->status &= ~BlkDirty; 970 } else { 971 assert(pkt->isInvalidate()); 972 invalidateBlock(blk); 973 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__, 974 pkt->print()); 975 } 976} 977 978///////////////////////////////////////////////////// 979// 980// Access path: requests coming in from the CPU side 981// 982///////////////////////////////////////////////////// 983Cycles 984BaseCache::calculateTagOnlyLatency(const uint32_t delay, 985 const Cycles lookup_lat) const 986{ 987 // A tag-only access has to wait for the packet to arrive in order to 988 // perform the tag lookup. 989 return ticksToCycles(delay) + lookup_lat; 990} 991 992Cycles 993BaseCache::calculateAccessLatency(const CacheBlk* blk, const uint32_t delay, 994 const Cycles lookup_lat) const 995{ 996 Cycles lat(0); 997 998 if (blk != nullptr) { 999 // As soon as the access arrives, for sequential accesses first access 1000 // tags, then the data entry. In the case of parallel accesses the 1001 // latency is dictated by the slowest of tag and data latencies. 1002 if (sequentialAccess) { 1003 lat = ticksToCycles(delay) + lookup_lat + dataLatency; 1004 } else { 1005 lat = ticksToCycles(delay) + std::max(lookup_lat, dataLatency); 1006 } 1007 1008 // Check if the block to be accessed is available. If not, apply the 1009 // access latency on top of when the block is ready to be accessed. 1010 const Tick tick = curTick() + delay; 1011 const Tick when_ready = blk->getWhenReady(); 1012 if (when_ready > tick && 1013 ticksToCycles(when_ready - tick) > lat) { 1014 lat += ticksToCycles(when_ready - tick); 1015 } 1016 } else { 1017 // In case of a miss, we neglect the data access in a parallel 1018 // configuration (i.e., the data access will be stopped as soon as 1019 // we find out it is a miss), and use the tag-only latency. 1020 lat = calculateTagOnlyLatency(delay, lookup_lat); 1021 } 1022 1023 return lat; 1024} 1025 1026bool 1027BaseCache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 1028 PacketList &writebacks) 1029{ 1030 // sanity check 1031 assert(pkt->isRequest()); 1032 1033 chatty_assert(!(isReadOnly && pkt->isWrite()), 1034 "Should never see a write in a read-only cache %s\n", 1035 name()); 1036 1037 // Access block in the tags 1038 Cycles tag_latency(0); 1039 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), tag_latency); 1040 1041 DPRINTF(Cache, "%s for %s %s\n", __func__, pkt->print(), 1042 blk ? "hit " + blk->print() : "miss"); 1043 1044 if (pkt->req->isCacheMaintenance()) { 1045 // A cache maintenance operation is always forwarded to the 1046 // memory below even if the block is found in dirty state. 1047 1048 // We defer any changes to the state of the block until we 1049 // create and mark as in service the mshr for the downstream 1050 // packet. 1051 1052 // Calculate access latency on top of when the packet arrives. This 1053 // takes into account the bus delay. 1054 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1055 1056 return false; 1057 } 1058 1059 if (pkt->isEviction()) { 1060 // We check for presence of block in above caches before issuing 1061 // Writeback or CleanEvict to write buffer. Therefore the only 1062 // possible cases can be of a CleanEvict packet coming from above 1063 // encountering a Writeback generated in this cache peer cache and 1064 // waiting in the write buffer. Cases of upper level peer caches 1065 // generating CleanEvict and Writeback or simply CleanEvict and 1066 // CleanEvict almost simultaneously will be caught by snoops sent out 1067 // by crossbar. 1068 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(), 1069 pkt->isSecure()); 1070 if (wb_entry) { 1071 assert(wb_entry->getNumTargets() == 1); 1072 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 1073 assert(wbPkt->isWriteback()); 1074 1075 if (pkt->isCleanEviction()) { 1076 // The CleanEvict and WritebackClean snoops into other 1077 // peer caches of the same level while traversing the 1078 // crossbar. If a copy of the block is found, the 1079 // packet is deleted in the crossbar. Hence, none of 1080 // the other upper level caches connected to this 1081 // cache have the block, so we can clear the 1082 // BLOCK_CACHED flag in the Writeback if set and 1083 // discard the CleanEvict by returning true. 1084 wbPkt->clearBlockCached(); 1085 1086 // A clean evict does not need to access the data array 1087 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1088 1089 return true; 1090 } else { 1091 assert(pkt->cmd == MemCmd::WritebackDirty); 1092 // Dirty writeback from above trumps our clean 1093 // writeback... discard here 1094 // Note: markInService will remove entry from writeback buffer. 1095 markInService(wb_entry); 1096 delete wbPkt; 1097 } 1098 } 1099 } 1100 1101 // Writeback handling is special case. We can write the block into 1102 // the cache without having a writeable copy (or any copy at all). 1103 if (pkt->isWriteback()) { 1104 assert(blkSize == pkt->getSize()); 1105 1106 // we could get a clean writeback while we are having 1107 // outstanding accesses to a block, do the simple thing for 1108 // now and drop the clean writeback so that we do not upset 1109 // any ordering/decisions about ownership already taken 1110 if (pkt->cmd == MemCmd::WritebackClean && 1111 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) { 1112 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, " 1113 "dropping\n", pkt->getAddr()); 1114 1115 // A writeback searches for the block, then writes the data. 1116 // As the writeback is being dropped, the data is not touched, 1117 // and we just had to wait for the time to find a match in the 1118 // MSHR. As of now assume a mshr queue search takes as long as 1119 // a tag lookup for simplicity. 1120 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1121 1122 return true; 1123 } 1124 1125 if (!blk) { 1126 // need to do a replacement 1127 blk = allocateBlock(pkt, writebacks); 1128 if (!blk) { 1129 // no replaceable block available: give up, fwd to next level. 1130 incMissCount(pkt); 1131 1132 // A writeback searches for the block, then writes the data. 1133 // As the block could not be found, it was a tag-only access. 1134 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1135 1136 return false; 1137 } 1138 1139 blk->status |= BlkReadable;
|
1039 } else { 1040 if (compressor) { 1041 // This is an overwrite to an existing block, therefore we need 1042 // to check for data expansion (i.e., block was compressed with 1043 // a smaller size, and now it doesn't fit the entry anymore). 1044 // If that is the case we might need to evict blocks. 1045 // @todo Update compression data
| 1140 } else if (compressor) { 1141 // This is an overwrite to an existing block, therefore we need 1142 // to check for data expansion (i.e., block was compressed with 1143 // a smaller size, and now it doesn't fit the entry anymore). 1144 // If that is the case we might need to evict blocks. 1145 if (!updateCompressionData(blk, pkt->getConstPtr<uint64_t>(), 1146 writebacks)) { 1147 // This is a failed data expansion (write), which happened 1148 // after finding the replacement entries and accessing the 1149 // block's data. There were no replaceable entries available 1150 // to make room for the expanded block, and since it does not 1151 // fit anymore and it has been properly updated to contain 1152 // the new data, forward it to the next level 1153 lat = calculateAccessLatency(blk, pkt->headerDelay, 1154 tag_latency); 1155 invalidateBlock(blk); 1156 return false;
|
1046 } 1047 } 1048 1049 // only mark the block dirty if we got a writeback command, 1050 // and leave it as is for a clean writeback 1051 if (pkt->cmd == MemCmd::WritebackDirty) { 1052 // TODO: the coherent cache can assert(!blk->isDirty()); 1053 blk->status |= BlkDirty; 1054 } 1055 // if the packet does not have sharers, it is passing 1056 // writable, and we got the writeback in Modified or Exclusive 1057 // state, if not we are in the Owned or Shared state 1058 if (!pkt->hasSharers()) { 1059 blk->status |= BlkWritable; 1060 } 1061 // nothing else to do; writeback doesn't expect response 1062 assert(!pkt->needsResponse()); 1063 pkt->writeDataToBlock(blk->data, blkSize); 1064 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 1065 incHitCount(pkt); 1066 1067 // A writeback searches for the block, then writes the data 1068 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1069 1070 // When the packet metadata arrives, the tag lookup will be done while 1071 // the payload is arriving. Then the block will be ready to access as 1072 // soon as the fill is done 1073 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1074 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay)); 1075 1076 return true; 1077 } else if (pkt->cmd == MemCmd::CleanEvict) { 1078 // A CleanEvict does not need to access the data array 1079 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1080 1081 if (blk) { 1082 // Found the block in the tags, need to stop CleanEvict from 1083 // propagating further down the hierarchy. Returning true will 1084 // treat the CleanEvict like a satisfied write request and delete 1085 // it. 1086 return true; 1087 } 1088 // We didn't find the block here, propagate the CleanEvict further 1089 // down the memory hierarchy. Returning false will treat the CleanEvict 1090 // like a Writeback which could not find a replaceable block so has to 1091 // go to next level. 1092 return false; 1093 } else if (pkt->cmd == MemCmd::WriteClean) { 1094 // WriteClean handling is a special case. We can allocate a 1095 // block directly if it doesn't exist and we can update the 1096 // block immediately. The WriteClean transfers the ownership 1097 // of the block as well. 1098 assert(blkSize == pkt->getSize()); 1099 1100 if (!blk) { 1101 if (pkt->writeThrough()) { 1102 // A writeback searches for the block, then writes the data. 1103 // As the block could not be found, it was a tag-only access. 1104 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1105 1106 // if this is a write through packet, we don't try to 1107 // allocate if the block is not present 1108 return false; 1109 } else { 1110 // a writeback that misses needs to allocate a new block 1111 blk = allocateBlock(pkt, writebacks); 1112 if (!blk) { 1113 // no replaceable block available: give up, fwd to 1114 // next level. 1115 incMissCount(pkt); 1116 1117 // A writeback searches for the block, then writes the 1118 // data. As the block could not be found, it was a tag-only 1119 // access. 1120 lat = calculateTagOnlyLatency(pkt->headerDelay, 1121 tag_latency); 1122 1123 return false; 1124 } 1125 1126 blk->status |= BlkReadable; 1127 }
| 1157 } 1158 } 1159 1160 // only mark the block dirty if we got a writeback command, 1161 // and leave it as is for a clean writeback 1162 if (pkt->cmd == MemCmd::WritebackDirty) { 1163 // TODO: the coherent cache can assert(!blk->isDirty()); 1164 blk->status |= BlkDirty; 1165 } 1166 // if the packet does not have sharers, it is passing 1167 // writable, and we got the writeback in Modified or Exclusive 1168 // state, if not we are in the Owned or Shared state 1169 if (!pkt->hasSharers()) { 1170 blk->status |= BlkWritable; 1171 } 1172 // nothing else to do; writeback doesn't expect response 1173 assert(!pkt->needsResponse()); 1174 pkt->writeDataToBlock(blk->data, blkSize); 1175 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 1176 incHitCount(pkt); 1177 1178 // A writeback searches for the block, then writes the data 1179 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1180 1181 // When the packet metadata arrives, the tag lookup will be done while 1182 // the payload is arriving. Then the block will be ready to access as 1183 // soon as the fill is done 1184 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1185 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay)); 1186 1187 return true; 1188 } else if (pkt->cmd == MemCmd::CleanEvict) { 1189 // A CleanEvict does not need to access the data array 1190 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1191 1192 if (blk) { 1193 // Found the block in the tags, need to stop CleanEvict from 1194 // propagating further down the hierarchy. Returning true will 1195 // treat the CleanEvict like a satisfied write request and delete 1196 // it. 1197 return true; 1198 } 1199 // We didn't find the block here, propagate the CleanEvict further 1200 // down the memory hierarchy. Returning false will treat the CleanEvict 1201 // like a Writeback which could not find a replaceable block so has to 1202 // go to next level. 1203 return false; 1204 } else if (pkt->cmd == MemCmd::WriteClean) { 1205 // WriteClean handling is a special case. We can allocate a 1206 // block directly if it doesn't exist and we can update the 1207 // block immediately. The WriteClean transfers the ownership 1208 // of the block as well. 1209 assert(blkSize == pkt->getSize()); 1210 1211 if (!blk) { 1212 if (pkt->writeThrough()) { 1213 // A writeback searches for the block, then writes the data. 1214 // As the block could not be found, it was a tag-only access. 1215 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1216 1217 // if this is a write through packet, we don't try to 1218 // allocate if the block is not present 1219 return false; 1220 } else { 1221 // a writeback that misses needs to allocate a new block 1222 blk = allocateBlock(pkt, writebacks); 1223 if (!blk) { 1224 // no replaceable block available: give up, fwd to 1225 // next level. 1226 incMissCount(pkt); 1227 1228 // A writeback searches for the block, then writes the 1229 // data. As the block could not be found, it was a tag-only 1230 // access. 1231 lat = calculateTagOnlyLatency(pkt->headerDelay, 1232 tag_latency); 1233 1234 return false; 1235 } 1236 1237 blk->status |= BlkReadable; 1238 }
|
1128 } else { 1129 if (compressor) { 1130 // @todo Update compression data
| 1239 } else if (compressor) { 1240 // This is an overwrite to an existing block, therefore we need 1241 // to check for data expansion (i.e., block was compressed with 1242 // a smaller size, and now it doesn't fit the entry anymore). 1243 // If that is the case we might need to evict blocks. 1244 if (!updateCompressionData(blk, pkt->getConstPtr<uint64_t>(), 1245 writebacks)) { 1246 // This is a failed data expansion (write), which happened 1247 // after finding the replacement entries and accessing the 1248 // block's data. There were no replaceable entries available 1249 // to make room for the expanded block, and since it does not 1250 // fit anymore and it has been properly updated to contain 1251 // the new data, forward it to the next level 1252 lat = calculateAccessLatency(blk, pkt->headerDelay, 1253 tag_latency); 1254 invalidateBlock(blk); 1255 return false;
|
1131 } 1132 } 1133 1134 // at this point either this is a writeback or a write-through 1135 // write clean operation and the block is already in this 1136 // cache, we need to update the data and the block flags 1137 assert(blk); 1138 // TODO: the coherent cache can assert(!blk->isDirty()); 1139 if (!pkt->writeThrough()) { 1140 blk->status |= BlkDirty; 1141 } 1142 // nothing else to do; writeback doesn't expect response 1143 assert(!pkt->needsResponse()); 1144 pkt->writeDataToBlock(blk->data, blkSize); 1145 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 1146 1147 incHitCount(pkt); 1148 1149 // A writeback searches for the block, then writes the data 1150 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1151 1152 // When the packet metadata arrives, the tag lookup will be done while 1153 // the payload is arriving. Then the block will be ready to access as 1154 // soon as the fill is done 1155 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1156 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay)); 1157
| 1256 } 1257 } 1258 1259 // at this point either this is a writeback or a write-through 1260 // write clean operation and the block is already in this 1261 // cache, we need to update the data and the block flags 1262 assert(blk); 1263 // TODO: the coherent cache can assert(!blk->isDirty()); 1264 if (!pkt->writeThrough()) { 1265 blk->status |= BlkDirty; 1266 } 1267 // nothing else to do; writeback doesn't expect response 1268 assert(!pkt->needsResponse()); 1269 pkt->writeDataToBlock(blk->data, blkSize); 1270 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 1271 1272 incHitCount(pkt); 1273 1274 // A writeback searches for the block, then writes the data 1275 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1276 1277 // When the packet metadata arrives, the tag lookup will be done while 1278 // the payload is arriving. Then the block will be ready to access as 1279 // soon as the fill is done 1280 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1281 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay)); 1282
|
1158 // if this a write-through packet it will be sent to cache 1159 // below
| 1283 // If this a write-through packet it will be sent to cache below
|
1160 return !pkt->writeThrough(); 1161 } else if (blk && (pkt->needsWritable() ? blk->isWritable() : 1162 blk->isReadable())) { 1163 // OK to satisfy access 1164 incHitCount(pkt); 1165 1166 // Calculate access latency based on the need to access the data array 1167 if (pkt->isRead() || pkt->isWrite()) { 1168 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1169 1170 // When a block is compressed, it must first be decompressed 1171 // before being read. This adds to the access latency. 1172 if (compressor && pkt->isRead()) { 1173 lat += compressor->getDecompressionLatency(blk); 1174 } 1175 } else { 1176 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1177 } 1178 1179 satisfyRequest(pkt, blk); 1180 maintainClusivity(pkt->fromCache(), blk); 1181 1182 return true; 1183 } 1184 1185 // Can't satisfy access normally... either no block (blk == nullptr) 1186 // or have block but need writable 1187 1188 incMissCount(pkt); 1189 1190 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1191 1192 if (!blk && pkt->isLLSC() && pkt->isWrite()) { 1193 // complete miss on store conditional... just give up now 1194 pkt->req->setExtraData(0); 1195 return true; 1196 } 1197 1198 return false; 1199} 1200 1201void 1202BaseCache::maintainClusivity(bool from_cache, CacheBlk *blk) 1203{ 1204 if (from_cache && blk && blk->isValid() && !blk->isDirty() && 1205 clusivity == Enums::mostly_excl) { 1206 // if we have responded to a cache, and our block is still 1207 // valid, but not dirty, and this cache is mostly exclusive 1208 // with respect to the cache above, drop the block 1209 invalidateBlock(blk); 1210 } 1211} 1212 1213CacheBlk* 1214BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks, 1215 bool allocate) 1216{ 1217 assert(pkt->isResponse()); 1218 Addr addr = pkt->getAddr(); 1219 bool is_secure = pkt->isSecure(); 1220#if TRACING_ON 1221 CacheBlk::State old_state = blk ? blk->status : 0; 1222#endif 1223 1224 // When handling a fill, we should have no writes to this line. 1225 assert(addr == pkt->getBlockAddr(blkSize)); 1226 assert(!writeBuffer.findMatch(addr, is_secure)); 1227 1228 if (!blk) { 1229 // better have read new data... 1230 assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp); 1231 1232 // need to do a replacement if allocating, otherwise we stick 1233 // with the temporary storage 1234 blk = allocate ? allocateBlock(pkt, writebacks) : nullptr; 1235 1236 if (!blk) { 1237 // No replaceable block or a mostly exclusive 1238 // cache... just use temporary storage to complete the 1239 // current request and then get rid of it 1240 blk = tempBlock; 1241 tempBlock->insert(addr, is_secure); 1242 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr, 1243 is_secure ? "s" : "ns"); 1244 } 1245 } else { 1246 // existing block... probably an upgrade 1247 // don't clear block status... if block is already dirty we 1248 // don't want to lose that 1249 } 1250 1251 // Block is guaranteed to be valid at this point 1252 assert(blk->isValid()); 1253 assert(blk->isSecure() == is_secure); 1254 assert(regenerateBlkAddr(blk) == addr); 1255 1256 blk->status |= BlkReadable; 1257 1258 // sanity check for whole-line writes, which should always be 1259 // marked as writable as part of the fill, and then later marked 1260 // dirty as part of satisfyRequest 1261 if (pkt->cmd == MemCmd::InvalidateResp) { 1262 assert(!pkt->hasSharers()); 1263 } 1264 1265 // here we deal with setting the appropriate state of the line, 1266 // and we start by looking at the hasSharers flag, and ignore the 1267 // cacheResponding flag (normally signalling dirty data) if the 1268 // packet has sharers, thus the line is never allocated as Owned 1269 // (dirty but not writable), and always ends up being either 1270 // Shared, Exclusive or Modified, see Packet::setCacheResponding 1271 // for more details 1272 if (!pkt->hasSharers()) { 1273 // we could get a writable line from memory (rather than a 1274 // cache) even in a read-only cache, note that we set this bit 1275 // even for a read-only cache, possibly revisit this decision 1276 blk->status |= BlkWritable; 1277 1278 // check if we got this via cache-to-cache transfer (i.e., from a 1279 // cache that had the block in Modified or Owned state) 1280 if (pkt->cacheResponding()) { 1281 // we got the block in Modified state, and invalidated the 1282 // owners copy 1283 blk->status |= BlkDirty; 1284 1285 chatty_assert(!isReadOnly, "Should never see dirty snoop response " 1286 "in read-only cache %s\n", name()); 1287 1288 } else if (pkt->cmd.isSWPrefetch() && pkt->needsWritable()) { 1289 // All other copies of the block were invalidated and we 1290 // have an exclusive copy. 1291 1292 // The coherence protocol assumes that if we fetched an 1293 // exclusive copy of the block, we have the intention to 1294 // modify it. Therefore the MSHR for the PrefetchExReq has 1295 // been the point of ordering and this cache has commited 1296 // to respond to snoops for the block. 1297 // 1298 // In most cases this is true anyway - a PrefetchExReq 1299 // will be followed by a WriteReq. However, if that 1300 // doesn't happen, the block is not marked as dirty and 1301 // the cache doesn't respond to snoops that has committed 1302 // to do so. 1303 // 1304 // To avoid deadlocks in cases where there is a snoop 1305 // between the PrefetchExReq and the expected WriteReq, we 1306 // proactively mark the block as Dirty. 1307 1308 blk->status |= BlkDirty; 1309 1310 panic_if(!isReadOnly, "Prefetch exclusive requests from read-only " 1311 "cache %s\n", name()); 1312 } 1313 } 1314 1315 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n", 1316 addr, is_secure ? "s" : "ns", old_state, blk->print()); 1317 1318 // if we got new data, copy it in (checking for a read response 1319 // and a response that has data is the same in the end) 1320 if (pkt->isRead()) { 1321 // sanity checks 1322 assert(pkt->hasData()); 1323 assert(pkt->getSize() == blkSize); 1324 1325 pkt->writeDataToBlock(blk->data, blkSize); 1326 } 1327 // The block will be ready when the payload arrives and the fill is done 1328 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1329 pkt->payloadDelay); 1330 1331 return blk; 1332} 1333 1334CacheBlk* 1335BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks) 1336{ 1337 // Get address 1338 const Addr addr = pkt->getAddr(); 1339 1340 // Get secure bit 1341 const bool is_secure = pkt->isSecure(); 1342 1343 // Block size and compression related access latency. Only relevant if 1344 // using a compressor, otherwise there is no extra delay, and the block 1345 // is fully sized 1346 std::size_t blk_size_bits = blkSize*8; 1347 Cycles compression_lat = Cycles(0); 1348 Cycles decompression_lat = Cycles(0); 1349 1350 // If a compressor is being used, it is called to compress data before 1351 // insertion. Although in Gem5 the data is stored uncompressed, even if a 1352 // compressor is used, the compression/decompression methods are called to 1353 // calculate the amount of extra cycles needed to read or write compressed 1354 // blocks. 1355 if (compressor) { 1356 compressor->compress(pkt->getConstPtr<uint64_t>(), compression_lat, 1357 decompression_lat, blk_size_bits); 1358 } 1359 1360 // Find replacement victim 1361 std::vector<CacheBlk*> evict_blks; 1362 CacheBlk *victim = tags->findVictim(addr, is_secure, blk_size_bits, 1363 evict_blks); 1364 1365 // It is valid to return nullptr if there is no victim 1366 if (!victim) 1367 return nullptr; 1368 1369 // Print victim block's information 1370 DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print()); 1371 1372 // Check for transient state allocations. If any of the entries listed 1373 // for eviction has a transient state, the allocation fails 1374 bool replacement = false; 1375 for (const auto& blk : evict_blks) { 1376 if (blk->isValid()) { 1377 replacement = true; 1378 1379 Addr repl_addr = regenerateBlkAddr(blk); 1380 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure()); 1381 if (repl_mshr) { 1382 // must be an outstanding upgrade or clean request 1383 // on a block we're about to replace... 1384 assert((!blk->isWritable() && repl_mshr->needsWritable()) || 1385 repl_mshr->isCleaning()); 1386 1387 // too hard to replace block with transient state 1388 // allocation failed, block not inserted 1389 return nullptr; 1390 } 1391 } 1392 } 1393 1394 // The victim will be replaced by a new entry, so increase the replacement 1395 // counter if a valid block is being replaced 1396 if (replacement) { 1397 // Evict valid blocks associated to this victim block 1398 for (const auto& blk : evict_blks) { 1399 if (blk->isValid()) { 1400 DPRINTF(CacheRepl, "Evicting %s (%#llx) to make room for " \ 1401 "%#llx (%s)\n", blk->print(), regenerateBlkAddr(blk), 1402 addr, is_secure); 1403 1404 if (blk->wasPrefetched()) { 1405 unusedPrefetches++; 1406 } 1407 1408 evictBlock(blk, writebacks); 1409 } 1410 } 1411 1412 replacements++; 1413 } 1414 1415 // If using a compressor, set compression data. This must be done before 1416 // block insertion, as compressed tags use this information. 1417 if (compressor) { 1418 compressor->setSizeBits(victim, blk_size_bits); 1419 compressor->setDecompressionLatency(victim, decompression_lat); 1420 } 1421 1422 // Insert new block at victimized entry 1423 tags->insertBlock(pkt, victim); 1424 1425 return victim; 1426} 1427 1428void 1429BaseCache::invalidateBlock(CacheBlk *blk) 1430{ 1431 // If handling a block present in the Tags, let it do its invalidation 1432 // process, which will update stats and invalidate the block itself 1433 if (blk != tempBlock) { 1434 tags->invalidate(blk); 1435 } else { 1436 tempBlock->invalidate(); 1437 } 1438} 1439 1440void 1441BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks) 1442{ 1443 PacketPtr pkt = evictBlock(blk); 1444 if (pkt) { 1445 writebacks.push_back(pkt); 1446 } 1447} 1448 1449PacketPtr 1450BaseCache::writebackBlk(CacheBlk *blk) 1451{ 1452 chatty_assert(!isReadOnly || writebackClean, 1453 "Writeback from read-only cache"); 1454 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean)); 1455 1456 writebacks[Request::wbMasterId]++; 1457 1458 RequestPtr req = std::make_shared<Request>( 1459 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1460 1461 if (blk->isSecure()) 1462 req->setFlags(Request::SECURE); 1463 1464 req->taskId(blk->task_id); 1465 1466 PacketPtr pkt = 1467 new Packet(req, blk->isDirty() ? 1468 MemCmd::WritebackDirty : MemCmd::WritebackClean); 1469 1470 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n", 1471 pkt->print(), blk->isWritable(), blk->isDirty()); 1472 1473 if (blk->isWritable()) { 1474 // not asserting shared means we pass the block in modified 1475 // state, mark our own block non-writeable 1476 blk->status &= ~BlkWritable; 1477 } else { 1478 // we are in the Owned state, tell the receiver 1479 pkt->setHasSharers(); 1480 } 1481 1482 // make sure the block is not marked dirty 1483 blk->status &= ~BlkDirty; 1484 1485 pkt->allocate(); 1486 pkt->setDataFromBlock(blk->data, blkSize); 1487 1488 // When a block is compressed, it must first be decompressed before being 1489 // sent for writeback. 1490 if (compressor) { 1491 pkt->payloadDelay = compressor->getDecompressionLatency(blk); 1492 } 1493 1494 return pkt; 1495} 1496 1497PacketPtr 1498BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id) 1499{ 1500 RequestPtr req = std::make_shared<Request>( 1501 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1502 1503 if (blk->isSecure()) { 1504 req->setFlags(Request::SECURE); 1505 } 1506 req->taskId(blk->task_id); 1507 1508 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id); 1509 1510 if (dest) { 1511 req->setFlags(dest); 1512 pkt->setWriteThrough(); 1513 } 1514 1515 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(), 1516 blk->isWritable(), blk->isDirty()); 1517 1518 if (blk->isWritable()) { 1519 // not asserting shared means we pass the block in modified 1520 // state, mark our own block non-writeable 1521 blk->status &= ~BlkWritable; 1522 } else { 1523 // we are in the Owned state, tell the receiver 1524 pkt->setHasSharers(); 1525 } 1526 1527 // make sure the block is not marked dirty 1528 blk->status &= ~BlkDirty; 1529 1530 pkt->allocate(); 1531 pkt->setDataFromBlock(blk->data, blkSize); 1532 1533 // When a block is compressed, it must first be decompressed before being 1534 // sent for writeback. 1535 if (compressor) { 1536 pkt->payloadDelay = compressor->getDecompressionLatency(blk); 1537 } 1538 1539 return pkt; 1540} 1541 1542 1543void 1544BaseCache::memWriteback() 1545{ 1546 tags->forEachBlk([this](CacheBlk &blk) { writebackVisitor(blk); }); 1547} 1548 1549void 1550BaseCache::memInvalidate() 1551{ 1552 tags->forEachBlk([this](CacheBlk &blk) { invalidateVisitor(blk); }); 1553} 1554 1555bool 1556BaseCache::isDirty() const 1557{ 1558 return tags->anyBlk([](CacheBlk &blk) { return blk.isDirty(); }); 1559} 1560 1561bool 1562BaseCache::coalesce() const 1563{ 1564 return writeAllocator && writeAllocator->coalesce(); 1565} 1566 1567void 1568BaseCache::writebackVisitor(CacheBlk &blk) 1569{ 1570 if (blk.isDirty()) { 1571 assert(blk.isValid()); 1572 1573 RequestPtr request = std::make_shared<Request>( 1574 regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId); 1575 1576 request->taskId(blk.task_id); 1577 if (blk.isSecure()) { 1578 request->setFlags(Request::SECURE); 1579 } 1580 1581 Packet packet(request, MemCmd::WriteReq); 1582 packet.dataStatic(blk.data); 1583 1584 memSidePort.sendFunctional(&packet); 1585 1586 blk.status &= ~BlkDirty; 1587 } 1588} 1589 1590void 1591BaseCache::invalidateVisitor(CacheBlk &blk) 1592{ 1593 if (blk.isDirty()) 1594 warn_once("Invalidating dirty cache lines. " \ 1595 "Expect things to break.\n"); 1596 1597 if (blk.isValid()) { 1598 assert(!blk.isDirty()); 1599 invalidateBlock(&blk); 1600 } 1601} 1602 1603Tick 1604BaseCache::nextQueueReadyTime() const 1605{ 1606 Tick nextReady = std::min(mshrQueue.nextReadyTime(), 1607 writeBuffer.nextReadyTime()); 1608 1609 // Don't signal prefetch ready time if no MSHRs available 1610 // Will signal once enoguh MSHRs are deallocated 1611 if (prefetcher && mshrQueue.canPrefetch()) { 1612 nextReady = std::min(nextReady, 1613 prefetcher->nextPrefetchReadyTime()); 1614 } 1615 1616 return nextReady; 1617} 1618 1619 1620bool 1621BaseCache::sendMSHRQueuePacket(MSHR* mshr) 1622{ 1623 assert(mshr); 1624 1625 // use request from 1st target 1626 PacketPtr tgt_pkt = mshr->getTarget()->pkt; 1627 1628 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print()); 1629 1630 // if the cache is in write coalescing mode or (additionally) in 1631 // no allocation mode, and we have a write packet with an MSHR 1632 // that is not a whole-line write (due to incompatible flags etc), 1633 // then reset the write mode 1634 if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) { 1635 if (!mshr->isWholeLineWrite()) { 1636 // if we are currently write coalescing, hold on the 1637 // MSHR as many cycles extra as we need to completely 1638 // write a cache line 1639 if (writeAllocator->delay(mshr->blkAddr)) { 1640 Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod(); 1641 DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow " 1642 "for write coalescing\n", tgt_pkt->print(), delay); 1643 mshrQueue.delay(mshr, delay); 1644 return false; 1645 } else { 1646 writeAllocator->reset(); 1647 } 1648 } else { 1649 writeAllocator->resetDelay(mshr->blkAddr); 1650 } 1651 } 1652 1653 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 1654 1655 // either a prefetch that is not present upstream, or a normal 1656 // MSHR request, proceed to get the packet to send downstream 1657 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(), 1658 mshr->isWholeLineWrite()); 1659 1660 mshr->isForward = (pkt == nullptr); 1661 1662 if (mshr->isForward) { 1663 // not a cache block request, but a response is expected 1664 // make copy of current packet to forward, keep current 1665 // copy for response handling 1666 pkt = new Packet(tgt_pkt, false, true); 1667 assert(!pkt->isWrite()); 1668 } 1669 1670 // play it safe and append (rather than set) the sender state, 1671 // as forwarded packets may already have existing state 1672 pkt->pushSenderState(mshr); 1673 1674 if (pkt->isClean() && blk && blk->isDirty()) { 1675 // A cache clean opearation is looking for a dirty block. Mark 1676 // the packet so that the destination xbar can determine that 1677 // there will be a follow-up write packet as well. 1678 pkt->setSatisfied(); 1679 } 1680 1681 if (!memSidePort.sendTimingReq(pkt)) { 1682 // we are awaiting a retry, but we 1683 // delete the packet and will be creating a new packet 1684 // when we get the opportunity 1685 delete pkt; 1686 1687 // note that we have now masked any requestBus and 1688 // schedSendEvent (we will wait for a retry before 1689 // doing anything), and this is so even if we do not 1690 // care about this packet and might override it before 1691 // it gets retried 1692 return true; 1693 } else { 1694 // As part of the call to sendTimingReq the packet is 1695 // forwarded to all neighbouring caches (and any caches 1696 // above them) as a snoop. Thus at this point we know if 1697 // any of the neighbouring caches are responding, and if 1698 // so, we know it is dirty, and we can determine if it is 1699 // being passed as Modified, making our MSHR the ordering 1700 // point 1701 bool pending_modified_resp = !pkt->hasSharers() && 1702 pkt->cacheResponding(); 1703 markInService(mshr, pending_modified_resp); 1704 1705 if (pkt->isClean() && blk && blk->isDirty()) { 1706 // A cache clean opearation is looking for a dirty 1707 // block. If a dirty block is encountered a WriteClean 1708 // will update any copies to the path to the memory 1709 // until the point of reference. 1710 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 1711 __func__, pkt->print(), blk->print()); 1712 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), 1713 pkt->id); 1714 PacketList writebacks; 1715 writebacks.push_back(wb_pkt); 1716 doWritebacks(writebacks, 0); 1717 } 1718 1719 return false; 1720 } 1721} 1722 1723bool 1724BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) 1725{ 1726 assert(wq_entry); 1727 1728 // always a single target for write queue entries 1729 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 1730 1731 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print()); 1732 1733 // forward as is, both for evictions and uncacheable writes 1734 if (!memSidePort.sendTimingReq(tgt_pkt)) { 1735 // note that we have now masked any requestBus and 1736 // schedSendEvent (we will wait for a retry before 1737 // doing anything), and this is so even if we do not 1738 // care about this packet and might override it before 1739 // it gets retried 1740 return true; 1741 } else { 1742 markInService(wq_entry); 1743 return false; 1744 } 1745} 1746 1747void 1748BaseCache::serialize(CheckpointOut &cp) const 1749{ 1750 bool dirty(isDirty()); 1751 1752 if (dirty) { 1753 warn("*** The cache still contains dirty data. ***\n"); 1754 warn(" Make sure to drain the system using the correct flags.\n"); 1755 warn(" This checkpoint will not restore correctly " \ 1756 "and dirty data in the cache will be lost!\n"); 1757 } 1758 1759 // Since we don't checkpoint the data in the cache, any dirty data 1760 // will be lost when restoring from a checkpoint of a system that 1761 // wasn't drained properly. Flag the checkpoint as invalid if the 1762 // cache contains dirty data. 1763 bool bad_checkpoint(dirty); 1764 SERIALIZE_SCALAR(bad_checkpoint); 1765} 1766 1767void 1768BaseCache::unserialize(CheckpointIn &cp) 1769{ 1770 bool bad_checkpoint; 1771 UNSERIALIZE_SCALAR(bad_checkpoint); 1772 if (bad_checkpoint) { 1773 fatal("Restoring from checkpoints with dirty caches is not " 1774 "supported in the classic memory system. Please remove any " 1775 "caches or drain them properly before taking checkpoints.\n"); 1776 } 1777} 1778 1779void 1780BaseCache::regStats() 1781{ 1782 ClockedObject::regStats(); 1783 1784 using namespace Stats; 1785 1786 // Hit statistics 1787 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1788 MemCmd cmd(access_idx); 1789 const string &cstr = cmd.toString(); 1790 1791 hits[access_idx] 1792 .init(system->maxMasters()) 1793 .name(name() + "." + cstr + "_hits") 1794 .desc("number of " + cstr + " hits") 1795 .flags(total | nozero | nonan) 1796 ; 1797 for (int i = 0; i < system->maxMasters(); i++) { 1798 hits[access_idx].subname(i, system->getMasterName(i)); 1799 } 1800 } 1801 1802// These macros make it easier to sum the right subset of commands and 1803// to change the subset of commands that are considered "demand" vs 1804// "non-demand" 1805#define SUM_DEMAND(s) \ 1806 (s[MemCmd::ReadReq] + s[MemCmd::WriteReq] + s[MemCmd::WriteLineReq] + \ 1807 s[MemCmd::ReadExReq] + s[MemCmd::ReadCleanReq] + s[MemCmd::ReadSharedReq]) 1808 1809// should writebacks be included here? prior code was inconsistent... 1810#define SUM_NON_DEMAND(s) \ 1811 (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq] + s[MemCmd::SoftPFExReq]) 1812 1813 demandHits 1814 .name(name() + ".demand_hits") 1815 .desc("number of demand (read+write) hits") 1816 .flags(total | nozero | nonan) 1817 ; 1818 demandHits = SUM_DEMAND(hits); 1819 for (int i = 0; i < system->maxMasters(); i++) { 1820 demandHits.subname(i, system->getMasterName(i)); 1821 } 1822 1823 overallHits 1824 .name(name() + ".overall_hits") 1825 .desc("number of overall hits") 1826 .flags(total | nozero | nonan) 1827 ; 1828 overallHits = demandHits + SUM_NON_DEMAND(hits); 1829 for (int i = 0; i < system->maxMasters(); i++) { 1830 overallHits.subname(i, system->getMasterName(i)); 1831 } 1832 1833 // Miss statistics 1834 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1835 MemCmd cmd(access_idx); 1836 const string &cstr = cmd.toString(); 1837 1838 misses[access_idx] 1839 .init(system->maxMasters()) 1840 .name(name() + "." + cstr + "_misses") 1841 .desc("number of " + cstr + " misses") 1842 .flags(total | nozero | nonan) 1843 ; 1844 for (int i = 0; i < system->maxMasters(); i++) { 1845 misses[access_idx].subname(i, system->getMasterName(i)); 1846 } 1847 } 1848 1849 demandMisses 1850 .name(name() + ".demand_misses") 1851 .desc("number of demand (read+write) misses") 1852 .flags(total | nozero | nonan) 1853 ; 1854 demandMisses = SUM_DEMAND(misses); 1855 for (int i = 0; i < system->maxMasters(); i++) { 1856 demandMisses.subname(i, system->getMasterName(i)); 1857 } 1858 1859 overallMisses 1860 .name(name() + ".overall_misses") 1861 .desc("number of overall misses") 1862 .flags(total | nozero | nonan) 1863 ; 1864 overallMisses = demandMisses + SUM_NON_DEMAND(misses); 1865 for (int i = 0; i < system->maxMasters(); i++) { 1866 overallMisses.subname(i, system->getMasterName(i)); 1867 } 1868 1869 // Miss latency statistics 1870 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1871 MemCmd cmd(access_idx); 1872 const string &cstr = cmd.toString(); 1873 1874 missLatency[access_idx] 1875 .init(system->maxMasters()) 1876 .name(name() + "." + cstr + "_miss_latency") 1877 .desc("number of " + cstr + " miss cycles") 1878 .flags(total | nozero | nonan) 1879 ; 1880 for (int i = 0; i < system->maxMasters(); i++) { 1881 missLatency[access_idx].subname(i, system->getMasterName(i)); 1882 } 1883 } 1884 1885 demandMissLatency 1886 .name(name() + ".demand_miss_latency") 1887 .desc("number of demand (read+write) miss cycles") 1888 .flags(total | nozero | nonan) 1889 ; 1890 demandMissLatency = SUM_DEMAND(missLatency); 1891 for (int i = 0; i < system->maxMasters(); i++) { 1892 demandMissLatency.subname(i, system->getMasterName(i)); 1893 } 1894 1895 overallMissLatency 1896 .name(name() + ".overall_miss_latency") 1897 .desc("number of overall miss cycles") 1898 .flags(total | nozero | nonan) 1899 ; 1900 overallMissLatency = demandMissLatency + SUM_NON_DEMAND(missLatency); 1901 for (int i = 0; i < system->maxMasters(); i++) { 1902 overallMissLatency.subname(i, system->getMasterName(i)); 1903 } 1904 1905 // access formulas 1906 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1907 MemCmd cmd(access_idx); 1908 const string &cstr = cmd.toString(); 1909 1910 accesses[access_idx] 1911 .name(name() + "." + cstr + "_accesses") 1912 .desc("number of " + cstr + " accesses(hits+misses)") 1913 .flags(total | nozero | nonan) 1914 ; 1915 accesses[access_idx] = hits[access_idx] + misses[access_idx]; 1916 1917 for (int i = 0; i < system->maxMasters(); i++) { 1918 accesses[access_idx].subname(i, system->getMasterName(i)); 1919 } 1920 } 1921 1922 demandAccesses 1923 .name(name() + ".demand_accesses") 1924 .desc("number of demand (read+write) accesses") 1925 .flags(total | nozero | nonan) 1926 ; 1927 demandAccesses = demandHits + demandMisses; 1928 for (int i = 0; i < system->maxMasters(); i++) { 1929 demandAccesses.subname(i, system->getMasterName(i)); 1930 } 1931 1932 overallAccesses 1933 .name(name() + ".overall_accesses") 1934 .desc("number of overall (read+write) accesses") 1935 .flags(total | nozero | nonan) 1936 ; 1937 overallAccesses = overallHits + overallMisses; 1938 for (int i = 0; i < system->maxMasters(); i++) { 1939 overallAccesses.subname(i, system->getMasterName(i)); 1940 } 1941 1942 // miss rate formulas 1943 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1944 MemCmd cmd(access_idx); 1945 const string &cstr = cmd.toString(); 1946 1947 missRate[access_idx] 1948 .name(name() + "." + cstr + "_miss_rate") 1949 .desc("miss rate for " + cstr + " accesses") 1950 .flags(total | nozero | nonan) 1951 ; 1952 missRate[access_idx] = misses[access_idx] / accesses[access_idx]; 1953 1954 for (int i = 0; i < system->maxMasters(); i++) { 1955 missRate[access_idx].subname(i, system->getMasterName(i)); 1956 } 1957 } 1958 1959 demandMissRate 1960 .name(name() + ".demand_miss_rate") 1961 .desc("miss rate for demand accesses") 1962 .flags(total | nozero | nonan) 1963 ; 1964 demandMissRate = demandMisses / demandAccesses; 1965 for (int i = 0; i < system->maxMasters(); i++) { 1966 demandMissRate.subname(i, system->getMasterName(i)); 1967 } 1968 1969 overallMissRate 1970 .name(name() + ".overall_miss_rate") 1971 .desc("miss rate for overall accesses") 1972 .flags(total | nozero | nonan) 1973 ; 1974 overallMissRate = overallMisses / overallAccesses; 1975 for (int i = 0; i < system->maxMasters(); i++) { 1976 overallMissRate.subname(i, system->getMasterName(i)); 1977 } 1978 1979 // miss latency formulas 1980 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1981 MemCmd cmd(access_idx); 1982 const string &cstr = cmd.toString(); 1983 1984 avgMissLatency[access_idx] 1985 .name(name() + "." + cstr + "_avg_miss_latency") 1986 .desc("average " + cstr + " miss latency") 1987 .flags(total | nozero | nonan) 1988 ; 1989 avgMissLatency[access_idx] = 1990 missLatency[access_idx] / misses[access_idx]; 1991 1992 for (int i = 0; i < system->maxMasters(); i++) { 1993 avgMissLatency[access_idx].subname(i, system->getMasterName(i)); 1994 } 1995 } 1996 1997 demandAvgMissLatency 1998 .name(name() + ".demand_avg_miss_latency") 1999 .desc("average overall miss latency") 2000 .flags(total | nozero | nonan) 2001 ; 2002 demandAvgMissLatency = demandMissLatency / demandMisses; 2003 for (int i = 0; i < system->maxMasters(); i++) { 2004 demandAvgMissLatency.subname(i, system->getMasterName(i)); 2005 } 2006 2007 overallAvgMissLatency 2008 .name(name() + ".overall_avg_miss_latency") 2009 .desc("average overall miss latency") 2010 .flags(total | nozero | nonan) 2011 ; 2012 overallAvgMissLatency = overallMissLatency / overallMisses; 2013 for (int i = 0; i < system->maxMasters(); i++) { 2014 overallAvgMissLatency.subname(i, system->getMasterName(i)); 2015 } 2016 2017 blocked_cycles.init(NUM_BLOCKED_CAUSES); 2018 blocked_cycles 2019 .name(name() + ".blocked_cycles") 2020 .desc("number of cycles access was blocked") 2021 .subname(Blocked_NoMSHRs, "no_mshrs") 2022 .subname(Blocked_NoTargets, "no_targets") 2023 ; 2024 2025 2026 blocked_causes.init(NUM_BLOCKED_CAUSES); 2027 blocked_causes 2028 .name(name() + ".blocked") 2029 .desc("number of cycles access was blocked") 2030 .subname(Blocked_NoMSHRs, "no_mshrs") 2031 .subname(Blocked_NoTargets, "no_targets") 2032 ; 2033 2034 avg_blocked 2035 .name(name() + ".avg_blocked_cycles") 2036 .desc("average number of cycles each access was blocked") 2037 .subname(Blocked_NoMSHRs, "no_mshrs") 2038 .subname(Blocked_NoTargets, "no_targets") 2039 ; 2040 2041 avg_blocked = blocked_cycles / blocked_causes; 2042 2043 unusedPrefetches 2044 .name(name() + ".unused_prefetches") 2045 .desc("number of HardPF blocks evicted w/o reference") 2046 .flags(nozero) 2047 ; 2048 2049 writebacks 2050 .init(system->maxMasters()) 2051 .name(name() + ".writebacks") 2052 .desc("number of writebacks") 2053 .flags(total | nozero | nonan) 2054 ; 2055 for (int i = 0; i < system->maxMasters(); i++) { 2056 writebacks.subname(i, system->getMasterName(i)); 2057 } 2058 2059 // MSHR statistics 2060 // MSHR hit statistics 2061 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2062 MemCmd cmd(access_idx); 2063 const string &cstr = cmd.toString(); 2064 2065 mshr_hits[access_idx] 2066 .init(system->maxMasters()) 2067 .name(name() + "." + cstr + "_mshr_hits") 2068 .desc("number of " + cstr + " MSHR hits") 2069 .flags(total | nozero | nonan) 2070 ; 2071 for (int i = 0; i < system->maxMasters(); i++) { 2072 mshr_hits[access_idx].subname(i, system->getMasterName(i)); 2073 } 2074 } 2075 2076 demandMshrHits 2077 .name(name() + ".demand_mshr_hits") 2078 .desc("number of demand (read+write) MSHR hits") 2079 .flags(total | nozero | nonan) 2080 ; 2081 demandMshrHits = SUM_DEMAND(mshr_hits); 2082 for (int i = 0; i < system->maxMasters(); i++) { 2083 demandMshrHits.subname(i, system->getMasterName(i)); 2084 } 2085 2086 overallMshrHits 2087 .name(name() + ".overall_mshr_hits") 2088 .desc("number of overall MSHR hits") 2089 .flags(total | nozero | nonan) 2090 ; 2091 overallMshrHits = demandMshrHits + SUM_NON_DEMAND(mshr_hits); 2092 for (int i = 0; i < system->maxMasters(); i++) { 2093 overallMshrHits.subname(i, system->getMasterName(i)); 2094 } 2095 2096 // MSHR miss statistics 2097 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2098 MemCmd cmd(access_idx); 2099 const string &cstr = cmd.toString(); 2100 2101 mshr_misses[access_idx] 2102 .init(system->maxMasters()) 2103 .name(name() + "." + cstr + "_mshr_misses") 2104 .desc("number of " + cstr + " MSHR misses") 2105 .flags(total | nozero | nonan) 2106 ; 2107 for (int i = 0; i < system->maxMasters(); i++) { 2108 mshr_misses[access_idx].subname(i, system->getMasterName(i)); 2109 } 2110 } 2111 2112 demandMshrMisses 2113 .name(name() + ".demand_mshr_misses") 2114 .desc("number of demand (read+write) MSHR misses") 2115 .flags(total | nozero | nonan) 2116 ; 2117 demandMshrMisses = SUM_DEMAND(mshr_misses); 2118 for (int i = 0; i < system->maxMasters(); i++) { 2119 demandMshrMisses.subname(i, system->getMasterName(i)); 2120 } 2121 2122 overallMshrMisses 2123 .name(name() + ".overall_mshr_misses") 2124 .desc("number of overall MSHR misses") 2125 .flags(total | nozero | nonan) 2126 ; 2127 overallMshrMisses = demandMshrMisses + SUM_NON_DEMAND(mshr_misses); 2128 for (int i = 0; i < system->maxMasters(); i++) { 2129 overallMshrMisses.subname(i, system->getMasterName(i)); 2130 } 2131 2132 // MSHR miss latency statistics 2133 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2134 MemCmd cmd(access_idx); 2135 const string &cstr = cmd.toString(); 2136 2137 mshr_miss_latency[access_idx] 2138 .init(system->maxMasters()) 2139 .name(name() + "." + cstr + "_mshr_miss_latency") 2140 .desc("number of " + cstr + " MSHR miss cycles") 2141 .flags(total | nozero | nonan) 2142 ; 2143 for (int i = 0; i < system->maxMasters(); i++) { 2144 mshr_miss_latency[access_idx].subname(i, system->getMasterName(i)); 2145 } 2146 } 2147 2148 demandMshrMissLatency 2149 .name(name() + ".demand_mshr_miss_latency") 2150 .desc("number of demand (read+write) MSHR miss cycles") 2151 .flags(total | nozero | nonan) 2152 ; 2153 demandMshrMissLatency = SUM_DEMAND(mshr_miss_latency); 2154 for (int i = 0; i < system->maxMasters(); i++) { 2155 demandMshrMissLatency.subname(i, system->getMasterName(i)); 2156 } 2157 2158 overallMshrMissLatency 2159 .name(name() + ".overall_mshr_miss_latency") 2160 .desc("number of overall MSHR miss cycles") 2161 .flags(total | nozero | nonan) 2162 ; 2163 overallMshrMissLatency = 2164 demandMshrMissLatency + SUM_NON_DEMAND(mshr_miss_latency); 2165 for (int i = 0; i < system->maxMasters(); i++) { 2166 overallMshrMissLatency.subname(i, system->getMasterName(i)); 2167 } 2168 2169 // MSHR uncacheable statistics 2170 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2171 MemCmd cmd(access_idx); 2172 const string &cstr = cmd.toString(); 2173 2174 mshr_uncacheable[access_idx] 2175 .init(system->maxMasters()) 2176 .name(name() + "." + cstr + "_mshr_uncacheable") 2177 .desc("number of " + cstr + " MSHR uncacheable") 2178 .flags(total | nozero | nonan) 2179 ; 2180 for (int i = 0; i < system->maxMasters(); i++) { 2181 mshr_uncacheable[access_idx].subname(i, system->getMasterName(i)); 2182 } 2183 } 2184 2185 overallMshrUncacheable 2186 .name(name() + ".overall_mshr_uncacheable_misses") 2187 .desc("number of overall MSHR uncacheable misses") 2188 .flags(total | nozero | nonan) 2189 ; 2190 overallMshrUncacheable = 2191 SUM_DEMAND(mshr_uncacheable) + SUM_NON_DEMAND(mshr_uncacheable); 2192 for (int i = 0; i < system->maxMasters(); i++) { 2193 overallMshrUncacheable.subname(i, system->getMasterName(i)); 2194 } 2195 2196 // MSHR miss latency statistics 2197 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2198 MemCmd cmd(access_idx); 2199 const string &cstr = cmd.toString(); 2200 2201 mshr_uncacheable_lat[access_idx] 2202 .init(system->maxMasters()) 2203 .name(name() + "." + cstr + "_mshr_uncacheable_latency") 2204 .desc("number of " + cstr + " MSHR uncacheable cycles") 2205 .flags(total | nozero | nonan) 2206 ; 2207 for (int i = 0; i < system->maxMasters(); i++) { 2208 mshr_uncacheable_lat[access_idx].subname( 2209 i, system->getMasterName(i)); 2210 } 2211 } 2212 2213 overallMshrUncacheableLatency 2214 .name(name() + ".overall_mshr_uncacheable_latency") 2215 .desc("number of overall MSHR uncacheable cycles") 2216 .flags(total | nozero | nonan) 2217 ; 2218 overallMshrUncacheableLatency = 2219 SUM_DEMAND(mshr_uncacheable_lat) + 2220 SUM_NON_DEMAND(mshr_uncacheable_lat); 2221 for (int i = 0; i < system->maxMasters(); i++) { 2222 overallMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2223 } 2224 2225#if 0 2226 // MSHR access formulas 2227 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2228 MemCmd cmd(access_idx); 2229 const string &cstr = cmd.toString(); 2230 2231 mshrAccesses[access_idx] 2232 .name(name() + "." + cstr + "_mshr_accesses") 2233 .desc("number of " + cstr + " mshr accesses(hits+misses)") 2234 .flags(total | nozero | nonan) 2235 ; 2236 mshrAccesses[access_idx] = 2237 mshr_hits[access_idx] + mshr_misses[access_idx] 2238 + mshr_uncacheable[access_idx]; 2239 } 2240 2241 demandMshrAccesses 2242 .name(name() + ".demand_mshr_accesses") 2243 .desc("number of demand (read+write) mshr accesses") 2244 .flags(total | nozero | nonan) 2245 ; 2246 demandMshrAccesses = demandMshrHits + demandMshrMisses; 2247 2248 overallMshrAccesses 2249 .name(name() + ".overall_mshr_accesses") 2250 .desc("number of overall (read+write) mshr accesses") 2251 .flags(total | nozero | nonan) 2252 ; 2253 overallMshrAccesses = overallMshrHits + overallMshrMisses 2254 + overallMshrUncacheable; 2255#endif 2256 2257 // MSHR miss rate formulas 2258 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2259 MemCmd cmd(access_idx); 2260 const string &cstr = cmd.toString(); 2261 2262 mshrMissRate[access_idx] 2263 .name(name() + "." + cstr + "_mshr_miss_rate") 2264 .desc("mshr miss rate for " + cstr + " accesses") 2265 .flags(total | nozero | nonan) 2266 ; 2267 mshrMissRate[access_idx] = 2268 mshr_misses[access_idx] / accesses[access_idx]; 2269 2270 for (int i = 0; i < system->maxMasters(); i++) { 2271 mshrMissRate[access_idx].subname(i, system->getMasterName(i)); 2272 } 2273 } 2274 2275 demandMshrMissRate 2276 .name(name() + ".demand_mshr_miss_rate") 2277 .desc("mshr miss rate for demand accesses") 2278 .flags(total | nozero | nonan) 2279 ; 2280 demandMshrMissRate = demandMshrMisses / demandAccesses; 2281 for (int i = 0; i < system->maxMasters(); i++) { 2282 demandMshrMissRate.subname(i, system->getMasterName(i)); 2283 } 2284 2285 overallMshrMissRate 2286 .name(name() + ".overall_mshr_miss_rate") 2287 .desc("mshr miss rate for overall accesses") 2288 .flags(total | nozero | nonan) 2289 ; 2290 overallMshrMissRate = overallMshrMisses / overallAccesses; 2291 for (int i = 0; i < system->maxMasters(); i++) { 2292 overallMshrMissRate.subname(i, system->getMasterName(i)); 2293 } 2294 2295 // mshrMiss latency formulas 2296 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2297 MemCmd cmd(access_idx); 2298 const string &cstr = cmd.toString(); 2299 2300 avgMshrMissLatency[access_idx] 2301 .name(name() + "." + cstr + "_avg_mshr_miss_latency") 2302 .desc("average " + cstr + " mshr miss latency") 2303 .flags(total | nozero | nonan) 2304 ; 2305 avgMshrMissLatency[access_idx] = 2306 mshr_miss_latency[access_idx] / mshr_misses[access_idx]; 2307 2308 for (int i = 0; i < system->maxMasters(); i++) { 2309 avgMshrMissLatency[access_idx].subname( 2310 i, system->getMasterName(i)); 2311 } 2312 } 2313 2314 demandAvgMshrMissLatency 2315 .name(name() + ".demand_avg_mshr_miss_latency") 2316 .desc("average overall mshr miss latency") 2317 .flags(total | nozero | nonan) 2318 ; 2319 demandAvgMshrMissLatency = demandMshrMissLatency / demandMshrMisses; 2320 for (int i = 0; i < system->maxMasters(); i++) { 2321 demandAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2322 } 2323 2324 overallAvgMshrMissLatency 2325 .name(name() + ".overall_avg_mshr_miss_latency") 2326 .desc("average overall mshr miss latency") 2327 .flags(total | nozero | nonan) 2328 ; 2329 overallAvgMshrMissLatency = overallMshrMissLatency / overallMshrMisses; 2330 for (int i = 0; i < system->maxMasters(); i++) { 2331 overallAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2332 } 2333 2334 // mshrUncacheable latency formulas 2335 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2336 MemCmd cmd(access_idx); 2337 const string &cstr = cmd.toString(); 2338 2339 avgMshrUncacheableLatency[access_idx] 2340 .name(name() + "." + cstr + "_avg_mshr_uncacheable_latency") 2341 .desc("average " + cstr + " mshr uncacheable latency") 2342 .flags(total | nozero | nonan) 2343 ; 2344 avgMshrUncacheableLatency[access_idx] = 2345 mshr_uncacheable_lat[access_idx] / mshr_uncacheable[access_idx]; 2346 2347 for (int i = 0; i < system->maxMasters(); i++) { 2348 avgMshrUncacheableLatency[access_idx].subname( 2349 i, system->getMasterName(i)); 2350 } 2351 } 2352 2353 overallAvgMshrUncacheableLatency 2354 .name(name() + ".overall_avg_mshr_uncacheable_latency") 2355 .desc("average overall mshr uncacheable latency") 2356 .flags(total | nozero | nonan) 2357 ; 2358 overallAvgMshrUncacheableLatency = 2359 overallMshrUncacheableLatency / overallMshrUncacheable; 2360 for (int i = 0; i < system->maxMasters(); i++) { 2361 overallAvgMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2362 } 2363 2364 replacements 2365 .name(name() + ".replacements") 2366 .desc("number of replacements") 2367 ;
| 1284 return !pkt->writeThrough(); 1285 } else if (blk && (pkt->needsWritable() ? blk->isWritable() : 1286 blk->isReadable())) { 1287 // OK to satisfy access 1288 incHitCount(pkt); 1289 1290 // Calculate access latency based on the need to access the data array 1291 if (pkt->isRead() || pkt->isWrite()) { 1292 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1293 1294 // When a block is compressed, it must first be decompressed 1295 // before being read. This adds to the access latency. 1296 if (compressor && pkt->isRead()) { 1297 lat += compressor->getDecompressionLatency(blk); 1298 } 1299 } else { 1300 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1301 } 1302 1303 satisfyRequest(pkt, blk); 1304 maintainClusivity(pkt->fromCache(), blk); 1305 1306 return true; 1307 } 1308 1309 // Can't satisfy access normally... either no block (blk == nullptr) 1310 // or have block but need writable 1311 1312 incMissCount(pkt); 1313 1314 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency); 1315 1316 if (!blk && pkt->isLLSC() && pkt->isWrite()) { 1317 // complete miss on store conditional... just give up now 1318 pkt->req->setExtraData(0); 1319 return true; 1320 } 1321 1322 return false; 1323} 1324 1325void 1326BaseCache::maintainClusivity(bool from_cache, CacheBlk *blk) 1327{ 1328 if (from_cache && blk && blk->isValid() && !blk->isDirty() && 1329 clusivity == Enums::mostly_excl) { 1330 // if we have responded to a cache, and our block is still 1331 // valid, but not dirty, and this cache is mostly exclusive 1332 // with respect to the cache above, drop the block 1333 invalidateBlock(blk); 1334 } 1335} 1336 1337CacheBlk* 1338BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks, 1339 bool allocate) 1340{ 1341 assert(pkt->isResponse()); 1342 Addr addr = pkt->getAddr(); 1343 bool is_secure = pkt->isSecure(); 1344#if TRACING_ON 1345 CacheBlk::State old_state = blk ? blk->status : 0; 1346#endif 1347 1348 // When handling a fill, we should have no writes to this line. 1349 assert(addr == pkt->getBlockAddr(blkSize)); 1350 assert(!writeBuffer.findMatch(addr, is_secure)); 1351 1352 if (!blk) { 1353 // better have read new data... 1354 assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp); 1355 1356 // need to do a replacement if allocating, otherwise we stick 1357 // with the temporary storage 1358 blk = allocate ? allocateBlock(pkt, writebacks) : nullptr; 1359 1360 if (!blk) { 1361 // No replaceable block or a mostly exclusive 1362 // cache... just use temporary storage to complete the 1363 // current request and then get rid of it 1364 blk = tempBlock; 1365 tempBlock->insert(addr, is_secure); 1366 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr, 1367 is_secure ? "s" : "ns"); 1368 } 1369 } else { 1370 // existing block... probably an upgrade 1371 // don't clear block status... if block is already dirty we 1372 // don't want to lose that 1373 } 1374 1375 // Block is guaranteed to be valid at this point 1376 assert(blk->isValid()); 1377 assert(blk->isSecure() == is_secure); 1378 assert(regenerateBlkAddr(blk) == addr); 1379 1380 blk->status |= BlkReadable; 1381 1382 // sanity check for whole-line writes, which should always be 1383 // marked as writable as part of the fill, and then later marked 1384 // dirty as part of satisfyRequest 1385 if (pkt->cmd == MemCmd::InvalidateResp) { 1386 assert(!pkt->hasSharers()); 1387 } 1388 1389 // here we deal with setting the appropriate state of the line, 1390 // and we start by looking at the hasSharers flag, and ignore the 1391 // cacheResponding flag (normally signalling dirty data) if the 1392 // packet has sharers, thus the line is never allocated as Owned 1393 // (dirty but not writable), and always ends up being either 1394 // Shared, Exclusive or Modified, see Packet::setCacheResponding 1395 // for more details 1396 if (!pkt->hasSharers()) { 1397 // we could get a writable line from memory (rather than a 1398 // cache) even in a read-only cache, note that we set this bit 1399 // even for a read-only cache, possibly revisit this decision 1400 blk->status |= BlkWritable; 1401 1402 // check if we got this via cache-to-cache transfer (i.e., from a 1403 // cache that had the block in Modified or Owned state) 1404 if (pkt->cacheResponding()) { 1405 // we got the block in Modified state, and invalidated the 1406 // owners copy 1407 blk->status |= BlkDirty; 1408 1409 chatty_assert(!isReadOnly, "Should never see dirty snoop response " 1410 "in read-only cache %s\n", name()); 1411 1412 } else if (pkt->cmd.isSWPrefetch() && pkt->needsWritable()) { 1413 // All other copies of the block were invalidated and we 1414 // have an exclusive copy. 1415 1416 // The coherence protocol assumes that if we fetched an 1417 // exclusive copy of the block, we have the intention to 1418 // modify it. Therefore the MSHR for the PrefetchExReq has 1419 // been the point of ordering and this cache has commited 1420 // to respond to snoops for the block. 1421 // 1422 // In most cases this is true anyway - a PrefetchExReq 1423 // will be followed by a WriteReq. However, if that 1424 // doesn't happen, the block is not marked as dirty and 1425 // the cache doesn't respond to snoops that has committed 1426 // to do so. 1427 // 1428 // To avoid deadlocks in cases where there is a snoop 1429 // between the PrefetchExReq and the expected WriteReq, we 1430 // proactively mark the block as Dirty. 1431 1432 blk->status |= BlkDirty; 1433 1434 panic_if(!isReadOnly, "Prefetch exclusive requests from read-only " 1435 "cache %s\n", name()); 1436 } 1437 } 1438 1439 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n", 1440 addr, is_secure ? "s" : "ns", old_state, blk->print()); 1441 1442 // if we got new data, copy it in (checking for a read response 1443 // and a response that has data is the same in the end) 1444 if (pkt->isRead()) { 1445 // sanity checks 1446 assert(pkt->hasData()); 1447 assert(pkt->getSize() == blkSize); 1448 1449 pkt->writeDataToBlock(blk->data, blkSize); 1450 } 1451 // The block will be ready when the payload arrives and the fill is done 1452 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1453 pkt->payloadDelay); 1454 1455 return blk; 1456} 1457 1458CacheBlk* 1459BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks) 1460{ 1461 // Get address 1462 const Addr addr = pkt->getAddr(); 1463 1464 // Get secure bit 1465 const bool is_secure = pkt->isSecure(); 1466 1467 // Block size and compression related access latency. Only relevant if 1468 // using a compressor, otherwise there is no extra delay, and the block 1469 // is fully sized 1470 std::size_t blk_size_bits = blkSize*8; 1471 Cycles compression_lat = Cycles(0); 1472 Cycles decompression_lat = Cycles(0); 1473 1474 // If a compressor is being used, it is called to compress data before 1475 // insertion. Although in Gem5 the data is stored uncompressed, even if a 1476 // compressor is used, the compression/decompression methods are called to 1477 // calculate the amount of extra cycles needed to read or write compressed 1478 // blocks. 1479 if (compressor) { 1480 compressor->compress(pkt->getConstPtr<uint64_t>(), compression_lat, 1481 decompression_lat, blk_size_bits); 1482 } 1483 1484 // Find replacement victim 1485 std::vector<CacheBlk*> evict_blks; 1486 CacheBlk *victim = tags->findVictim(addr, is_secure, blk_size_bits, 1487 evict_blks); 1488 1489 // It is valid to return nullptr if there is no victim 1490 if (!victim) 1491 return nullptr; 1492 1493 // Print victim block's information 1494 DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print()); 1495 1496 // Check for transient state allocations. If any of the entries listed 1497 // for eviction has a transient state, the allocation fails 1498 bool replacement = false; 1499 for (const auto& blk : evict_blks) { 1500 if (blk->isValid()) { 1501 replacement = true; 1502 1503 Addr repl_addr = regenerateBlkAddr(blk); 1504 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure()); 1505 if (repl_mshr) { 1506 // must be an outstanding upgrade or clean request 1507 // on a block we're about to replace... 1508 assert((!blk->isWritable() && repl_mshr->needsWritable()) || 1509 repl_mshr->isCleaning()); 1510 1511 // too hard to replace block with transient state 1512 // allocation failed, block not inserted 1513 return nullptr; 1514 } 1515 } 1516 } 1517 1518 // The victim will be replaced by a new entry, so increase the replacement 1519 // counter if a valid block is being replaced 1520 if (replacement) { 1521 // Evict valid blocks associated to this victim block 1522 for (const auto& blk : evict_blks) { 1523 if (blk->isValid()) { 1524 DPRINTF(CacheRepl, "Evicting %s (%#llx) to make room for " \ 1525 "%#llx (%s)\n", blk->print(), regenerateBlkAddr(blk), 1526 addr, is_secure); 1527 1528 if (blk->wasPrefetched()) { 1529 unusedPrefetches++; 1530 } 1531 1532 evictBlock(blk, writebacks); 1533 } 1534 } 1535 1536 replacements++; 1537 } 1538 1539 // If using a compressor, set compression data. This must be done before 1540 // block insertion, as compressed tags use this information. 1541 if (compressor) { 1542 compressor->setSizeBits(victim, blk_size_bits); 1543 compressor->setDecompressionLatency(victim, decompression_lat); 1544 } 1545 1546 // Insert new block at victimized entry 1547 tags->insertBlock(pkt, victim); 1548 1549 return victim; 1550} 1551 1552void 1553BaseCache::invalidateBlock(CacheBlk *blk) 1554{ 1555 // If handling a block present in the Tags, let it do its invalidation 1556 // process, which will update stats and invalidate the block itself 1557 if (blk != tempBlock) { 1558 tags->invalidate(blk); 1559 } else { 1560 tempBlock->invalidate(); 1561 } 1562} 1563 1564void 1565BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks) 1566{ 1567 PacketPtr pkt = evictBlock(blk); 1568 if (pkt) { 1569 writebacks.push_back(pkt); 1570 } 1571} 1572 1573PacketPtr 1574BaseCache::writebackBlk(CacheBlk *blk) 1575{ 1576 chatty_assert(!isReadOnly || writebackClean, 1577 "Writeback from read-only cache"); 1578 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean)); 1579 1580 writebacks[Request::wbMasterId]++; 1581 1582 RequestPtr req = std::make_shared<Request>( 1583 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1584 1585 if (blk->isSecure()) 1586 req->setFlags(Request::SECURE); 1587 1588 req->taskId(blk->task_id); 1589 1590 PacketPtr pkt = 1591 new Packet(req, blk->isDirty() ? 1592 MemCmd::WritebackDirty : MemCmd::WritebackClean); 1593 1594 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n", 1595 pkt->print(), blk->isWritable(), blk->isDirty()); 1596 1597 if (blk->isWritable()) { 1598 // not asserting shared means we pass the block in modified 1599 // state, mark our own block non-writeable 1600 blk->status &= ~BlkWritable; 1601 } else { 1602 // we are in the Owned state, tell the receiver 1603 pkt->setHasSharers(); 1604 } 1605 1606 // make sure the block is not marked dirty 1607 blk->status &= ~BlkDirty; 1608 1609 pkt->allocate(); 1610 pkt->setDataFromBlock(blk->data, blkSize); 1611 1612 // When a block is compressed, it must first be decompressed before being 1613 // sent for writeback. 1614 if (compressor) { 1615 pkt->payloadDelay = compressor->getDecompressionLatency(blk); 1616 } 1617 1618 return pkt; 1619} 1620 1621PacketPtr 1622BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id) 1623{ 1624 RequestPtr req = std::make_shared<Request>( 1625 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1626 1627 if (blk->isSecure()) { 1628 req->setFlags(Request::SECURE); 1629 } 1630 req->taskId(blk->task_id); 1631 1632 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id); 1633 1634 if (dest) { 1635 req->setFlags(dest); 1636 pkt->setWriteThrough(); 1637 } 1638 1639 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(), 1640 blk->isWritable(), blk->isDirty()); 1641 1642 if (blk->isWritable()) { 1643 // not asserting shared means we pass the block in modified 1644 // state, mark our own block non-writeable 1645 blk->status &= ~BlkWritable; 1646 } else { 1647 // we are in the Owned state, tell the receiver 1648 pkt->setHasSharers(); 1649 } 1650 1651 // make sure the block is not marked dirty 1652 blk->status &= ~BlkDirty; 1653 1654 pkt->allocate(); 1655 pkt->setDataFromBlock(blk->data, blkSize); 1656 1657 // When a block is compressed, it must first be decompressed before being 1658 // sent for writeback. 1659 if (compressor) { 1660 pkt->payloadDelay = compressor->getDecompressionLatency(blk); 1661 } 1662 1663 return pkt; 1664} 1665 1666 1667void 1668BaseCache::memWriteback() 1669{ 1670 tags->forEachBlk([this](CacheBlk &blk) { writebackVisitor(blk); }); 1671} 1672 1673void 1674BaseCache::memInvalidate() 1675{ 1676 tags->forEachBlk([this](CacheBlk &blk) { invalidateVisitor(blk); }); 1677} 1678 1679bool 1680BaseCache::isDirty() const 1681{ 1682 return tags->anyBlk([](CacheBlk &blk) { return blk.isDirty(); }); 1683} 1684 1685bool 1686BaseCache::coalesce() const 1687{ 1688 return writeAllocator && writeAllocator->coalesce(); 1689} 1690 1691void 1692BaseCache::writebackVisitor(CacheBlk &blk) 1693{ 1694 if (blk.isDirty()) { 1695 assert(blk.isValid()); 1696 1697 RequestPtr request = std::make_shared<Request>( 1698 regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId); 1699 1700 request->taskId(blk.task_id); 1701 if (blk.isSecure()) { 1702 request->setFlags(Request::SECURE); 1703 } 1704 1705 Packet packet(request, MemCmd::WriteReq); 1706 packet.dataStatic(blk.data); 1707 1708 memSidePort.sendFunctional(&packet); 1709 1710 blk.status &= ~BlkDirty; 1711 } 1712} 1713 1714void 1715BaseCache::invalidateVisitor(CacheBlk &blk) 1716{ 1717 if (blk.isDirty()) 1718 warn_once("Invalidating dirty cache lines. " \ 1719 "Expect things to break.\n"); 1720 1721 if (blk.isValid()) { 1722 assert(!blk.isDirty()); 1723 invalidateBlock(&blk); 1724 } 1725} 1726 1727Tick 1728BaseCache::nextQueueReadyTime() const 1729{ 1730 Tick nextReady = std::min(mshrQueue.nextReadyTime(), 1731 writeBuffer.nextReadyTime()); 1732 1733 // Don't signal prefetch ready time if no MSHRs available 1734 // Will signal once enoguh MSHRs are deallocated 1735 if (prefetcher && mshrQueue.canPrefetch()) { 1736 nextReady = std::min(nextReady, 1737 prefetcher->nextPrefetchReadyTime()); 1738 } 1739 1740 return nextReady; 1741} 1742 1743 1744bool 1745BaseCache::sendMSHRQueuePacket(MSHR* mshr) 1746{ 1747 assert(mshr); 1748 1749 // use request from 1st target 1750 PacketPtr tgt_pkt = mshr->getTarget()->pkt; 1751 1752 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print()); 1753 1754 // if the cache is in write coalescing mode or (additionally) in 1755 // no allocation mode, and we have a write packet with an MSHR 1756 // that is not a whole-line write (due to incompatible flags etc), 1757 // then reset the write mode 1758 if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) { 1759 if (!mshr->isWholeLineWrite()) { 1760 // if we are currently write coalescing, hold on the 1761 // MSHR as many cycles extra as we need to completely 1762 // write a cache line 1763 if (writeAllocator->delay(mshr->blkAddr)) { 1764 Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod(); 1765 DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow " 1766 "for write coalescing\n", tgt_pkt->print(), delay); 1767 mshrQueue.delay(mshr, delay); 1768 return false; 1769 } else { 1770 writeAllocator->reset(); 1771 } 1772 } else { 1773 writeAllocator->resetDelay(mshr->blkAddr); 1774 } 1775 } 1776 1777 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 1778 1779 // either a prefetch that is not present upstream, or a normal 1780 // MSHR request, proceed to get the packet to send downstream 1781 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(), 1782 mshr->isWholeLineWrite()); 1783 1784 mshr->isForward = (pkt == nullptr); 1785 1786 if (mshr->isForward) { 1787 // not a cache block request, but a response is expected 1788 // make copy of current packet to forward, keep current 1789 // copy for response handling 1790 pkt = new Packet(tgt_pkt, false, true); 1791 assert(!pkt->isWrite()); 1792 } 1793 1794 // play it safe and append (rather than set) the sender state, 1795 // as forwarded packets may already have existing state 1796 pkt->pushSenderState(mshr); 1797 1798 if (pkt->isClean() && blk && blk->isDirty()) { 1799 // A cache clean opearation is looking for a dirty block. Mark 1800 // the packet so that the destination xbar can determine that 1801 // there will be a follow-up write packet as well. 1802 pkt->setSatisfied(); 1803 } 1804 1805 if (!memSidePort.sendTimingReq(pkt)) { 1806 // we are awaiting a retry, but we 1807 // delete the packet and will be creating a new packet 1808 // when we get the opportunity 1809 delete pkt; 1810 1811 // note that we have now masked any requestBus and 1812 // schedSendEvent (we will wait for a retry before 1813 // doing anything), and this is so even if we do not 1814 // care about this packet and might override it before 1815 // it gets retried 1816 return true; 1817 } else { 1818 // As part of the call to sendTimingReq the packet is 1819 // forwarded to all neighbouring caches (and any caches 1820 // above them) as a snoop. Thus at this point we know if 1821 // any of the neighbouring caches are responding, and if 1822 // so, we know it is dirty, and we can determine if it is 1823 // being passed as Modified, making our MSHR the ordering 1824 // point 1825 bool pending_modified_resp = !pkt->hasSharers() && 1826 pkt->cacheResponding(); 1827 markInService(mshr, pending_modified_resp); 1828 1829 if (pkt->isClean() && blk && blk->isDirty()) { 1830 // A cache clean opearation is looking for a dirty 1831 // block. If a dirty block is encountered a WriteClean 1832 // will update any copies to the path to the memory 1833 // until the point of reference. 1834 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 1835 __func__, pkt->print(), blk->print()); 1836 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), 1837 pkt->id); 1838 PacketList writebacks; 1839 writebacks.push_back(wb_pkt); 1840 doWritebacks(writebacks, 0); 1841 } 1842 1843 return false; 1844 } 1845} 1846 1847bool 1848BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) 1849{ 1850 assert(wq_entry); 1851 1852 // always a single target for write queue entries 1853 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 1854 1855 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print()); 1856 1857 // forward as is, both for evictions and uncacheable writes 1858 if (!memSidePort.sendTimingReq(tgt_pkt)) { 1859 // note that we have now masked any requestBus and 1860 // schedSendEvent (we will wait for a retry before 1861 // doing anything), and this is so even if we do not 1862 // care about this packet and might override it before 1863 // it gets retried 1864 return true; 1865 } else { 1866 markInService(wq_entry); 1867 return false; 1868 } 1869} 1870 1871void 1872BaseCache::serialize(CheckpointOut &cp) const 1873{ 1874 bool dirty(isDirty()); 1875 1876 if (dirty) { 1877 warn("*** The cache still contains dirty data. ***\n"); 1878 warn(" Make sure to drain the system using the correct flags.\n"); 1879 warn(" This checkpoint will not restore correctly " \ 1880 "and dirty data in the cache will be lost!\n"); 1881 } 1882 1883 // Since we don't checkpoint the data in the cache, any dirty data 1884 // will be lost when restoring from a checkpoint of a system that 1885 // wasn't drained properly. Flag the checkpoint as invalid if the 1886 // cache contains dirty data. 1887 bool bad_checkpoint(dirty); 1888 SERIALIZE_SCALAR(bad_checkpoint); 1889} 1890 1891void 1892BaseCache::unserialize(CheckpointIn &cp) 1893{ 1894 bool bad_checkpoint; 1895 UNSERIALIZE_SCALAR(bad_checkpoint); 1896 if (bad_checkpoint) { 1897 fatal("Restoring from checkpoints with dirty caches is not " 1898 "supported in the classic memory system. Please remove any " 1899 "caches or drain them properly before taking checkpoints.\n"); 1900 } 1901} 1902 1903void 1904BaseCache::regStats() 1905{ 1906 ClockedObject::regStats(); 1907 1908 using namespace Stats; 1909 1910 // Hit statistics 1911 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1912 MemCmd cmd(access_idx); 1913 const string &cstr = cmd.toString(); 1914 1915 hits[access_idx] 1916 .init(system->maxMasters()) 1917 .name(name() + "." + cstr + "_hits") 1918 .desc("number of " + cstr + " hits") 1919 .flags(total | nozero | nonan) 1920 ; 1921 for (int i = 0; i < system->maxMasters(); i++) { 1922 hits[access_idx].subname(i, system->getMasterName(i)); 1923 } 1924 } 1925 1926// These macros make it easier to sum the right subset of commands and 1927// to change the subset of commands that are considered "demand" vs 1928// "non-demand" 1929#define SUM_DEMAND(s) \ 1930 (s[MemCmd::ReadReq] + s[MemCmd::WriteReq] + s[MemCmd::WriteLineReq] + \ 1931 s[MemCmd::ReadExReq] + s[MemCmd::ReadCleanReq] + s[MemCmd::ReadSharedReq]) 1932 1933// should writebacks be included here? prior code was inconsistent... 1934#define SUM_NON_DEMAND(s) \ 1935 (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq] + s[MemCmd::SoftPFExReq]) 1936 1937 demandHits 1938 .name(name() + ".demand_hits") 1939 .desc("number of demand (read+write) hits") 1940 .flags(total | nozero | nonan) 1941 ; 1942 demandHits = SUM_DEMAND(hits); 1943 for (int i = 0; i < system->maxMasters(); i++) { 1944 demandHits.subname(i, system->getMasterName(i)); 1945 } 1946 1947 overallHits 1948 .name(name() + ".overall_hits") 1949 .desc("number of overall hits") 1950 .flags(total | nozero | nonan) 1951 ; 1952 overallHits = demandHits + SUM_NON_DEMAND(hits); 1953 for (int i = 0; i < system->maxMasters(); i++) { 1954 overallHits.subname(i, system->getMasterName(i)); 1955 } 1956 1957 // Miss statistics 1958 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1959 MemCmd cmd(access_idx); 1960 const string &cstr = cmd.toString(); 1961 1962 misses[access_idx] 1963 .init(system->maxMasters()) 1964 .name(name() + "." + cstr + "_misses") 1965 .desc("number of " + cstr + " misses") 1966 .flags(total | nozero | nonan) 1967 ; 1968 for (int i = 0; i < system->maxMasters(); i++) { 1969 misses[access_idx].subname(i, system->getMasterName(i)); 1970 } 1971 } 1972 1973 demandMisses 1974 .name(name() + ".demand_misses") 1975 .desc("number of demand (read+write) misses") 1976 .flags(total | nozero | nonan) 1977 ; 1978 demandMisses = SUM_DEMAND(misses); 1979 for (int i = 0; i < system->maxMasters(); i++) { 1980 demandMisses.subname(i, system->getMasterName(i)); 1981 } 1982 1983 overallMisses 1984 .name(name() + ".overall_misses") 1985 .desc("number of overall misses") 1986 .flags(total | nozero | nonan) 1987 ; 1988 overallMisses = demandMisses + SUM_NON_DEMAND(misses); 1989 for (int i = 0; i < system->maxMasters(); i++) { 1990 overallMisses.subname(i, system->getMasterName(i)); 1991 } 1992 1993 // Miss latency statistics 1994 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1995 MemCmd cmd(access_idx); 1996 const string &cstr = cmd.toString(); 1997 1998 missLatency[access_idx] 1999 .init(system->maxMasters()) 2000 .name(name() + "." + cstr + "_miss_latency") 2001 .desc("number of " + cstr + " miss cycles") 2002 .flags(total | nozero | nonan) 2003 ; 2004 for (int i = 0; i < system->maxMasters(); i++) { 2005 missLatency[access_idx].subname(i, system->getMasterName(i)); 2006 } 2007 } 2008 2009 demandMissLatency 2010 .name(name() + ".demand_miss_latency") 2011 .desc("number of demand (read+write) miss cycles") 2012 .flags(total | nozero | nonan) 2013 ; 2014 demandMissLatency = SUM_DEMAND(missLatency); 2015 for (int i = 0; i < system->maxMasters(); i++) { 2016 demandMissLatency.subname(i, system->getMasterName(i)); 2017 } 2018 2019 overallMissLatency 2020 .name(name() + ".overall_miss_latency") 2021 .desc("number of overall miss cycles") 2022 .flags(total | nozero | nonan) 2023 ; 2024 overallMissLatency = demandMissLatency + SUM_NON_DEMAND(missLatency); 2025 for (int i = 0; i < system->maxMasters(); i++) { 2026 overallMissLatency.subname(i, system->getMasterName(i)); 2027 } 2028 2029 // access formulas 2030 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2031 MemCmd cmd(access_idx); 2032 const string &cstr = cmd.toString(); 2033 2034 accesses[access_idx] 2035 .name(name() + "." + cstr + "_accesses") 2036 .desc("number of " + cstr + " accesses(hits+misses)") 2037 .flags(total | nozero | nonan) 2038 ; 2039 accesses[access_idx] = hits[access_idx] + misses[access_idx]; 2040 2041 for (int i = 0; i < system->maxMasters(); i++) { 2042 accesses[access_idx].subname(i, system->getMasterName(i)); 2043 } 2044 } 2045 2046 demandAccesses 2047 .name(name() + ".demand_accesses") 2048 .desc("number of demand (read+write) accesses") 2049 .flags(total | nozero | nonan) 2050 ; 2051 demandAccesses = demandHits + demandMisses; 2052 for (int i = 0; i < system->maxMasters(); i++) { 2053 demandAccesses.subname(i, system->getMasterName(i)); 2054 } 2055 2056 overallAccesses 2057 .name(name() + ".overall_accesses") 2058 .desc("number of overall (read+write) accesses") 2059 .flags(total | nozero | nonan) 2060 ; 2061 overallAccesses = overallHits + overallMisses; 2062 for (int i = 0; i < system->maxMasters(); i++) { 2063 overallAccesses.subname(i, system->getMasterName(i)); 2064 } 2065 2066 // miss rate formulas 2067 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2068 MemCmd cmd(access_idx); 2069 const string &cstr = cmd.toString(); 2070 2071 missRate[access_idx] 2072 .name(name() + "." + cstr + "_miss_rate") 2073 .desc("miss rate for " + cstr + " accesses") 2074 .flags(total | nozero | nonan) 2075 ; 2076 missRate[access_idx] = misses[access_idx] / accesses[access_idx]; 2077 2078 for (int i = 0; i < system->maxMasters(); i++) { 2079 missRate[access_idx].subname(i, system->getMasterName(i)); 2080 } 2081 } 2082 2083 demandMissRate 2084 .name(name() + ".demand_miss_rate") 2085 .desc("miss rate for demand accesses") 2086 .flags(total | nozero | nonan) 2087 ; 2088 demandMissRate = demandMisses / demandAccesses; 2089 for (int i = 0; i < system->maxMasters(); i++) { 2090 demandMissRate.subname(i, system->getMasterName(i)); 2091 } 2092 2093 overallMissRate 2094 .name(name() + ".overall_miss_rate") 2095 .desc("miss rate for overall accesses") 2096 .flags(total | nozero | nonan) 2097 ; 2098 overallMissRate = overallMisses / overallAccesses; 2099 for (int i = 0; i < system->maxMasters(); i++) { 2100 overallMissRate.subname(i, system->getMasterName(i)); 2101 } 2102 2103 // miss latency formulas 2104 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2105 MemCmd cmd(access_idx); 2106 const string &cstr = cmd.toString(); 2107 2108 avgMissLatency[access_idx] 2109 .name(name() + "." + cstr + "_avg_miss_latency") 2110 .desc("average " + cstr + " miss latency") 2111 .flags(total | nozero | nonan) 2112 ; 2113 avgMissLatency[access_idx] = 2114 missLatency[access_idx] / misses[access_idx]; 2115 2116 for (int i = 0; i < system->maxMasters(); i++) { 2117 avgMissLatency[access_idx].subname(i, system->getMasterName(i)); 2118 } 2119 } 2120 2121 demandAvgMissLatency 2122 .name(name() + ".demand_avg_miss_latency") 2123 .desc("average overall miss latency") 2124 .flags(total | nozero | nonan) 2125 ; 2126 demandAvgMissLatency = demandMissLatency / demandMisses; 2127 for (int i = 0; i < system->maxMasters(); i++) { 2128 demandAvgMissLatency.subname(i, system->getMasterName(i)); 2129 } 2130 2131 overallAvgMissLatency 2132 .name(name() + ".overall_avg_miss_latency") 2133 .desc("average overall miss latency") 2134 .flags(total | nozero | nonan) 2135 ; 2136 overallAvgMissLatency = overallMissLatency / overallMisses; 2137 for (int i = 0; i < system->maxMasters(); i++) { 2138 overallAvgMissLatency.subname(i, system->getMasterName(i)); 2139 } 2140 2141 blocked_cycles.init(NUM_BLOCKED_CAUSES); 2142 blocked_cycles 2143 .name(name() + ".blocked_cycles") 2144 .desc("number of cycles access was blocked") 2145 .subname(Blocked_NoMSHRs, "no_mshrs") 2146 .subname(Blocked_NoTargets, "no_targets") 2147 ; 2148 2149 2150 blocked_causes.init(NUM_BLOCKED_CAUSES); 2151 blocked_causes 2152 .name(name() + ".blocked") 2153 .desc("number of cycles access was blocked") 2154 .subname(Blocked_NoMSHRs, "no_mshrs") 2155 .subname(Blocked_NoTargets, "no_targets") 2156 ; 2157 2158 avg_blocked 2159 .name(name() + ".avg_blocked_cycles") 2160 .desc("average number of cycles each access was blocked") 2161 .subname(Blocked_NoMSHRs, "no_mshrs") 2162 .subname(Blocked_NoTargets, "no_targets") 2163 ; 2164 2165 avg_blocked = blocked_cycles / blocked_causes; 2166 2167 unusedPrefetches 2168 .name(name() + ".unused_prefetches") 2169 .desc("number of HardPF blocks evicted w/o reference") 2170 .flags(nozero) 2171 ; 2172 2173 writebacks 2174 .init(system->maxMasters()) 2175 .name(name() + ".writebacks") 2176 .desc("number of writebacks") 2177 .flags(total | nozero | nonan) 2178 ; 2179 for (int i = 0; i < system->maxMasters(); i++) { 2180 writebacks.subname(i, system->getMasterName(i)); 2181 } 2182 2183 // MSHR statistics 2184 // MSHR hit statistics 2185 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2186 MemCmd cmd(access_idx); 2187 const string &cstr = cmd.toString(); 2188 2189 mshr_hits[access_idx] 2190 .init(system->maxMasters()) 2191 .name(name() + "." + cstr + "_mshr_hits") 2192 .desc("number of " + cstr + " MSHR hits") 2193 .flags(total | nozero | nonan) 2194 ; 2195 for (int i = 0; i < system->maxMasters(); i++) { 2196 mshr_hits[access_idx].subname(i, system->getMasterName(i)); 2197 } 2198 } 2199 2200 demandMshrHits 2201 .name(name() + ".demand_mshr_hits") 2202 .desc("number of demand (read+write) MSHR hits") 2203 .flags(total | nozero | nonan) 2204 ; 2205 demandMshrHits = SUM_DEMAND(mshr_hits); 2206 for (int i = 0; i < system->maxMasters(); i++) { 2207 demandMshrHits.subname(i, system->getMasterName(i)); 2208 } 2209 2210 overallMshrHits 2211 .name(name() + ".overall_mshr_hits") 2212 .desc("number of overall MSHR hits") 2213 .flags(total | nozero | nonan) 2214 ; 2215 overallMshrHits = demandMshrHits + SUM_NON_DEMAND(mshr_hits); 2216 for (int i = 0; i < system->maxMasters(); i++) { 2217 overallMshrHits.subname(i, system->getMasterName(i)); 2218 } 2219 2220 // MSHR miss statistics 2221 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2222 MemCmd cmd(access_idx); 2223 const string &cstr = cmd.toString(); 2224 2225 mshr_misses[access_idx] 2226 .init(system->maxMasters()) 2227 .name(name() + "." + cstr + "_mshr_misses") 2228 .desc("number of " + cstr + " MSHR misses") 2229 .flags(total | nozero | nonan) 2230 ; 2231 for (int i = 0; i < system->maxMasters(); i++) { 2232 mshr_misses[access_idx].subname(i, system->getMasterName(i)); 2233 } 2234 } 2235 2236 demandMshrMisses 2237 .name(name() + ".demand_mshr_misses") 2238 .desc("number of demand (read+write) MSHR misses") 2239 .flags(total | nozero | nonan) 2240 ; 2241 demandMshrMisses = SUM_DEMAND(mshr_misses); 2242 for (int i = 0; i < system->maxMasters(); i++) { 2243 demandMshrMisses.subname(i, system->getMasterName(i)); 2244 } 2245 2246 overallMshrMisses 2247 .name(name() + ".overall_mshr_misses") 2248 .desc("number of overall MSHR misses") 2249 .flags(total | nozero | nonan) 2250 ; 2251 overallMshrMisses = demandMshrMisses + SUM_NON_DEMAND(mshr_misses); 2252 for (int i = 0; i < system->maxMasters(); i++) { 2253 overallMshrMisses.subname(i, system->getMasterName(i)); 2254 } 2255 2256 // MSHR miss latency statistics 2257 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2258 MemCmd cmd(access_idx); 2259 const string &cstr = cmd.toString(); 2260 2261 mshr_miss_latency[access_idx] 2262 .init(system->maxMasters()) 2263 .name(name() + "." + cstr + "_mshr_miss_latency") 2264 .desc("number of " + cstr + " MSHR miss cycles") 2265 .flags(total | nozero | nonan) 2266 ; 2267 for (int i = 0; i < system->maxMasters(); i++) { 2268 mshr_miss_latency[access_idx].subname(i, system->getMasterName(i)); 2269 } 2270 } 2271 2272 demandMshrMissLatency 2273 .name(name() + ".demand_mshr_miss_latency") 2274 .desc("number of demand (read+write) MSHR miss cycles") 2275 .flags(total | nozero | nonan) 2276 ; 2277 demandMshrMissLatency = SUM_DEMAND(mshr_miss_latency); 2278 for (int i = 0; i < system->maxMasters(); i++) { 2279 demandMshrMissLatency.subname(i, system->getMasterName(i)); 2280 } 2281 2282 overallMshrMissLatency 2283 .name(name() + ".overall_mshr_miss_latency") 2284 .desc("number of overall MSHR miss cycles") 2285 .flags(total | nozero | nonan) 2286 ; 2287 overallMshrMissLatency = 2288 demandMshrMissLatency + SUM_NON_DEMAND(mshr_miss_latency); 2289 for (int i = 0; i < system->maxMasters(); i++) { 2290 overallMshrMissLatency.subname(i, system->getMasterName(i)); 2291 } 2292 2293 // MSHR uncacheable statistics 2294 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2295 MemCmd cmd(access_idx); 2296 const string &cstr = cmd.toString(); 2297 2298 mshr_uncacheable[access_idx] 2299 .init(system->maxMasters()) 2300 .name(name() + "." + cstr + "_mshr_uncacheable") 2301 .desc("number of " + cstr + " MSHR uncacheable") 2302 .flags(total | nozero | nonan) 2303 ; 2304 for (int i = 0; i < system->maxMasters(); i++) { 2305 mshr_uncacheable[access_idx].subname(i, system->getMasterName(i)); 2306 } 2307 } 2308 2309 overallMshrUncacheable 2310 .name(name() + ".overall_mshr_uncacheable_misses") 2311 .desc("number of overall MSHR uncacheable misses") 2312 .flags(total | nozero | nonan) 2313 ; 2314 overallMshrUncacheable = 2315 SUM_DEMAND(mshr_uncacheable) + SUM_NON_DEMAND(mshr_uncacheable); 2316 for (int i = 0; i < system->maxMasters(); i++) { 2317 overallMshrUncacheable.subname(i, system->getMasterName(i)); 2318 } 2319 2320 // MSHR miss latency statistics 2321 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2322 MemCmd cmd(access_idx); 2323 const string &cstr = cmd.toString(); 2324 2325 mshr_uncacheable_lat[access_idx] 2326 .init(system->maxMasters()) 2327 .name(name() + "." + cstr + "_mshr_uncacheable_latency") 2328 .desc("number of " + cstr + " MSHR uncacheable cycles") 2329 .flags(total | nozero | nonan) 2330 ; 2331 for (int i = 0; i < system->maxMasters(); i++) { 2332 mshr_uncacheable_lat[access_idx].subname( 2333 i, system->getMasterName(i)); 2334 } 2335 } 2336 2337 overallMshrUncacheableLatency 2338 .name(name() + ".overall_mshr_uncacheable_latency") 2339 .desc("number of overall MSHR uncacheable cycles") 2340 .flags(total | nozero | nonan) 2341 ; 2342 overallMshrUncacheableLatency = 2343 SUM_DEMAND(mshr_uncacheable_lat) + 2344 SUM_NON_DEMAND(mshr_uncacheable_lat); 2345 for (int i = 0; i < system->maxMasters(); i++) { 2346 overallMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2347 } 2348 2349#if 0 2350 // MSHR access formulas 2351 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2352 MemCmd cmd(access_idx); 2353 const string &cstr = cmd.toString(); 2354 2355 mshrAccesses[access_idx] 2356 .name(name() + "." + cstr + "_mshr_accesses") 2357 .desc("number of " + cstr + " mshr accesses(hits+misses)") 2358 .flags(total | nozero | nonan) 2359 ; 2360 mshrAccesses[access_idx] = 2361 mshr_hits[access_idx] + mshr_misses[access_idx] 2362 + mshr_uncacheable[access_idx]; 2363 } 2364 2365 demandMshrAccesses 2366 .name(name() + ".demand_mshr_accesses") 2367 .desc("number of demand (read+write) mshr accesses") 2368 .flags(total | nozero | nonan) 2369 ; 2370 demandMshrAccesses = demandMshrHits + demandMshrMisses; 2371 2372 overallMshrAccesses 2373 .name(name() + ".overall_mshr_accesses") 2374 .desc("number of overall (read+write) mshr accesses") 2375 .flags(total | nozero | nonan) 2376 ; 2377 overallMshrAccesses = overallMshrHits + overallMshrMisses 2378 + overallMshrUncacheable; 2379#endif 2380 2381 // MSHR miss rate formulas 2382 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2383 MemCmd cmd(access_idx); 2384 const string &cstr = cmd.toString(); 2385 2386 mshrMissRate[access_idx] 2387 .name(name() + "." + cstr + "_mshr_miss_rate") 2388 .desc("mshr miss rate for " + cstr + " accesses") 2389 .flags(total | nozero | nonan) 2390 ; 2391 mshrMissRate[access_idx] = 2392 mshr_misses[access_idx] / accesses[access_idx]; 2393 2394 for (int i = 0; i < system->maxMasters(); i++) { 2395 mshrMissRate[access_idx].subname(i, system->getMasterName(i)); 2396 } 2397 } 2398 2399 demandMshrMissRate 2400 .name(name() + ".demand_mshr_miss_rate") 2401 .desc("mshr miss rate for demand accesses") 2402 .flags(total | nozero | nonan) 2403 ; 2404 demandMshrMissRate = demandMshrMisses / demandAccesses; 2405 for (int i = 0; i < system->maxMasters(); i++) { 2406 demandMshrMissRate.subname(i, system->getMasterName(i)); 2407 } 2408 2409 overallMshrMissRate 2410 .name(name() + ".overall_mshr_miss_rate") 2411 .desc("mshr miss rate for overall accesses") 2412 .flags(total | nozero | nonan) 2413 ; 2414 overallMshrMissRate = overallMshrMisses / overallAccesses; 2415 for (int i = 0; i < system->maxMasters(); i++) { 2416 overallMshrMissRate.subname(i, system->getMasterName(i)); 2417 } 2418 2419 // mshrMiss latency formulas 2420 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2421 MemCmd cmd(access_idx); 2422 const string &cstr = cmd.toString(); 2423 2424 avgMshrMissLatency[access_idx] 2425 .name(name() + "." + cstr + "_avg_mshr_miss_latency") 2426 .desc("average " + cstr + " mshr miss latency") 2427 .flags(total | nozero | nonan) 2428 ; 2429 avgMshrMissLatency[access_idx] = 2430 mshr_miss_latency[access_idx] / mshr_misses[access_idx]; 2431 2432 for (int i = 0; i < system->maxMasters(); i++) { 2433 avgMshrMissLatency[access_idx].subname( 2434 i, system->getMasterName(i)); 2435 } 2436 } 2437 2438 demandAvgMshrMissLatency 2439 .name(name() + ".demand_avg_mshr_miss_latency") 2440 .desc("average overall mshr miss latency") 2441 .flags(total | nozero | nonan) 2442 ; 2443 demandAvgMshrMissLatency = demandMshrMissLatency / demandMshrMisses; 2444 for (int i = 0; i < system->maxMasters(); i++) { 2445 demandAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2446 } 2447 2448 overallAvgMshrMissLatency 2449 .name(name() + ".overall_avg_mshr_miss_latency") 2450 .desc("average overall mshr miss latency") 2451 .flags(total | nozero | nonan) 2452 ; 2453 overallAvgMshrMissLatency = overallMshrMissLatency / overallMshrMisses; 2454 for (int i = 0; i < system->maxMasters(); i++) { 2455 overallAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2456 } 2457 2458 // mshrUncacheable latency formulas 2459 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2460 MemCmd cmd(access_idx); 2461 const string &cstr = cmd.toString(); 2462 2463 avgMshrUncacheableLatency[access_idx] 2464 .name(name() + "." + cstr + "_avg_mshr_uncacheable_latency") 2465 .desc("average " + cstr + " mshr uncacheable latency") 2466 .flags(total | nozero | nonan) 2467 ; 2468 avgMshrUncacheableLatency[access_idx] = 2469 mshr_uncacheable_lat[access_idx] / mshr_uncacheable[access_idx]; 2470 2471 for (int i = 0; i < system->maxMasters(); i++) { 2472 avgMshrUncacheableLatency[access_idx].subname( 2473 i, system->getMasterName(i)); 2474 } 2475 } 2476 2477 overallAvgMshrUncacheableLatency 2478 .name(name() + ".overall_avg_mshr_uncacheable_latency") 2479 .desc("average overall mshr uncacheable latency") 2480 .flags(total | nozero | nonan) 2481 ; 2482 overallAvgMshrUncacheableLatency = 2483 overallMshrUncacheableLatency / overallMshrUncacheable; 2484 for (int i = 0; i < system->maxMasters(); i++) { 2485 overallAvgMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2486 } 2487 2488 replacements 2489 .name(name() + ".replacements") 2490 .desc("number of replacements") 2491 ;
|
| 2492 2493 dataExpansions 2494 .name(name() + ".data_expansions") 2495 .desc("number of data expansions") 2496 .flags(nozero | nonan) 2497 ;
|
2368} 2369 2370void 2371BaseCache::regProbePoints() 2372{ 2373 ppHit = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Hit"); 2374 ppMiss = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Miss"); 2375 ppFill = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Fill"); 2376} 2377 2378/////////////// 2379// 2380// CpuSidePort 2381// 2382/////////////// 2383bool 2384BaseCache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt) 2385{ 2386 // Snoops shouldn't happen when bypassing caches 2387 assert(!cache->system->bypassCaches()); 2388 2389 assert(pkt->isResponse()); 2390 2391 // Express snoop responses from master to slave, e.g., from L1 to L2 2392 cache->recvTimingSnoopResp(pkt); 2393 return true; 2394} 2395 2396 2397bool 2398BaseCache::CpuSidePort::tryTiming(PacketPtr pkt) 2399{ 2400 if (cache->system->bypassCaches() || pkt->isExpressSnoop()) { 2401 // always let express snoop packets through even if blocked 2402 return true; 2403 } else if (blocked || mustSendRetry) { 2404 // either already committed to send a retry, or blocked 2405 mustSendRetry = true; 2406 return false; 2407 } 2408 mustSendRetry = false; 2409 return true; 2410} 2411 2412bool 2413BaseCache::CpuSidePort::recvTimingReq(PacketPtr pkt) 2414{ 2415 assert(pkt->isRequest()); 2416 2417 if (cache->system->bypassCaches()) { 2418 // Just forward the packet if caches are disabled. 2419 // @todo This should really enqueue the packet rather 2420 bool M5_VAR_USED success = cache->memSidePort.sendTimingReq(pkt); 2421 assert(success); 2422 return true; 2423 } else if (tryTiming(pkt)) { 2424 cache->recvTimingReq(pkt); 2425 return true; 2426 } 2427 return false; 2428} 2429 2430Tick 2431BaseCache::CpuSidePort::recvAtomic(PacketPtr pkt) 2432{ 2433 if (cache->system->bypassCaches()) { 2434 // Forward the request if the system is in cache bypass mode. 2435 return cache->memSidePort.sendAtomic(pkt); 2436 } else { 2437 return cache->recvAtomic(pkt); 2438 } 2439} 2440 2441void 2442BaseCache::CpuSidePort::recvFunctional(PacketPtr pkt) 2443{ 2444 if (cache->system->bypassCaches()) { 2445 // The cache should be flushed if we are in cache bypass mode, 2446 // so we don't need to check if we need to update anything. 2447 cache->memSidePort.sendFunctional(pkt); 2448 return; 2449 } 2450 2451 // functional request 2452 cache->functionalAccess(pkt, true); 2453} 2454 2455AddrRangeList 2456BaseCache::CpuSidePort::getAddrRanges() const 2457{ 2458 return cache->getAddrRanges(); 2459} 2460 2461 2462BaseCache:: 2463CpuSidePort::CpuSidePort(const std::string &_name, BaseCache *_cache, 2464 const std::string &_label) 2465 : CacheSlavePort(_name, _cache, _label), cache(_cache) 2466{ 2467} 2468 2469/////////////// 2470// 2471// MemSidePort 2472// 2473/////////////// 2474bool 2475BaseCache::MemSidePort::recvTimingResp(PacketPtr pkt) 2476{ 2477 cache->recvTimingResp(pkt); 2478 return true; 2479} 2480 2481// Express snooping requests to memside port 2482void 2483BaseCache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt) 2484{ 2485 // Snoops shouldn't happen when bypassing caches 2486 assert(!cache->system->bypassCaches()); 2487 2488 // handle snooping requests 2489 cache->recvTimingSnoopReq(pkt); 2490} 2491 2492Tick 2493BaseCache::MemSidePort::recvAtomicSnoop(PacketPtr pkt) 2494{ 2495 // Snoops shouldn't happen when bypassing caches 2496 assert(!cache->system->bypassCaches()); 2497 2498 return cache->recvAtomicSnoop(pkt); 2499} 2500 2501void 2502BaseCache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt) 2503{ 2504 // Snoops shouldn't happen when bypassing caches 2505 assert(!cache->system->bypassCaches()); 2506 2507 // functional snoop (note that in contrast to atomic we don't have 2508 // a specific functionalSnoop method, as they have the same 2509 // behaviour regardless) 2510 cache->functionalAccess(pkt, false); 2511} 2512 2513void 2514BaseCache::CacheReqPacketQueue::sendDeferredPacket() 2515{ 2516 // sanity check 2517 assert(!waitingOnRetry); 2518 2519 // there should never be any deferred request packets in the 2520 // queue, instead we resly on the cache to provide the packets 2521 // from the MSHR queue or write queue 2522 assert(deferredPacketReadyTime() == MaxTick); 2523 2524 // check for request packets (requests & writebacks) 2525 QueueEntry* entry = cache.getNextQueueEntry(); 2526 2527 if (!entry) { 2528 // can happen if e.g. we attempt a writeback and fail, but 2529 // before the retry, the writeback is eliminated because 2530 // we snoop another cache's ReadEx. 2531 } else { 2532 // let our snoop responses go first if there are responses to 2533 // the same addresses 2534 if (checkConflictingSnoop(entry->getTarget()->pkt)) { 2535 return; 2536 } 2537 waitingOnRetry = entry->sendPacket(cache); 2538 } 2539 2540 // if we succeeded and are not waiting for a retry, schedule the 2541 // next send considering when the next queue is ready, note that 2542 // snoop responses have their own packet queue and thus schedule 2543 // their own events 2544 if (!waitingOnRetry) { 2545 schedSendEvent(cache.nextQueueReadyTime()); 2546 } 2547} 2548 2549BaseCache::MemSidePort::MemSidePort(const std::string &_name, 2550 BaseCache *_cache, 2551 const std::string &_label) 2552 : CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2553 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2554 _snoopRespQueue(*_cache, *this, true, _label), cache(_cache) 2555{ 2556} 2557 2558void 2559WriteAllocator::updateMode(Addr write_addr, unsigned write_size, 2560 Addr blk_addr) 2561{ 2562 // check if we are continuing where the last write ended 2563 if (nextAddr == write_addr) { 2564 delayCtr[blk_addr] = delayThreshold; 2565 // stop if we have already saturated 2566 if (mode != WriteMode::NO_ALLOCATE) { 2567 byteCount += write_size; 2568 // switch to streaming mode if we have passed the lower 2569 // threshold 2570 if (mode == WriteMode::ALLOCATE && 2571 byteCount > coalesceLimit) { 2572 mode = WriteMode::COALESCE; 2573 DPRINTF(Cache, "Switched to write coalescing\n"); 2574 } else if (mode == WriteMode::COALESCE && 2575 byteCount > noAllocateLimit) { 2576 // and continue and switch to non-allocating mode if we 2577 // pass the upper threshold 2578 mode = WriteMode::NO_ALLOCATE; 2579 DPRINTF(Cache, "Switched to write-no-allocate\n"); 2580 } 2581 } 2582 } else { 2583 // we did not see a write matching the previous one, start 2584 // over again 2585 byteCount = write_size; 2586 mode = WriteMode::ALLOCATE; 2587 resetDelay(blk_addr); 2588 } 2589 nextAddr = write_addr + write_size; 2590} 2591 2592WriteAllocator* 2593WriteAllocatorParams::create() 2594{ 2595 return new WriteAllocator(this); 2596}
| 2498} 2499 2500void 2501BaseCache::regProbePoints() 2502{ 2503 ppHit = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Hit"); 2504 ppMiss = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Miss"); 2505 ppFill = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Fill"); 2506} 2507 2508/////////////// 2509// 2510// CpuSidePort 2511// 2512/////////////// 2513bool 2514BaseCache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt) 2515{ 2516 // Snoops shouldn't happen when bypassing caches 2517 assert(!cache->system->bypassCaches()); 2518 2519 assert(pkt->isResponse()); 2520 2521 // Express snoop responses from master to slave, e.g., from L1 to L2 2522 cache->recvTimingSnoopResp(pkt); 2523 return true; 2524} 2525 2526 2527bool 2528BaseCache::CpuSidePort::tryTiming(PacketPtr pkt) 2529{ 2530 if (cache->system->bypassCaches() || pkt->isExpressSnoop()) { 2531 // always let express snoop packets through even if blocked 2532 return true; 2533 } else if (blocked || mustSendRetry) { 2534 // either already committed to send a retry, or blocked 2535 mustSendRetry = true; 2536 return false; 2537 } 2538 mustSendRetry = false; 2539 return true; 2540} 2541 2542bool 2543BaseCache::CpuSidePort::recvTimingReq(PacketPtr pkt) 2544{ 2545 assert(pkt->isRequest()); 2546 2547 if (cache->system->bypassCaches()) { 2548 // Just forward the packet if caches are disabled. 2549 // @todo This should really enqueue the packet rather 2550 bool M5_VAR_USED success = cache->memSidePort.sendTimingReq(pkt); 2551 assert(success); 2552 return true; 2553 } else if (tryTiming(pkt)) { 2554 cache->recvTimingReq(pkt); 2555 return true; 2556 } 2557 return false; 2558} 2559 2560Tick 2561BaseCache::CpuSidePort::recvAtomic(PacketPtr pkt) 2562{ 2563 if (cache->system->bypassCaches()) { 2564 // Forward the request if the system is in cache bypass mode. 2565 return cache->memSidePort.sendAtomic(pkt); 2566 } else { 2567 return cache->recvAtomic(pkt); 2568 } 2569} 2570 2571void 2572BaseCache::CpuSidePort::recvFunctional(PacketPtr pkt) 2573{ 2574 if (cache->system->bypassCaches()) { 2575 // The cache should be flushed if we are in cache bypass mode, 2576 // so we don't need to check if we need to update anything. 2577 cache->memSidePort.sendFunctional(pkt); 2578 return; 2579 } 2580 2581 // functional request 2582 cache->functionalAccess(pkt, true); 2583} 2584 2585AddrRangeList 2586BaseCache::CpuSidePort::getAddrRanges() const 2587{ 2588 return cache->getAddrRanges(); 2589} 2590 2591 2592BaseCache:: 2593CpuSidePort::CpuSidePort(const std::string &_name, BaseCache *_cache, 2594 const std::string &_label) 2595 : CacheSlavePort(_name, _cache, _label), cache(_cache) 2596{ 2597} 2598 2599/////////////// 2600// 2601// MemSidePort 2602// 2603/////////////// 2604bool 2605BaseCache::MemSidePort::recvTimingResp(PacketPtr pkt) 2606{ 2607 cache->recvTimingResp(pkt); 2608 return true; 2609} 2610 2611// Express snooping requests to memside port 2612void 2613BaseCache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt) 2614{ 2615 // Snoops shouldn't happen when bypassing caches 2616 assert(!cache->system->bypassCaches()); 2617 2618 // handle snooping requests 2619 cache->recvTimingSnoopReq(pkt); 2620} 2621 2622Tick 2623BaseCache::MemSidePort::recvAtomicSnoop(PacketPtr pkt) 2624{ 2625 // Snoops shouldn't happen when bypassing caches 2626 assert(!cache->system->bypassCaches()); 2627 2628 return cache->recvAtomicSnoop(pkt); 2629} 2630 2631void 2632BaseCache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt) 2633{ 2634 // Snoops shouldn't happen when bypassing caches 2635 assert(!cache->system->bypassCaches()); 2636 2637 // functional snoop (note that in contrast to atomic we don't have 2638 // a specific functionalSnoop method, as they have the same 2639 // behaviour regardless) 2640 cache->functionalAccess(pkt, false); 2641} 2642 2643void 2644BaseCache::CacheReqPacketQueue::sendDeferredPacket() 2645{ 2646 // sanity check 2647 assert(!waitingOnRetry); 2648 2649 // there should never be any deferred request packets in the 2650 // queue, instead we resly on the cache to provide the packets 2651 // from the MSHR queue or write queue 2652 assert(deferredPacketReadyTime() == MaxTick); 2653 2654 // check for request packets (requests & writebacks) 2655 QueueEntry* entry = cache.getNextQueueEntry(); 2656 2657 if (!entry) { 2658 // can happen if e.g. we attempt a writeback and fail, but 2659 // before the retry, the writeback is eliminated because 2660 // we snoop another cache's ReadEx. 2661 } else { 2662 // let our snoop responses go first if there are responses to 2663 // the same addresses 2664 if (checkConflictingSnoop(entry->getTarget()->pkt)) { 2665 return; 2666 } 2667 waitingOnRetry = entry->sendPacket(cache); 2668 } 2669 2670 // if we succeeded and are not waiting for a retry, schedule the 2671 // next send considering when the next queue is ready, note that 2672 // snoop responses have their own packet queue and thus schedule 2673 // their own events 2674 if (!waitingOnRetry) { 2675 schedSendEvent(cache.nextQueueReadyTime()); 2676 } 2677} 2678 2679BaseCache::MemSidePort::MemSidePort(const std::string &_name, 2680 BaseCache *_cache, 2681 const std::string &_label) 2682 : CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2683 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2684 _snoopRespQueue(*_cache, *this, true, _label), cache(_cache) 2685{ 2686} 2687 2688void 2689WriteAllocator::updateMode(Addr write_addr, unsigned write_size, 2690 Addr blk_addr) 2691{ 2692 // check if we are continuing where the last write ended 2693 if (nextAddr == write_addr) { 2694 delayCtr[blk_addr] = delayThreshold; 2695 // stop if we have already saturated 2696 if (mode != WriteMode::NO_ALLOCATE) { 2697 byteCount += write_size; 2698 // switch to streaming mode if we have passed the lower 2699 // threshold 2700 if (mode == WriteMode::ALLOCATE && 2701 byteCount > coalesceLimit) { 2702 mode = WriteMode::COALESCE; 2703 DPRINTF(Cache, "Switched to write coalescing\n"); 2704 } else if (mode == WriteMode::COALESCE && 2705 byteCount > noAllocateLimit) { 2706 // and continue and switch to non-allocating mode if we 2707 // pass the upper threshold 2708 mode = WriteMode::NO_ALLOCATE; 2709 DPRINTF(Cache, "Switched to write-no-allocate\n"); 2710 } 2711 } 2712 } else { 2713 // we did not see a write matching the previous one, start 2714 // over again 2715 byteCount = write_size; 2716 mode = WriteMode::ALLOCATE; 2717 resetDelay(blk_addr); 2718 } 2719 nextAddr = write_addr + write_size; 2720} 2721 2722WriteAllocator* 2723WriteAllocatorParams::create() 2724{ 2725 return new WriteAllocator(this); 2726}
|