base.cc revision 13478
113531Sjairo.balart@metempsy.com/* 214167Sgiacomo.travaglini@arm.com * Copyright (c) 2012-2013, 2018 ARM Limited 314167Sgiacomo.travaglini@arm.com * All rights reserved. 414167Sgiacomo.travaglini@arm.com * 514167Sgiacomo.travaglini@arm.com * The license below extends only to copyright in the software and shall 614167Sgiacomo.travaglini@arm.com * not be construed as granting a license to any other intellectual 714167Sgiacomo.travaglini@arm.com * property including but not limited to intellectual property relating 814167Sgiacomo.travaglini@arm.com * to a hardware implementation of the functionality of the software 914167Sgiacomo.travaglini@arm.com * licensed hereunder. You may use the software subject to the license 1014167Sgiacomo.travaglini@arm.com * terms below provided that you ensure that this notice is replicated 1114167Sgiacomo.travaglini@arm.com * unmodified and in its entirety in all distributions of the software, 1214167Sgiacomo.travaglini@arm.com * modified or unmodified, in source code or in binary form. 1314167Sgiacomo.travaglini@arm.com * 1413531Sjairo.balart@metempsy.com * Copyright (c) 2003-2005 The Regents of The University of Michigan 1513531Sjairo.balart@metempsy.com * All rights reserved. 1613531Sjairo.balart@metempsy.com * 1713531Sjairo.balart@metempsy.com * Redistribution and use in source and binary forms, with or without 1813531Sjairo.balart@metempsy.com * modification, are permitted provided that the following conditions are 1913531Sjairo.balart@metempsy.com * met: redistributions of source code must retain the above copyright 2013531Sjairo.balart@metempsy.com * notice, this list of conditions and the following disclaimer; 2113531Sjairo.balart@metempsy.com * redistributions in binary form must reproduce the above copyright 2213531Sjairo.balart@metempsy.com * notice, this list of conditions and the following disclaimer in the 2313531Sjairo.balart@metempsy.com * documentation and/or other materials provided with the distribution; 2413531Sjairo.balart@metempsy.com * neither the name of the copyright holders nor the names of its 2513531Sjairo.balart@metempsy.com * contributors may be used to endorse or promote products derived from 2613531Sjairo.balart@metempsy.com * this software without specific prior written permission. 2713531Sjairo.balart@metempsy.com * 2813531Sjairo.balart@metempsy.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2913531Sjairo.balart@metempsy.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3013531Sjairo.balart@metempsy.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3113531Sjairo.balart@metempsy.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3213531Sjairo.balart@metempsy.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3313531Sjairo.balart@metempsy.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3413531Sjairo.balart@metempsy.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3513531Sjairo.balart@metempsy.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3613531Sjairo.balart@metempsy.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3713531Sjairo.balart@metempsy.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3813531Sjairo.balart@metempsy.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3913531Sjairo.balart@metempsy.com * 4013531Sjairo.balart@metempsy.com * Authors: Erik Hallnor 4113531Sjairo.balart@metempsy.com * Nikos Nikoleris 4213756Sjairo.balart@metempsy.com */ 4313531Sjairo.balart@metempsy.com 4413531Sjairo.balart@metempsy.com/** 4513531Sjairo.balart@metempsy.com * @file 4613531Sjairo.balart@metempsy.com * Definition of BaseCache functions. 4713531Sjairo.balart@metempsy.com */ 4813531Sjairo.balart@metempsy.com 4913531Sjairo.balart@metempsy.com#include "mem/cache/base.hh" 5013531Sjairo.balart@metempsy.com 5113531Sjairo.balart@metempsy.com#include "base/compiler.hh" 5213756Sjairo.balart@metempsy.com#include "base/logging.hh" 5313756Sjairo.balart@metempsy.com#include "debug/Cache.hh" 5413756Sjairo.balart@metempsy.com#include "debug/CachePort.hh" 5513756Sjairo.balart@metempsy.com#include "debug/CacheRepl.hh" 5613756Sjairo.balart@metempsy.com#include "debug/CacheVerbose.hh" 5713756Sjairo.balart@metempsy.com#include "mem/cache/mshr.hh" 5813756Sjairo.balart@metempsy.com#include "mem/cache/prefetch/base.hh" 5913531Sjairo.balart@metempsy.com#include "mem/cache/queue_entry.hh" 6013756Sjairo.balart@metempsy.com#include "params/BaseCache.hh" 6113756Sjairo.balart@metempsy.com#include "params/WriteAllocator.hh" 6213756Sjairo.balart@metempsy.com#include "sim/core.hh" 6313756Sjairo.balart@metempsy.com 6413756Sjairo.balart@metempsy.comclass BaseMasterPort; 6513756Sjairo.balart@metempsy.comclass BaseSlavePort; 6613756Sjairo.balart@metempsy.com 6713531Sjairo.balart@metempsy.comusing namespace std; 6813531Sjairo.balart@metempsy.com 6913531Sjairo.balart@metempsy.comBaseCache::CacheSlavePort::CacheSlavePort(const std::string &_name, 7013531Sjairo.balart@metempsy.com BaseCache *_cache, 7113531Sjairo.balart@metempsy.com const std::string &_label) 7213531Sjairo.balart@metempsy.com : QueuedSlavePort(_name, _cache, queue), queue(*_cache, *this, _label), 7313531Sjairo.balart@metempsy.com blocked(false), mustSendRetry(false), 7413531Sjairo.balart@metempsy.com sendRetryEvent([this]{ processSendRetry(); }, _name) 7513531Sjairo.balart@metempsy.com{ 7613531Sjairo.balart@metempsy.com} 7713531Sjairo.balart@metempsy.com 7813531Sjairo.balart@metempsy.comBaseCache::BaseCache(const BaseCacheParams *p, unsigned blk_size) 7914167Sgiacomo.travaglini@arm.com : MemObject(p), 8014167Sgiacomo.travaglini@arm.com cpuSidePort (p->name + ".cpu_side", this, "CpuSidePort"), 8114167Sgiacomo.travaglini@arm.com memSidePort(p->name + ".mem_side", this, "MemSidePort"), 8214167Sgiacomo.travaglini@arm.com mshrQueue("MSHRs", p->mshrs, 0, p->demand_mshr_reserve), // see below 8314167Sgiacomo.travaglini@arm.com writeBuffer("write buffer", p->write_buffers, p->mshrs), // see below 8414167Sgiacomo.travaglini@arm.com tags(p->tags), 8513531Sjairo.balart@metempsy.com prefetcher(p->prefetcher), 8613531Sjairo.balart@metempsy.com writeAllocator(p->write_allocator), 8713531Sjairo.balart@metempsy.com writebackClean(p->writeback_clean), 8813531Sjairo.balart@metempsy.com tempBlockWriteback(nullptr), 8913531Sjairo.balart@metempsy.com writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); }, 9013531Sjairo.balart@metempsy.com name(), false, 9113531Sjairo.balart@metempsy.com EventBase::Delayed_Writeback_Pri), 9213531Sjairo.balart@metempsy.com blkSize(blk_size), 9313531Sjairo.balart@metempsy.com lookupLatency(p->tag_latency), 9413531Sjairo.balart@metempsy.com dataLatency(p->data_latency), 9513531Sjairo.balart@metempsy.com forwardLatency(p->tag_latency), 9613531Sjairo.balart@metempsy.com fillLatency(p->data_latency), 9713531Sjairo.balart@metempsy.com responseLatency(p->response_latency), 9813531Sjairo.balart@metempsy.com sequentialAccess(p->sequential_access), 9913531Sjairo.balart@metempsy.com numTarget(p->tgts_per_mshr), 10013531Sjairo.balart@metempsy.com forwardSnoops(true), 10113531Sjairo.balart@metempsy.com clusivity(p->clusivity), 10213531Sjairo.balart@metempsy.com isReadOnly(p->is_read_only), 10313531Sjairo.balart@metempsy.com blocked(0), 10413531Sjairo.balart@metempsy.com order(0), 10513531Sjairo.balart@metempsy.com noTargetMSHR(nullptr), 10613531Sjairo.balart@metempsy.com missCount(p->max_miss_count), 10713531Sjairo.balart@metempsy.com addrRanges(p->addr_ranges.begin(), p->addr_ranges.end()), 10813531Sjairo.balart@metempsy.com system(p->system) 10913531Sjairo.balart@metempsy.com{ 11013531Sjairo.balart@metempsy.com // the MSHR queue has no reserve entries as we check the MSHR 11113531Sjairo.balart@metempsy.com // queue on every single allocation, whereas the write queue has 11213531Sjairo.balart@metempsy.com // as many reserve entries as we have MSHRs, since every MSHR may 11313531Sjairo.balart@metempsy.com // eventually require a writeback, and we do not check the write 11413531Sjairo.balart@metempsy.com // buffer before committing to an MSHR 11513531Sjairo.balart@metempsy.com 11613531Sjairo.balart@metempsy.com // forward snoops is overridden in init() once we can query 11713531Sjairo.balart@metempsy.com // whether the connected master is actually snooping or not 11813531Sjairo.balart@metempsy.com 11913531Sjairo.balart@metempsy.com tempBlock = new TempCacheBlk(blkSize); 12013531Sjairo.balart@metempsy.com 12113531Sjairo.balart@metempsy.com tags->tagsInit(); 12213531Sjairo.balart@metempsy.com if (prefetcher) 12313531Sjairo.balart@metempsy.com prefetcher->setCache(this); 12413531Sjairo.balart@metempsy.com} 12513531Sjairo.balart@metempsy.com 12613531Sjairo.balart@metempsy.comBaseCache::~BaseCache() 12713531Sjairo.balart@metempsy.com{ 12813531Sjairo.balart@metempsy.com delete tempBlock; 12913531Sjairo.balart@metempsy.com} 13013531Sjairo.balart@metempsy.com 13113531Sjairo.balart@metempsy.comvoid 13213531Sjairo.balart@metempsy.comBaseCache::CacheSlavePort::setBlocked() 13313531Sjairo.balart@metempsy.com{ 13413531Sjairo.balart@metempsy.com assert(!blocked); 13513531Sjairo.balart@metempsy.com DPRINTF(CachePort, "Port is blocking new requests\n"); 13613531Sjairo.balart@metempsy.com blocked = true; 13713531Sjairo.balart@metempsy.com // if we already scheduled a retry in this cycle, but it has not yet 13813531Sjairo.balart@metempsy.com // happened, cancel it 13913531Sjairo.balart@metempsy.com if (sendRetryEvent.scheduled()) { 14013531Sjairo.balart@metempsy.com owner.deschedule(sendRetryEvent); 14113531Sjairo.balart@metempsy.com DPRINTF(CachePort, "Port descheduled retry\n"); 14213531Sjairo.balart@metempsy.com mustSendRetry = true; 14313531Sjairo.balart@metempsy.com } 14413531Sjairo.balart@metempsy.com} 14513531Sjairo.balart@metempsy.com 14613531Sjairo.balart@metempsy.comvoid 14713531Sjairo.balart@metempsy.comBaseCache::CacheSlavePort::clearBlocked() 14813531Sjairo.balart@metempsy.com{ 14913756Sjairo.balart@metempsy.com assert(blocked); 15013531Sjairo.balart@metempsy.com DPRINTF(CachePort, "Port is accepting new requests\n"); 15113531Sjairo.balart@metempsy.com blocked = false; 15213531Sjairo.balart@metempsy.com if (mustSendRetry) { 15313531Sjairo.balart@metempsy.com // @TODO: need to find a better time (next cycle?) 15413756Sjairo.balart@metempsy.com owner.schedule(sendRetryEvent, curTick() + 1); 15513531Sjairo.balart@metempsy.com } 15613531Sjairo.balart@metempsy.com} 15713531Sjairo.balart@metempsy.com 15813531Sjairo.balart@metempsy.comvoid 15913531Sjairo.balart@metempsy.comBaseCache::CacheSlavePort::processSendRetry() 16013531Sjairo.balart@metempsy.com{ 16113531Sjairo.balart@metempsy.com DPRINTF(CachePort, "Port is sending retry\n"); 16213531Sjairo.balart@metempsy.com 16313531Sjairo.balart@metempsy.com // reset the flag and call retry 16413756Sjairo.balart@metempsy.com mustSendRetry = false; 16513756Sjairo.balart@metempsy.com sendRetryReq(); 16613531Sjairo.balart@metempsy.com} 16713531Sjairo.balart@metempsy.com 16813531Sjairo.balart@metempsy.comAddr 16913531Sjairo.balart@metempsy.comBaseCache::regenerateBlkAddr(CacheBlk* blk) 17013531Sjairo.balart@metempsy.com{ 17113531Sjairo.balart@metempsy.com if (blk != tempBlock) { 17213531Sjairo.balart@metempsy.com return tags->regenerateBlkAddr(blk); 17313531Sjairo.balart@metempsy.com } else { 17413531Sjairo.balart@metempsy.com return tempBlock->getAddr(); 17513531Sjairo.balart@metempsy.com } 17613531Sjairo.balart@metempsy.com} 17713531Sjairo.balart@metempsy.com 17813531Sjairo.balart@metempsy.comvoid 17913531Sjairo.balart@metempsy.comBaseCache::init() 18013531Sjairo.balart@metempsy.com{ 18113531Sjairo.balart@metempsy.com if (!cpuSidePort.isConnected() || !memSidePort.isConnected()) 18213531Sjairo.balart@metempsy.com fatal("Cache ports on %s are not connected\n", name()); 18313531Sjairo.balart@metempsy.com cpuSidePort.sendRangeChange(); 18413531Sjairo.balart@metempsy.com forwardSnoops = cpuSidePort.isSnooping(); 18513756Sjairo.balart@metempsy.com} 18613756Sjairo.balart@metempsy.com 18713531Sjairo.balart@metempsy.comBaseMasterPort & 18813531Sjairo.balart@metempsy.comBaseCache::getMasterPort(const std::string &if_name, PortID idx) 18913531Sjairo.balart@metempsy.com{ 19013531Sjairo.balart@metempsy.com if (if_name == "mem_side") { 19113531Sjairo.balart@metempsy.com return memSidePort; 19213531Sjairo.balart@metempsy.com } else { 19313531Sjairo.balart@metempsy.com return MemObject::getMasterPort(if_name, idx); 19413531Sjairo.balart@metempsy.com } 19513531Sjairo.balart@metempsy.com} 19613531Sjairo.balart@metempsy.com 19713756Sjairo.balart@metempsy.comBaseSlavePort & 19813531Sjairo.balart@metempsy.comBaseCache::getSlavePort(const std::string &if_name, PortID idx) 19913531Sjairo.balart@metempsy.com{ 20013531Sjairo.balart@metempsy.com if (if_name == "cpu_side") { 20113531Sjairo.balart@metempsy.com return cpuSidePort; 20213531Sjairo.balart@metempsy.com } else { 20313531Sjairo.balart@metempsy.com return MemObject::getSlavePort(if_name, idx); 20413531Sjairo.balart@metempsy.com } 20513531Sjairo.balart@metempsy.com} 20613756Sjairo.balart@metempsy.com 20713756Sjairo.balart@metempsy.combool 20813531Sjairo.balart@metempsy.comBaseCache::inRange(Addr addr) const 20913531Sjairo.balart@metempsy.com{ 21013531Sjairo.balart@metempsy.com for (const auto& r : addrRanges) { 21113531Sjairo.balart@metempsy.com if (r.contains(addr)) { 21213531Sjairo.balart@metempsy.com return true; 21313531Sjairo.balart@metempsy.com } 21413531Sjairo.balart@metempsy.com } 21513531Sjairo.balart@metempsy.com return false; 21613531Sjairo.balart@metempsy.com} 21713531Sjairo.balart@metempsy.com 21813531Sjairo.balart@metempsy.comvoid 21913531Sjairo.balart@metempsy.comBaseCache::handleTimingReqHit(PacketPtr pkt, CacheBlk *blk, Tick request_time) 22013531Sjairo.balart@metempsy.com{ 22113756Sjairo.balart@metempsy.com if (pkt->needsResponse()) { 22213531Sjairo.balart@metempsy.com pkt->makeTimingResponse(); 22313531Sjairo.balart@metempsy.com // @todo: Make someone pay for this 22413531Sjairo.balart@metempsy.com pkt->headerDelay = pkt->payloadDelay = 0; 22513531Sjairo.balart@metempsy.com 22613531Sjairo.balart@metempsy.com // In this case we are considering request_time that takes 22713531Sjairo.balart@metempsy.com // into account the delay of the xbar, if any, and just 22813531Sjairo.balart@metempsy.com // lat, neglecting responseLatency, modelling hit latency 22913531Sjairo.balart@metempsy.com // just as the value of lat overriden by access(), which calls 23013756Sjairo.balart@metempsy.com // the calculateAccessLatency() function. 23113756Sjairo.balart@metempsy.com cpuSidePort.schedTimingResp(pkt, request_time, true); 23213531Sjairo.balart@metempsy.com } else { 23313531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__, 23413531Sjairo.balart@metempsy.com pkt->print()); 23513531Sjairo.balart@metempsy.com 23613531Sjairo.balart@metempsy.com // queue the packet for deletion, as the sending cache is 23713531Sjairo.balart@metempsy.com // still relying on it; if the block is found in access(), 23813531Sjairo.balart@metempsy.com // CleanEvict and Writeback messages will be deleted 23913531Sjairo.balart@metempsy.com // here as well 24013531Sjairo.balart@metempsy.com pendingDelete.reset(pkt); 24113531Sjairo.balart@metempsy.com } 24213531Sjairo.balart@metempsy.com} 24313531Sjairo.balart@metempsy.com 24413756Sjairo.balart@metempsy.comvoid 24513531Sjairo.balart@metempsy.comBaseCache::handleTimingReqMiss(PacketPtr pkt, MSHR *mshr, CacheBlk *blk, 24613531Sjairo.balart@metempsy.com Tick forward_time, Tick request_time) 24713531Sjairo.balart@metempsy.com{ 24813531Sjairo.balart@metempsy.com if (writeAllocator && 24913531Sjairo.balart@metempsy.com pkt && pkt->isWrite() && !pkt->req->isUncacheable()) { 25013531Sjairo.balart@metempsy.com writeAllocator->updateMode(pkt->getAddr(), pkt->getSize(), 25113531Sjairo.balart@metempsy.com pkt->getBlockAddr(blkSize)); 25213531Sjairo.balart@metempsy.com } 25313531Sjairo.balart@metempsy.com 25413531Sjairo.balart@metempsy.com if (mshr) { 25513756Sjairo.balart@metempsy.com /// MSHR hit 25613756Sjairo.balart@metempsy.com /// @note writebacks will be checked in getNextMSHR() 25713531Sjairo.balart@metempsy.com /// for any conflicting requests to the same block 25813531Sjairo.balart@metempsy.com 25913531Sjairo.balart@metempsy.com //@todo remove hw_pf here 26013531Sjairo.balart@metempsy.com 26113531Sjairo.balart@metempsy.com // Coalesce unless it was a software prefetch (see above). 26213531Sjairo.balart@metempsy.com if (pkt) { 26313531Sjairo.balart@metempsy.com assert(!pkt->isWriteback()); 26413531Sjairo.balart@metempsy.com // CleanEvicts corresponding to blocks which have 26513531Sjairo.balart@metempsy.com // outstanding requests in MSHRs are simply sunk here 26613531Sjairo.balart@metempsy.com if (pkt->cmd == MemCmd::CleanEvict) { 26713531Sjairo.balart@metempsy.com pendingDelete.reset(pkt); 26813531Sjairo.balart@metempsy.com } else if (pkt->cmd == MemCmd::WriteClean) { 26913756Sjairo.balart@metempsy.com // A WriteClean should never coalesce with any 27013531Sjairo.balart@metempsy.com // outstanding cache maintenance requests. 27113531Sjairo.balart@metempsy.com 27213531Sjairo.balart@metempsy.com // We use forward_time here because there is an 27313531Sjairo.balart@metempsy.com // uncached memory write, forwarded to WriteBuffer. 27413531Sjairo.balart@metempsy.com allocateWriteBuffer(pkt, forward_time); 27513531Sjairo.balart@metempsy.com } else { 27613531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__, 27713531Sjairo.balart@metempsy.com pkt->print()); 27813531Sjairo.balart@metempsy.com 27913531Sjairo.balart@metempsy.com assert(pkt->req->masterId() < system->maxMasters()); 28013756Sjairo.balart@metempsy.com mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++; 28113756Sjairo.balart@metempsy.com 28213531Sjairo.balart@metempsy.com // We use forward_time here because it is the same 28313531Sjairo.balart@metempsy.com // considering new targets. We have multiple 28413531Sjairo.balart@metempsy.com // requests for the same address here. It 28513531Sjairo.balart@metempsy.com // specifies the latency to allocate an internal 28613531Sjairo.balart@metempsy.com // buffer and to schedule an event to the queued 28713531Sjairo.balart@metempsy.com // port and also takes into account the additional 28813531Sjairo.balart@metempsy.com // delay of the xbar. 28913531Sjairo.balart@metempsy.com mshr->allocateTarget(pkt, forward_time, order++, 29013531Sjairo.balart@metempsy.com allocOnFill(pkt->cmd)); 29113531Sjairo.balart@metempsy.com if (mshr->getNumTargets() == numTarget) { 29213531Sjairo.balart@metempsy.com noTargetMSHR = mshr; 29313756Sjairo.balart@metempsy.com setBlocked(Blocked_NoTargets); 29413531Sjairo.balart@metempsy.com // need to be careful with this... if this mshr isn't 29513531Sjairo.balart@metempsy.com // ready yet (i.e. time > curTick()), we don't want to 29613531Sjairo.balart@metempsy.com // move it ahead of mshrs that are ready 29713531Sjairo.balart@metempsy.com // mshrQueue.moveToFront(mshr); 29813531Sjairo.balart@metempsy.com } 29913531Sjairo.balart@metempsy.com } 30013531Sjairo.balart@metempsy.com } 30113531Sjairo.balart@metempsy.com } else { 30213531Sjairo.balart@metempsy.com // no MSHR 30313756Sjairo.balart@metempsy.com assert(pkt->req->masterId() < system->maxMasters()); 30413756Sjairo.balart@metempsy.com mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 30513531Sjairo.balart@metempsy.com 30613531Sjairo.balart@metempsy.com if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean) { 30713531Sjairo.balart@metempsy.com // We use forward_time here because there is an 30813531Sjairo.balart@metempsy.com // writeback or writeclean, forwarded to WriteBuffer. 30913531Sjairo.balart@metempsy.com allocateWriteBuffer(pkt, forward_time); 31013531Sjairo.balart@metempsy.com } else { 31113531Sjairo.balart@metempsy.com if (blk && blk->isValid()) { 31213531Sjairo.balart@metempsy.com // If we have a write miss to a valid block, we 31313531Sjairo.balart@metempsy.com // need to mark the block non-readable. Otherwise 31413531Sjairo.balart@metempsy.com // if we allow reads while there's an outstanding 31513531Sjairo.balart@metempsy.com // write miss, the read could return stale data 31613531Sjairo.balart@metempsy.com // out of the cache block... a more aggressive 31713531Sjairo.balart@metempsy.com // system could detect the overlap (if any) and 31813531Sjairo.balart@metempsy.com // forward data out of the MSHRs, but we don't do 31913531Sjairo.balart@metempsy.com // that yet. Note that we do need to leave the 32013531Sjairo.balart@metempsy.com // block valid so that it stays in the cache, in 32113531Sjairo.balart@metempsy.com // case we get an upgrade response (and hence no 32213531Sjairo.balart@metempsy.com // new data) when the write miss completes. 32313531Sjairo.balart@metempsy.com // As long as CPUs do proper store/load forwarding 32413531Sjairo.balart@metempsy.com // internally, and have a sufficiently weak memory 32513531Sjairo.balart@metempsy.com // model, this is probably unnecessary, but at some 32613531Sjairo.balart@metempsy.com // point it must have seemed like we needed it... 32713756Sjairo.balart@metempsy.com assert((pkt->needsWritable() && !blk->isWritable()) || 32813531Sjairo.balart@metempsy.com pkt->req->isCacheMaintenance()); 32913531Sjairo.balart@metempsy.com blk->status &= ~BlkReadable; 33013531Sjairo.balart@metempsy.com } 33113531Sjairo.balart@metempsy.com // Here we are using forward_time, modelling the latency of 33213531Sjairo.balart@metempsy.com // a miss (outbound) just as forwardLatency, neglecting the 33313531Sjairo.balart@metempsy.com // lookupLatency component. 33413531Sjairo.balart@metempsy.com allocateMissBuffer(pkt, forward_time); 33513531Sjairo.balart@metempsy.com } 33613531Sjairo.balart@metempsy.com } 33713531Sjairo.balart@metempsy.com} 33813756Sjairo.balart@metempsy.com 33913756Sjairo.balart@metempsy.comvoid 34013531Sjairo.balart@metempsy.comBaseCache::recvTimingReq(PacketPtr pkt) 34113531Sjairo.balart@metempsy.com{ 34213531Sjairo.balart@metempsy.com // anything that is merely forwarded pays for the forward latency and 34313531Sjairo.balart@metempsy.com // the delay provided by the crossbar 34413531Sjairo.balart@metempsy.com Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 34513531Sjairo.balart@metempsy.com 34613531Sjairo.balart@metempsy.com Cycles lat; 34713531Sjairo.balart@metempsy.com CacheBlk *blk = nullptr; 34813531Sjairo.balart@metempsy.com bool satisfied = false; 34913531Sjairo.balart@metempsy.com { 35013531Sjairo.balart@metempsy.com PacketList writebacks; 35113531Sjairo.balart@metempsy.com // Note that lat is passed by reference here. The function 35213531Sjairo.balart@metempsy.com // access() will set the lat value. 35313531Sjairo.balart@metempsy.com satisfied = access(pkt, blk, lat, writebacks); 35413531Sjairo.balart@metempsy.com 35513531Sjairo.balart@metempsy.com // copy writebacks to write buffer here to ensure they logically 35613531Sjairo.balart@metempsy.com // precede anything happening below 35713531Sjairo.balart@metempsy.com doWritebacks(writebacks, forward_time); 35813531Sjairo.balart@metempsy.com } 35913531Sjairo.balart@metempsy.com 36013531Sjairo.balart@metempsy.com // Here we charge the headerDelay that takes into account the latencies 36113531Sjairo.balart@metempsy.com // of the bus, if the packet comes from it. 36213531Sjairo.balart@metempsy.com // The latency charged is just the value set by the access() function. 36313531Sjairo.balart@metempsy.com // In case of a hit we are neglecting response latency. 36413531Sjairo.balart@metempsy.com // In case of a miss we are neglecting forward latency. 36513531Sjairo.balart@metempsy.com Tick request_time = clockEdge(lat) + pkt->headerDelay; 36613531Sjairo.balart@metempsy.com // Here we reset the timing of the packet. 36713531Sjairo.balart@metempsy.com pkt->headerDelay = pkt->payloadDelay = 0; 36813531Sjairo.balart@metempsy.com 36913531Sjairo.balart@metempsy.com if (satisfied) { 37013756Sjairo.balart@metempsy.com // notify before anything else as later handleTimingReqHit might turn 37113531Sjairo.balart@metempsy.com // the packet in a response 37213531Sjairo.balart@metempsy.com ppHit->notify(pkt); 37313531Sjairo.balart@metempsy.com 37413531Sjairo.balart@metempsy.com if (prefetcher && blk && blk->wasPrefetched()) { 37513531Sjairo.balart@metempsy.com blk->status &= ~BlkHWPrefetched; 37613531Sjairo.balart@metempsy.com } 37713756Sjairo.balart@metempsy.com 37813531Sjairo.balart@metempsy.com handleTimingReqHit(pkt, blk, request_time); 37913531Sjairo.balart@metempsy.com } else { 38013531Sjairo.balart@metempsy.com handleTimingReqMiss(pkt, blk, forward_time, request_time); 38113531Sjairo.balart@metempsy.com 38213531Sjairo.balart@metempsy.com ppMiss->notify(pkt); 38313531Sjairo.balart@metempsy.com } 38413531Sjairo.balart@metempsy.com 38513531Sjairo.balart@metempsy.com if (prefetcher) { 38613531Sjairo.balart@metempsy.com // track time of availability of next prefetch, if any 38713531Sjairo.balart@metempsy.com Tick next_pf_time = prefetcher->nextPrefetchReadyTime(); 38813531Sjairo.balart@metempsy.com if (next_pf_time != MaxTick) { 38913531Sjairo.balart@metempsy.com schedMemSideSendEvent(next_pf_time); 39013531Sjairo.balart@metempsy.com } 39113531Sjairo.balart@metempsy.com } 39213531Sjairo.balart@metempsy.com} 39313756Sjairo.balart@metempsy.com 39413531Sjairo.balart@metempsy.comvoid 39513531Sjairo.balart@metempsy.comBaseCache::handleUncacheableWriteResp(PacketPtr pkt) 39613531Sjairo.balart@metempsy.com{ 39713531Sjairo.balart@metempsy.com Tick completion_time = clockEdge(responseLatency) + 39813531Sjairo.balart@metempsy.com pkt->headerDelay + pkt->payloadDelay; 39913531Sjairo.balart@metempsy.com 40013531Sjairo.balart@metempsy.com // Reset the bus additional time as it is now accounted for 40113531Sjairo.balart@metempsy.com pkt->headerDelay = pkt->payloadDelay = 0; 40213531Sjairo.balart@metempsy.com 40313531Sjairo.balart@metempsy.com cpuSidePort.schedTimingResp(pkt, completion_time, true); 40413531Sjairo.balart@metempsy.com} 40513531Sjairo.balart@metempsy.com 40613531Sjairo.balart@metempsy.comvoid 40713531Sjairo.balart@metempsy.comBaseCache::recvTimingResp(PacketPtr pkt) 40813531Sjairo.balart@metempsy.com{ 40913531Sjairo.balart@metempsy.com assert(pkt->isResponse()); 41013531Sjairo.balart@metempsy.com 41113531Sjairo.balart@metempsy.com // all header delay should be paid for by the crossbar, unless 41213531Sjairo.balart@metempsy.com // this is a prefetch response from above 41313531Sjairo.balart@metempsy.com panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 41413531Sjairo.balart@metempsy.com "%s saw a non-zero packet delay\n", name()); 41513531Sjairo.balart@metempsy.com 41613531Sjairo.balart@metempsy.com const bool is_error = pkt->isError(); 41713531Sjairo.balart@metempsy.com 41813531Sjairo.balart@metempsy.com if (is_error) { 41913531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s: Cache received %s with error\n", __func__, 42013531Sjairo.balart@metempsy.com pkt->print()); 42113531Sjairo.balart@metempsy.com } 42213531Sjairo.balart@metempsy.com 42313531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s: Handling response %s\n", __func__, 42413531Sjairo.balart@metempsy.com pkt->print()); 42513531Sjairo.balart@metempsy.com 42613531Sjairo.balart@metempsy.com // if this is a write, we should be looking at an uncacheable 42713531Sjairo.balart@metempsy.com // write 42813531Sjairo.balart@metempsy.com if (pkt->isWrite()) { 42913531Sjairo.balart@metempsy.com assert(pkt->req->isUncacheable()); 43013531Sjairo.balart@metempsy.com handleUncacheableWriteResp(pkt); 43113531Sjairo.balart@metempsy.com return; 43213531Sjairo.balart@metempsy.com } 43313531Sjairo.balart@metempsy.com 43413531Sjairo.balart@metempsy.com // we have dealt with any (uncacheable) writes above, from here on 43513531Sjairo.balart@metempsy.com // we know we are dealing with an MSHR due to a miss or a prefetch 43613531Sjairo.balart@metempsy.com MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState()); 43713531Sjairo.balart@metempsy.com assert(mshr); 43813531Sjairo.balart@metempsy.com 43913531Sjairo.balart@metempsy.com if (mshr == noTargetMSHR) { 44013531Sjairo.balart@metempsy.com // we always clear at least one target 44113531Sjairo.balart@metempsy.com clearBlocked(Blocked_NoTargets); 44213531Sjairo.balart@metempsy.com noTargetMSHR = nullptr; 44313531Sjairo.balart@metempsy.com } 44413531Sjairo.balart@metempsy.com 44513531Sjairo.balart@metempsy.com // Initial target is used just for stats 44613531Sjairo.balart@metempsy.com MSHR::Target *initial_tgt = mshr->getTarget(); 44713531Sjairo.balart@metempsy.com int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 44813531Sjairo.balart@metempsy.com Tick miss_latency = curTick() - initial_tgt->recvTime; 44913531Sjairo.balart@metempsy.com 45013531Sjairo.balart@metempsy.com if (pkt->req->isUncacheable()) { 45113531Sjairo.balart@metempsy.com assert(pkt->req->masterId() < system->maxMasters()); 45213531Sjairo.balart@metempsy.com mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 45313531Sjairo.balart@metempsy.com miss_latency; 45413531Sjairo.balart@metempsy.com } else { 45513531Sjairo.balart@metempsy.com assert(pkt->req->masterId() < system->maxMasters()); 45613531Sjairo.balart@metempsy.com mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 45713531Sjairo.balart@metempsy.com miss_latency; 45813531Sjairo.balart@metempsy.com } 45913531Sjairo.balart@metempsy.com 46013531Sjairo.balart@metempsy.com PacketList writebacks; 46113531Sjairo.balart@metempsy.com 46213531Sjairo.balart@metempsy.com bool is_fill = !mshr->isForward && 46313531Sjairo.balart@metempsy.com (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp || 46413531Sjairo.balart@metempsy.com mshr->wasWholeLineWrite); 46513531Sjairo.balart@metempsy.com 46613531Sjairo.balart@metempsy.com // make sure that if the mshr was due to a whole line write then 46713531Sjairo.balart@metempsy.com // the response is an invalidation 46813531Sjairo.balart@metempsy.com assert(!mshr->wasWholeLineWrite || pkt->isInvalidate()); 46913531Sjairo.balart@metempsy.com 47013531Sjairo.balart@metempsy.com CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 47113531Sjairo.balart@metempsy.com 47213531Sjairo.balart@metempsy.com if (is_fill && !is_error) { 47313531Sjairo.balart@metempsy.com DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n", 47413531Sjairo.balart@metempsy.com pkt->getAddr()); 47513531Sjairo.balart@metempsy.com 47613531Sjairo.balart@metempsy.com const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ? 47713690Sjairo.balart@metempsy.com writeAllocator->allocate() : mshr->allocOnFill(); 47813531Sjairo.balart@metempsy.com blk = handleFill(pkt, blk, writebacks, allocate); 47913531Sjairo.balart@metempsy.com assert(blk != nullptr); 48013531Sjairo.balart@metempsy.com } 48113531Sjairo.balart@metempsy.com 48213531Sjairo.balart@metempsy.com if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) { 48313531Sjairo.balart@metempsy.com // The block was marked not readable while there was a pending 48413531Sjairo.balart@metempsy.com // cache maintenance operation, restore its flag. 48513531Sjairo.balart@metempsy.com blk->status |= BlkReadable; 48613531Sjairo.balart@metempsy.com 48713531Sjairo.balart@metempsy.com // This was a cache clean operation (without invalidate) 48813531Sjairo.balart@metempsy.com // and we have a copy of the block already. Since there 48913531Sjairo.balart@metempsy.com // is no invalidation, we can promote targets that don't 49013531Sjairo.balart@metempsy.com // require a writable copy 49113531Sjairo.balart@metempsy.com mshr->promoteReadable(); 49213927Sgiacomo.travaglini@arm.com } 49313690Sjairo.balart@metempsy.com 49413531Sjairo.balart@metempsy.com if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) { 49513531Sjairo.balart@metempsy.com // If at this point the referenced block is writable and the 49613531Sjairo.balart@metempsy.com // response is not a cache invalidate, we promote targets that 49713531Sjairo.balart@metempsy.com // were deferred as we couldn't guarrantee a writable copy 49813531Sjairo.balart@metempsy.com mshr->promoteWritable(); 49913531Sjairo.balart@metempsy.com } 50013531Sjairo.balart@metempsy.com 50113531Sjairo.balart@metempsy.com serviceMSHRTargets(mshr, pkt, blk); 50213531Sjairo.balart@metempsy.com 50313531Sjairo.balart@metempsy.com if (mshr->promoteDeferredTargets()) { 50413531Sjairo.balart@metempsy.com // avoid later read getting stale data while write miss is 50513756Sjairo.balart@metempsy.com // outstanding.. see comment in timingAccess() 50614167Sgiacomo.travaglini@arm.com if (blk) { 50713531Sjairo.balart@metempsy.com blk->status &= ~BlkReadable; 50814167Sgiacomo.travaglini@arm.com } 50914167Sgiacomo.travaglini@arm.com mshrQueue.markPending(mshr); 51013531Sjairo.balart@metempsy.com schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 51114167Sgiacomo.travaglini@arm.com } else { 51214167Sgiacomo.travaglini@arm.com // while we deallocate an mshr from the queue we still have to 51313531Sjairo.balart@metempsy.com // check the isFull condition before and after as we might 51413531Sjairo.balart@metempsy.com // have been using the reserved entries already 51514167Sgiacomo.travaglini@arm.com const bool was_full = mshrQueue.isFull(); 51613531Sjairo.balart@metempsy.com mshrQueue.deallocate(mshr); 51714167Sgiacomo.travaglini@arm.com if (was_full && !mshrQueue.isFull()) { 51814167Sgiacomo.travaglini@arm.com clearBlocked(Blocked_NoMSHRs); 51913531Sjairo.balart@metempsy.com } 52013531Sjairo.balart@metempsy.com 52113531Sjairo.balart@metempsy.com // Request the bus for a prefetch if this deallocation freed enough 52213531Sjairo.balart@metempsy.com // MSHRs for a prefetch to take place 52313531Sjairo.balart@metempsy.com if (prefetcher && mshrQueue.canPrefetch()) { 52413531Sjairo.balart@metempsy.com Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 52513531Sjairo.balart@metempsy.com clockEdge()); 52613531Sjairo.balart@metempsy.com if (next_pf_time != MaxTick) 52713531Sjairo.balart@metempsy.com schedMemSideSendEvent(next_pf_time); 52813531Sjairo.balart@metempsy.com } 52913531Sjairo.balart@metempsy.com } 53013531Sjairo.balart@metempsy.com 53113531Sjairo.balart@metempsy.com // if we used temp block, check to see if its valid and then clear it out 53213531Sjairo.balart@metempsy.com if (blk == tempBlock && tempBlock->isValid()) { 53313531Sjairo.balart@metempsy.com evictBlock(blk, writebacks); 53413531Sjairo.balart@metempsy.com } 53513531Sjairo.balart@metempsy.com 53613531Sjairo.balart@metempsy.com const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 53713531Sjairo.balart@metempsy.com // copy writebacks to write buffer 53813531Sjairo.balart@metempsy.com doWritebacks(writebacks, forward_time); 53913531Sjairo.balart@metempsy.com 54013531Sjairo.balart@metempsy.com DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print()); 54113531Sjairo.balart@metempsy.com delete pkt; 54213531Sjairo.balart@metempsy.com} 54313531Sjairo.balart@metempsy.com 54413531Sjairo.balart@metempsy.com 54513531Sjairo.balart@metempsy.comTick 54613531Sjairo.balart@metempsy.comBaseCache::recvAtomic(PacketPtr pkt) 54713531Sjairo.balart@metempsy.com{ 54813756Sjairo.balart@metempsy.com // should assert here that there are no outstanding MSHRs or 54913531Sjairo.balart@metempsy.com // writebacks... that would mean that someone used an atomic 55013531Sjairo.balart@metempsy.com // access in timing mode 55113531Sjairo.balart@metempsy.com 55213531Sjairo.balart@metempsy.com // We use lookupLatency here because it is used to specify the latency 55313531Sjairo.balart@metempsy.com // to access. 55413531Sjairo.balart@metempsy.com Cycles lat = lookupLatency; 55513756Sjairo.balart@metempsy.com 55613531Sjairo.balart@metempsy.com CacheBlk *blk = nullptr; 55713531Sjairo.balart@metempsy.com PacketList writebacks; 55813531Sjairo.balart@metempsy.com bool satisfied = access(pkt, blk, lat, writebacks); 55913531Sjairo.balart@metempsy.com 56013531Sjairo.balart@metempsy.com if (pkt->isClean() && blk && blk->isDirty()) { 56113531Sjairo.balart@metempsy.com // A cache clean opearation is looking for a dirty 56213531Sjairo.balart@metempsy.com // block. If a dirty block is encountered a WriteClean 56313531Sjairo.balart@metempsy.com // will update any copies to the path to the memory 56413756Sjairo.balart@metempsy.com // until the point of reference. 56513756Sjairo.balart@metempsy.com DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 56613531Sjairo.balart@metempsy.com __func__, pkt->print(), blk->print()); 56713531Sjairo.balart@metempsy.com PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id); 56813531Sjairo.balart@metempsy.com writebacks.push_back(wb_pkt); 56913531Sjairo.balart@metempsy.com pkt->setSatisfied(); 57013531Sjairo.balart@metempsy.com } 57113531Sjairo.balart@metempsy.com 57213531Sjairo.balart@metempsy.com // handle writebacks resulting from the access here to ensure they 57313531Sjairo.balart@metempsy.com // logically precede anything happening below 57413531Sjairo.balart@metempsy.com doWritebacksAtomic(writebacks); 57513531Sjairo.balart@metempsy.com assert(writebacks.empty()); 57613531Sjairo.balart@metempsy.com 57713531Sjairo.balart@metempsy.com if (!satisfied) { 57813531Sjairo.balart@metempsy.com lat += handleAtomicReqMiss(pkt, blk, writebacks); 57913531Sjairo.balart@metempsy.com } 58013531Sjairo.balart@metempsy.com 58113531Sjairo.balart@metempsy.com // Note that we don't invoke the prefetcher at all in atomic mode. 58213531Sjairo.balart@metempsy.com // It's not clear how to do it properly, particularly for 58313531Sjairo.balart@metempsy.com // prefetchers that aggressively generate prefetch candidates and 58413531Sjairo.balart@metempsy.com // rely on bandwidth contention to throttle them; these will tend 58513531Sjairo.balart@metempsy.com // to pollute the cache in atomic mode since there is no bandwidth 58613531Sjairo.balart@metempsy.com // contention. If we ever do want to enable prefetching in atomic 58713531Sjairo.balart@metempsy.com // mode, though, this is the place to do it... see timingAccess() 58813812Sgiacomo.travaglini@arm.com // for an example (though we'd want to issue the prefetch(es) 58913812Sgiacomo.travaglini@arm.com // immediately rather than calling requestMemSideBus() as we do 59013812Sgiacomo.travaglini@arm.com // there). 59113812Sgiacomo.travaglini@arm.com 59213531Sjairo.balart@metempsy.com // do any writebacks resulting from the response handling 59313756Sjairo.balart@metempsy.com doWritebacksAtomic(writebacks); 59413756Sjairo.balart@metempsy.com 59513531Sjairo.balart@metempsy.com // if we used temp block, check to see if its valid and if so 59613531Sjairo.balart@metempsy.com // clear it out, but only do so after the call to recvAtomic is 59713531Sjairo.balart@metempsy.com // finished so that any downstream observers (such as a snoop 59813531Sjairo.balart@metempsy.com // filter), first see the fill, and only then see the eviction 59913531Sjairo.balart@metempsy.com if (blk == tempBlock && tempBlock->isValid()) { 60013531Sjairo.balart@metempsy.com // the atomic CPU calls recvAtomic for fetch and load/store 60113531Sjairo.balart@metempsy.com // sequentuially, and we may already have a tempBlock 60213531Sjairo.balart@metempsy.com // writeback from the fetch that we have not yet sent 60313531Sjairo.balart@metempsy.com if (tempBlockWriteback) { 60413531Sjairo.balart@metempsy.com // if that is the case, write the prevoius one back, and 60513531Sjairo.balart@metempsy.com // do not schedule any new event 60613531Sjairo.balart@metempsy.com writebackTempBlockAtomic(); 60713531Sjairo.balart@metempsy.com } else { 60813531Sjairo.balart@metempsy.com // the writeback/clean eviction happens after the call to 60913531Sjairo.balart@metempsy.com // recvAtomic has finished (but before any successive 61013531Sjairo.balart@metempsy.com // calls), so that the response handling from the fill is 61113531Sjairo.balart@metempsy.com // allowed to happen first 61213531Sjairo.balart@metempsy.com schedule(writebackTempBlockAtomicEvent, curTick()); 61313531Sjairo.balart@metempsy.com } 61413756Sjairo.balart@metempsy.com 61513531Sjairo.balart@metempsy.com tempBlockWriteback = evictBlock(blk); 61613531Sjairo.balart@metempsy.com } 61713531Sjairo.balart@metempsy.com 61813531Sjairo.balart@metempsy.com if (pkt->needsResponse()) { 61913531Sjairo.balart@metempsy.com pkt->makeAtomicResponse(); 62013531Sjairo.balart@metempsy.com } 62113531Sjairo.balart@metempsy.com 62213756Sjairo.balart@metempsy.com return lat * clockPeriod(); 62313756Sjairo.balart@metempsy.com} 62413531Sjairo.balart@metempsy.com 62513531Sjairo.balart@metempsy.comvoid 62613531Sjairo.balart@metempsy.comBaseCache::functionalAccess(PacketPtr pkt, bool from_cpu_side) 62713531Sjairo.balart@metempsy.com{ 62813531Sjairo.balart@metempsy.com Addr blk_addr = pkt->getBlockAddr(blkSize); 62913531Sjairo.balart@metempsy.com bool is_secure = pkt->isSecure(); 63013531Sjairo.balart@metempsy.com CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure); 63113531Sjairo.balart@metempsy.com MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure); 63213531Sjairo.balart@metempsy.com 63313531Sjairo.balart@metempsy.com pkt->pushLabel(name()); 63413531Sjairo.balart@metempsy.com 63513531Sjairo.balart@metempsy.com CacheBlkPrintWrapper cbpw(blk); 63613531Sjairo.balart@metempsy.com 63713531Sjairo.balart@metempsy.com // Note that just because an L2/L3 has valid data doesn't mean an 63813531Sjairo.balart@metempsy.com // L1 doesn't have a more up-to-date modified copy that still 63913531Sjairo.balart@metempsy.com // needs to be found. As a result we always update the request if 64013531Sjairo.balart@metempsy.com // we have it, but only declare it satisfied if we are the owner. 64114231Sgiacomo.travaglini@arm.com 64213531Sjairo.balart@metempsy.com // see if we have data at all (owned or otherwise) 64313531Sjairo.balart@metempsy.com bool have_data = blk && blk->isValid() 64413756Sjairo.balart@metempsy.com && pkt->trySatisfyFunctional(&cbpw, blk_addr, is_secure, blkSize, 64513531Sjairo.balart@metempsy.com blk->data); 64613531Sjairo.balart@metempsy.com 64713531Sjairo.balart@metempsy.com // data we have is dirty if marked as such or if we have an 64813531Sjairo.balart@metempsy.com // in-service MSHR that is pending a modified line 64913531Sjairo.balart@metempsy.com bool have_dirty = 65013531Sjairo.balart@metempsy.com have_data && (blk->isDirty() || 65113531Sjairo.balart@metempsy.com (mshr && mshr->inService && mshr->isPendingModified())); 65213756Sjairo.balart@metempsy.com 65313756Sjairo.balart@metempsy.com bool done = have_dirty || 65413531Sjairo.balart@metempsy.com cpuSidePort.trySatisfyFunctional(pkt) || 65513531Sjairo.balart@metempsy.com mshrQueue.trySatisfyFunctional(pkt, blk_addr) || 65613531Sjairo.balart@metempsy.com writeBuffer.trySatisfyFunctional(pkt, blk_addr) || 65713531Sjairo.balart@metempsy.com memSidePort.trySatisfyFunctional(pkt); 65813531Sjairo.balart@metempsy.com 65913531Sjairo.balart@metempsy.com DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(), 66013531Sjairo.balart@metempsy.com (blk && blk->isValid()) ? "valid " : "", 66113531Sjairo.balart@metempsy.com have_data ? "data " : "", done ? "done " : ""); 66213531Sjairo.balart@metempsy.com 66313531Sjairo.balart@metempsy.com // We're leaving the cache, so pop cache->name() label 66413531Sjairo.balart@metempsy.com pkt->popLabel(); 66513531Sjairo.balart@metempsy.com 66614231Sgiacomo.travaglini@arm.com if (done) { 66713531Sjairo.balart@metempsy.com pkt->makeResponse(); 66813531Sjairo.balart@metempsy.com } else { 66913531Sjairo.balart@metempsy.com // if it came as a request from the CPU side then make sure it 67014231Sgiacomo.travaglini@arm.com // continues towards the memory side 67113531Sjairo.balart@metempsy.com if (from_cpu_side) { 67213756Sjairo.balart@metempsy.com memSidePort.sendFunctional(pkt); 67313531Sjairo.balart@metempsy.com } else if (cpuSidePort.isSnooping()) { 67413531Sjairo.balart@metempsy.com // if it came from the memory side, it must be a snoop request 67513531Sjairo.balart@metempsy.com // and we should only forward it if we are forwarding snoops 67613531Sjairo.balart@metempsy.com cpuSidePort.sendFunctionalSnoop(pkt); 67713531Sjairo.balart@metempsy.com } 67813531Sjairo.balart@metempsy.com } 67913531Sjairo.balart@metempsy.com} 68013531Sjairo.balart@metempsy.com 68113756Sjairo.balart@metempsy.com 68213756Sjairo.balart@metempsy.comvoid 68313531Sjairo.balart@metempsy.comBaseCache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt) 68413531Sjairo.balart@metempsy.com{ 68513531Sjairo.balart@metempsy.com assert(pkt->isRequest()); 68613531Sjairo.balart@metempsy.com 68713531Sjairo.balart@metempsy.com uint64_t overwrite_val; 68813531Sjairo.balart@metempsy.com bool overwrite_mem; 68913531Sjairo.balart@metempsy.com uint64_t condition_val64; 69013531Sjairo.balart@metempsy.com uint32_t condition_val32; 69113531Sjairo.balart@metempsy.com 69213531Sjairo.balart@metempsy.com int offset = pkt->getOffset(blkSize); 69313531Sjairo.balart@metempsy.com uint8_t *blk_data = blk->data + offset; 69413531Sjairo.balart@metempsy.com 69513531Sjairo.balart@metempsy.com assert(sizeof(uint64_t) >= pkt->getSize()); 69613531Sjairo.balart@metempsy.com 69713531Sjairo.balart@metempsy.com overwrite_mem = true; 69813531Sjairo.balart@metempsy.com // keep a copy of our possible write value, and copy what is at the 69913531Sjairo.balart@metempsy.com // memory address into the packet 70013531Sjairo.balart@metempsy.com pkt->writeData((uint8_t *)&overwrite_val); 70113531Sjairo.balart@metempsy.com pkt->setData(blk_data); 70213531Sjairo.balart@metempsy.com 70313531Sjairo.balart@metempsy.com if (pkt->req->isCondSwap()) { 70413531Sjairo.balart@metempsy.com if (pkt->getSize() == sizeof(uint64_t)) { 70513756Sjairo.balart@metempsy.com condition_val64 = pkt->req->getExtraData(); 70613756Sjairo.balart@metempsy.com overwrite_mem = !std::memcmp(&condition_val64, blk_data, 70713531Sjairo.balart@metempsy.com sizeof(uint64_t)); 70813531Sjairo.balart@metempsy.com } else if (pkt->getSize() == sizeof(uint32_t)) { 70913531Sjairo.balart@metempsy.com condition_val32 = (uint32_t)pkt->req->getExtraData(); 71013531Sjairo.balart@metempsy.com overwrite_mem = !std::memcmp(&condition_val32, blk_data, 71113531Sjairo.balart@metempsy.com sizeof(uint32_t)); 71213531Sjairo.balart@metempsy.com } else 71313531Sjairo.balart@metempsy.com panic("Invalid size for conditional read/write\n"); 71413531Sjairo.balart@metempsy.com } 71513531Sjairo.balart@metempsy.com 71613531Sjairo.balart@metempsy.com if (overwrite_mem) { 71713531Sjairo.balart@metempsy.com std::memcpy(blk_data, &overwrite_val, pkt->getSize()); 71813531Sjairo.balart@metempsy.com blk->status |= BlkDirty; 71913531Sjairo.balart@metempsy.com } 72013531Sjairo.balart@metempsy.com} 72113531Sjairo.balart@metempsy.com 72213531Sjairo.balart@metempsy.comQueueEntry* 72313531Sjairo.balart@metempsy.comBaseCache::getNextQueueEntry() 72413531Sjairo.balart@metempsy.com{ 72513756Sjairo.balart@metempsy.com // Check both MSHR queue and write buffer for potential requests, 72613531Sjairo.balart@metempsy.com // note that null does not mean there is no request, it could 72713531Sjairo.balart@metempsy.com // simply be that it is not ready 72813531Sjairo.balart@metempsy.com MSHR *miss_mshr = mshrQueue.getNext(); 72913531Sjairo.balart@metempsy.com WriteQueueEntry *wq_entry = writeBuffer.getNext(); 73013531Sjairo.balart@metempsy.com 73113531Sjairo.balart@metempsy.com // If we got a write buffer request ready, first priority is a 73213531Sjairo.balart@metempsy.com // full write buffer, otherwise we favour the miss requests 73313531Sjairo.balart@metempsy.com if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) { 73413531Sjairo.balart@metempsy.com // need to search MSHR queue for conflicting earlier miss. 73513531Sjairo.balart@metempsy.com MSHR *conflict_mshr = 73613531Sjairo.balart@metempsy.com mshrQueue.findPending(wq_entry->blkAddr, 73713531Sjairo.balart@metempsy.com wq_entry->isSecure); 73813531Sjairo.balart@metempsy.com 73913531Sjairo.balart@metempsy.com if (conflict_mshr && conflict_mshr->order < wq_entry->order) { 74013531Sjairo.balart@metempsy.com // Service misses in order until conflict is cleared. 74113531Sjairo.balart@metempsy.com return conflict_mshr; 74213531Sjairo.balart@metempsy.com 74313531Sjairo.balart@metempsy.com // @todo Note that we ignore the ready time of the conflict here 74413531Sjairo.balart@metempsy.com } 74513531Sjairo.balart@metempsy.com 74613531Sjairo.balart@metempsy.com // No conflicts; issue write 74713531Sjairo.balart@metempsy.com return wq_entry; 74813531Sjairo.balart@metempsy.com } else if (miss_mshr) { 74913531Sjairo.balart@metempsy.com // need to check for conflicting earlier writeback 75013531Sjairo.balart@metempsy.com WriteQueueEntry *conflict_mshr = 75113531Sjairo.balart@metempsy.com writeBuffer.findPending(miss_mshr->blkAddr, 75213531Sjairo.balart@metempsy.com miss_mshr->isSecure); 75313531Sjairo.balart@metempsy.com if (conflict_mshr) { 75413531Sjairo.balart@metempsy.com // not sure why we don't check order here... it was in the 75513531Sjairo.balart@metempsy.com // original code but commented out. 75613531Sjairo.balart@metempsy.com 75713531Sjairo.balart@metempsy.com // The only way this happens is if we are 75813756Sjairo.balart@metempsy.com // doing a write and we didn't have permissions 75913531Sjairo.balart@metempsy.com // then subsequently saw a writeback (owned got evicted) 76013756Sjairo.balart@metempsy.com // We need to make sure to perform the writeback first 76113756Sjairo.balart@metempsy.com // To preserve the dirty data, then we can issue the write 76213756Sjairo.balart@metempsy.com 76313756Sjairo.balart@metempsy.com // should we return wq_entry here instead? I.e. do we 76413756Sjairo.balart@metempsy.com // have to flush writes in order? I don't think so... not 76513531Sjairo.balart@metempsy.com // for Alpha anyway. Maybe for x86? 76613531Sjairo.balart@metempsy.com return conflict_mshr; 76713531Sjairo.balart@metempsy.com 76813531Sjairo.balart@metempsy.com // @todo Note that we ignore the ready time of the conflict here 76913531Sjairo.balart@metempsy.com } 77013531Sjairo.balart@metempsy.com 77113531Sjairo.balart@metempsy.com // No conflicts; issue read 77213756Sjairo.balart@metempsy.com return miss_mshr; 77313531Sjairo.balart@metempsy.com } 77413531Sjairo.balart@metempsy.com 77513531Sjairo.balart@metempsy.com // fall through... no pending requests. Try a prefetch. 77613531Sjairo.balart@metempsy.com assert(!miss_mshr && !wq_entry); 77713531Sjairo.balart@metempsy.com if (prefetcher && mshrQueue.canPrefetch()) { 77813531Sjairo.balart@metempsy.com // If we have a miss queue slot, we can try a prefetch 77913531Sjairo.balart@metempsy.com PacketPtr pkt = prefetcher->getPacket(); 78013531Sjairo.balart@metempsy.com if (pkt) { 78113531Sjairo.balart@metempsy.com Addr pf_addr = pkt->getBlockAddr(blkSize); 78213531Sjairo.balart@metempsy.com if (!tags->findBlock(pf_addr, pkt->isSecure()) && 78313531Sjairo.balart@metempsy.com !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 78413531Sjairo.balart@metempsy.com !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { 78513531Sjairo.balart@metempsy.com // Update statistic on number of prefetches issued 78613531Sjairo.balart@metempsy.com // (hwpf_mshr_misses) 78713531Sjairo.balart@metempsy.com assert(pkt->req->masterId() < system->maxMasters()); 78813531Sjairo.balart@metempsy.com mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 78913531Sjairo.balart@metempsy.com 79013531Sjairo.balart@metempsy.com // allocate an MSHR and return it, note 79113531Sjairo.balart@metempsy.com // that we send the packet straight away, so do not 79213531Sjairo.balart@metempsy.com // schedule the send 79313531Sjairo.balart@metempsy.com return allocateMissBuffer(pkt, curTick(), false); 79413531Sjairo.balart@metempsy.com } else { 79513531Sjairo.balart@metempsy.com // free the request and packet 79613531Sjairo.balart@metempsy.com delete pkt; 79713756Sjairo.balart@metempsy.com } 79814230Sgiacomo.travaglini@arm.com } 79913531Sjairo.balart@metempsy.com } 80013531Sjairo.balart@metempsy.com 80113531Sjairo.balart@metempsy.com return nullptr; 80213531Sjairo.balart@metempsy.com} 80313531Sjairo.balart@metempsy.com 80413531Sjairo.balart@metempsy.comvoid 80513756Sjairo.balart@metempsy.comBaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) 80613531Sjairo.balart@metempsy.com{ 80713531Sjairo.balart@metempsy.com assert(pkt->isRequest()); 80813531Sjairo.balart@metempsy.com 80913531Sjairo.balart@metempsy.com assert(blk && blk->isValid()); 81013531Sjairo.balart@metempsy.com // Occasionally this is not true... if we are a lower-level cache 81113531Sjairo.balart@metempsy.com // satisfying a string of Read and ReadEx requests from 81213531Sjairo.balart@metempsy.com // upper-level caches, a Read will mark the block as shared but we 81313531Sjairo.balart@metempsy.com // can satisfy a following ReadEx anyway since we can rely on the 81413531Sjairo.balart@metempsy.com // Read requester(s) to have buffered the ReadEx snoop and to 81513531Sjairo.balart@metempsy.com // invalidate their blocks after receiving them. 81613531Sjairo.balart@metempsy.com // assert(!pkt->needsWritable() || blk->isWritable()); 81713531Sjairo.balart@metempsy.com assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize); 81813531Sjairo.balart@metempsy.com 81913756Sjairo.balart@metempsy.com // Check RMW operations first since both isRead() and 82013531Sjairo.balart@metempsy.com // isWrite() will be true for them 82113531Sjairo.balart@metempsy.com if (pkt->cmd == MemCmd::SwapReq) { 82213531Sjairo.balart@metempsy.com if (pkt->isAtomicOp()) { 82313531Sjairo.balart@metempsy.com // extract data from cache and save it into the data field in 82413531Sjairo.balart@metempsy.com // the packet as a return value from this atomic op 82513531Sjairo.balart@metempsy.com int offset = tags->extractBlkOffset(pkt->getAddr()); 82613531Sjairo.balart@metempsy.com uint8_t *blk_data = blk->data + offset; 82713531Sjairo.balart@metempsy.com pkt->setData(blk_data); 82813531Sjairo.balart@metempsy.com 82913531Sjairo.balart@metempsy.com // execute AMO operation 83013531Sjairo.balart@metempsy.com (*(pkt->getAtomicOp()))(blk_data); 83113531Sjairo.balart@metempsy.com 83213531Sjairo.balart@metempsy.com // set block status to dirty 83313531Sjairo.balart@metempsy.com blk->status |= BlkDirty; 83413531Sjairo.balart@metempsy.com } else { 83513531Sjairo.balart@metempsy.com cmpAndSwap(blk, pkt); 83613531Sjairo.balart@metempsy.com } 83713531Sjairo.balart@metempsy.com } else if (pkt->isWrite()) { 83813531Sjairo.balart@metempsy.com // we have the block in a writable state and can go ahead, 83913531Sjairo.balart@metempsy.com // note that the line may be also be considered writable in 84013531Sjairo.balart@metempsy.com // downstream caches along the path to memory, but always 84113531Sjairo.balart@metempsy.com // Exclusive, and never Modified 84213531Sjairo.balart@metempsy.com assert(blk->isWritable()); 84313531Sjairo.balart@metempsy.com // Write or WriteLine at the first cache with block in writable state 84413531Sjairo.balart@metempsy.com if (blk->checkWrite(pkt)) { 84513531Sjairo.balart@metempsy.com pkt->writeDataToBlock(blk->data, blkSize); 84613531Sjairo.balart@metempsy.com } 84713531Sjairo.balart@metempsy.com // Always mark the line as dirty (and thus transition to the 84813531Sjairo.balart@metempsy.com // Modified state) even if we are a failed StoreCond so we 84913531Sjairo.balart@metempsy.com // supply data to any snoops that have appended themselves to 85013531Sjairo.balart@metempsy.com // this cache before knowing the store will fail. 85113531Sjairo.balart@metempsy.com blk->status |= BlkDirty; 85213531Sjairo.balart@metempsy.com DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print()); 85313531Sjairo.balart@metempsy.com } else if (pkt->isRead()) { 85413531Sjairo.balart@metempsy.com if (pkt->isLLSC()) { 85513531Sjairo.balart@metempsy.com blk->trackLoadLocked(pkt); 85613531Sjairo.balart@metempsy.com } 85713531Sjairo.balart@metempsy.com 85813531Sjairo.balart@metempsy.com // all read responses have a data payload 85913531Sjairo.balart@metempsy.com assert(pkt->hasRespData()); 86013531Sjairo.balart@metempsy.com pkt->setDataFromBlock(blk->data, blkSize); 86113531Sjairo.balart@metempsy.com } else if (pkt->isUpgrade()) { 86213531Sjairo.balart@metempsy.com // sanity check 86313531Sjairo.balart@metempsy.com assert(!pkt->hasSharers()); 86413531Sjairo.balart@metempsy.com 86513531Sjairo.balart@metempsy.com if (blk->isDirty()) { 86613531Sjairo.balart@metempsy.com // we were in the Owned state, and a cache above us that 86713531Sjairo.balart@metempsy.com // has the line in Shared state needs to be made aware 86813531Sjairo.balart@metempsy.com // that the data it already has is in fact dirty 86913531Sjairo.balart@metempsy.com pkt->setCacheResponding(); 87013531Sjairo.balart@metempsy.com blk->status &= ~BlkDirty; 87113531Sjairo.balart@metempsy.com } 87213531Sjairo.balart@metempsy.com } else if (pkt->isClean()) { 87313531Sjairo.balart@metempsy.com blk->status &= ~BlkDirty; 87413531Sjairo.balart@metempsy.com } else { 87513531Sjairo.balart@metempsy.com assert(pkt->isInvalidate()); 87613531Sjairo.balart@metempsy.com invalidateBlock(blk); 87713531Sjairo.balart@metempsy.com DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__, 87813531Sjairo.balart@metempsy.com pkt->print()); 87913531Sjairo.balart@metempsy.com } 88013531Sjairo.balart@metempsy.com} 88113531Sjairo.balart@metempsy.com 88213531Sjairo.balart@metempsy.com///////////////////////////////////////////////////// 88313531Sjairo.balart@metempsy.com// 88413531Sjairo.balart@metempsy.com// Access path: requests coming in from the CPU side 88513531Sjairo.balart@metempsy.com// 88613531Sjairo.balart@metempsy.com///////////////////////////////////////////////////// 88713531Sjairo.balart@metempsy.comCycles 88813531Sjairo.balart@metempsy.comBaseCache::calculateAccessLatency(const CacheBlk* blk, 88913531Sjairo.balart@metempsy.com const Cycles lookup_lat) const 89013531Sjairo.balart@metempsy.com{ 89113531Sjairo.balart@metempsy.com Cycles lat(lookup_lat); 89213531Sjairo.balart@metempsy.com 89313531Sjairo.balart@metempsy.com if (blk != nullptr) { 89413531Sjairo.balart@metempsy.com // First access tags, then data 89513531Sjairo.balart@metempsy.com if (sequentialAccess) { 89613531Sjairo.balart@metempsy.com lat += dataLatency; 89713531Sjairo.balart@metempsy.com // Latency is dictated by the slowest of tag and data latencies 89813531Sjairo.balart@metempsy.com } else { 89913531Sjairo.balart@metempsy.com lat = std::max(lookup_lat, dataLatency); 90013531Sjairo.balart@metempsy.com } 90113531Sjairo.balart@metempsy.com 90213531Sjairo.balart@metempsy.com // Check if the block to be accessed is available. If not, apply the 90313531Sjairo.balart@metempsy.com // access latency on top of when the block is ready to be accessed. 90413531Sjairo.balart@metempsy.com const Tick when_ready = blk->getWhenReady(); 90513531Sjairo.balart@metempsy.com if (when_ready > curTick() && 90613531Sjairo.balart@metempsy.com ticksToCycles(when_ready - curTick()) > lat) { 90713531Sjairo.balart@metempsy.com lat += ticksToCycles(when_ready - curTick()); 90813531Sjairo.balart@metempsy.com } 90913531Sjairo.balart@metempsy.com } 91013531Sjairo.balart@metempsy.com 91113531Sjairo.balart@metempsy.com return lat; 91213531Sjairo.balart@metempsy.com} 91313531Sjairo.balart@metempsy.com 91413531Sjairo.balart@metempsy.combool 91513531Sjairo.balart@metempsy.comBaseCache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 91613531Sjairo.balart@metempsy.com PacketList &writebacks) 91713531Sjairo.balart@metempsy.com{ 91813531Sjairo.balart@metempsy.com // sanity check 91913531Sjairo.balart@metempsy.com assert(pkt->isRequest()); 92013531Sjairo.balart@metempsy.com 92113531Sjairo.balart@metempsy.com chatty_assert(!(isReadOnly && pkt->isWrite()), 92213531Sjairo.balart@metempsy.com "Should never see a write in a read-only cache %s\n", 92313531Sjairo.balart@metempsy.com name()); 92413531Sjairo.balart@metempsy.com 92513531Sjairo.balart@metempsy.com // Access block in the tags 92613531Sjairo.balart@metempsy.com Cycles tag_latency(0); 92713531Sjairo.balart@metempsy.com blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), tag_latency); 92813531Sjairo.balart@metempsy.com 92913531Sjairo.balart@metempsy.com // Calculate access latency 93013531Sjairo.balart@metempsy.com lat = calculateAccessLatency(blk, tag_latency); 93113531Sjairo.balart@metempsy.com 93213531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s for %s %s\n", __func__, pkt->print(), 93313531Sjairo.balart@metempsy.com blk ? "hit " + blk->print() : "miss"); 93413531Sjairo.balart@metempsy.com 93514232Sgiacomo.travaglini@arm.com if (pkt->req->isCacheMaintenance()) { 93614232Sgiacomo.travaglini@arm.com // A cache maintenance operation is always forwarded to the 93713531Sjairo.balart@metempsy.com // memory below even if the block is found in dirty state. 93813531Sjairo.balart@metempsy.com 93913531Sjairo.balart@metempsy.com // We defer any changes to the state of the block until we 94013531Sjairo.balart@metempsy.com // create and mark as in service the mshr for the downstream 94113531Sjairo.balart@metempsy.com // packet. 94213531Sjairo.balart@metempsy.com return false; 94313531Sjairo.balart@metempsy.com } 94413531Sjairo.balart@metempsy.com 94513531Sjairo.balart@metempsy.com if (pkt->isEviction()) { 94613531Sjairo.balart@metempsy.com // We check for presence of block in above caches before issuing 94713531Sjairo.balart@metempsy.com // Writeback or CleanEvict to write buffer. Therefore the only 94813531Sjairo.balart@metempsy.com // possible cases can be of a CleanEvict packet coming from above 94913531Sjairo.balart@metempsy.com // encountering a Writeback generated in this cache peer cache and 95013531Sjairo.balart@metempsy.com // waiting in the write buffer. Cases of upper level peer caches 95113531Sjairo.balart@metempsy.com // generating CleanEvict and Writeback or simply CleanEvict and 95213531Sjairo.balart@metempsy.com // CleanEvict almost simultaneously will be caught by snoops sent out 95314231Sgiacomo.travaglini@arm.com // by crossbar. 95413531Sjairo.balart@metempsy.com WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(), 95513531Sjairo.balart@metempsy.com pkt->isSecure()); 95613531Sjairo.balart@metempsy.com if (wb_entry) { 95713756Sjairo.balart@metempsy.com assert(wb_entry->getNumTargets() == 1); 95813531Sjairo.balart@metempsy.com PacketPtr wbPkt = wb_entry->getTarget()->pkt; 95913531Sjairo.balart@metempsy.com assert(wbPkt->isWriteback()); 96013531Sjairo.balart@metempsy.com 96113531Sjairo.balart@metempsy.com if (pkt->isCleanEviction()) { 96214231Sgiacomo.travaglini@arm.com // The CleanEvict and WritebackClean snoops into other 96314231Sgiacomo.travaglini@arm.com // peer caches of the same level while traversing the 96414231Sgiacomo.travaglini@arm.com // crossbar. If a copy of the block is found, the 96513531Sjairo.balart@metempsy.com // packet is deleted in the crossbar. Hence, none of 96613531Sjairo.balart@metempsy.com // the other upper level caches connected to this 96714231Sgiacomo.travaglini@arm.com // cache have the block, so we can clear the 96814231Sgiacomo.travaglini@arm.com // BLOCK_CACHED flag in the Writeback if set and 96913531Sjairo.balart@metempsy.com // discard the CleanEvict by returning true. 97014231Sgiacomo.travaglini@arm.com wbPkt->clearBlockCached(); 97114231Sgiacomo.travaglini@arm.com return true; 97213531Sjairo.balart@metempsy.com } else { 97314231Sgiacomo.travaglini@arm.com assert(pkt->cmd == MemCmd::WritebackDirty); 97414231Sgiacomo.travaglini@arm.com // Dirty writeback from above trumps our clean 97514231Sgiacomo.travaglini@arm.com // writeback... discard here 97614231Sgiacomo.travaglini@arm.com // Note: markInService will remove entry from writeback buffer. 97714231Sgiacomo.travaglini@arm.com markInService(wb_entry); 97814231Sgiacomo.travaglini@arm.com delete wbPkt; 97914231Sgiacomo.travaglini@arm.com } 98014231Sgiacomo.travaglini@arm.com } 98114231Sgiacomo.travaglini@arm.com } 98214231Sgiacomo.travaglini@arm.com 98314231Sgiacomo.travaglini@arm.com // Writeback handling is special case. We can write the block into 98414231Sgiacomo.travaglini@arm.com // the cache without having a writeable copy (or any copy at all). 98514231Sgiacomo.travaglini@arm.com if (pkt->isWriteback()) { 98614231Sgiacomo.travaglini@arm.com assert(blkSize == pkt->getSize()); 98714231Sgiacomo.travaglini@arm.com 98814231Sgiacomo.travaglini@arm.com // we could get a clean writeback while we are having 98914231Sgiacomo.travaglini@arm.com // outstanding accesses to a block, do the simple thing for 99014231Sgiacomo.travaglini@arm.com // now and drop the clean writeback so that we do not upset 99114231Sgiacomo.travaglini@arm.com // any ordering/decisions about ownership already taken 99214231Sgiacomo.travaglini@arm.com if (pkt->cmd == MemCmd::WritebackClean && 99314231Sgiacomo.travaglini@arm.com mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) { 99414231Sgiacomo.travaglini@arm.com DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, " 99514231Sgiacomo.travaglini@arm.com "dropping\n", pkt->getAddr()); 99614231Sgiacomo.travaglini@arm.com return true; 99714231Sgiacomo.travaglini@arm.com } 99814231Sgiacomo.travaglini@arm.com 99914231Sgiacomo.travaglini@arm.com if (!blk) { 100014231Sgiacomo.travaglini@arm.com // need to do a replacement 100113531Sjairo.balart@metempsy.com blk = allocateBlock(pkt, writebacks); 100213531Sjairo.balart@metempsy.com if (!blk) { 100313531Sjairo.balart@metempsy.com // no replaceable block available: give up, fwd to next level. 100413531Sjairo.balart@metempsy.com incMissCount(pkt); 100514231Sgiacomo.travaglini@arm.com return false; 100613531Sjairo.balart@metempsy.com } 100714231Sgiacomo.travaglini@arm.com 100814231Sgiacomo.travaglini@arm.com blk->status |= BlkReadable; 100914231Sgiacomo.travaglini@arm.com } 101013531Sjairo.balart@metempsy.com // only mark the block dirty if we got a writeback command, 101113531Sjairo.balart@metempsy.com // and leave it as is for a clean writeback 101213531Sjairo.balart@metempsy.com if (pkt->cmd == MemCmd::WritebackDirty) { 101313531Sjairo.balart@metempsy.com // TODO: the coherent cache can assert(!blk->isDirty()); 101413531Sjairo.balart@metempsy.com blk->status |= BlkDirty; 101513531Sjairo.balart@metempsy.com } 101613531Sjairo.balart@metempsy.com // if the packet does not have sharers, it is passing 101713756Sjairo.balart@metempsy.com // writable, and we got the writeback in Modified or Exclusive 101813531Sjairo.balart@metempsy.com // state, if not we are in the Owned or Shared state 101913531Sjairo.balart@metempsy.com if (!pkt->hasSharers()) { 102013531Sjairo.balart@metempsy.com blk->status |= BlkWritable; 102113531Sjairo.balart@metempsy.com } 102213756Sjairo.balart@metempsy.com // nothing else to do; writeback doesn't expect response 102313531Sjairo.balart@metempsy.com assert(!pkt->needsResponse()); 102414231Sgiacomo.travaglini@arm.com pkt->writeDataToBlock(blk->data, blkSize); 102514231Sgiacomo.travaglini@arm.com DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 102613531Sjairo.balart@metempsy.com incHitCount(pkt); 102714231Sgiacomo.travaglini@arm.com // populate the time when the block will be ready to access. 102814231Sgiacomo.travaglini@arm.com blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 102913531Sjairo.balart@metempsy.com pkt->payloadDelay); 103013531Sjairo.balart@metempsy.com return true; 103113756Sjairo.balart@metempsy.com } else if (pkt->cmd == MemCmd::CleanEvict) { 103213756Sjairo.balart@metempsy.com if (blk) { 103314231Sgiacomo.travaglini@arm.com // Found the block in the tags, need to stop CleanEvict from 103413531Sjairo.balart@metempsy.com // propagating further down the hierarchy. Returning true will 103513531Sjairo.balart@metempsy.com // treat the CleanEvict like a satisfied write request and delete 103613531Sjairo.balart@metempsy.com // it. 103713531Sjairo.balart@metempsy.com return true; 103813531Sjairo.balart@metempsy.com } 103913531Sjairo.balart@metempsy.com // We didn't find the block here, propagate the CleanEvict further 104013531Sjairo.balart@metempsy.com // down the memory hierarchy. Returning false will treat the CleanEvict 104114231Sgiacomo.travaglini@arm.com // like a Writeback which could not find a replaceable block so has to 104213531Sjairo.balart@metempsy.com // go to next level. 104314231Sgiacomo.travaglini@arm.com return false; 104413531Sjairo.balart@metempsy.com } else if (pkt->cmd == MemCmd::WriteClean) { 104513531Sjairo.balart@metempsy.com // WriteClean handling is a special case. We can allocate a 104613531Sjairo.balart@metempsy.com // block directly if it doesn't exist and we can update the 104713531Sjairo.balart@metempsy.com // block immediately. The WriteClean transfers the ownership 104813756Sjairo.balart@metempsy.com // of the block as well. 104913531Sjairo.balart@metempsy.com assert(blkSize == pkt->getSize()); 105013531Sjairo.balart@metempsy.com 105113531Sjairo.balart@metempsy.com if (!blk) { 105213531Sjairo.balart@metempsy.com if (pkt->writeThrough()) { 105313531Sjairo.balart@metempsy.com // if this is a write through packet, we don't try to 105413531Sjairo.balart@metempsy.com // allocate if the block is not present 105513531Sjairo.balart@metempsy.com return false; 105613531Sjairo.balart@metempsy.com } else { 105713531Sjairo.balart@metempsy.com // a writeback that misses needs to allocate a new block 105813531Sjairo.balart@metempsy.com blk = allocateBlock(pkt, writebacks); 105913531Sjairo.balart@metempsy.com if (!blk) { 106013531Sjairo.balart@metempsy.com // no replaceable block available: give up, fwd to 106113531Sjairo.balart@metempsy.com // next level. 106213531Sjairo.balart@metempsy.com incMissCount(pkt); 106313531Sjairo.balart@metempsy.com return false; 106413531Sjairo.balart@metempsy.com } 106513531Sjairo.balart@metempsy.com 106613531Sjairo.balart@metempsy.com blk->status |= BlkReadable; 106713756Sjairo.balart@metempsy.com } 106813531Sjairo.balart@metempsy.com } 106913531Sjairo.balart@metempsy.com 107013531Sjairo.balart@metempsy.com // at this point either this is a writeback or a write-through 107113531Sjairo.balart@metempsy.com // write clean operation and the block is already in this 107213531Sjairo.balart@metempsy.com // cache, we need to update the data and the block flags 107313531Sjairo.balart@metempsy.com assert(blk); 107413531Sjairo.balart@metempsy.com // TODO: the coherent cache can assert(!blk->isDirty()); 107513531Sjairo.balart@metempsy.com if (!pkt->writeThrough()) { 107613531Sjairo.balart@metempsy.com blk->status |= BlkDirty; 107713531Sjairo.balart@metempsy.com } 107813531Sjairo.balart@metempsy.com // nothing else to do; writeback doesn't expect response 107913531Sjairo.balart@metempsy.com assert(!pkt->needsResponse()); 108013531Sjairo.balart@metempsy.com pkt->writeDataToBlock(blk->data, blkSize); 108113531Sjairo.balart@metempsy.com DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 108213531Sjairo.balart@metempsy.com 108313531Sjairo.balart@metempsy.com incHitCount(pkt); 108413531Sjairo.balart@metempsy.com // populate the time when the block will be ready to access. 108513531Sjairo.balart@metempsy.com blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 108613531Sjairo.balart@metempsy.com pkt->payloadDelay); 108713531Sjairo.balart@metempsy.com // if this a write-through packet it will be sent to cache 108813531Sjairo.balart@metempsy.com // below 108913531Sjairo.balart@metempsy.com return !pkt->writeThrough(); 109013531Sjairo.balart@metempsy.com } else if (blk && (pkt->needsWritable() ? blk->isWritable() : 109113531Sjairo.balart@metempsy.com blk->isReadable())) { 109213531Sjairo.balart@metempsy.com // OK to satisfy access 109313531Sjairo.balart@metempsy.com incHitCount(pkt); 109413531Sjairo.balart@metempsy.com satisfyRequest(pkt, blk); 109513531Sjairo.balart@metempsy.com maintainClusivity(pkt->fromCache(), blk); 109613531Sjairo.balart@metempsy.com 109713531Sjairo.balart@metempsy.com return true; 109813531Sjairo.balart@metempsy.com } 109913531Sjairo.balart@metempsy.com 110013531Sjairo.balart@metempsy.com // Can't satisfy access normally... either no block (blk == nullptr) 110113531Sjairo.balart@metempsy.com // or have block but need writable 110213531Sjairo.balart@metempsy.com 110313531Sjairo.balart@metempsy.com incMissCount(pkt); 110413531Sjairo.balart@metempsy.com 110513531Sjairo.balart@metempsy.com if (!blk && pkt->isLLSC() && pkt->isWrite()) { 110613531Sjairo.balart@metempsy.com // complete miss on store conditional... just give up now 110713531Sjairo.balart@metempsy.com pkt->req->setExtraData(0); 110813531Sjairo.balart@metempsy.com return true; 110913531Sjairo.balart@metempsy.com } 111013531Sjairo.balart@metempsy.com 111113531Sjairo.balart@metempsy.com return false; 111213531Sjairo.balart@metempsy.com} 111313531Sjairo.balart@metempsy.com 111413531Sjairo.balart@metempsy.comvoid 111513531Sjairo.balart@metempsy.comBaseCache::maintainClusivity(bool from_cache, CacheBlk *blk) 111613531Sjairo.balart@metempsy.com{ 111713531Sjairo.balart@metempsy.com if (from_cache && blk && blk->isValid() && !blk->isDirty() && 111813531Sjairo.balart@metempsy.com clusivity == Enums::mostly_excl) { 111913531Sjairo.balart@metempsy.com // if we have responded to a cache, and our block is still 112013531Sjairo.balart@metempsy.com // valid, but not dirty, and this cache is mostly exclusive 112113531Sjairo.balart@metempsy.com // with respect to the cache above, drop the block 112213531Sjairo.balart@metempsy.com invalidateBlock(blk); 112313531Sjairo.balart@metempsy.com } 112413531Sjairo.balart@metempsy.com} 112513531Sjairo.balart@metempsy.com 112613531Sjairo.balart@metempsy.comCacheBlk* 112713531Sjairo.balart@metempsy.comBaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks, 112813531Sjairo.balart@metempsy.com bool allocate) 112913531Sjairo.balart@metempsy.com{ 113013531Sjairo.balart@metempsy.com assert(pkt->isResponse()); 113113531Sjairo.balart@metempsy.com Addr addr = pkt->getAddr(); 113213531Sjairo.balart@metempsy.com bool is_secure = pkt->isSecure(); 113313531Sjairo.balart@metempsy.com#if TRACING_ON 113413531Sjairo.balart@metempsy.com CacheBlk::State old_state = blk ? blk->status : 0; 113513531Sjairo.balart@metempsy.com#endif 113613531Sjairo.balart@metempsy.com 113713531Sjairo.balart@metempsy.com // When handling a fill, we should have no writes to this line. 113813531Sjairo.balart@metempsy.com assert(addr == pkt->getBlockAddr(blkSize)); 113913531Sjairo.balart@metempsy.com assert(!writeBuffer.findMatch(addr, is_secure)); 114013531Sjairo.balart@metempsy.com 114113531Sjairo.balart@metempsy.com if (!blk) { 114213531Sjairo.balart@metempsy.com // better have read new data... 1143 assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp); 1144 1145 // need to do a replacement if allocating, otherwise we stick 1146 // with the temporary storage 1147 blk = allocate ? allocateBlock(pkt, writebacks) : nullptr; 1148 1149 if (!blk) { 1150 // No replaceable block or a mostly exclusive 1151 // cache... just use temporary storage to complete the 1152 // current request and then get rid of it 1153 blk = tempBlock; 1154 tempBlock->insert(addr, is_secure); 1155 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr, 1156 is_secure ? "s" : "ns"); 1157 } 1158 } else { 1159 // existing block... probably an upgrade 1160 // don't clear block status... if block is already dirty we 1161 // don't want to lose that 1162 } 1163 1164 // Block is guaranteed to be valid at this point 1165 assert(blk->isValid()); 1166 assert(blk->isSecure() == is_secure); 1167 assert(regenerateBlkAddr(blk) == addr); 1168 1169 blk->status |= BlkReadable; 1170 1171 // sanity check for whole-line writes, which should always be 1172 // marked as writable as part of the fill, and then later marked 1173 // dirty as part of satisfyRequest 1174 if (pkt->cmd == MemCmd::InvalidateResp) { 1175 assert(!pkt->hasSharers()); 1176 } 1177 1178 // here we deal with setting the appropriate state of the line, 1179 // and we start by looking at the hasSharers flag, and ignore the 1180 // cacheResponding flag (normally signalling dirty data) if the 1181 // packet has sharers, thus the line is never allocated as Owned 1182 // (dirty but not writable), and always ends up being either 1183 // Shared, Exclusive or Modified, see Packet::setCacheResponding 1184 // for more details 1185 if (!pkt->hasSharers()) { 1186 // we could get a writable line from memory (rather than a 1187 // cache) even in a read-only cache, note that we set this bit 1188 // even for a read-only cache, possibly revisit this decision 1189 blk->status |= BlkWritable; 1190 1191 // check if we got this via cache-to-cache transfer (i.e., from a 1192 // cache that had the block in Modified or Owned state) 1193 if (pkt->cacheResponding()) { 1194 // we got the block in Modified state, and invalidated the 1195 // owners copy 1196 blk->status |= BlkDirty; 1197 1198 chatty_assert(!isReadOnly, "Should never see dirty snoop response " 1199 "in read-only cache %s\n", name()); 1200 } 1201 } 1202 1203 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n", 1204 addr, is_secure ? "s" : "ns", old_state, blk->print()); 1205 1206 // if we got new data, copy it in (checking for a read response 1207 // and a response that has data is the same in the end) 1208 if (pkt->isRead()) { 1209 // sanity checks 1210 assert(pkt->hasData()); 1211 assert(pkt->getSize() == blkSize); 1212 1213 pkt->writeDataToBlock(blk->data, blkSize); 1214 } 1215 // We pay for fillLatency here. 1216 blk->setWhenReady(clockEdge(fillLatency) + pkt->payloadDelay); 1217 1218 return blk; 1219} 1220 1221CacheBlk* 1222BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks) 1223{ 1224 // Get address 1225 const Addr addr = pkt->getAddr(); 1226 1227 // Get secure bit 1228 const bool is_secure = pkt->isSecure(); 1229 1230 // Find replacement victim 1231 std::vector<CacheBlk*> evict_blks; 1232 CacheBlk *victim = tags->findVictim(addr, is_secure, evict_blks); 1233 1234 // It is valid to return nullptr if there is no victim 1235 if (!victim) 1236 return nullptr; 1237 1238 // Print victim block's information 1239 DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print()); 1240 1241 // Check for transient state allocations. If any of the entries listed 1242 // for eviction has a transient state, the allocation fails 1243 for (const auto& blk : evict_blks) { 1244 if (blk->isValid()) { 1245 Addr repl_addr = regenerateBlkAddr(blk); 1246 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure()); 1247 if (repl_mshr) { 1248 // must be an outstanding upgrade or clean request 1249 // on a block we're about to replace... 1250 assert((!blk->isWritable() && repl_mshr->needsWritable()) || 1251 repl_mshr->isCleaning()); 1252 1253 // too hard to replace block with transient state 1254 // allocation failed, block not inserted 1255 return nullptr; 1256 } 1257 } 1258 } 1259 1260 // The victim will be replaced by a new entry, so increase the replacement 1261 // counter if a valid block is being replaced 1262 if (victim->isValid()) { 1263 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx " 1264 "(%s): %s\n", regenerateBlkAddr(victim), 1265 victim->isSecure() ? "s" : "ns", 1266 addr, is_secure ? "s" : "ns", 1267 victim->isDirty() ? "writeback" : "clean"); 1268 1269 replacements++; 1270 } 1271 1272 // Evict valid blocks associated to this victim block 1273 for (const auto& blk : evict_blks) { 1274 if (blk->isValid()) { 1275 if (blk->wasPrefetched()) { 1276 unusedPrefetches++; 1277 } 1278 1279 evictBlock(blk, writebacks); 1280 } 1281 } 1282 1283 // Insert new block at victimized entry 1284 tags->insertBlock(addr, is_secure, pkt->req->masterId(), 1285 pkt->req->taskId(), victim); 1286 1287 return victim; 1288} 1289 1290void 1291BaseCache::invalidateBlock(CacheBlk *blk) 1292{ 1293 // If handling a block present in the Tags, let it do its invalidation 1294 // process, which will update stats and invalidate the block itself 1295 if (blk != tempBlock) { 1296 tags->invalidate(blk); 1297 } else { 1298 tempBlock->invalidate(); 1299 } 1300} 1301 1302void 1303BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks) 1304{ 1305 PacketPtr pkt = evictBlock(blk); 1306 if (pkt) { 1307 writebacks.push_back(pkt); 1308 } 1309} 1310 1311PacketPtr 1312BaseCache::writebackBlk(CacheBlk *blk) 1313{ 1314 chatty_assert(!isReadOnly || writebackClean, 1315 "Writeback from read-only cache"); 1316 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean)); 1317 1318 writebacks[Request::wbMasterId]++; 1319 1320 RequestPtr req = std::make_shared<Request>( 1321 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1322 1323 if (blk->isSecure()) 1324 req->setFlags(Request::SECURE); 1325 1326 req->taskId(blk->task_id); 1327 1328 PacketPtr pkt = 1329 new Packet(req, blk->isDirty() ? 1330 MemCmd::WritebackDirty : MemCmd::WritebackClean); 1331 1332 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n", 1333 pkt->print(), blk->isWritable(), blk->isDirty()); 1334 1335 if (blk->isWritable()) { 1336 // not asserting shared means we pass the block in modified 1337 // state, mark our own block non-writeable 1338 blk->status &= ~BlkWritable; 1339 } else { 1340 // we are in the Owned state, tell the receiver 1341 pkt->setHasSharers(); 1342 } 1343 1344 // make sure the block is not marked dirty 1345 blk->status &= ~BlkDirty; 1346 1347 pkt->allocate(); 1348 pkt->setDataFromBlock(blk->data, blkSize); 1349 1350 return pkt; 1351} 1352 1353PacketPtr 1354BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id) 1355{ 1356 RequestPtr req = std::make_shared<Request>( 1357 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId); 1358 1359 if (blk->isSecure()) { 1360 req->setFlags(Request::SECURE); 1361 } 1362 req->taskId(blk->task_id); 1363 1364 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id); 1365 1366 if (dest) { 1367 req->setFlags(dest); 1368 pkt->setWriteThrough(); 1369 } 1370 1371 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(), 1372 blk->isWritable(), blk->isDirty()); 1373 1374 if (blk->isWritable()) { 1375 // not asserting shared means we pass the block in modified 1376 // state, mark our own block non-writeable 1377 blk->status &= ~BlkWritable; 1378 } else { 1379 // we are in the Owned state, tell the receiver 1380 pkt->setHasSharers(); 1381 } 1382 1383 // make sure the block is not marked dirty 1384 blk->status &= ~BlkDirty; 1385 1386 pkt->allocate(); 1387 pkt->setDataFromBlock(blk->data, blkSize); 1388 1389 return pkt; 1390} 1391 1392 1393void 1394BaseCache::memWriteback() 1395{ 1396 tags->forEachBlk([this](CacheBlk &blk) { writebackVisitor(blk); }); 1397} 1398 1399void 1400BaseCache::memInvalidate() 1401{ 1402 tags->forEachBlk([this](CacheBlk &blk) { invalidateVisitor(blk); }); 1403} 1404 1405bool 1406BaseCache::isDirty() const 1407{ 1408 return tags->anyBlk([](CacheBlk &blk) { return blk.isDirty(); }); 1409} 1410 1411bool 1412BaseCache::coalesce() const 1413{ 1414 return writeAllocator && writeAllocator->coalesce(); 1415} 1416 1417void 1418BaseCache::writebackVisitor(CacheBlk &blk) 1419{ 1420 if (blk.isDirty()) { 1421 assert(blk.isValid()); 1422 1423 RequestPtr request = std::make_shared<Request>( 1424 regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId); 1425 1426 request->taskId(blk.task_id); 1427 if (blk.isSecure()) { 1428 request->setFlags(Request::SECURE); 1429 } 1430 1431 Packet packet(request, MemCmd::WriteReq); 1432 packet.dataStatic(blk.data); 1433 1434 memSidePort.sendFunctional(&packet); 1435 1436 blk.status &= ~BlkDirty; 1437 } 1438} 1439 1440void 1441BaseCache::invalidateVisitor(CacheBlk &blk) 1442{ 1443 if (blk.isDirty()) 1444 warn_once("Invalidating dirty cache lines. " \ 1445 "Expect things to break.\n"); 1446 1447 if (blk.isValid()) { 1448 assert(!blk.isDirty()); 1449 invalidateBlock(&blk); 1450 } 1451} 1452 1453Tick 1454BaseCache::nextQueueReadyTime() const 1455{ 1456 Tick nextReady = std::min(mshrQueue.nextReadyTime(), 1457 writeBuffer.nextReadyTime()); 1458 1459 // Don't signal prefetch ready time if no MSHRs available 1460 // Will signal once enoguh MSHRs are deallocated 1461 if (prefetcher && mshrQueue.canPrefetch()) { 1462 nextReady = std::min(nextReady, 1463 prefetcher->nextPrefetchReadyTime()); 1464 } 1465 1466 return nextReady; 1467} 1468 1469 1470bool 1471BaseCache::sendMSHRQueuePacket(MSHR* mshr) 1472{ 1473 assert(mshr); 1474 1475 // use request from 1st target 1476 PacketPtr tgt_pkt = mshr->getTarget()->pkt; 1477 1478 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print()); 1479 1480 // if the cache is in write coalescing mode or (additionally) in 1481 // no allocation mode, and we have a write packet with an MSHR 1482 // that is not a whole-line write (due to incompatible flags etc), 1483 // then reset the write mode 1484 if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) { 1485 if (!mshr->isWholeLineWrite()) { 1486 // if we are currently write coalescing, hold on the 1487 // MSHR as many cycles extra as we need to completely 1488 // write a cache line 1489 if (writeAllocator->delay(mshr->blkAddr)) { 1490 Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod(); 1491 DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow " 1492 "for write coalescing\n", tgt_pkt->print(), delay); 1493 mshrQueue.delay(mshr, delay); 1494 return false; 1495 } else { 1496 writeAllocator->reset(); 1497 } 1498 } else { 1499 writeAllocator->resetDelay(mshr->blkAddr); 1500 } 1501 } 1502 1503 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 1504 1505 // either a prefetch that is not present upstream, or a normal 1506 // MSHR request, proceed to get the packet to send downstream 1507 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(), 1508 mshr->isWholeLineWrite()); 1509 1510 mshr->isForward = (pkt == nullptr); 1511 1512 if (mshr->isForward) { 1513 // not a cache block request, but a response is expected 1514 // make copy of current packet to forward, keep current 1515 // copy for response handling 1516 pkt = new Packet(tgt_pkt, false, true); 1517 assert(!pkt->isWrite()); 1518 } 1519 1520 // play it safe and append (rather than set) the sender state, 1521 // as forwarded packets may already have existing state 1522 pkt->pushSenderState(mshr); 1523 1524 if (pkt->isClean() && blk && blk->isDirty()) { 1525 // A cache clean opearation is looking for a dirty block. Mark 1526 // the packet so that the destination xbar can determine that 1527 // there will be a follow-up write packet as well. 1528 pkt->setSatisfied(); 1529 } 1530 1531 if (!memSidePort.sendTimingReq(pkt)) { 1532 // we are awaiting a retry, but we 1533 // delete the packet and will be creating a new packet 1534 // when we get the opportunity 1535 delete pkt; 1536 1537 // note that we have now masked any requestBus and 1538 // schedSendEvent (we will wait for a retry before 1539 // doing anything), and this is so even if we do not 1540 // care about this packet and might override it before 1541 // it gets retried 1542 return true; 1543 } else { 1544 // As part of the call to sendTimingReq the packet is 1545 // forwarded to all neighbouring caches (and any caches 1546 // above them) as a snoop. Thus at this point we know if 1547 // any of the neighbouring caches are responding, and if 1548 // so, we know it is dirty, and we can determine if it is 1549 // being passed as Modified, making our MSHR the ordering 1550 // point 1551 bool pending_modified_resp = !pkt->hasSharers() && 1552 pkt->cacheResponding(); 1553 markInService(mshr, pending_modified_resp); 1554 1555 if (pkt->isClean() && blk && blk->isDirty()) { 1556 // A cache clean opearation is looking for a dirty 1557 // block. If a dirty block is encountered a WriteClean 1558 // will update any copies to the path to the memory 1559 // until the point of reference. 1560 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 1561 __func__, pkt->print(), blk->print()); 1562 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), 1563 pkt->id); 1564 PacketList writebacks; 1565 writebacks.push_back(wb_pkt); 1566 doWritebacks(writebacks, 0); 1567 } 1568 1569 return false; 1570 } 1571} 1572 1573bool 1574BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) 1575{ 1576 assert(wq_entry); 1577 1578 // always a single target for write queue entries 1579 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 1580 1581 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print()); 1582 1583 // forward as is, both for evictions and uncacheable writes 1584 if (!memSidePort.sendTimingReq(tgt_pkt)) { 1585 // note that we have now masked any requestBus and 1586 // schedSendEvent (we will wait for a retry before 1587 // doing anything), and this is so even if we do not 1588 // care about this packet and might override it before 1589 // it gets retried 1590 return true; 1591 } else { 1592 markInService(wq_entry); 1593 return false; 1594 } 1595} 1596 1597void 1598BaseCache::serialize(CheckpointOut &cp) const 1599{ 1600 bool dirty(isDirty()); 1601 1602 if (dirty) { 1603 warn("*** The cache still contains dirty data. ***\n"); 1604 warn(" Make sure to drain the system using the correct flags.\n"); 1605 warn(" This checkpoint will not restore correctly " \ 1606 "and dirty data in the cache will be lost!\n"); 1607 } 1608 1609 // Since we don't checkpoint the data in the cache, any dirty data 1610 // will be lost when restoring from a checkpoint of a system that 1611 // wasn't drained properly. Flag the checkpoint as invalid if the 1612 // cache contains dirty data. 1613 bool bad_checkpoint(dirty); 1614 SERIALIZE_SCALAR(bad_checkpoint); 1615} 1616 1617void 1618BaseCache::unserialize(CheckpointIn &cp) 1619{ 1620 bool bad_checkpoint; 1621 UNSERIALIZE_SCALAR(bad_checkpoint); 1622 if (bad_checkpoint) { 1623 fatal("Restoring from checkpoints with dirty caches is not " 1624 "supported in the classic memory system. Please remove any " 1625 "caches or drain them properly before taking checkpoints.\n"); 1626 } 1627} 1628 1629void 1630BaseCache::regStats() 1631{ 1632 MemObject::regStats(); 1633 1634 using namespace Stats; 1635 1636 // Hit statistics 1637 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1638 MemCmd cmd(access_idx); 1639 const string &cstr = cmd.toString(); 1640 1641 hits[access_idx] 1642 .init(system->maxMasters()) 1643 .name(name() + "." + cstr + "_hits") 1644 .desc("number of " + cstr + " hits") 1645 .flags(total | nozero | nonan) 1646 ; 1647 for (int i = 0; i < system->maxMasters(); i++) { 1648 hits[access_idx].subname(i, system->getMasterName(i)); 1649 } 1650 } 1651 1652// These macros make it easier to sum the right subset of commands and 1653// to change the subset of commands that are considered "demand" vs 1654// "non-demand" 1655#define SUM_DEMAND(s) \ 1656 (s[MemCmd::ReadReq] + s[MemCmd::WriteReq] + s[MemCmd::WriteLineReq] + \ 1657 s[MemCmd::ReadExReq] + s[MemCmd::ReadCleanReq] + s[MemCmd::ReadSharedReq]) 1658 1659// should writebacks be included here? prior code was inconsistent... 1660#define SUM_NON_DEMAND(s) \ 1661 (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq] + s[MemCmd::SoftPFExReq]) 1662 1663 demandHits 1664 .name(name() + ".demand_hits") 1665 .desc("number of demand (read+write) hits") 1666 .flags(total | nozero | nonan) 1667 ; 1668 demandHits = SUM_DEMAND(hits); 1669 for (int i = 0; i < system->maxMasters(); i++) { 1670 demandHits.subname(i, system->getMasterName(i)); 1671 } 1672 1673 overallHits 1674 .name(name() + ".overall_hits") 1675 .desc("number of overall hits") 1676 .flags(total | nozero | nonan) 1677 ; 1678 overallHits = demandHits + SUM_NON_DEMAND(hits); 1679 for (int i = 0; i < system->maxMasters(); i++) { 1680 overallHits.subname(i, system->getMasterName(i)); 1681 } 1682 1683 // Miss statistics 1684 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1685 MemCmd cmd(access_idx); 1686 const string &cstr = cmd.toString(); 1687 1688 misses[access_idx] 1689 .init(system->maxMasters()) 1690 .name(name() + "." + cstr + "_misses") 1691 .desc("number of " + cstr + " misses") 1692 .flags(total | nozero | nonan) 1693 ; 1694 for (int i = 0; i < system->maxMasters(); i++) { 1695 misses[access_idx].subname(i, system->getMasterName(i)); 1696 } 1697 } 1698 1699 demandMisses 1700 .name(name() + ".demand_misses") 1701 .desc("number of demand (read+write) misses") 1702 .flags(total | nozero | nonan) 1703 ; 1704 demandMisses = SUM_DEMAND(misses); 1705 for (int i = 0; i < system->maxMasters(); i++) { 1706 demandMisses.subname(i, system->getMasterName(i)); 1707 } 1708 1709 overallMisses 1710 .name(name() + ".overall_misses") 1711 .desc("number of overall misses") 1712 .flags(total | nozero | nonan) 1713 ; 1714 overallMisses = demandMisses + SUM_NON_DEMAND(misses); 1715 for (int i = 0; i < system->maxMasters(); i++) { 1716 overallMisses.subname(i, system->getMasterName(i)); 1717 } 1718 1719 // Miss latency statistics 1720 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1721 MemCmd cmd(access_idx); 1722 const string &cstr = cmd.toString(); 1723 1724 missLatency[access_idx] 1725 .init(system->maxMasters()) 1726 .name(name() + "." + cstr + "_miss_latency") 1727 .desc("number of " + cstr + " miss cycles") 1728 .flags(total | nozero | nonan) 1729 ; 1730 for (int i = 0; i < system->maxMasters(); i++) { 1731 missLatency[access_idx].subname(i, system->getMasterName(i)); 1732 } 1733 } 1734 1735 demandMissLatency 1736 .name(name() + ".demand_miss_latency") 1737 .desc("number of demand (read+write) miss cycles") 1738 .flags(total | nozero | nonan) 1739 ; 1740 demandMissLatency = SUM_DEMAND(missLatency); 1741 for (int i = 0; i < system->maxMasters(); i++) { 1742 demandMissLatency.subname(i, system->getMasterName(i)); 1743 } 1744 1745 overallMissLatency 1746 .name(name() + ".overall_miss_latency") 1747 .desc("number of overall miss cycles") 1748 .flags(total | nozero | nonan) 1749 ; 1750 overallMissLatency = demandMissLatency + SUM_NON_DEMAND(missLatency); 1751 for (int i = 0; i < system->maxMasters(); i++) { 1752 overallMissLatency.subname(i, system->getMasterName(i)); 1753 } 1754 1755 // access formulas 1756 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1757 MemCmd cmd(access_idx); 1758 const string &cstr = cmd.toString(); 1759 1760 accesses[access_idx] 1761 .name(name() + "." + cstr + "_accesses") 1762 .desc("number of " + cstr + " accesses(hits+misses)") 1763 .flags(total | nozero | nonan) 1764 ; 1765 accesses[access_idx] = hits[access_idx] + misses[access_idx]; 1766 1767 for (int i = 0; i < system->maxMasters(); i++) { 1768 accesses[access_idx].subname(i, system->getMasterName(i)); 1769 } 1770 } 1771 1772 demandAccesses 1773 .name(name() + ".demand_accesses") 1774 .desc("number of demand (read+write) accesses") 1775 .flags(total | nozero | nonan) 1776 ; 1777 demandAccesses = demandHits + demandMisses; 1778 for (int i = 0; i < system->maxMasters(); i++) { 1779 demandAccesses.subname(i, system->getMasterName(i)); 1780 } 1781 1782 overallAccesses 1783 .name(name() + ".overall_accesses") 1784 .desc("number of overall (read+write) accesses") 1785 .flags(total | nozero | nonan) 1786 ; 1787 overallAccesses = overallHits + overallMisses; 1788 for (int i = 0; i < system->maxMasters(); i++) { 1789 overallAccesses.subname(i, system->getMasterName(i)); 1790 } 1791 1792 // miss rate formulas 1793 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1794 MemCmd cmd(access_idx); 1795 const string &cstr = cmd.toString(); 1796 1797 missRate[access_idx] 1798 .name(name() + "." + cstr + "_miss_rate") 1799 .desc("miss rate for " + cstr + " accesses") 1800 .flags(total | nozero | nonan) 1801 ; 1802 missRate[access_idx] = misses[access_idx] / accesses[access_idx]; 1803 1804 for (int i = 0; i < system->maxMasters(); i++) { 1805 missRate[access_idx].subname(i, system->getMasterName(i)); 1806 } 1807 } 1808 1809 demandMissRate 1810 .name(name() + ".demand_miss_rate") 1811 .desc("miss rate for demand accesses") 1812 .flags(total | nozero | nonan) 1813 ; 1814 demandMissRate = demandMisses / demandAccesses; 1815 for (int i = 0; i < system->maxMasters(); i++) { 1816 demandMissRate.subname(i, system->getMasterName(i)); 1817 } 1818 1819 overallMissRate 1820 .name(name() + ".overall_miss_rate") 1821 .desc("miss rate for overall accesses") 1822 .flags(total | nozero | nonan) 1823 ; 1824 overallMissRate = overallMisses / overallAccesses; 1825 for (int i = 0; i < system->maxMasters(); i++) { 1826 overallMissRate.subname(i, system->getMasterName(i)); 1827 } 1828 1829 // miss latency formulas 1830 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1831 MemCmd cmd(access_idx); 1832 const string &cstr = cmd.toString(); 1833 1834 avgMissLatency[access_idx] 1835 .name(name() + "." + cstr + "_avg_miss_latency") 1836 .desc("average " + cstr + " miss latency") 1837 .flags(total | nozero | nonan) 1838 ; 1839 avgMissLatency[access_idx] = 1840 missLatency[access_idx] / misses[access_idx]; 1841 1842 for (int i = 0; i < system->maxMasters(); i++) { 1843 avgMissLatency[access_idx].subname(i, system->getMasterName(i)); 1844 } 1845 } 1846 1847 demandAvgMissLatency 1848 .name(name() + ".demand_avg_miss_latency") 1849 .desc("average overall miss latency") 1850 .flags(total | nozero | nonan) 1851 ; 1852 demandAvgMissLatency = demandMissLatency / demandMisses; 1853 for (int i = 0; i < system->maxMasters(); i++) { 1854 demandAvgMissLatency.subname(i, system->getMasterName(i)); 1855 } 1856 1857 overallAvgMissLatency 1858 .name(name() + ".overall_avg_miss_latency") 1859 .desc("average overall miss latency") 1860 .flags(total | nozero | nonan) 1861 ; 1862 overallAvgMissLatency = overallMissLatency / overallMisses; 1863 for (int i = 0; i < system->maxMasters(); i++) { 1864 overallAvgMissLatency.subname(i, system->getMasterName(i)); 1865 } 1866 1867 blocked_cycles.init(NUM_BLOCKED_CAUSES); 1868 blocked_cycles 1869 .name(name() + ".blocked_cycles") 1870 .desc("number of cycles access was blocked") 1871 .subname(Blocked_NoMSHRs, "no_mshrs") 1872 .subname(Blocked_NoTargets, "no_targets") 1873 ; 1874 1875 1876 blocked_causes.init(NUM_BLOCKED_CAUSES); 1877 blocked_causes 1878 .name(name() + ".blocked") 1879 .desc("number of cycles access was blocked") 1880 .subname(Blocked_NoMSHRs, "no_mshrs") 1881 .subname(Blocked_NoTargets, "no_targets") 1882 ; 1883 1884 avg_blocked 1885 .name(name() + ".avg_blocked_cycles") 1886 .desc("average number of cycles each access was blocked") 1887 .subname(Blocked_NoMSHRs, "no_mshrs") 1888 .subname(Blocked_NoTargets, "no_targets") 1889 ; 1890 1891 avg_blocked = blocked_cycles / blocked_causes; 1892 1893 unusedPrefetches 1894 .name(name() + ".unused_prefetches") 1895 .desc("number of HardPF blocks evicted w/o reference") 1896 .flags(nozero) 1897 ; 1898 1899 writebacks 1900 .init(system->maxMasters()) 1901 .name(name() + ".writebacks") 1902 .desc("number of writebacks") 1903 .flags(total | nozero | nonan) 1904 ; 1905 for (int i = 0; i < system->maxMasters(); i++) { 1906 writebacks.subname(i, system->getMasterName(i)); 1907 } 1908 1909 // MSHR statistics 1910 // MSHR hit statistics 1911 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1912 MemCmd cmd(access_idx); 1913 const string &cstr = cmd.toString(); 1914 1915 mshr_hits[access_idx] 1916 .init(system->maxMasters()) 1917 .name(name() + "." + cstr + "_mshr_hits") 1918 .desc("number of " + cstr + " MSHR hits") 1919 .flags(total | nozero | nonan) 1920 ; 1921 for (int i = 0; i < system->maxMasters(); i++) { 1922 mshr_hits[access_idx].subname(i, system->getMasterName(i)); 1923 } 1924 } 1925 1926 demandMshrHits 1927 .name(name() + ".demand_mshr_hits") 1928 .desc("number of demand (read+write) MSHR hits") 1929 .flags(total | nozero | nonan) 1930 ; 1931 demandMshrHits = SUM_DEMAND(mshr_hits); 1932 for (int i = 0; i < system->maxMasters(); i++) { 1933 demandMshrHits.subname(i, system->getMasterName(i)); 1934 } 1935 1936 overallMshrHits 1937 .name(name() + ".overall_mshr_hits") 1938 .desc("number of overall MSHR hits") 1939 .flags(total | nozero | nonan) 1940 ; 1941 overallMshrHits = demandMshrHits + SUM_NON_DEMAND(mshr_hits); 1942 for (int i = 0; i < system->maxMasters(); i++) { 1943 overallMshrHits.subname(i, system->getMasterName(i)); 1944 } 1945 1946 // MSHR miss statistics 1947 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1948 MemCmd cmd(access_idx); 1949 const string &cstr = cmd.toString(); 1950 1951 mshr_misses[access_idx] 1952 .init(system->maxMasters()) 1953 .name(name() + "." + cstr + "_mshr_misses") 1954 .desc("number of " + cstr + " MSHR misses") 1955 .flags(total | nozero | nonan) 1956 ; 1957 for (int i = 0; i < system->maxMasters(); i++) { 1958 mshr_misses[access_idx].subname(i, system->getMasterName(i)); 1959 } 1960 } 1961 1962 demandMshrMisses 1963 .name(name() + ".demand_mshr_misses") 1964 .desc("number of demand (read+write) MSHR misses") 1965 .flags(total | nozero | nonan) 1966 ; 1967 demandMshrMisses = SUM_DEMAND(mshr_misses); 1968 for (int i = 0; i < system->maxMasters(); i++) { 1969 demandMshrMisses.subname(i, system->getMasterName(i)); 1970 } 1971 1972 overallMshrMisses 1973 .name(name() + ".overall_mshr_misses") 1974 .desc("number of overall MSHR misses") 1975 .flags(total | nozero | nonan) 1976 ; 1977 overallMshrMisses = demandMshrMisses + SUM_NON_DEMAND(mshr_misses); 1978 for (int i = 0; i < system->maxMasters(); i++) { 1979 overallMshrMisses.subname(i, system->getMasterName(i)); 1980 } 1981 1982 // MSHR miss latency statistics 1983 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 1984 MemCmd cmd(access_idx); 1985 const string &cstr = cmd.toString(); 1986 1987 mshr_miss_latency[access_idx] 1988 .init(system->maxMasters()) 1989 .name(name() + "." + cstr + "_mshr_miss_latency") 1990 .desc("number of " + cstr + " MSHR miss cycles") 1991 .flags(total | nozero | nonan) 1992 ; 1993 for (int i = 0; i < system->maxMasters(); i++) { 1994 mshr_miss_latency[access_idx].subname(i, system->getMasterName(i)); 1995 } 1996 } 1997 1998 demandMshrMissLatency 1999 .name(name() + ".demand_mshr_miss_latency") 2000 .desc("number of demand (read+write) MSHR miss cycles") 2001 .flags(total | nozero | nonan) 2002 ; 2003 demandMshrMissLatency = SUM_DEMAND(mshr_miss_latency); 2004 for (int i = 0; i < system->maxMasters(); i++) { 2005 demandMshrMissLatency.subname(i, system->getMasterName(i)); 2006 } 2007 2008 overallMshrMissLatency 2009 .name(name() + ".overall_mshr_miss_latency") 2010 .desc("number of overall MSHR miss cycles") 2011 .flags(total | nozero | nonan) 2012 ; 2013 overallMshrMissLatency = 2014 demandMshrMissLatency + SUM_NON_DEMAND(mshr_miss_latency); 2015 for (int i = 0; i < system->maxMasters(); i++) { 2016 overallMshrMissLatency.subname(i, system->getMasterName(i)); 2017 } 2018 2019 // MSHR uncacheable statistics 2020 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2021 MemCmd cmd(access_idx); 2022 const string &cstr = cmd.toString(); 2023 2024 mshr_uncacheable[access_idx] 2025 .init(system->maxMasters()) 2026 .name(name() + "." + cstr + "_mshr_uncacheable") 2027 .desc("number of " + cstr + " MSHR uncacheable") 2028 .flags(total | nozero | nonan) 2029 ; 2030 for (int i = 0; i < system->maxMasters(); i++) { 2031 mshr_uncacheable[access_idx].subname(i, system->getMasterName(i)); 2032 } 2033 } 2034 2035 overallMshrUncacheable 2036 .name(name() + ".overall_mshr_uncacheable_misses") 2037 .desc("number of overall MSHR uncacheable misses") 2038 .flags(total | nozero | nonan) 2039 ; 2040 overallMshrUncacheable = 2041 SUM_DEMAND(mshr_uncacheable) + SUM_NON_DEMAND(mshr_uncacheable); 2042 for (int i = 0; i < system->maxMasters(); i++) { 2043 overallMshrUncacheable.subname(i, system->getMasterName(i)); 2044 } 2045 2046 // MSHR miss latency statistics 2047 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2048 MemCmd cmd(access_idx); 2049 const string &cstr = cmd.toString(); 2050 2051 mshr_uncacheable_lat[access_idx] 2052 .init(system->maxMasters()) 2053 .name(name() + "." + cstr + "_mshr_uncacheable_latency") 2054 .desc("number of " + cstr + " MSHR uncacheable cycles") 2055 .flags(total | nozero | nonan) 2056 ; 2057 for (int i = 0; i < system->maxMasters(); i++) { 2058 mshr_uncacheable_lat[access_idx].subname( 2059 i, system->getMasterName(i)); 2060 } 2061 } 2062 2063 overallMshrUncacheableLatency 2064 .name(name() + ".overall_mshr_uncacheable_latency") 2065 .desc("number of overall MSHR uncacheable cycles") 2066 .flags(total | nozero | nonan) 2067 ; 2068 overallMshrUncacheableLatency = 2069 SUM_DEMAND(mshr_uncacheable_lat) + 2070 SUM_NON_DEMAND(mshr_uncacheable_lat); 2071 for (int i = 0; i < system->maxMasters(); i++) { 2072 overallMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2073 } 2074 2075#if 0 2076 // MSHR access formulas 2077 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2078 MemCmd cmd(access_idx); 2079 const string &cstr = cmd.toString(); 2080 2081 mshrAccesses[access_idx] 2082 .name(name() + "." + cstr + "_mshr_accesses") 2083 .desc("number of " + cstr + " mshr accesses(hits+misses)") 2084 .flags(total | nozero | nonan) 2085 ; 2086 mshrAccesses[access_idx] = 2087 mshr_hits[access_idx] + mshr_misses[access_idx] 2088 + mshr_uncacheable[access_idx]; 2089 } 2090 2091 demandMshrAccesses 2092 .name(name() + ".demand_mshr_accesses") 2093 .desc("number of demand (read+write) mshr accesses") 2094 .flags(total | nozero | nonan) 2095 ; 2096 demandMshrAccesses = demandMshrHits + demandMshrMisses; 2097 2098 overallMshrAccesses 2099 .name(name() + ".overall_mshr_accesses") 2100 .desc("number of overall (read+write) mshr accesses") 2101 .flags(total | nozero | nonan) 2102 ; 2103 overallMshrAccesses = overallMshrHits + overallMshrMisses 2104 + overallMshrUncacheable; 2105#endif 2106 2107 // MSHR miss rate formulas 2108 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2109 MemCmd cmd(access_idx); 2110 const string &cstr = cmd.toString(); 2111 2112 mshrMissRate[access_idx] 2113 .name(name() + "." + cstr + "_mshr_miss_rate") 2114 .desc("mshr miss rate for " + cstr + " accesses") 2115 .flags(total | nozero | nonan) 2116 ; 2117 mshrMissRate[access_idx] = 2118 mshr_misses[access_idx] / accesses[access_idx]; 2119 2120 for (int i = 0; i < system->maxMasters(); i++) { 2121 mshrMissRate[access_idx].subname(i, system->getMasterName(i)); 2122 } 2123 } 2124 2125 demandMshrMissRate 2126 .name(name() + ".demand_mshr_miss_rate") 2127 .desc("mshr miss rate for demand accesses") 2128 .flags(total | nozero | nonan) 2129 ; 2130 demandMshrMissRate = demandMshrMisses / demandAccesses; 2131 for (int i = 0; i < system->maxMasters(); i++) { 2132 demandMshrMissRate.subname(i, system->getMasterName(i)); 2133 } 2134 2135 overallMshrMissRate 2136 .name(name() + ".overall_mshr_miss_rate") 2137 .desc("mshr miss rate for overall accesses") 2138 .flags(total | nozero | nonan) 2139 ; 2140 overallMshrMissRate = overallMshrMisses / overallAccesses; 2141 for (int i = 0; i < system->maxMasters(); i++) { 2142 overallMshrMissRate.subname(i, system->getMasterName(i)); 2143 } 2144 2145 // mshrMiss latency formulas 2146 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2147 MemCmd cmd(access_idx); 2148 const string &cstr = cmd.toString(); 2149 2150 avgMshrMissLatency[access_idx] 2151 .name(name() + "." + cstr + "_avg_mshr_miss_latency") 2152 .desc("average " + cstr + " mshr miss latency") 2153 .flags(total | nozero | nonan) 2154 ; 2155 avgMshrMissLatency[access_idx] = 2156 mshr_miss_latency[access_idx] / mshr_misses[access_idx]; 2157 2158 for (int i = 0; i < system->maxMasters(); i++) { 2159 avgMshrMissLatency[access_idx].subname( 2160 i, system->getMasterName(i)); 2161 } 2162 } 2163 2164 demandAvgMshrMissLatency 2165 .name(name() + ".demand_avg_mshr_miss_latency") 2166 .desc("average overall mshr miss latency") 2167 .flags(total | nozero | nonan) 2168 ; 2169 demandAvgMshrMissLatency = demandMshrMissLatency / demandMshrMisses; 2170 for (int i = 0; i < system->maxMasters(); i++) { 2171 demandAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2172 } 2173 2174 overallAvgMshrMissLatency 2175 .name(name() + ".overall_avg_mshr_miss_latency") 2176 .desc("average overall mshr miss latency") 2177 .flags(total | nozero | nonan) 2178 ; 2179 overallAvgMshrMissLatency = overallMshrMissLatency / overallMshrMisses; 2180 for (int i = 0; i < system->maxMasters(); i++) { 2181 overallAvgMshrMissLatency.subname(i, system->getMasterName(i)); 2182 } 2183 2184 // mshrUncacheable latency formulas 2185 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) { 2186 MemCmd cmd(access_idx); 2187 const string &cstr = cmd.toString(); 2188 2189 avgMshrUncacheableLatency[access_idx] 2190 .name(name() + "." + cstr + "_avg_mshr_uncacheable_latency") 2191 .desc("average " + cstr + " mshr uncacheable latency") 2192 .flags(total | nozero | nonan) 2193 ; 2194 avgMshrUncacheableLatency[access_idx] = 2195 mshr_uncacheable_lat[access_idx] / mshr_uncacheable[access_idx]; 2196 2197 for (int i = 0; i < system->maxMasters(); i++) { 2198 avgMshrUncacheableLatency[access_idx].subname( 2199 i, system->getMasterName(i)); 2200 } 2201 } 2202 2203 overallAvgMshrUncacheableLatency 2204 .name(name() + ".overall_avg_mshr_uncacheable_latency") 2205 .desc("average overall mshr uncacheable latency") 2206 .flags(total | nozero | nonan) 2207 ; 2208 overallAvgMshrUncacheableLatency = 2209 overallMshrUncacheableLatency / overallMshrUncacheable; 2210 for (int i = 0; i < system->maxMasters(); i++) { 2211 overallAvgMshrUncacheableLatency.subname(i, system->getMasterName(i)); 2212 } 2213 2214 replacements 2215 .name(name() + ".replacements") 2216 .desc("number of replacements") 2217 ; 2218} 2219 2220void 2221BaseCache::regProbePoints() 2222{ 2223 ppHit = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Hit"); 2224 ppMiss = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Miss"); 2225} 2226 2227/////////////// 2228// 2229// CpuSidePort 2230// 2231/////////////// 2232bool 2233BaseCache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt) 2234{ 2235 // Snoops shouldn't happen when bypassing caches 2236 assert(!cache->system->bypassCaches()); 2237 2238 assert(pkt->isResponse()); 2239 2240 // Express snoop responses from master to slave, e.g., from L1 to L2 2241 cache->recvTimingSnoopResp(pkt); 2242 return true; 2243} 2244 2245 2246bool 2247BaseCache::CpuSidePort::tryTiming(PacketPtr pkt) 2248{ 2249 if (cache->system->bypassCaches() || pkt->isExpressSnoop()) { 2250 // always let express snoop packets through even if blocked 2251 return true; 2252 } else if (blocked || mustSendRetry) { 2253 // either already committed to send a retry, or blocked 2254 mustSendRetry = true; 2255 return false; 2256 } 2257 mustSendRetry = false; 2258 return true; 2259} 2260 2261bool 2262BaseCache::CpuSidePort::recvTimingReq(PacketPtr pkt) 2263{ 2264 assert(pkt->isRequest()); 2265 2266 if (cache->system->bypassCaches()) { 2267 // Just forward the packet if caches are disabled. 2268 // @todo This should really enqueue the packet rather 2269 bool M5_VAR_USED success = cache->memSidePort.sendTimingReq(pkt); 2270 assert(success); 2271 return true; 2272 } else if (tryTiming(pkt)) { 2273 cache->recvTimingReq(pkt); 2274 return true; 2275 } 2276 return false; 2277} 2278 2279Tick 2280BaseCache::CpuSidePort::recvAtomic(PacketPtr pkt) 2281{ 2282 if (cache->system->bypassCaches()) { 2283 // Forward the request if the system is in cache bypass mode. 2284 return cache->memSidePort.sendAtomic(pkt); 2285 } else { 2286 return cache->recvAtomic(pkt); 2287 } 2288} 2289 2290void 2291BaseCache::CpuSidePort::recvFunctional(PacketPtr pkt) 2292{ 2293 if (cache->system->bypassCaches()) { 2294 // The cache should be flushed if we are in cache bypass mode, 2295 // so we don't need to check if we need to update anything. 2296 cache->memSidePort.sendFunctional(pkt); 2297 return; 2298 } 2299 2300 // functional request 2301 cache->functionalAccess(pkt, true); 2302} 2303 2304AddrRangeList 2305BaseCache::CpuSidePort::getAddrRanges() const 2306{ 2307 return cache->getAddrRanges(); 2308} 2309 2310 2311BaseCache:: 2312CpuSidePort::CpuSidePort(const std::string &_name, BaseCache *_cache, 2313 const std::string &_label) 2314 : CacheSlavePort(_name, _cache, _label), cache(_cache) 2315{ 2316} 2317 2318/////////////// 2319// 2320// MemSidePort 2321// 2322/////////////// 2323bool 2324BaseCache::MemSidePort::recvTimingResp(PacketPtr pkt) 2325{ 2326 cache->recvTimingResp(pkt); 2327 return true; 2328} 2329 2330// Express snooping requests to memside port 2331void 2332BaseCache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt) 2333{ 2334 // Snoops shouldn't happen when bypassing caches 2335 assert(!cache->system->bypassCaches()); 2336 2337 // handle snooping requests 2338 cache->recvTimingSnoopReq(pkt); 2339} 2340 2341Tick 2342BaseCache::MemSidePort::recvAtomicSnoop(PacketPtr pkt) 2343{ 2344 // Snoops shouldn't happen when bypassing caches 2345 assert(!cache->system->bypassCaches()); 2346 2347 return cache->recvAtomicSnoop(pkt); 2348} 2349 2350void 2351BaseCache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt) 2352{ 2353 // Snoops shouldn't happen when bypassing caches 2354 assert(!cache->system->bypassCaches()); 2355 2356 // functional snoop (note that in contrast to atomic we don't have 2357 // a specific functionalSnoop method, as they have the same 2358 // behaviour regardless) 2359 cache->functionalAccess(pkt, false); 2360} 2361 2362void 2363BaseCache::CacheReqPacketQueue::sendDeferredPacket() 2364{ 2365 // sanity check 2366 assert(!waitingOnRetry); 2367 2368 // there should never be any deferred request packets in the 2369 // queue, instead we resly on the cache to provide the packets 2370 // from the MSHR queue or write queue 2371 assert(deferredPacketReadyTime() == MaxTick); 2372 2373 // check for request packets (requests & writebacks) 2374 QueueEntry* entry = cache.getNextQueueEntry(); 2375 2376 if (!entry) { 2377 // can happen if e.g. we attempt a writeback and fail, but 2378 // before the retry, the writeback is eliminated because 2379 // we snoop another cache's ReadEx. 2380 } else { 2381 // let our snoop responses go first if there are responses to 2382 // the same addresses 2383 if (checkConflictingSnoop(entry->blkAddr)) { 2384 return; 2385 } 2386 waitingOnRetry = entry->sendPacket(cache); 2387 } 2388 2389 // if we succeeded and are not waiting for a retry, schedule the 2390 // next send considering when the next queue is ready, note that 2391 // snoop responses have their own packet queue and thus schedule 2392 // their own events 2393 if (!waitingOnRetry) { 2394 schedSendEvent(cache.nextQueueReadyTime()); 2395 } 2396} 2397 2398BaseCache::MemSidePort::MemSidePort(const std::string &_name, 2399 BaseCache *_cache, 2400 const std::string &_label) 2401 : CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2402 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2403 _snoopRespQueue(*_cache, *this, _label), cache(_cache) 2404{ 2405} 2406 2407void 2408WriteAllocator::updateMode(Addr write_addr, unsigned write_size, 2409 Addr blk_addr) 2410{ 2411 // check if we are continuing where the last write ended 2412 if (nextAddr == write_addr) { 2413 delayCtr[blk_addr] = delayThreshold; 2414 // stop if we have already saturated 2415 if (mode != WriteMode::NO_ALLOCATE) { 2416 byteCount += write_size; 2417 // switch to streaming mode if we have passed the lower 2418 // threshold 2419 if (mode == WriteMode::ALLOCATE && 2420 byteCount > coalesceLimit) { 2421 mode = WriteMode::COALESCE; 2422 DPRINTF(Cache, "Switched to write coalescing\n"); 2423 } else if (mode == WriteMode::COALESCE && 2424 byteCount > noAllocateLimit) { 2425 // and continue and switch to non-allocating mode if we 2426 // pass the upper threshold 2427 mode = WriteMode::NO_ALLOCATE; 2428 DPRINTF(Cache, "Switched to write-no-allocate\n"); 2429 } 2430 } 2431 } else { 2432 // we did not see a write matching the previous one, start 2433 // over again 2434 byteCount = write_size; 2435 mode = WriteMode::ALLOCATE; 2436 resetDelay(blk_addr); 2437 } 2438 nextAddr = write_addr + write_size; 2439} 2440 2441WriteAllocator* 2442WriteAllocatorParams::create() 2443{ 2444 return new WriteAllocator(this); 2445} 2446