mshr.cc revision 13349:20890038e8a0
112841Sgabeblack@google.com/* 212841Sgabeblack@google.com * Copyright (c) 2012-2013, 2015-2018 ARM Limited 312841Sgabeblack@google.com * All rights reserved. 412841Sgabeblack@google.com * 512841Sgabeblack@google.com * The license below extends only to copyright in the software and shall 612841Sgabeblack@google.com * not be construed as granting a license to any other intellectual 712841Sgabeblack@google.com * property including but not limited to intellectual property relating 812841Sgabeblack@google.com * to a hardware implementation of the functionality of the software 912841Sgabeblack@google.com * licensed hereunder. You may use the software subject to the license 1012841Sgabeblack@google.com * terms below provided that you ensure that this notice is replicated 1112841Sgabeblack@google.com * unmodified and in its entirety in all distributions of the software, 1212841Sgabeblack@google.com * modified or unmodified, in source code or in binary form. 1312841Sgabeblack@google.com * 1412841Sgabeblack@google.com * Copyright (c) 2002-2005 The Regents of The University of Michigan 1512841Sgabeblack@google.com * Copyright (c) 2010 Advanced Micro Devices, Inc. 1612841Sgabeblack@google.com * All rights reserved. 1712841Sgabeblack@google.com * 1812841Sgabeblack@google.com * Redistribution and use in source and binary forms, with or without 1912841Sgabeblack@google.com * modification, are permitted provided that the following conditions are 2012841Sgabeblack@google.com * met: redistributions of source code must retain the above copyright 2112841Sgabeblack@google.com * notice, this list of conditions and the following disclaimer; 2212841Sgabeblack@google.com * redistributions in binary form must reproduce the above copyright 2312841Sgabeblack@google.com * notice, this list of conditions and the following disclaimer in the 2412841Sgabeblack@google.com * documentation and/or other materials provided with the distribution; 2512841Sgabeblack@google.com * neither the name of the copyright holders nor the names of its 2612841Sgabeblack@google.com * contributors may be used to endorse or promote products derived from 2712841Sgabeblack@google.com * this software without specific prior written permission. 2812841Sgabeblack@google.com * 2912841Sgabeblack@google.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 3012841Sgabeblack@google.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3112841Sgabeblack@google.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3212841Sgabeblack@google.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3312841Sgabeblack@google.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3412841Sgabeblack@google.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3512841Sgabeblack@google.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3612841Sgabeblack@google.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3712841Sgabeblack@google.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3812841Sgabeblack@google.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3912841Sgabeblack@google.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 4012841Sgabeblack@google.com * 4112841Sgabeblack@google.com * Authors: Erik Hallnor 4212841Sgabeblack@google.com * Dave Greene 4312841Sgabeblack@google.com * Nikos Nikoleris 4412841Sgabeblack@google.com */ 4512841Sgabeblack@google.com 4612841Sgabeblack@google.com/** 4712841Sgabeblack@google.com * @file 4812841Sgabeblack@google.com * Miss Status and Handling Register (MSHR) definitions. 4912841Sgabeblack@google.com */ 5012841Sgabeblack@google.com 5112841Sgabeblack@google.com#include "mem/cache/mshr.hh" 5212841Sgabeblack@google.com 5312841Sgabeblack@google.com#include <cassert> 5412841Sgabeblack@google.com#include <string> 5512841Sgabeblack@google.com 5612912Sgabeblack@google.com#include "base/logging.hh" 5712912Sgabeblack@google.com#include "base/trace.hh" 5812912Sgabeblack@google.com#include "base/types.hh" 5912912Sgabeblack@google.com#include "debug/Cache.hh" 6012912Sgabeblack@google.com#include "mem/cache/base.hh" 6112912Sgabeblack@google.com#include "mem/request.hh" 6212841Sgabeblack@google.com#include "sim/core.hh" 6312841Sgabeblack@google.com 6412841Sgabeblack@google.comMSHR::MSHR() : downstreamPending(false), 6512841Sgabeblack@google.com pendingModified(false), 6612841Sgabeblack@google.com postInvalidate(false), postDowngrade(false), 6712841Sgabeblack@google.com wasWholeLineWrite(false), isForward(false) 6812841Sgabeblack@google.com{ 6912841Sgabeblack@google.com} 7012841Sgabeblack@google.com 7112841Sgabeblack@google.comMSHR::TargetList::TargetList() 7212841Sgabeblack@google.com : needsWritable(false), hasUpgrade(false), allocOnFill(false), 7312841Sgabeblack@google.com hasFromCache(false) 7412841Sgabeblack@google.com{} 7512841Sgabeblack@google.com 7612841Sgabeblack@google.com 7712841Sgabeblack@google.comvoid 7812841Sgabeblack@google.comMSHR::TargetList::updateFlags(PacketPtr pkt, Target::Source source, 7912841Sgabeblack@google.com bool alloc_on_fill) 8012841Sgabeblack@google.com{ 8112841Sgabeblack@google.com if (source != Target::FromSnoop) { 8212841Sgabeblack@google.com if (pkt->needsWritable()) { 8312841Sgabeblack@google.com needsWritable = true; 8412841Sgabeblack@google.com } 8512841Sgabeblack@google.com 8612841Sgabeblack@google.com // StoreCondReq is effectively an upgrade if it's in an MSHR 8712841Sgabeblack@google.com // since it would have been failed already if we didn't have a 8812841Sgabeblack@google.com // read-only copy 8912841Sgabeblack@google.com if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 9012841Sgabeblack@google.com hasUpgrade = true; 9112841Sgabeblack@google.com } 9212841Sgabeblack@google.com 9312841Sgabeblack@google.com // potentially re-evaluate whether we should allocate on a fill or 9412841Sgabeblack@google.com // not 9512841Sgabeblack@google.com allocOnFill = allocOnFill || alloc_on_fill; 9612841Sgabeblack@google.com 9712841Sgabeblack@google.com if (source != Target::FromPrefetcher) { 9812841Sgabeblack@google.com hasFromCache = hasFromCache || pkt->fromCache(); 9912841Sgabeblack@google.com 10012841Sgabeblack@google.com updateWriteFlags(pkt); 10112841Sgabeblack@google.com } 10212841Sgabeblack@google.com } 10312841Sgabeblack@google.com} 10412841Sgabeblack@google.com 10512841Sgabeblack@google.comvoid 10612841Sgabeblack@google.comMSHR::TargetList::populateFlags() 10712841Sgabeblack@google.com{ 10812841Sgabeblack@google.com resetFlags(); 10912841Sgabeblack@google.com for (auto& t: *this) { 11012841Sgabeblack@google.com updateFlags(t.pkt, t.source, t.allocOnFill); 11112841Sgabeblack@google.com } 11212841Sgabeblack@google.com} 11312841Sgabeblack@google.com 11412841Sgabeblack@google.cominline void 11512841Sgabeblack@google.comMSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 11612841Sgabeblack@google.com Counter order, Target::Source source, bool markPending, 11712841Sgabeblack@google.com bool alloc_on_fill) 11812841Sgabeblack@google.com{ 11912841Sgabeblack@google.com updateFlags(pkt, source, alloc_on_fill); 12012841Sgabeblack@google.com if (markPending) { 12112841Sgabeblack@google.com // Iterate over the SenderState stack and see if we find 12212841Sgabeblack@google.com // an MSHR entry. If we do, set the downstreamPending 12312841Sgabeblack@google.com // flag. Otherwise, do nothing. 12412841Sgabeblack@google.com MSHR *mshr = pkt->findNextSenderState<MSHR>(); 12512841Sgabeblack@google.com if (mshr != nullptr) { 12612841Sgabeblack@google.com assert(!mshr->downstreamPending); 12712841Sgabeblack@google.com mshr->downstreamPending = true; 12812841Sgabeblack@google.com } else { 12912841Sgabeblack@google.com // No need to clear downstreamPending later 13012841Sgabeblack@google.com markPending = false; 13112841Sgabeblack@google.com } 13212841Sgabeblack@google.com } 13312841Sgabeblack@google.com 13412841Sgabeblack@google.com emplace_back(pkt, readyTime, order, source, markPending, alloc_on_fill); 13512841Sgabeblack@google.com} 13612841Sgabeblack@google.com 13712841Sgabeblack@google.com 13812841Sgabeblack@google.comstatic void 13912841Sgabeblack@google.comreplaceUpgrade(PacketPtr pkt) 14012841Sgabeblack@google.com{ 14112841Sgabeblack@google.com // remember if the current packet has data allocated 14212841Sgabeblack@google.com bool has_data = pkt->hasData() || pkt->hasRespData(); 14312841Sgabeblack@google.com 14412841Sgabeblack@google.com if (pkt->cmd == MemCmd::UpgradeReq) { 14512841Sgabeblack@google.com pkt->cmd = MemCmd::ReadExReq; 14612841Sgabeblack@google.com DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); 14712841Sgabeblack@google.com } else if (pkt->cmd == MemCmd::SCUpgradeReq) { 14812841Sgabeblack@google.com pkt->cmd = MemCmd::SCUpgradeFailReq; 14912841Sgabeblack@google.com DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); 15012841Sgabeblack@google.com } else if (pkt->cmd == MemCmd::StoreCondReq) { 15112841Sgabeblack@google.com pkt->cmd = MemCmd::StoreCondFailReq; 15212841Sgabeblack@google.com DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n"); 15312841Sgabeblack@google.com } 15412841Sgabeblack@google.com 15512841Sgabeblack@google.com if (!has_data) { 15612841Sgabeblack@google.com // there is no sensible way of setting the data field if the 15712841Sgabeblack@google.com // new command actually would carry data 15812841Sgabeblack@google.com assert(!pkt->hasData()); 15912841Sgabeblack@google.com 16012841Sgabeblack@google.com if (pkt->hasRespData()) { 16112841Sgabeblack@google.com // we went from a packet that had no data (neither request, 16212841Sgabeblack@google.com // nor response), to one that does, and therefore we need to 16312841Sgabeblack@google.com // actually allocate space for the data payload 16412841Sgabeblack@google.com pkt->allocate(); 16512841Sgabeblack@google.com } 16612841Sgabeblack@google.com } 16712841Sgabeblack@google.com} 16812841Sgabeblack@google.com 16912841Sgabeblack@google.com 17012841Sgabeblack@google.comvoid 17112912Sgabeblack@google.comMSHR::TargetList::replaceUpgrades() 17212912Sgabeblack@google.com{ 17312912Sgabeblack@google.com if (!hasUpgrade) 17412912Sgabeblack@google.com return; 17512912Sgabeblack@google.com 17612912Sgabeblack@google.com for (auto& t : *this) { 17712841Sgabeblack@google.com replaceUpgrade(t.pkt); 17812841Sgabeblack@google.com } 17912841Sgabeblack@google.com 18012841Sgabeblack@google.com hasUpgrade = false; 18112841Sgabeblack@google.com} 18212841Sgabeblack@google.com 18312841Sgabeblack@google.com 18412841Sgabeblack@google.comvoid 18512841Sgabeblack@google.comMSHR::TargetList::clearDownstreamPending(MSHR::TargetList::iterator begin, 18612841Sgabeblack@google.com MSHR::TargetList::iterator end) 18712841Sgabeblack@google.com{ 18812841Sgabeblack@google.com for (auto t = begin; t != end; t++) { 18912841Sgabeblack@google.com if (t->markedPending) { 19012841Sgabeblack@google.com // Iterate over the SenderState stack and see if we find 19112841Sgabeblack@google.com // an MSHR entry. If we find one, clear the 19212841Sgabeblack@google.com // downstreamPending flag by calling 19312841Sgabeblack@google.com // clearDownstreamPending(). This recursively clears the 19412841Sgabeblack@google.com // downstreamPending flag in all caches this packet has 19512841Sgabeblack@google.com // passed through. 19612841Sgabeblack@google.com MSHR *mshr = t->pkt->findNextSenderState<MSHR>(); 19712841Sgabeblack@google.com if (mshr != nullptr) { 19812841Sgabeblack@google.com mshr->clearDownstreamPending(); 19912841Sgabeblack@google.com } 20012841Sgabeblack@google.com t->markedPending = false; 20112841Sgabeblack@google.com } 20212841Sgabeblack@google.com } 20312841Sgabeblack@google.com} 20412841Sgabeblack@google.com 20512841Sgabeblack@google.comvoid 20612841Sgabeblack@google.comMSHR::TargetList::clearDownstreamPending() 20712841Sgabeblack@google.com{ 20812841Sgabeblack@google.com clearDownstreamPending(begin(), end()); 20912841Sgabeblack@google.com} 21012841Sgabeblack@google.com 21112841Sgabeblack@google.com 21212841Sgabeblack@google.combool 21312841Sgabeblack@google.comMSHR::TargetList::trySatisfyFunctional(PacketPtr pkt) 21412841Sgabeblack@google.com{ 21512841Sgabeblack@google.com for (auto& t : *this) { 21612841Sgabeblack@google.com if (pkt->trySatisfyFunctional(t.pkt)) { 21712841Sgabeblack@google.com return true; 21812841Sgabeblack@google.com } 21912841Sgabeblack@google.com } 22012841Sgabeblack@google.com 22112841Sgabeblack@google.com return false; 22212841Sgabeblack@google.com} 22312841Sgabeblack@google.com 22412841Sgabeblack@google.com 22512841Sgabeblack@google.comvoid 22612841Sgabeblack@google.comMSHR::TargetList::print(std::ostream &os, int verbosity, 22712841Sgabeblack@google.com const std::string &prefix) const 22812841Sgabeblack@google.com{ 22912841Sgabeblack@google.com for (auto& t : *this) { 23012841Sgabeblack@google.com const char *s; 23112841Sgabeblack@google.com switch (t.source) { 23212841Sgabeblack@google.com case Target::FromCPU: 23312841Sgabeblack@google.com s = "FromCPU"; 23412841Sgabeblack@google.com break; 23512841Sgabeblack@google.com case Target::FromSnoop: 23612841Sgabeblack@google.com s = "FromSnoop"; 23712841Sgabeblack@google.com break; 23812841Sgabeblack@google.com case Target::FromPrefetcher: 23912841Sgabeblack@google.com s = "FromPrefetcher"; 24012841Sgabeblack@google.com break; 24112841Sgabeblack@google.com default: 24212841Sgabeblack@google.com s = ""; 24312841Sgabeblack@google.com break; 24412841Sgabeblack@google.com } 24512841Sgabeblack@google.com ccprintf(os, "%s%s: ", prefix, s); 24612841Sgabeblack@google.com t.pkt->print(os, verbosity, ""); 24712841Sgabeblack@google.com ccprintf(os, "\n"); 24812841Sgabeblack@google.com } 24912841Sgabeblack@google.com} 25012841Sgabeblack@google.com 25112841Sgabeblack@google.com 25212841Sgabeblack@google.comvoid 25312841Sgabeblack@google.comMSHR::allocate(Addr blk_addr, unsigned blk_size, PacketPtr target, 25412841Sgabeblack@google.com Tick when_ready, Counter _order, bool alloc_on_fill) 25512841Sgabeblack@google.com{ 25612841Sgabeblack@google.com blkAddr = blk_addr; 25712841Sgabeblack@google.com blkSize = blk_size; 25812841Sgabeblack@google.com isSecure = target->isSecure(); 25912841Sgabeblack@google.com readyTime = when_ready; 26012841Sgabeblack@google.com order = _order; 26112841Sgabeblack@google.com assert(target); 26212841Sgabeblack@google.com isForward = false; 26312841Sgabeblack@google.com wasWholeLineWrite = false; 26412841Sgabeblack@google.com _isUncacheable = target->req->isUncacheable(); 26512841Sgabeblack@google.com inService = false; 26612841Sgabeblack@google.com downstreamPending = false; 26712841Sgabeblack@google.com 26812841Sgabeblack@google.com targets.init(blkAddr, blkSize); 26912841Sgabeblack@google.com deferredTargets.init(blkAddr, blkSize); 27012841Sgabeblack@google.com 27112841Sgabeblack@google.com // Don't know of a case where we would allocate a new MSHR for a 27212841Sgabeblack@google.com // snoop (mem-side request), so set source according to request here 27312841Sgabeblack@google.com Target::Source source = (target->cmd == MemCmd::HardPFReq) ? 27412841Sgabeblack@google.com Target::FromPrefetcher : Target::FromCPU; 27512841Sgabeblack@google.com targets.add(target, when_ready, _order, source, true, alloc_on_fill); 27612841Sgabeblack@google.com} 27712841Sgabeblack@google.com 27812841Sgabeblack@google.com 27912841Sgabeblack@google.comvoid 28012841Sgabeblack@google.comMSHR::clearDownstreamPending() 28112841Sgabeblack@google.com{ 28212841Sgabeblack@google.com assert(downstreamPending); 28312841Sgabeblack@google.com downstreamPending = false; 28412841Sgabeblack@google.com // recursively clear flag on any MSHRs we will be forwarding 28512841Sgabeblack@google.com // responses to 28612841Sgabeblack@google.com targets.clearDownstreamPending(); 28712841Sgabeblack@google.com} 28812841Sgabeblack@google.com 28912841Sgabeblack@google.comvoid 29012841Sgabeblack@google.comMSHR::markInService(bool pending_modified_resp) 29112841Sgabeblack@google.com{ 29212841Sgabeblack@google.com assert(!inService); 29312841Sgabeblack@google.com 29412841Sgabeblack@google.com inService = true; 29512841Sgabeblack@google.com pendingModified = targets.needsWritable || pending_modified_resp; 29612841Sgabeblack@google.com postInvalidate = postDowngrade = false; 29712841Sgabeblack@google.com 29812841Sgabeblack@google.com if (!downstreamPending) { 29912841Sgabeblack@google.com // let upstream caches know that the request has made it to a 30012841Sgabeblack@google.com // level where it's going to get a response 30112841Sgabeblack@google.com targets.clearDownstreamPending(); 30212841Sgabeblack@google.com } 30312841Sgabeblack@google.com // if the line is not considered a whole-line write when sent 30412841Sgabeblack@google.com // downstream, make sure it is also not considered a whole-line 30512841Sgabeblack@google.com // write when receiving the response, and vice versa 30612841Sgabeblack@google.com wasWholeLineWrite = isWholeLineWrite(); 30712912Sgabeblack@google.com} 30812912Sgabeblack@google.com 30912912Sgabeblack@google.com 31012912Sgabeblack@google.comvoid 31112912Sgabeblack@google.comMSHR::deallocate() 31212912Sgabeblack@google.com{ 31312912Sgabeblack@google.com assert(targets.empty()); 31412841Sgabeblack@google.com targets.resetFlags(); 31512841Sgabeblack@google.com assert(deferredTargets.isReset()); 31612841Sgabeblack@google.com inService = false; 31712841Sgabeblack@google.com} 31812841Sgabeblack@google.com 31912841Sgabeblack@google.com/* 32012841Sgabeblack@google.com * Adds a target to an MSHR 32112841Sgabeblack@google.com */ 32212841Sgabeblack@google.comvoid 32312841Sgabeblack@google.comMSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order, 32412841Sgabeblack@google.com bool alloc_on_fill) 32512841Sgabeblack@google.com{ 32612841Sgabeblack@google.com // assume we'd never issue a prefetch when we've got an 32712841Sgabeblack@google.com // outstanding miss 32812841Sgabeblack@google.com assert(pkt->cmd != MemCmd::HardPFReq); 32912841Sgabeblack@google.com 33012841Sgabeblack@google.com // if there's a request already in service for this MSHR, we will 33112841Sgabeblack@google.com // have to defer the new target until after the response if any of 33212841Sgabeblack@google.com // the following are true: 33312841Sgabeblack@google.com // - there are other targets already deferred 33412841Sgabeblack@google.com // - there's a pending invalidate to be applied after the response 33512841Sgabeblack@google.com // comes back (but before this target is processed) 33612841Sgabeblack@google.com // - the MSHR's first (and only) non-deferred target is a cache 33712841Sgabeblack@google.com // maintenance packet 33812841Sgabeblack@google.com // - the new target is a cache maintenance packet (this is probably 33912841Sgabeblack@google.com // overly conservative but certainly safe) 34012841Sgabeblack@google.com // - this target requires a writable block and either we're not 34112841Sgabeblack@google.com // getting a writable block back or we have already snooped 34212841Sgabeblack@google.com // another read request that will downgrade our writable block 34312841Sgabeblack@google.com // to non-writable (Shared or Owned) 34412841Sgabeblack@google.com PacketPtr tgt_pkt = targets.front().pkt; 34512841Sgabeblack@google.com if (pkt->req->isCacheMaintenance() || 34612841Sgabeblack@google.com tgt_pkt->req->isCacheMaintenance() || 34712841Sgabeblack@google.com !deferredTargets.empty() || 34812841Sgabeblack@google.com (inService && 34912841Sgabeblack@google.com (hasPostInvalidate() || 35012841Sgabeblack@google.com (pkt->needsWritable() && 35112841Sgabeblack@google.com (!isPendingModified() || hasPostDowngrade() || isForward))))) { 35212841Sgabeblack@google.com // need to put on deferred list 35312841Sgabeblack@google.com if (inService && hasPostInvalidate()) 35412841Sgabeblack@google.com replaceUpgrade(pkt); 35512841Sgabeblack@google.com deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true, 35612841Sgabeblack@google.com alloc_on_fill); 35712841Sgabeblack@google.com } else { 35812841Sgabeblack@google.com // No request outstanding, or still OK to append to 35912841Sgabeblack@google.com // outstanding request: append to regular target list. Only 36012841Sgabeblack@google.com // mark pending if current request hasn't been issued yet 36112841Sgabeblack@google.com // (isn't in service). 36212841Sgabeblack@google.com targets.add(pkt, whenReady, _order, Target::FromCPU, !inService, 36312841Sgabeblack@google.com alloc_on_fill); 36412841Sgabeblack@google.com } 36512841Sgabeblack@google.com} 36612841Sgabeblack@google.com 36712841Sgabeblack@google.combool 36812841Sgabeblack@google.comMSHR::handleSnoop(PacketPtr pkt, Counter _order) 36912841Sgabeblack@google.com{ 37012841Sgabeblack@google.com DPRINTF(Cache, "%s for %s\n", __func__, pkt->print()); 37112841Sgabeblack@google.com 37212841Sgabeblack@google.com // when we snoop packets the needsWritable and isInvalidate flags 37312841Sgabeblack@google.com // should always be the same, however, this assumes that we never 37412841Sgabeblack@google.com // snoop writes as they are currently not marked as invalidations 37512841Sgabeblack@google.com panic_if((pkt->needsWritable() != pkt->isInvalidate()) && 37612841Sgabeblack@google.com !pkt->req->isCacheMaintenance(), 37712841Sgabeblack@google.com "%s got snoop %s where needsWritable, " 37812841Sgabeblack@google.com "does not match isInvalidate", name(), pkt->print()); 37912841Sgabeblack@google.com 38012841Sgabeblack@google.com if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 38112841Sgabeblack@google.com // Request has not been issued yet, or it's been issued 38212841Sgabeblack@google.com // locally but is buffered unissued at some downstream cache 38312841Sgabeblack@google.com // which is forwarding us this snoop. Either way, the packet 38412841Sgabeblack@google.com // we're snooping logically precedes this MSHR's request, so 38512841Sgabeblack@google.com // the snoop has no impact on the MSHR, but must be processed 38612841Sgabeblack@google.com // in the standard way by the cache. The only exception is 38712841Sgabeblack@google.com // that if we're an L2+ cache buffering an UpgradeReq from a 38812841Sgabeblack@google.com // higher-level cache, and the snoop is invalidating, then our 38912841Sgabeblack@google.com // buffered upgrades must be converted to read exclusives, 39012841Sgabeblack@google.com // since the upper-level cache no longer has a valid copy. 39112841Sgabeblack@google.com // That is, even though the upper-level cache got out on its 39212841Sgabeblack@google.com // local bus first, some other invalidating transaction 39312841Sgabeblack@google.com // reached the global bus before the upgrade did. 39412841Sgabeblack@google.com if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 39512841Sgabeblack@google.com targets.replaceUpgrades(); 39612841Sgabeblack@google.com deferredTargets.replaceUpgrades(); 39712841Sgabeblack@google.com } 39812841Sgabeblack@google.com 39912841Sgabeblack@google.com return false; 40012841Sgabeblack@google.com } 40112841Sgabeblack@google.com 40212841Sgabeblack@google.com // From here on down, the request issued by this MSHR logically 40312841Sgabeblack@google.com // precedes the request we're snooping. 40412841Sgabeblack@google.com if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 40512841Sgabeblack@google.com // snooped request still precedes the re-request we'll have to 40612841Sgabeblack@google.com // issue for deferred targets, if any... 40712841Sgabeblack@google.com deferredTargets.replaceUpgrades(); 40812841Sgabeblack@google.com } 40912841Sgabeblack@google.com 41012841Sgabeblack@google.com PacketPtr tgt_pkt = targets.front().pkt; 41112841Sgabeblack@google.com if (hasPostInvalidate() || tgt_pkt->req->isCacheInvalidate()) { 41212841Sgabeblack@google.com // a prior snoop has already appended an invalidation or a 41312841Sgabeblack@google.com // cache invalidation operation is in progress, so logically 41412841Sgabeblack@google.com // we don't have the block anymore; no need for further 41512841Sgabeblack@google.com // snooping. 41612841Sgabeblack@google.com return true; 41712841Sgabeblack@google.com } 41812841Sgabeblack@google.com 41912841Sgabeblack@google.com if (isPendingModified() || pkt->isInvalidate()) { 42012841Sgabeblack@google.com // We need to save and replay the packet in two cases: 42112841Sgabeblack@google.com // 1. We're awaiting a writable copy (Modified or Exclusive), 42212841Sgabeblack@google.com // so this MSHR is the orgering point, and we need to respond 42312841Sgabeblack@google.com // after we receive data. 42412841Sgabeblack@google.com // 2. It's an invalidation (e.g., UpgradeReq), and we need 42512841Sgabeblack@google.com // to forward the snoop up the hierarchy after the current 42612841Sgabeblack@google.com // transaction completes. 42712841Sgabeblack@google.com 42812841Sgabeblack@google.com // Start by determining if we will eventually respond or not, 42912841Sgabeblack@google.com // matching the conditions checked in Cache::handleSnoop 43012841Sgabeblack@google.com bool will_respond = isPendingModified() && pkt->needsResponse() && 43112841Sgabeblack@google.com !pkt->isClean(); 43212841Sgabeblack@google.com 43312841Sgabeblack@google.com // The packet we are snooping may be deleted by the time we 434 // actually process the target, and we consequently need to 435 // save a copy here. Clear flags and also allocate new data as 436 // the original packet data storage may have been deleted by 437 // the time we get to process this packet. In the cases where 438 // we are not responding after handling the snoop we also need 439 // to create a copy of the request to be on the safe side. In 440 // the latter case the cache is responsible for deleting both 441 // the packet and the request as part of handling the deferred 442 // snoop. 443 PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : 444 new Packet(std::make_shared<Request>(*pkt->req), pkt->cmd, 445 blkSize, pkt->id); 446 447 if (will_respond) { 448 // we are the ordering point, and will consequently 449 // respond, and depending on whether the packet 450 // needsWritable or not we either pass a Shared line or a 451 // Modified line 452 pkt->setCacheResponding(); 453 454 // inform the cache hierarchy that this cache had the line 455 // in the Modified state, even if the response is passed 456 // as Shared (and thus non-writable) 457 pkt->setResponderHadWritable(); 458 459 // in the case of an uncacheable request there is no need 460 // to set the responderHadWritable flag, but since the 461 // recipient does not care there is no harm in doing so 462 } 463 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, 464 downstreamPending && targets.needsWritable, false); 465 466 if (pkt->needsWritable() || pkt->isInvalidate()) { 467 // This transaction will take away our pending copy 468 postInvalidate = true; 469 } 470 471 if (isPendingModified() && pkt->isClean()) { 472 pkt->setSatisfied(); 473 } 474 } 475 476 if (!pkt->needsWritable() && !pkt->req->isUncacheable()) { 477 // This transaction will get a read-shared copy, downgrading 478 // our copy if we had a writable one 479 postDowngrade = true; 480 // make sure that any downstream cache does not respond with a 481 // writable (and dirty) copy even if it has one, unless it was 482 // explicitly asked for one 483 pkt->setHasSharers(); 484 } 485 486 return true; 487} 488 489MSHR::TargetList 490MSHR::extractServiceableTargets(PacketPtr pkt) 491{ 492 TargetList ready_targets; 493 ready_targets.init(blkAddr, blkSize); 494 // If the downstream MSHR got an invalidation request then we only 495 // service the first of the FromCPU targets and any other 496 // non-FromCPU target. This way the remaining FromCPU targets 497 // issue a new request and get a fresh copy of the block and we 498 // avoid memory consistency violations. 499 if (pkt->cmd == MemCmd::ReadRespWithInvalidate) { 500 auto it = targets.begin(); 501 assert((it->source == Target::FromCPU) || 502 (it->source == Target::FromPrefetcher)); 503 ready_targets.push_back(*it); 504 it = targets.erase(it); 505 while (it != targets.end()) { 506 if (it->source == Target::FromCPU) { 507 it++; 508 } else { 509 assert(it->source == Target::FromSnoop); 510 ready_targets.push_back(*it); 511 it = targets.erase(it); 512 } 513 } 514 ready_targets.populateFlags(); 515 } else { 516 std::swap(ready_targets, targets); 517 } 518 targets.populateFlags(); 519 520 return ready_targets; 521} 522 523bool 524MSHR::promoteDeferredTargets() 525{ 526 if (targets.empty() && deferredTargets.empty()) { 527 // nothing to promote 528 return false; 529 } 530 531 // the deferred targets can be generally promoted unless they 532 // contain a cache maintenance request 533 534 // find the first target that is a cache maintenance request 535 auto it = std::find_if(deferredTargets.begin(), deferredTargets.end(), 536 [](MSHR::Target &t) { 537 return t.pkt->req->isCacheMaintenance(); 538 }); 539 if (it == deferredTargets.begin()) { 540 // if the first deferred target is a cache maintenance packet 541 // then we can promote provided the targets list is empty and 542 // we can service it on its own 543 if (targets.empty()) { 544 targets.splice(targets.end(), deferredTargets, it); 545 } 546 } else { 547 // if a cache maintenance operation exists, we promote all the 548 // deferred targets that precede it, or all deferred targets 549 // otherwise 550 targets.splice(targets.end(), deferredTargets, 551 deferredTargets.begin(), it); 552 } 553 554 deferredTargets.populateFlags(); 555 targets.populateFlags(); 556 order = targets.front().order; 557 readyTime = std::max(curTick(), targets.front().readyTime); 558 559 return true; 560} 561 562void 563MSHR::promoteIf(const std::function<bool (Target &)>& pred) 564{ 565 // if any of the deferred targets were upper-level cache 566 // requests marked downstreamPending, need to clear that 567 assert(!downstreamPending); // not pending here anymore 568 569 // find the first target does not satisfy the condition 570 auto last_it = std::find_if_not(deferredTargets.begin(), 571 deferredTargets.end(), 572 pred); 573 574 // for the prefix of the deferredTargets [begin(), last_it) clear 575 // the downstreamPending flag and move them to the target list 576 deferredTargets.clearDownstreamPending(deferredTargets.begin(), 577 last_it); 578 targets.splice(targets.end(), deferredTargets, 579 deferredTargets.begin(), last_it); 580 // We need to update the flags for the target lists after the 581 // modifications 582 deferredTargets.populateFlags(); 583} 584 585void 586MSHR::promoteReadable() 587{ 588 if (!deferredTargets.empty() && !hasPostInvalidate()) { 589 // We got a non invalidating response, and we have the block 590 // but we have deferred targets which are waiting and they do 591 // not need writable. This can happen if the original request 592 // was for a cache clean operation and we had a copy of the 593 // block. Since we serviced the cache clean operation and we 594 // have the block, there's no need to defer the targets, so 595 // move them up to the regular target list. 596 597 auto pred = [](Target &t) { 598 assert(t.source == Target::FromCPU); 599 return !t.pkt->req->isCacheInvalidate() && 600 !t.pkt->needsWritable(); 601 }; 602 promoteIf(pred); 603 } 604} 605 606void 607MSHR::promoteWritable() 608{ 609 if (deferredTargets.needsWritable && 610 !(hasPostInvalidate() || hasPostDowngrade())) { 611 // We got a writable response, but we have deferred targets 612 // which are waiting to request a writable copy (not because 613 // of a pending invalidate). This can happen if the original 614 // request was for a read-only block, but we got a writable 615 // response anyway. Since we got the writable copy there's no 616 // need to defer the targets, so move them up to the regular 617 // target list. 618 assert(!targets.needsWritable); 619 targets.needsWritable = true; 620 621 auto pred = [](Target &t) { 622 assert(t.source == Target::FromCPU); 623 return !t.pkt->req->isCacheInvalidate(); 624 }; 625 626 promoteIf(pred); 627 } 628} 629 630 631bool 632MSHR::trySatisfyFunctional(PacketPtr pkt) 633{ 634 // For printing, we treat the MSHR as a whole as single entity. 635 // For other requests, we iterate over the individual targets 636 // since that's where the actual data lies. 637 if (pkt->isPrint()) { 638 pkt->trySatisfyFunctional(this, blkAddr, isSecure, blkSize, nullptr); 639 return false; 640 } else { 641 return (targets.trySatisfyFunctional(pkt) || 642 deferredTargets.trySatisfyFunctional(pkt)); 643 } 644} 645 646bool 647MSHR::sendPacket(BaseCache &cache) 648{ 649 return cache.sendMSHRQueuePacket(this); 650} 651 652void 653MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 654{ 655 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s %s\n", 656 prefix, blkAddr, blkAddr + blkSize - 1, 657 isSecure ? "s" : "ns", 658 isForward ? "Forward" : "", 659 allocOnFill() ? "AllocOnFill" : "", 660 needsWritable() ? "Wrtbl" : "", 661 _isUncacheable ? "Unc" : "", 662 inService ? "InSvc" : "", 663 downstreamPending ? "DwnPend" : "", 664 postInvalidate ? "PostInv" : "", 665 postDowngrade ? "PostDowngr" : "", 666 hasFromCache() ? "HasFromCache" : ""); 667 668 if (!targets.empty()) { 669 ccprintf(os, "%s Targets:\n", prefix); 670 targets.print(os, verbosity, prefix + " "); 671 } 672 if (!deferredTargets.empty()) { 673 ccprintf(os, "%s Deferred Targets:\n", prefix); 674 deferredTargets.print(os, verbosity, prefix + " "); 675 } 676} 677 678std::string 679MSHR::print() const 680{ 681 std::ostringstream str; 682 print(str); 683 return str.str(); 684} 685