lsq.cc revision 14297
110259SAndrew.Bardsley@arm.com/* 213954Sgiacomo.gabrielli@arm.com * Copyright (c) 2013-2014,2017-2018 ARM Limited 310259SAndrew.Bardsley@arm.com * All rights reserved 410259SAndrew.Bardsley@arm.com * 510259SAndrew.Bardsley@arm.com * The license below extends only to copyright in the software and shall 610259SAndrew.Bardsley@arm.com * not be construed as granting a license to any other intellectual 710259SAndrew.Bardsley@arm.com * property including but not limited to intellectual property relating 810259SAndrew.Bardsley@arm.com * to a hardware implementation of the functionality of the software 910259SAndrew.Bardsley@arm.com * licensed hereunder. You may use the software subject to the license 1010259SAndrew.Bardsley@arm.com * terms below provided that you ensure that this notice is replicated 1110259SAndrew.Bardsley@arm.com * unmodified and in its entirety in all distributions of the software, 1210259SAndrew.Bardsley@arm.com * modified or unmodified, in source code or in binary form. 1310259SAndrew.Bardsley@arm.com * 1410259SAndrew.Bardsley@arm.com * Redistribution and use in source and binary forms, with or without 1510259SAndrew.Bardsley@arm.com * modification, are permitted provided that the following conditions are 1610259SAndrew.Bardsley@arm.com * met: redistributions of source code must retain the above copyright 1710259SAndrew.Bardsley@arm.com * notice, this list of conditions and the following disclaimer; 1810259SAndrew.Bardsley@arm.com * redistributions in binary form must reproduce the above copyright 1910259SAndrew.Bardsley@arm.com * notice, this list of conditions and the following disclaimer in the 2010259SAndrew.Bardsley@arm.com * documentation and/or other materials provided with the distribution; 2110259SAndrew.Bardsley@arm.com * neither the name of the copyright holders nor the names of its 2210259SAndrew.Bardsley@arm.com * contributors may be used to endorse or promote products derived from 2310259SAndrew.Bardsley@arm.com * this software without specific prior written permission. 2410259SAndrew.Bardsley@arm.com * 2510259SAndrew.Bardsley@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2610259SAndrew.Bardsley@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2710259SAndrew.Bardsley@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2810259SAndrew.Bardsley@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2910259SAndrew.Bardsley@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3010259SAndrew.Bardsley@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3110259SAndrew.Bardsley@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3210259SAndrew.Bardsley@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3310259SAndrew.Bardsley@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3410259SAndrew.Bardsley@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3510259SAndrew.Bardsley@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3610259SAndrew.Bardsley@arm.com * 3710259SAndrew.Bardsley@arm.com * Authors: Andrew Bardsley 3810259SAndrew.Bardsley@arm.com */ 3910259SAndrew.Bardsley@arm.com 4011793Sbrandon.potter@amd.com#include "cpu/minor/lsq.hh" 4111793Sbrandon.potter@amd.com 4210259SAndrew.Bardsley@arm.com#include <iomanip> 4310259SAndrew.Bardsley@arm.com#include <sstream> 4410259SAndrew.Bardsley@arm.com 4510259SAndrew.Bardsley@arm.com#include "arch/locked_mem.hh" 4610259SAndrew.Bardsley@arm.com#include "arch/mmapped_ipr.hh" 4713449Sgabeblack@google.com#include "base/logging.hh" 4810259SAndrew.Bardsley@arm.com#include "cpu/minor/cpu.hh" 4910259SAndrew.Bardsley@arm.com#include "cpu/minor/exec_context.hh" 5010259SAndrew.Bardsley@arm.com#include "cpu/minor/execute.hh" 5110259SAndrew.Bardsley@arm.com#include "cpu/minor/pipeline.hh" 5213954Sgiacomo.gabrielli@arm.com#include "cpu/utils.hh" 5310259SAndrew.Bardsley@arm.com#include "debug/Activity.hh" 5410259SAndrew.Bardsley@arm.com#include "debug/MinorMem.hh" 5510259SAndrew.Bardsley@arm.com 5610259SAndrew.Bardsley@arm.comnamespace Minor 5710259SAndrew.Bardsley@arm.com{ 5810259SAndrew.Bardsley@arm.com 5910259SAndrew.Bardsley@arm.comLSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_, 6010259SAndrew.Bardsley@arm.com PacketDataPtr data_, uint64_t *res_) : 6110259SAndrew.Bardsley@arm.com SenderState(), 6210259SAndrew.Bardsley@arm.com port(port_), 6310259SAndrew.Bardsley@arm.com inst(inst_), 6410259SAndrew.Bardsley@arm.com isLoad(isLoad_), 6510259SAndrew.Bardsley@arm.com data(data_), 6610259SAndrew.Bardsley@arm.com packet(NULL), 6710259SAndrew.Bardsley@arm.com request(), 6810259SAndrew.Bardsley@arm.com res(res_), 6910259SAndrew.Bardsley@arm.com skipped(false), 7010259SAndrew.Bardsley@arm.com issuedToMemory(false), 7114105Sgabor.dozsa@arm.com isTranslationDelayed(false), 7210259SAndrew.Bardsley@arm.com state(NotIssued) 7312749Sgiacomo.travaglini@arm.com{ 7412749Sgiacomo.travaglini@arm.com request = std::make_shared<Request>(); 7512749Sgiacomo.travaglini@arm.com} 7610259SAndrew.Bardsley@arm.com 7713954Sgiacomo.gabrielli@arm.comvoid 7814105Sgabor.dozsa@arm.comLSQ::LSQRequest::tryToSuppressFault() 7914105Sgabor.dozsa@arm.com{ 8014105Sgabor.dozsa@arm.com SimpleThread &thread = *port.cpu.threads[inst->id.threadId]; 8114105Sgabor.dozsa@arm.com TheISA::PCState old_pc = thread.pcState(); 8214105Sgabor.dozsa@arm.com ExecContext context(port.cpu, thread, port.execute, inst); 8314105Sgabor.dozsa@arm.com Fault M5_VAR_USED fault = inst->translationFault; 8414105Sgabor.dozsa@arm.com 8514105Sgabor.dozsa@arm.com // Give the instruction a chance to suppress a translation fault 8614105Sgabor.dozsa@arm.com inst->translationFault = inst->staticInst->initiateAcc(&context, nullptr); 8714105Sgabor.dozsa@arm.com if (inst->translationFault == NoFault) { 8814105Sgabor.dozsa@arm.com DPRINTFS(MinorMem, (&port), 8914105Sgabor.dozsa@arm.com "Translation fault suppressed for inst:%s\n", *inst); 9014105Sgabor.dozsa@arm.com } else { 9114105Sgabor.dozsa@arm.com assert(inst->translationFault == fault); 9214105Sgabor.dozsa@arm.com } 9314105Sgabor.dozsa@arm.com thread.pcState(old_pc); 9414105Sgabor.dozsa@arm.com} 9514105Sgabor.dozsa@arm.com 9614105Sgabor.dozsa@arm.comvoid 9714105Sgabor.dozsa@arm.comLSQ::LSQRequest::completeDisabledMemAccess() 9814105Sgabor.dozsa@arm.com{ 9914105Sgabor.dozsa@arm.com DPRINTFS(MinorMem, (&port), "Complete disabled mem access for inst:%s\n", 10014105Sgabor.dozsa@arm.com *inst); 10114105Sgabor.dozsa@arm.com 10214105Sgabor.dozsa@arm.com SimpleThread &thread = *port.cpu.threads[inst->id.threadId]; 10314105Sgabor.dozsa@arm.com TheISA::PCState old_pc = thread.pcState(); 10414105Sgabor.dozsa@arm.com 10514105Sgabor.dozsa@arm.com ExecContext context(port.cpu, thread, port.execute, inst); 10614105Sgabor.dozsa@arm.com 10714105Sgabor.dozsa@arm.com context.setMemAccPredicate(false); 10814105Sgabor.dozsa@arm.com inst->staticInst->completeAcc(nullptr, &context, inst->traceData); 10914105Sgabor.dozsa@arm.com 11014105Sgabor.dozsa@arm.com thread.pcState(old_pc); 11114105Sgabor.dozsa@arm.com} 11214105Sgabor.dozsa@arm.com 11314105Sgabor.dozsa@arm.comvoid 11413954Sgiacomo.gabrielli@arm.comLSQ::LSQRequest::disableMemAccess() 11513954Sgiacomo.gabrielli@arm.com{ 11613954Sgiacomo.gabrielli@arm.com port.cpu.threads[inst->id.threadId]->setMemAccPredicate(false); 11713954Sgiacomo.gabrielli@arm.com DPRINTFS(MinorMem, (&port), "Disable mem access for inst:%s\n", *inst); 11813954Sgiacomo.gabrielli@arm.com} 11913954Sgiacomo.gabrielli@arm.com 12010259SAndrew.Bardsley@arm.comLSQ::AddrRangeCoverage 12110259SAndrew.Bardsley@arm.comLSQ::LSQRequest::containsAddrRangeOf( 12210259SAndrew.Bardsley@arm.com Addr req1_addr, unsigned int req1_size, 12310259SAndrew.Bardsley@arm.com Addr req2_addr, unsigned int req2_size) 12410259SAndrew.Bardsley@arm.com{ 12510259SAndrew.Bardsley@arm.com /* 'end' here means the address of the byte just past the request 12610259SAndrew.Bardsley@arm.com * blocks */ 12710259SAndrew.Bardsley@arm.com Addr req2_end_addr = req2_addr + req2_size; 12810259SAndrew.Bardsley@arm.com Addr req1_end_addr = req1_addr + req1_size; 12910259SAndrew.Bardsley@arm.com 13010259SAndrew.Bardsley@arm.com AddrRangeCoverage ret; 13110259SAndrew.Bardsley@arm.com 13212179Spau.cabre@metempsy.com if (req1_addr >= req2_end_addr || req1_end_addr <= req2_addr) 13310259SAndrew.Bardsley@arm.com ret = NoAddrRangeCoverage; 13410259SAndrew.Bardsley@arm.com else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr) 13510259SAndrew.Bardsley@arm.com ret = FullAddrRangeCoverage; 13610259SAndrew.Bardsley@arm.com else 13710259SAndrew.Bardsley@arm.com ret = PartialAddrRangeCoverage; 13810259SAndrew.Bardsley@arm.com 13910259SAndrew.Bardsley@arm.com return ret; 14010259SAndrew.Bardsley@arm.com} 14110259SAndrew.Bardsley@arm.com 14210259SAndrew.Bardsley@arm.comLSQ::AddrRangeCoverage 14310259SAndrew.Bardsley@arm.comLSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request) 14410259SAndrew.Bardsley@arm.com{ 14512749Sgiacomo.travaglini@arm.com return containsAddrRangeOf(request->getPaddr(), request->getSize(), 14612749Sgiacomo.travaglini@arm.com other_request->request->getPaddr(), other_request->request->getSize()); 14710259SAndrew.Bardsley@arm.com} 14810259SAndrew.Bardsley@arm.com 14910259SAndrew.Bardsley@arm.combool 15010259SAndrew.Bardsley@arm.comLSQ::LSQRequest::isBarrier() 15110259SAndrew.Bardsley@arm.com{ 15210259SAndrew.Bardsley@arm.com return inst->isInst() && inst->staticInst->isMemBarrier(); 15310259SAndrew.Bardsley@arm.com} 15410259SAndrew.Bardsley@arm.com 15510259SAndrew.Bardsley@arm.combool 15610259SAndrew.Bardsley@arm.comLSQ::LSQRequest::needsToBeSentToStoreBuffer() 15710259SAndrew.Bardsley@arm.com{ 15810259SAndrew.Bardsley@arm.com return state == StoreToStoreBuffer; 15910259SAndrew.Bardsley@arm.com} 16010259SAndrew.Bardsley@arm.com 16110259SAndrew.Bardsley@arm.comvoid 16210259SAndrew.Bardsley@arm.comLSQ::LSQRequest::setState(LSQRequestState new_state) 16310259SAndrew.Bardsley@arm.com{ 16410259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:" 16510259SAndrew.Bardsley@arm.com " %s\n", state, new_state, *inst); 16610259SAndrew.Bardsley@arm.com state = new_state; 16710259SAndrew.Bardsley@arm.com} 16810259SAndrew.Bardsley@arm.com 16910259SAndrew.Bardsley@arm.combool 17010259SAndrew.Bardsley@arm.comLSQ::LSQRequest::isComplete() const 17110259SAndrew.Bardsley@arm.com{ 17210259SAndrew.Bardsley@arm.com /* @todo, There is currently only one 'completed' state. This 17310259SAndrew.Bardsley@arm.com * may not be a good choice */ 17410259SAndrew.Bardsley@arm.com return state == Complete; 17510259SAndrew.Bardsley@arm.com} 17610259SAndrew.Bardsley@arm.com 17710259SAndrew.Bardsley@arm.comvoid 17810259SAndrew.Bardsley@arm.comLSQ::LSQRequest::reportData(std::ostream &os) const 17910259SAndrew.Bardsley@arm.com{ 18010259SAndrew.Bardsley@arm.com os << (isLoad ? 'R' : 'W') << ';'; 18110259SAndrew.Bardsley@arm.com inst->reportData(os); 18210259SAndrew.Bardsley@arm.com os << ';' << state; 18310259SAndrew.Bardsley@arm.com} 18410259SAndrew.Bardsley@arm.com 18510259SAndrew.Bardsley@arm.comstd::ostream & 18610259SAndrew.Bardsley@arm.comoperator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage) 18710259SAndrew.Bardsley@arm.com{ 18810259SAndrew.Bardsley@arm.com switch (coverage) { 18910259SAndrew.Bardsley@arm.com case LSQ::PartialAddrRangeCoverage: 19010259SAndrew.Bardsley@arm.com os << "PartialAddrRangeCoverage"; 19110259SAndrew.Bardsley@arm.com break; 19210259SAndrew.Bardsley@arm.com case LSQ::FullAddrRangeCoverage: 19310259SAndrew.Bardsley@arm.com os << "FullAddrRangeCoverage"; 19410259SAndrew.Bardsley@arm.com break; 19510259SAndrew.Bardsley@arm.com case LSQ::NoAddrRangeCoverage: 19610259SAndrew.Bardsley@arm.com os << "NoAddrRangeCoverage"; 19710259SAndrew.Bardsley@arm.com break; 19810259SAndrew.Bardsley@arm.com default: 19910259SAndrew.Bardsley@arm.com os << "AddrRangeCoverage-" << static_cast<int>(coverage); 20010259SAndrew.Bardsley@arm.com break; 20110259SAndrew.Bardsley@arm.com } 20210259SAndrew.Bardsley@arm.com return os; 20310259SAndrew.Bardsley@arm.com} 20410259SAndrew.Bardsley@arm.com 20510259SAndrew.Bardsley@arm.comstd::ostream & 20610259SAndrew.Bardsley@arm.comoperator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state) 20710259SAndrew.Bardsley@arm.com{ 20810259SAndrew.Bardsley@arm.com switch (state) { 20910259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::NotIssued: 21010259SAndrew.Bardsley@arm.com os << "NotIssued"; 21110259SAndrew.Bardsley@arm.com break; 21210259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::InTranslation: 21310259SAndrew.Bardsley@arm.com os << "InTranslation"; 21410259SAndrew.Bardsley@arm.com break; 21510259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::Translated: 21610259SAndrew.Bardsley@arm.com os << "Translated"; 21710259SAndrew.Bardsley@arm.com break; 21810259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::Failed: 21910259SAndrew.Bardsley@arm.com os << "Failed"; 22010259SAndrew.Bardsley@arm.com break; 22110259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::RequestIssuing: 22210259SAndrew.Bardsley@arm.com os << "RequestIssuing"; 22310259SAndrew.Bardsley@arm.com break; 22410259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::StoreToStoreBuffer: 22510259SAndrew.Bardsley@arm.com os << "StoreToStoreBuffer"; 22610259SAndrew.Bardsley@arm.com break; 22710259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::StoreInStoreBuffer: 22810259SAndrew.Bardsley@arm.com os << "StoreInStoreBuffer"; 22910259SAndrew.Bardsley@arm.com break; 23010259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::StoreBufferIssuing: 23110259SAndrew.Bardsley@arm.com os << "StoreBufferIssuing"; 23210259SAndrew.Bardsley@arm.com break; 23310259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::RequestNeedsRetry: 23410259SAndrew.Bardsley@arm.com os << "RequestNeedsRetry"; 23510259SAndrew.Bardsley@arm.com break; 23610259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::StoreBufferNeedsRetry: 23710259SAndrew.Bardsley@arm.com os << "StoreBufferNeedsRetry"; 23810259SAndrew.Bardsley@arm.com break; 23910259SAndrew.Bardsley@arm.com case LSQ::LSQRequest::Complete: 24010259SAndrew.Bardsley@arm.com os << "Complete"; 24110259SAndrew.Bardsley@arm.com break; 24210259SAndrew.Bardsley@arm.com default: 24310259SAndrew.Bardsley@arm.com os << "LSQRequestState-" << static_cast<int>(state); 24410259SAndrew.Bardsley@arm.com break; 24510259SAndrew.Bardsley@arm.com } 24610259SAndrew.Bardsley@arm.com return os; 24710259SAndrew.Bardsley@arm.com} 24810259SAndrew.Bardsley@arm.com 24910259SAndrew.Bardsley@arm.comvoid 25010259SAndrew.Bardsley@arm.comLSQ::clearMemBarrier(MinorDynInstPtr inst) 25110259SAndrew.Bardsley@arm.com{ 25211567Smitch.hayenga@arm.com bool is_last_barrier = 25311567Smitch.hayenga@arm.com inst->id.execSeqNum >= lastMemBarrier[inst->id.threadId]; 25410259SAndrew.Bardsley@arm.com 25510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n", 25610259SAndrew.Bardsley@arm.com (is_last_barrier ? "last" : "a"), *inst); 25710259SAndrew.Bardsley@arm.com 25810259SAndrew.Bardsley@arm.com if (is_last_barrier) 25911567Smitch.hayenga@arm.com lastMemBarrier[inst->id.threadId] = 0; 26010259SAndrew.Bardsley@arm.com} 26110259SAndrew.Bardsley@arm.com 26210259SAndrew.Bardsley@arm.comvoid 26312749Sgiacomo.travaglini@arm.comLSQ::SingleDataRequest::finish(const Fault &fault_, const RequestPtr &request_, 26410379Sandreas.hansson@arm.com ThreadContext *tc, BaseTLB::Mode mode) 26510259SAndrew.Bardsley@arm.com{ 26610259SAndrew.Bardsley@arm.com port.numAccessesInDTLB--; 26710259SAndrew.Bardsley@arm.com 26810259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Received translation response for" 26914105Sgabor.dozsa@arm.com " request: %s delayed:%d %s\n", *inst, isTranslationDelayed, 27014105Sgabor.dozsa@arm.com fault_ != NoFault ? fault_->name() : ""); 27110259SAndrew.Bardsley@arm.com 27214105Sgabor.dozsa@arm.com if (fault_ != NoFault) { 27314105Sgabor.dozsa@arm.com inst->translationFault = fault_; 27414105Sgabor.dozsa@arm.com if (isTranslationDelayed) { 27514105Sgabor.dozsa@arm.com tryToSuppressFault(); 27614105Sgabor.dozsa@arm.com if (inst->translationFault == NoFault) { 27714105Sgabor.dozsa@arm.com completeDisabledMemAccess(); 27814105Sgabor.dozsa@arm.com setState(Complete); 27914105Sgabor.dozsa@arm.com } 28014105Sgabor.dozsa@arm.com } 28114105Sgabor.dozsa@arm.com setState(Translated); 28214105Sgabor.dozsa@arm.com } else { 28314105Sgabor.dozsa@arm.com setState(Translated); 28414105Sgabor.dozsa@arm.com makePacket(); 28514105Sgabor.dozsa@arm.com } 28610259SAndrew.Bardsley@arm.com port.tryToSendToTransfers(this); 28710259SAndrew.Bardsley@arm.com 28810259SAndrew.Bardsley@arm.com /* Let's try and wake up the processor for the next cycle */ 28910259SAndrew.Bardsley@arm.com port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 29010259SAndrew.Bardsley@arm.com} 29110259SAndrew.Bardsley@arm.com 29210259SAndrew.Bardsley@arm.comvoid 29310259SAndrew.Bardsley@arm.comLSQ::SingleDataRequest::startAddrTranslation() 29410259SAndrew.Bardsley@arm.com{ 29510259SAndrew.Bardsley@arm.com ThreadContext *thread = port.cpu.getContext( 29610259SAndrew.Bardsley@arm.com inst->id.threadId); 29710259SAndrew.Bardsley@arm.com 29813954Sgiacomo.gabrielli@arm.com const auto &byteEnable = request->getByteEnable(); 29913954Sgiacomo.gabrielli@arm.com if (byteEnable.size() == 0 || 30013954Sgiacomo.gabrielli@arm.com isAnyActiveElement(byteEnable.cbegin(), byteEnable.cend())) { 30113954Sgiacomo.gabrielli@arm.com port.numAccessesInDTLB++; 30210259SAndrew.Bardsley@arm.com 30313954Sgiacomo.gabrielli@arm.com setState(LSQ::LSQRequest::InTranslation); 30410259SAndrew.Bardsley@arm.com 30513954Sgiacomo.gabrielli@arm.com DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n"); 30613954Sgiacomo.gabrielli@arm.com /* Submit the translation request. The response will come through 30713954Sgiacomo.gabrielli@arm.com * finish/markDelayed on the LSQRequest as it bears the Translation 30813954Sgiacomo.gabrielli@arm.com * interface */ 30913954Sgiacomo.gabrielli@arm.com thread->getDTBPtr()->translateTiming( 31013954Sgiacomo.gabrielli@arm.com request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write)); 31113954Sgiacomo.gabrielli@arm.com } else { 31213954Sgiacomo.gabrielli@arm.com disableMemAccess(); 31313954Sgiacomo.gabrielli@arm.com setState(LSQ::LSQRequest::Complete); 31413954Sgiacomo.gabrielli@arm.com } 31510259SAndrew.Bardsley@arm.com} 31610259SAndrew.Bardsley@arm.com 31710259SAndrew.Bardsley@arm.comvoid 31810259SAndrew.Bardsley@arm.comLSQ::SingleDataRequest::retireResponse(PacketPtr packet_) 31910259SAndrew.Bardsley@arm.com{ 32010259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Retiring packet\n"); 32110259SAndrew.Bardsley@arm.com packet = packet_; 32210259SAndrew.Bardsley@arm.com packetInFlight = false; 32310259SAndrew.Bardsley@arm.com setState(Complete); 32410259SAndrew.Bardsley@arm.com} 32510259SAndrew.Bardsley@arm.com 32610259SAndrew.Bardsley@arm.comvoid 32712749Sgiacomo.travaglini@arm.comLSQ::SplitDataRequest::finish(const Fault &fault_, const RequestPtr &request_, 32810379Sandreas.hansson@arm.com ThreadContext *tc, BaseTLB::Mode mode) 32910259SAndrew.Bardsley@arm.com{ 33010259SAndrew.Bardsley@arm.com port.numAccessesInDTLB--; 33110259SAndrew.Bardsley@arm.com 33210259SAndrew.Bardsley@arm.com unsigned int M5_VAR_USED expected_fragment_index = 33310259SAndrew.Bardsley@arm.com numTranslatedFragments; 33410259SAndrew.Bardsley@arm.com 33510259SAndrew.Bardsley@arm.com numInTranslationFragments--; 33610259SAndrew.Bardsley@arm.com numTranslatedFragments++; 33710259SAndrew.Bardsley@arm.com 33810259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Received translation response for fragment" 33914105Sgabor.dozsa@arm.com " %d of request: %s delayed:%d %s\n", expected_fragment_index, 34014105Sgabor.dozsa@arm.com *inst, isTranslationDelayed, 34114105Sgabor.dozsa@arm.com fault_ != NoFault ? fault_->name() : ""); 34210259SAndrew.Bardsley@arm.com 34310259SAndrew.Bardsley@arm.com assert(request_ == fragmentRequests[expected_fragment_index]); 34410259SAndrew.Bardsley@arm.com 34510259SAndrew.Bardsley@arm.com /* Wake up next cycle to get things going again in case the 34610259SAndrew.Bardsley@arm.com * tryToSendToTransfers does take */ 34710259SAndrew.Bardsley@arm.com port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 34810259SAndrew.Bardsley@arm.com 34914105Sgabor.dozsa@arm.com if (fault_ != NoFault) { 35010259SAndrew.Bardsley@arm.com /* tryToSendToTransfers will handle the fault */ 35114105Sgabor.dozsa@arm.com inst->translationFault = fault_; 35210259SAndrew.Bardsley@arm.com 35310259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:" 35410259SAndrew.Bardsley@arm.com " %d of request: %s\n", 35510259SAndrew.Bardsley@arm.com expected_fragment_index, *inst); 35610259SAndrew.Bardsley@arm.com 35714105Sgabor.dozsa@arm.com if (expected_fragment_index > 0 || isTranslationDelayed) 35814105Sgabor.dozsa@arm.com tryToSuppressFault(); 35914105Sgabor.dozsa@arm.com if (expected_fragment_index == 0) { 36014105Sgabor.dozsa@arm.com if (isTranslationDelayed && inst->translationFault == NoFault) { 36114105Sgabor.dozsa@arm.com completeDisabledMemAccess(); 36214105Sgabor.dozsa@arm.com setState(Complete); 36314105Sgabor.dozsa@arm.com } else { 36414105Sgabor.dozsa@arm.com setState(Translated); 36514105Sgabor.dozsa@arm.com } 36614105Sgabor.dozsa@arm.com } else if (inst->translationFault == NoFault) { 36714105Sgabor.dozsa@arm.com setState(Translated); 36814105Sgabor.dozsa@arm.com numTranslatedFragments--; 36914105Sgabor.dozsa@arm.com makeFragmentPackets(); 37014105Sgabor.dozsa@arm.com } else { 37114105Sgabor.dozsa@arm.com setState(Translated); 37214105Sgabor.dozsa@arm.com } 37310259SAndrew.Bardsley@arm.com port.tryToSendToTransfers(this); 37410259SAndrew.Bardsley@arm.com } else if (numTranslatedFragments == numFragments) { 37510259SAndrew.Bardsley@arm.com makeFragmentPackets(); 37610259SAndrew.Bardsley@arm.com setState(Translated); 37710259SAndrew.Bardsley@arm.com port.tryToSendToTransfers(this); 37810259SAndrew.Bardsley@arm.com } else { 37910259SAndrew.Bardsley@arm.com /* Avoid calling translateTiming from within ::finish */ 38010259SAndrew.Bardsley@arm.com assert(!translationEvent.scheduled()); 38110259SAndrew.Bardsley@arm.com port.cpu.schedule(translationEvent, curTick()); 38210259SAndrew.Bardsley@arm.com } 38310259SAndrew.Bardsley@arm.com} 38410259SAndrew.Bardsley@arm.com 38510259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_, 38610259SAndrew.Bardsley@arm.com bool isLoad_, PacketDataPtr data_, uint64_t *res_) : 38710259SAndrew.Bardsley@arm.com LSQRequest(port_, inst_, isLoad_, data_, res_), 38812127Sspwilson2@wisc.edu translationEvent([this]{ sendNextFragmentToTranslation(); }, 38912127Sspwilson2@wisc.edu "translationEvent"), 39010259SAndrew.Bardsley@arm.com numFragments(0), 39110259SAndrew.Bardsley@arm.com numInTranslationFragments(0), 39210259SAndrew.Bardsley@arm.com numTranslatedFragments(0), 39310259SAndrew.Bardsley@arm.com numIssuedFragments(0), 39410259SAndrew.Bardsley@arm.com numRetiredFragments(0), 39510259SAndrew.Bardsley@arm.com fragmentRequests(), 39610259SAndrew.Bardsley@arm.com fragmentPackets() 39710259SAndrew.Bardsley@arm.com{ 39810259SAndrew.Bardsley@arm.com /* Don't know how many elements are needed until the request is 39910259SAndrew.Bardsley@arm.com * populated by the caller. */ 40010259SAndrew.Bardsley@arm.com} 40110259SAndrew.Bardsley@arm.com 40210259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::~SplitDataRequest() 40310259SAndrew.Bardsley@arm.com{ 40410259SAndrew.Bardsley@arm.com for (auto i = fragmentPackets.begin(); 40510259SAndrew.Bardsley@arm.com i != fragmentPackets.end(); i++) 40610259SAndrew.Bardsley@arm.com { 40710259SAndrew.Bardsley@arm.com delete *i; 40810259SAndrew.Bardsley@arm.com } 40910259SAndrew.Bardsley@arm.com} 41010259SAndrew.Bardsley@arm.com 41110259SAndrew.Bardsley@arm.comvoid 41210259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::makeFragmentRequests() 41310259SAndrew.Bardsley@arm.com{ 41412749Sgiacomo.travaglini@arm.com Addr base_addr = request->getVaddr(); 41512749Sgiacomo.travaglini@arm.com unsigned int whole_size = request->getSize(); 41610259SAndrew.Bardsley@arm.com unsigned int line_width = port.lineWidth; 41710259SAndrew.Bardsley@arm.com 41810259SAndrew.Bardsley@arm.com unsigned int fragment_size; 41910259SAndrew.Bardsley@arm.com Addr fragment_addr; 42010259SAndrew.Bardsley@arm.com 42113954Sgiacomo.gabrielli@arm.com std::vector<bool> fragment_write_byte_en; 42213954Sgiacomo.gabrielli@arm.com 42310259SAndrew.Bardsley@arm.com /* Assume that this transfer is across potentially many block snap 42410259SAndrew.Bardsley@arm.com * boundaries: 42510259SAndrew.Bardsley@arm.com * 42610259SAndrew.Bardsley@arm.com * | _|________|________|________|___ | 42710259SAndrew.Bardsley@arm.com * | |0| 1 | 2 | 3 | 4 | | 42810259SAndrew.Bardsley@arm.com * | |_|________|________|________|___| | 42910259SAndrew.Bardsley@arm.com * | | | | | | 43010259SAndrew.Bardsley@arm.com * 43110259SAndrew.Bardsley@arm.com * The first transfer (0) can be up to lineWidth in size. 43210259SAndrew.Bardsley@arm.com * All the middle transfers (1-3) are lineWidth in size 43310259SAndrew.Bardsley@arm.com * The last transfer (4) can be from zero to lineWidth - 1 in size 43410259SAndrew.Bardsley@arm.com */ 43510259SAndrew.Bardsley@arm.com unsigned int first_fragment_offset = 43610259SAndrew.Bardsley@arm.com addrBlockOffset(base_addr, line_width); 43710259SAndrew.Bardsley@arm.com unsigned int last_fragment_size = 43810259SAndrew.Bardsley@arm.com addrBlockOffset(base_addr + whole_size, line_width); 43910259SAndrew.Bardsley@arm.com unsigned int first_fragment_size = 44010259SAndrew.Bardsley@arm.com line_width - first_fragment_offset; 44110259SAndrew.Bardsley@arm.com 44210259SAndrew.Bardsley@arm.com unsigned int middle_fragments_total_size = 44310259SAndrew.Bardsley@arm.com whole_size - (first_fragment_size + last_fragment_size); 44410259SAndrew.Bardsley@arm.com 44510259SAndrew.Bardsley@arm.com assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0); 44610259SAndrew.Bardsley@arm.com 44710259SAndrew.Bardsley@arm.com unsigned int middle_fragment_count = 44810259SAndrew.Bardsley@arm.com middle_fragments_total_size / line_width; 44910259SAndrew.Bardsley@arm.com 45010259SAndrew.Bardsley@arm.com numFragments = 1 /* first */ + middle_fragment_count + 45110259SAndrew.Bardsley@arm.com (last_fragment_size == 0 ? 0 : 1); 45210259SAndrew.Bardsley@arm.com 45310259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests." 45410259SAndrew.Bardsley@arm.com " First fragment size: %d Last fragment size: %d\n", 45510259SAndrew.Bardsley@arm.com numFragments, first_fragment_size, 45610259SAndrew.Bardsley@arm.com (last_fragment_size == 0 ? line_width : last_fragment_size)); 45710259SAndrew.Bardsley@arm.com 45810259SAndrew.Bardsley@arm.com assert(((middle_fragment_count * line_width) + 45910259SAndrew.Bardsley@arm.com first_fragment_size + last_fragment_size) == whole_size); 46010259SAndrew.Bardsley@arm.com 46110259SAndrew.Bardsley@arm.com fragment_addr = base_addr; 46210259SAndrew.Bardsley@arm.com fragment_size = first_fragment_size; 46310259SAndrew.Bardsley@arm.com 46410259SAndrew.Bardsley@arm.com /* Just past the last address in the request */ 46510259SAndrew.Bardsley@arm.com Addr end_addr = base_addr + whole_size; 46610259SAndrew.Bardsley@arm.com 46713954Sgiacomo.gabrielli@arm.com auto& byte_enable = request->getByteEnable(); 46813954Sgiacomo.gabrielli@arm.com unsigned int num_disabled_fragments = 0; 46913954Sgiacomo.gabrielli@arm.com 47010259SAndrew.Bardsley@arm.com for (unsigned int fragment_index = 0; fragment_index < numFragments; 47110259SAndrew.Bardsley@arm.com fragment_index++) 47210259SAndrew.Bardsley@arm.com { 47310259SAndrew.Bardsley@arm.com bool M5_VAR_USED is_last_fragment = false; 47410259SAndrew.Bardsley@arm.com 47510259SAndrew.Bardsley@arm.com if (fragment_addr == base_addr) { 47610259SAndrew.Bardsley@arm.com /* First fragment */ 47710259SAndrew.Bardsley@arm.com fragment_size = first_fragment_size; 47810259SAndrew.Bardsley@arm.com } else { 47910259SAndrew.Bardsley@arm.com if ((fragment_addr + line_width) > end_addr) { 48010259SAndrew.Bardsley@arm.com /* Adjust size of last fragment */ 48110259SAndrew.Bardsley@arm.com fragment_size = end_addr - fragment_addr; 48210259SAndrew.Bardsley@arm.com is_last_fragment = true; 48310259SAndrew.Bardsley@arm.com } else { 48410259SAndrew.Bardsley@arm.com /* Middle fragments */ 48510259SAndrew.Bardsley@arm.com fragment_size = line_width; 48610259SAndrew.Bardsley@arm.com } 48710259SAndrew.Bardsley@arm.com } 48810259SAndrew.Bardsley@arm.com 48912749Sgiacomo.travaglini@arm.com RequestPtr fragment = std::make_shared<Request>(); 49013954Sgiacomo.gabrielli@arm.com bool disabled_fragment = false; 49110259SAndrew.Bardsley@arm.com 49212749Sgiacomo.travaglini@arm.com fragment->setContext(request->contextId()); 49313954Sgiacomo.gabrielli@arm.com if (byte_enable.empty()) { 49413954Sgiacomo.gabrielli@arm.com fragment->setVirt(0 /* asid */, 49513954Sgiacomo.gabrielli@arm.com fragment_addr, fragment_size, request->getFlags(), 49613954Sgiacomo.gabrielli@arm.com request->masterId(), 49713954Sgiacomo.gabrielli@arm.com request->getPC()); 49813954Sgiacomo.gabrielli@arm.com } else { 49913954Sgiacomo.gabrielli@arm.com // Set up byte-enable mask for the current fragment 50013954Sgiacomo.gabrielli@arm.com auto it_start = byte_enable.begin() + 50113954Sgiacomo.gabrielli@arm.com (fragment_addr - base_addr); 50213954Sgiacomo.gabrielli@arm.com auto it_end = byte_enable.begin() + 50313954Sgiacomo.gabrielli@arm.com (fragment_addr - base_addr) + fragment_size; 50413954Sgiacomo.gabrielli@arm.com if (isAnyActiveElement(it_start, it_end)) { 50513954Sgiacomo.gabrielli@arm.com fragment->setVirt(0 /* asid */, 50613954Sgiacomo.gabrielli@arm.com fragment_addr, fragment_size, request->getFlags(), 50713954Sgiacomo.gabrielli@arm.com request->masterId(), 50813954Sgiacomo.gabrielli@arm.com request->getPC()); 50913954Sgiacomo.gabrielli@arm.com fragment->setByteEnable(std::vector<bool>(it_start, it_end)); 51013954Sgiacomo.gabrielli@arm.com } else { 51113954Sgiacomo.gabrielli@arm.com disabled_fragment = true; 51213954Sgiacomo.gabrielli@arm.com } 51313954Sgiacomo.gabrielli@arm.com } 51410259SAndrew.Bardsley@arm.com 51513954Sgiacomo.gabrielli@arm.com if (!disabled_fragment) { 51613954Sgiacomo.gabrielli@arm.com DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x" 51713954Sgiacomo.gabrielli@arm.com " size: %d (whole request addr: 0x%x size: %d) %s\n", 51813954Sgiacomo.gabrielli@arm.com fragment_addr, fragment_size, base_addr, whole_size, 51913954Sgiacomo.gabrielli@arm.com (is_last_fragment ? "last fragment" : "")); 52013954Sgiacomo.gabrielli@arm.com 52113954Sgiacomo.gabrielli@arm.com fragmentRequests.push_back(fragment); 52213954Sgiacomo.gabrielli@arm.com } else { 52313954Sgiacomo.gabrielli@arm.com num_disabled_fragments++; 52413954Sgiacomo.gabrielli@arm.com } 52510259SAndrew.Bardsley@arm.com 52610259SAndrew.Bardsley@arm.com fragment_addr += fragment_size; 52710259SAndrew.Bardsley@arm.com } 52813954Sgiacomo.gabrielli@arm.com assert(numFragments >= num_disabled_fragments); 52913954Sgiacomo.gabrielli@arm.com numFragments -= num_disabled_fragments; 53010259SAndrew.Bardsley@arm.com} 53110259SAndrew.Bardsley@arm.com 53210259SAndrew.Bardsley@arm.comvoid 53310259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::makeFragmentPackets() 53410259SAndrew.Bardsley@arm.com{ 53513954Sgiacomo.gabrielli@arm.com assert(numTranslatedFragments > 0); 53612749Sgiacomo.travaglini@arm.com Addr base_addr = request->getVaddr(); 53710259SAndrew.Bardsley@arm.com 53810259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst); 53910259SAndrew.Bardsley@arm.com 54013954Sgiacomo.gabrielli@arm.com for (unsigned int fragment_index = 0; 54113954Sgiacomo.gabrielli@arm.com fragment_index < numTranslatedFragments; 54210259SAndrew.Bardsley@arm.com fragment_index++) 54310259SAndrew.Bardsley@arm.com { 54412748Sgiacomo.travaglini@arm.com RequestPtr fragment = fragmentRequests[fragment_index]; 54510259SAndrew.Bardsley@arm.com 54610259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s" 54710259SAndrew.Bardsley@arm.com " (%d, 0x%x)\n", 54810259SAndrew.Bardsley@arm.com fragment_index, *inst, 54910259SAndrew.Bardsley@arm.com (fragment->hasPaddr() ? "has paddr" : "no paddr"), 55010259SAndrew.Bardsley@arm.com (fragment->hasPaddr() ? fragment->getPaddr() : 0)); 55110259SAndrew.Bardsley@arm.com 55210259SAndrew.Bardsley@arm.com Addr fragment_addr = fragment->getVaddr(); 55310259SAndrew.Bardsley@arm.com unsigned int fragment_size = fragment->getSize(); 55410259SAndrew.Bardsley@arm.com 55510259SAndrew.Bardsley@arm.com uint8_t *request_data = NULL; 55610259SAndrew.Bardsley@arm.com 55710259SAndrew.Bardsley@arm.com if (!isLoad) { 55810259SAndrew.Bardsley@arm.com /* Split data for Packets. Will become the property of the 55910259SAndrew.Bardsley@arm.com * outgoing Packets */ 56010259SAndrew.Bardsley@arm.com request_data = new uint8_t[fragment_size]; 56110259SAndrew.Bardsley@arm.com std::memcpy(request_data, data + (fragment_addr - base_addr), 56210259SAndrew.Bardsley@arm.com fragment_size); 56310259SAndrew.Bardsley@arm.com } 56410259SAndrew.Bardsley@arm.com 56510259SAndrew.Bardsley@arm.com assert(fragment->hasPaddr()); 56610259SAndrew.Bardsley@arm.com 56710259SAndrew.Bardsley@arm.com PacketPtr fragment_packet = 56812749Sgiacomo.travaglini@arm.com makePacketForRequest(fragment, isLoad, this, request_data); 56910259SAndrew.Bardsley@arm.com 57010259SAndrew.Bardsley@arm.com fragmentPackets.push_back(fragment_packet); 57110368SAndrew.Bardsley@arm.com /* Accumulate flags in parent request */ 57212749Sgiacomo.travaglini@arm.com request->setFlags(fragment->getFlags()); 57310259SAndrew.Bardsley@arm.com } 57410259SAndrew.Bardsley@arm.com 57510259SAndrew.Bardsley@arm.com /* Might as well make the overall/response packet here */ 57610259SAndrew.Bardsley@arm.com /* Get the physical address for the whole request/packet from the first 57710259SAndrew.Bardsley@arm.com * fragment */ 57812749Sgiacomo.travaglini@arm.com request->setPaddr(fragmentRequests[0]->getPaddr()); 57910259SAndrew.Bardsley@arm.com makePacket(); 58010259SAndrew.Bardsley@arm.com} 58110259SAndrew.Bardsley@arm.com 58210259SAndrew.Bardsley@arm.comvoid 58310259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::startAddrTranslation() 58410259SAndrew.Bardsley@arm.com{ 58510259SAndrew.Bardsley@arm.com makeFragmentRequests(); 58610259SAndrew.Bardsley@arm.com 58713954Sgiacomo.gabrielli@arm.com if (numFragments > 0) { 58813954Sgiacomo.gabrielli@arm.com setState(LSQ::LSQRequest::InTranslation); 58913954Sgiacomo.gabrielli@arm.com numInTranslationFragments = 0; 59013954Sgiacomo.gabrielli@arm.com numTranslatedFragments = 0; 59110259SAndrew.Bardsley@arm.com 59213954Sgiacomo.gabrielli@arm.com /* @todo, just do these in sequence for now with 59313954Sgiacomo.gabrielli@arm.com * a loop of: 59413954Sgiacomo.gabrielli@arm.com * do { 59513954Sgiacomo.gabrielli@arm.com * sendNextFragmentToTranslation ; translateTiming ; finish 59613954Sgiacomo.gabrielli@arm.com * } while (numTranslatedFragments != numFragments); 59713954Sgiacomo.gabrielli@arm.com */ 59810259SAndrew.Bardsley@arm.com 59913954Sgiacomo.gabrielli@arm.com /* Do first translation */ 60013954Sgiacomo.gabrielli@arm.com sendNextFragmentToTranslation(); 60113954Sgiacomo.gabrielli@arm.com } else { 60213954Sgiacomo.gabrielli@arm.com disableMemAccess(); 60313954Sgiacomo.gabrielli@arm.com setState(LSQ::LSQRequest::Complete); 60413954Sgiacomo.gabrielli@arm.com } 60510259SAndrew.Bardsley@arm.com} 60610259SAndrew.Bardsley@arm.com 60710259SAndrew.Bardsley@arm.comPacketPtr 60810259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::getHeadPacket() 60910259SAndrew.Bardsley@arm.com{ 61013954Sgiacomo.gabrielli@arm.com assert(numIssuedFragments < numTranslatedFragments); 61110259SAndrew.Bardsley@arm.com 61210259SAndrew.Bardsley@arm.com return fragmentPackets[numIssuedFragments]; 61310259SAndrew.Bardsley@arm.com} 61410259SAndrew.Bardsley@arm.com 61510259SAndrew.Bardsley@arm.comvoid 61610259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::stepToNextPacket() 61710259SAndrew.Bardsley@arm.com{ 61813954Sgiacomo.gabrielli@arm.com assert(numIssuedFragments < numTranslatedFragments); 61910259SAndrew.Bardsley@arm.com 62010259SAndrew.Bardsley@arm.com numIssuedFragments++; 62110259SAndrew.Bardsley@arm.com} 62210259SAndrew.Bardsley@arm.com 62310259SAndrew.Bardsley@arm.comvoid 62410259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::retireResponse(PacketPtr response) 62510259SAndrew.Bardsley@arm.com{ 62614105Sgabor.dozsa@arm.com assert(inst->translationFault == NoFault); 62713954Sgiacomo.gabrielli@arm.com assert(numRetiredFragments < numTranslatedFragments); 62810259SAndrew.Bardsley@arm.com 62910259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d" 63013954Sgiacomo.gabrielli@arm.com " offset: 0x%x (retired fragment num: %d)\n", 63110259SAndrew.Bardsley@arm.com response->req->getVaddr(), response->req->getSize(), 63212749Sgiacomo.travaglini@arm.com request->getVaddr() - response->req->getVaddr(), 63313954Sgiacomo.gabrielli@arm.com numRetiredFragments); 63410259SAndrew.Bardsley@arm.com 63510259SAndrew.Bardsley@arm.com numRetiredFragments++; 63610259SAndrew.Bardsley@arm.com 63710259SAndrew.Bardsley@arm.com if (skipped) { 63810259SAndrew.Bardsley@arm.com /* Skip because we already knew the request had faulted or been 63910259SAndrew.Bardsley@arm.com * skipped */ 64010259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Skipping this fragment\n"); 64110259SAndrew.Bardsley@arm.com } else if (response->isError()) { 64210259SAndrew.Bardsley@arm.com /* Mark up the error and leave to execute to handle it */ 64310259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n"); 64410259SAndrew.Bardsley@arm.com setSkipped(); 64510259SAndrew.Bardsley@arm.com packet->copyError(response); 64610259SAndrew.Bardsley@arm.com } else { 64710259SAndrew.Bardsley@arm.com if (isLoad) { 64810259SAndrew.Bardsley@arm.com if (!data) { 64910259SAndrew.Bardsley@arm.com /* For a split transfer, a Packet must be constructed 65010259SAndrew.Bardsley@arm.com * to contain all returning data. This is that packet's 65110259SAndrew.Bardsley@arm.com * data */ 65212749Sgiacomo.travaglini@arm.com data = new uint8_t[request->getSize()]; 65310259SAndrew.Bardsley@arm.com } 65410259SAndrew.Bardsley@arm.com 65510259SAndrew.Bardsley@arm.com /* Populate the portion of the overall response data represented 65610259SAndrew.Bardsley@arm.com * by the response fragment */ 65710259SAndrew.Bardsley@arm.com std::memcpy( 65812749Sgiacomo.travaglini@arm.com data + (response->req->getVaddr() - request->getVaddr()), 65910563Sandreas.hansson@arm.com response->getConstPtr<uint8_t>(), 66010259SAndrew.Bardsley@arm.com response->req->getSize()); 66110259SAndrew.Bardsley@arm.com } 66210259SAndrew.Bardsley@arm.com } 66310259SAndrew.Bardsley@arm.com 66410259SAndrew.Bardsley@arm.com /* Complete early if we're skipping are no more in-flight accesses */ 66510259SAndrew.Bardsley@arm.com if (skipped && !hasPacketsInMemSystem()) { 66610259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Completed skipped burst\n"); 66710259SAndrew.Bardsley@arm.com setState(Complete); 66810259SAndrew.Bardsley@arm.com if (packet->needsResponse()) 66910259SAndrew.Bardsley@arm.com packet->makeResponse(); 67010259SAndrew.Bardsley@arm.com } 67110259SAndrew.Bardsley@arm.com 67213954Sgiacomo.gabrielli@arm.com if (numRetiredFragments == numTranslatedFragments) 67310259SAndrew.Bardsley@arm.com setState(Complete); 67410259SAndrew.Bardsley@arm.com 67510259SAndrew.Bardsley@arm.com if (!skipped && isComplete()) { 67610259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL); 67710259SAndrew.Bardsley@arm.com 67810259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d" 67910259SAndrew.Bardsley@arm.com " needsResponse: %d packetSize: %s requestSize: %s responseSize:" 68010259SAndrew.Bardsley@arm.com " %s\n", packet->isRead(), packet->isWrite(), 68112749Sgiacomo.travaglini@arm.com packet->needsResponse(), packet->getSize(), request->getSize(), 68210259SAndrew.Bardsley@arm.com response->getSize()); 68310259SAndrew.Bardsley@arm.com 68410259SAndrew.Bardsley@arm.com /* A request can become complete by several paths, this is a sanity 68510259SAndrew.Bardsley@arm.com * check to make sure the packet's data is created */ 68610259SAndrew.Bardsley@arm.com if (!data) { 68712749Sgiacomo.travaglini@arm.com data = new uint8_t[request->getSize()]; 68810259SAndrew.Bardsley@arm.com } 68910259SAndrew.Bardsley@arm.com 69010259SAndrew.Bardsley@arm.com if (isLoad) { 69110259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Copying read data\n"); 69212749Sgiacomo.travaglini@arm.com std::memcpy(packet->getPtr<uint8_t>(), data, request->getSize()); 69310259SAndrew.Bardsley@arm.com } 69410259SAndrew.Bardsley@arm.com packet->makeResponse(); 69510259SAndrew.Bardsley@arm.com } 69610259SAndrew.Bardsley@arm.com 69710259SAndrew.Bardsley@arm.com /* Packets are all deallocated together in ~SplitLSQRequest */ 69810259SAndrew.Bardsley@arm.com} 69910259SAndrew.Bardsley@arm.com 70010259SAndrew.Bardsley@arm.comvoid 70110259SAndrew.Bardsley@arm.comLSQ::SplitDataRequest::sendNextFragmentToTranslation() 70210259SAndrew.Bardsley@arm.com{ 70310259SAndrew.Bardsley@arm.com unsigned int fragment_index = numTranslatedFragments; 70410259SAndrew.Bardsley@arm.com 70510259SAndrew.Bardsley@arm.com ThreadContext *thread = port.cpu.getContext( 70610259SAndrew.Bardsley@arm.com inst->id.threadId); 70710259SAndrew.Bardsley@arm.com 70810259SAndrew.Bardsley@arm.com DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n", 70910259SAndrew.Bardsley@arm.com fragment_index); 71010259SAndrew.Bardsley@arm.com 71110259SAndrew.Bardsley@arm.com port.numAccessesInDTLB++; 71210259SAndrew.Bardsley@arm.com numInTranslationFragments++; 71310259SAndrew.Bardsley@arm.com 71410259SAndrew.Bardsley@arm.com thread->getDTBPtr()->translateTiming( 71510259SAndrew.Bardsley@arm.com fragmentRequests[fragment_index], thread, this, (isLoad ? 71610259SAndrew.Bardsley@arm.com BaseTLB::Read : BaseTLB::Write)); 71710259SAndrew.Bardsley@arm.com} 71810259SAndrew.Bardsley@arm.com 71910259SAndrew.Bardsley@arm.combool 72010259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::canInsert() const 72110259SAndrew.Bardsley@arm.com{ 72210259SAndrew.Bardsley@arm.com /* @todo, support store amalgamation */ 72310259SAndrew.Bardsley@arm.com return slots.size() < numSlots; 72410259SAndrew.Bardsley@arm.com} 72510259SAndrew.Bardsley@arm.com 72610259SAndrew.Bardsley@arm.comvoid 72710259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::deleteRequest(LSQRequestPtr request) 72810259SAndrew.Bardsley@arm.com{ 72910259SAndrew.Bardsley@arm.com auto found = std::find(slots.begin(), slots.end(), request); 73010259SAndrew.Bardsley@arm.com 73110259SAndrew.Bardsley@arm.com if (found != slots.end()) { 73210259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n", 73310259SAndrew.Bardsley@arm.com request, *found, *(request->inst)); 73410259SAndrew.Bardsley@arm.com slots.erase(found); 73510259SAndrew.Bardsley@arm.com 73610259SAndrew.Bardsley@arm.com delete request; 73710259SAndrew.Bardsley@arm.com } 73810259SAndrew.Bardsley@arm.com} 73910259SAndrew.Bardsley@arm.com 74010259SAndrew.Bardsley@arm.comvoid 74110259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::insert(LSQRequestPtr request) 74210259SAndrew.Bardsley@arm.com{ 74310259SAndrew.Bardsley@arm.com if (!canInsert()) { 74410259SAndrew.Bardsley@arm.com warn("%s: store buffer insertion without space to insert from" 74510259SAndrew.Bardsley@arm.com " inst: %s\n", name(), *(request->inst)); 74610259SAndrew.Bardsley@arm.com } 74710259SAndrew.Bardsley@arm.com 74810259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request); 74910259SAndrew.Bardsley@arm.com 75010259SAndrew.Bardsley@arm.com numUnissuedAccesses++; 75110259SAndrew.Bardsley@arm.com 75210259SAndrew.Bardsley@arm.com if (request->state != LSQRequest::Complete) 75310259SAndrew.Bardsley@arm.com request->setState(LSQRequest::StoreInStoreBuffer); 75410259SAndrew.Bardsley@arm.com 75510259SAndrew.Bardsley@arm.com slots.push_back(request); 75610259SAndrew.Bardsley@arm.com 75710259SAndrew.Bardsley@arm.com /* Let's try and wake up the processor for the next cycle to step 75810259SAndrew.Bardsley@arm.com * the store buffer */ 75910259SAndrew.Bardsley@arm.com lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 76010259SAndrew.Bardsley@arm.com} 76110259SAndrew.Bardsley@arm.com 76210259SAndrew.Bardsley@arm.comLSQ::AddrRangeCoverage 76310259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request, 76410259SAndrew.Bardsley@arm.com unsigned int &found_slot) 76510259SAndrew.Bardsley@arm.com{ 76610259SAndrew.Bardsley@arm.com unsigned int slot_index = slots.size() - 1; 76710259SAndrew.Bardsley@arm.com auto i = slots.rbegin(); 76810259SAndrew.Bardsley@arm.com AddrRangeCoverage ret = NoAddrRangeCoverage; 76910259SAndrew.Bardsley@arm.com 77010259SAndrew.Bardsley@arm.com /* Traverse the store buffer in reverse order (most to least recent) 77110259SAndrew.Bardsley@arm.com * and try to find a slot whose address range overlaps this request */ 77210259SAndrew.Bardsley@arm.com while (ret == NoAddrRangeCoverage && i != slots.rend()) { 77310259SAndrew.Bardsley@arm.com LSQRequestPtr slot = *i; 77410259SAndrew.Bardsley@arm.com 77513652Sqtt2@cornell.edu /* Cache maintenance instructions go down via the store path but 77613652Sqtt2@cornell.edu * they carry no data and they shouldn't be considered 77713652Sqtt2@cornell.edu * for forwarding */ 77811567Smitch.hayenga@arm.com if (slot->packet && 77912355Snikos.nikoleris@arm.com slot->inst->id.threadId == request->inst->id.threadId && 78012355Snikos.nikoleris@arm.com !slot->packet->req->isCacheMaintenance()) { 78110259SAndrew.Bardsley@arm.com AddrRangeCoverage coverage = slot->containsAddrRangeOf(request); 78210259SAndrew.Bardsley@arm.com 78310259SAndrew.Bardsley@arm.com if (coverage != NoAddrRangeCoverage) { 78410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:" 78510259SAndrew.Bardsley@arm.com " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n", 78610259SAndrew.Bardsley@arm.com slot_index, coverage, 78712749Sgiacomo.travaglini@arm.com request->request->getPaddr(), request->request->getSize(), 78812749Sgiacomo.travaglini@arm.com slot->request->getPaddr(), slot->request->getSize()); 78910259SAndrew.Bardsley@arm.com 79010259SAndrew.Bardsley@arm.com found_slot = slot_index; 79110259SAndrew.Bardsley@arm.com ret = coverage; 79210259SAndrew.Bardsley@arm.com } 79310259SAndrew.Bardsley@arm.com } 79410259SAndrew.Bardsley@arm.com 79510259SAndrew.Bardsley@arm.com i++; 79610259SAndrew.Bardsley@arm.com slot_index--; 79710259SAndrew.Bardsley@arm.com } 79810259SAndrew.Bardsley@arm.com 79910259SAndrew.Bardsley@arm.com return ret; 80010259SAndrew.Bardsley@arm.com} 80110259SAndrew.Bardsley@arm.com 80210259SAndrew.Bardsley@arm.com/** Fill the given packet with appropriate date from slot slot_number */ 80310259SAndrew.Bardsley@arm.comvoid 80410259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load, 80510259SAndrew.Bardsley@arm.com unsigned int slot_number) 80610259SAndrew.Bardsley@arm.com{ 80710259SAndrew.Bardsley@arm.com assert(slot_number < slots.size()); 80810259SAndrew.Bardsley@arm.com assert(load->packet); 80910259SAndrew.Bardsley@arm.com assert(load->isLoad); 81010259SAndrew.Bardsley@arm.com 81110259SAndrew.Bardsley@arm.com LSQRequestPtr store = slots[slot_number]; 81210259SAndrew.Bardsley@arm.com 81310259SAndrew.Bardsley@arm.com assert(store->packet); 81410259SAndrew.Bardsley@arm.com assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage); 81510259SAndrew.Bardsley@arm.com 81612749Sgiacomo.travaglini@arm.com Addr load_addr = load->request->getPaddr(); 81712749Sgiacomo.travaglini@arm.com Addr store_addr = store->request->getPaddr(); 81810259SAndrew.Bardsley@arm.com Addr addr_offset = load_addr - store_addr; 81910259SAndrew.Bardsley@arm.com 82012749Sgiacomo.travaglini@arm.com unsigned int load_size = load->request->getSize(); 82110259SAndrew.Bardsley@arm.com 82210259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer" 82310259SAndrew.Bardsley@arm.com " slot: %d addr: 0x%x addressOffset: 0x%x\n", 82410259SAndrew.Bardsley@arm.com load_size, load_addr, slot_number, 82510259SAndrew.Bardsley@arm.com store_addr, addr_offset); 82610259SAndrew.Bardsley@arm.com 82710259SAndrew.Bardsley@arm.com void *load_packet_data = load->packet->getPtr<void>(); 82810259SAndrew.Bardsley@arm.com void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset; 82910259SAndrew.Bardsley@arm.com 83010259SAndrew.Bardsley@arm.com std::memcpy(load_packet_data, store_packet_data, load_size); 83110259SAndrew.Bardsley@arm.com} 83210259SAndrew.Bardsley@arm.com 83310259SAndrew.Bardsley@arm.comvoid 83410581SAndrew.Bardsley@arm.comLSQ::StoreBuffer::countIssuedStore(LSQRequestPtr request) 83510581SAndrew.Bardsley@arm.com{ 83610581SAndrew.Bardsley@arm.com /* Barriers are accounted for as they are cleared from 83710581SAndrew.Bardsley@arm.com * the queue, not after their transfers are complete */ 83810581SAndrew.Bardsley@arm.com if (!request->isBarrier()) 83910581SAndrew.Bardsley@arm.com numUnissuedAccesses--; 84010581SAndrew.Bardsley@arm.com} 84110581SAndrew.Bardsley@arm.com 84210581SAndrew.Bardsley@arm.comvoid 84310259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::step() 84410259SAndrew.Bardsley@arm.com{ 84510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n", 84610259SAndrew.Bardsley@arm.com numUnissuedAccesses); 84710259SAndrew.Bardsley@arm.com 84810259SAndrew.Bardsley@arm.com if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) { 84910259SAndrew.Bardsley@arm.com /* Clear all the leading barriers */ 85010259SAndrew.Bardsley@arm.com while (!slots.empty() && 85110259SAndrew.Bardsley@arm.com slots.front()->isComplete() && slots.front()->isBarrier()) 85210259SAndrew.Bardsley@arm.com { 85310259SAndrew.Bardsley@arm.com LSQRequestPtr barrier = slots.front(); 85410259SAndrew.Bardsley@arm.com 85510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Clearing barrier for inst: %s\n", 85610259SAndrew.Bardsley@arm.com *(barrier->inst)); 85710259SAndrew.Bardsley@arm.com 85810259SAndrew.Bardsley@arm.com numUnissuedAccesses--; 85910259SAndrew.Bardsley@arm.com lsq.clearMemBarrier(barrier->inst); 86010259SAndrew.Bardsley@arm.com slots.pop_front(); 86110259SAndrew.Bardsley@arm.com 86210259SAndrew.Bardsley@arm.com delete barrier; 86310259SAndrew.Bardsley@arm.com } 86410259SAndrew.Bardsley@arm.com 86510259SAndrew.Bardsley@arm.com auto i = slots.begin(); 86610259SAndrew.Bardsley@arm.com bool issued = true; 86710259SAndrew.Bardsley@arm.com unsigned int issue_count = 0; 86810259SAndrew.Bardsley@arm.com 86910259SAndrew.Bardsley@arm.com /* Skip trying if the memory system is busy */ 87010259SAndrew.Bardsley@arm.com if (lsq.state == LSQ::MemoryNeedsRetry) 87110259SAndrew.Bardsley@arm.com issued = false; 87210259SAndrew.Bardsley@arm.com 87310259SAndrew.Bardsley@arm.com /* Try to issue all stores in order starting from the head 87410259SAndrew.Bardsley@arm.com * of the queue. Responses are allowed to be retired 87510259SAndrew.Bardsley@arm.com * out of order */ 87610259SAndrew.Bardsley@arm.com while (issued && 87710259SAndrew.Bardsley@arm.com issue_count < storeLimitPerCycle && 87810259SAndrew.Bardsley@arm.com lsq.canSendToMemorySystem() && 87910259SAndrew.Bardsley@arm.com i != slots.end()) 88010259SAndrew.Bardsley@arm.com { 88110259SAndrew.Bardsley@arm.com LSQRequestPtr request = *i; 88210259SAndrew.Bardsley@arm.com 88310259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d" 88410259SAndrew.Bardsley@arm.com " state: %s\n", 88510259SAndrew.Bardsley@arm.com *(request->inst), request->sentAllPackets(), 88610259SAndrew.Bardsley@arm.com request->state); 88710259SAndrew.Bardsley@arm.com 88810259SAndrew.Bardsley@arm.com if (request->isBarrier() && request->isComplete()) { 88910259SAndrew.Bardsley@arm.com /* Give up at barriers */ 89010259SAndrew.Bardsley@arm.com issued = false; 89110259SAndrew.Bardsley@arm.com } else if (!(request->state == LSQRequest::StoreBufferIssuing && 89210259SAndrew.Bardsley@arm.com request->sentAllPackets())) 89310259SAndrew.Bardsley@arm.com { 89410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Trying to send request: %s to memory" 89510259SAndrew.Bardsley@arm.com " system\n", *(request->inst)); 89610259SAndrew.Bardsley@arm.com 89710259SAndrew.Bardsley@arm.com if (lsq.tryToSend(request)) { 89810581SAndrew.Bardsley@arm.com countIssuedStore(request); 89910259SAndrew.Bardsley@arm.com issue_count++; 90010259SAndrew.Bardsley@arm.com } else { 90110259SAndrew.Bardsley@arm.com /* Don't step on to the next store buffer entry if this 90210259SAndrew.Bardsley@arm.com * one hasn't issued all its packets as the store 90310259SAndrew.Bardsley@arm.com * buffer must still enforce ordering */ 90410259SAndrew.Bardsley@arm.com issued = false; 90510259SAndrew.Bardsley@arm.com } 90610259SAndrew.Bardsley@arm.com } 90710259SAndrew.Bardsley@arm.com i++; 90810259SAndrew.Bardsley@arm.com } 90910259SAndrew.Bardsley@arm.com } 91010259SAndrew.Bardsley@arm.com} 91110259SAndrew.Bardsley@arm.com 91210259SAndrew.Bardsley@arm.comvoid 91310259SAndrew.Bardsley@arm.comLSQ::completeMemBarrierInst(MinorDynInstPtr inst, 91410259SAndrew.Bardsley@arm.com bool committed) 91510259SAndrew.Bardsley@arm.com{ 91610259SAndrew.Bardsley@arm.com if (committed) { 91710259SAndrew.Bardsley@arm.com /* Not already sent to the store buffer as a store request? */ 91810259SAndrew.Bardsley@arm.com if (!inst->inStoreBuffer) { 91910259SAndrew.Bardsley@arm.com /* Insert an entry into the store buffer to tick off barriers 92010259SAndrew.Bardsley@arm.com * until there are none in flight */ 92110259SAndrew.Bardsley@arm.com storeBuffer.insert(new BarrierDataRequest(*this, inst)); 92210259SAndrew.Bardsley@arm.com } 92310259SAndrew.Bardsley@arm.com } else { 92410259SAndrew.Bardsley@arm.com /* Clear the barrier anyway if it wasn't actually committed */ 92510259SAndrew.Bardsley@arm.com clearMemBarrier(inst); 92610259SAndrew.Bardsley@arm.com } 92710259SAndrew.Bardsley@arm.com} 92810259SAndrew.Bardsley@arm.com 92910259SAndrew.Bardsley@arm.comvoid 93010259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::minorTrace() const 93110259SAndrew.Bardsley@arm.com{ 93210259SAndrew.Bardsley@arm.com unsigned int size = slots.size(); 93310259SAndrew.Bardsley@arm.com unsigned int i = 0; 93410259SAndrew.Bardsley@arm.com std::ostringstream os; 93510259SAndrew.Bardsley@arm.com 93610259SAndrew.Bardsley@arm.com while (i < size) { 93710259SAndrew.Bardsley@arm.com LSQRequestPtr request = slots[i]; 93810259SAndrew.Bardsley@arm.com 93910259SAndrew.Bardsley@arm.com request->reportData(os); 94010259SAndrew.Bardsley@arm.com 94110259SAndrew.Bardsley@arm.com i++; 94210259SAndrew.Bardsley@arm.com if (i < numSlots) 94310259SAndrew.Bardsley@arm.com os << ','; 94410259SAndrew.Bardsley@arm.com } 94510259SAndrew.Bardsley@arm.com 94610259SAndrew.Bardsley@arm.com while (i < numSlots) { 94710259SAndrew.Bardsley@arm.com os << '-'; 94810259SAndrew.Bardsley@arm.com 94910259SAndrew.Bardsley@arm.com i++; 95010259SAndrew.Bardsley@arm.com if (i < numSlots) 95110259SAndrew.Bardsley@arm.com os << ','; 95210259SAndrew.Bardsley@arm.com } 95310259SAndrew.Bardsley@arm.com 95410259SAndrew.Bardsley@arm.com MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(), 95510259SAndrew.Bardsley@arm.com numUnissuedAccesses); 95610259SAndrew.Bardsley@arm.com} 95710259SAndrew.Bardsley@arm.com 95810259SAndrew.Bardsley@arm.comvoid 95910259SAndrew.Bardsley@arm.comLSQ::tryToSendToTransfers(LSQRequestPtr request) 96010259SAndrew.Bardsley@arm.com{ 96110259SAndrew.Bardsley@arm.com if (state == MemoryNeedsRetry) { 96210259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Request needs retry, not issuing to" 96310259SAndrew.Bardsley@arm.com " memory until retry arrives\n"); 96410259SAndrew.Bardsley@arm.com return; 96510259SAndrew.Bardsley@arm.com } 96610259SAndrew.Bardsley@arm.com 96710259SAndrew.Bardsley@arm.com if (request->state == LSQRequest::InTranslation) { 96810259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Request still in translation, not issuing to" 96910259SAndrew.Bardsley@arm.com " memory\n"); 97010259SAndrew.Bardsley@arm.com return; 97110259SAndrew.Bardsley@arm.com } 97210259SAndrew.Bardsley@arm.com 97310259SAndrew.Bardsley@arm.com assert(request->state == LSQRequest::Translated || 97410259SAndrew.Bardsley@arm.com request->state == LSQRequest::RequestIssuing || 97510259SAndrew.Bardsley@arm.com request->state == LSQRequest::Failed || 97610259SAndrew.Bardsley@arm.com request->state == LSQRequest::Complete); 97710259SAndrew.Bardsley@arm.com 97810259SAndrew.Bardsley@arm.com if (requests.empty() || requests.front() != request) { 97910259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Request not at front of requests queue, can't" 98010259SAndrew.Bardsley@arm.com " issue to memory\n"); 98110259SAndrew.Bardsley@arm.com return; 98210259SAndrew.Bardsley@arm.com } 98310259SAndrew.Bardsley@arm.com 98410259SAndrew.Bardsley@arm.com if (transfers.unreservedRemainingSpace() == 0) { 98510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "No space to insert request into transfers" 98610259SAndrew.Bardsley@arm.com " queue\n"); 98710259SAndrew.Bardsley@arm.com return; 98810259SAndrew.Bardsley@arm.com } 98910259SAndrew.Bardsley@arm.com 99010259SAndrew.Bardsley@arm.com if (request->isComplete() || request->state == LSQRequest::Failed) { 99110259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Passing a %s transfer on to transfers" 99210259SAndrew.Bardsley@arm.com " queue\n", (request->isComplete() ? "completed" : "failed")); 99310259SAndrew.Bardsley@arm.com request->setState(LSQRequest::Complete); 99410259SAndrew.Bardsley@arm.com request->setSkipped(); 99510259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 99610259SAndrew.Bardsley@arm.com return; 99710259SAndrew.Bardsley@arm.com } 99810259SAndrew.Bardsley@arm.com 99910259SAndrew.Bardsley@arm.com if (!execute.instIsRightStream(request->inst)) { 100010259SAndrew.Bardsley@arm.com /* Wrong stream, try to abort the transfer but only do so if 100110259SAndrew.Bardsley@arm.com * there are no packets in flight */ 100210259SAndrew.Bardsley@arm.com if (request->hasPacketsInMemSystem()) { 100310259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 100410259SAndrew.Bardsley@arm.com " waiting for responses before aborting request\n"); 100510259SAndrew.Bardsley@arm.com } else { 100610259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 100710259SAndrew.Bardsley@arm.com " aborting request\n"); 100810259SAndrew.Bardsley@arm.com request->setState(LSQRequest::Complete); 100910259SAndrew.Bardsley@arm.com request->setSkipped(); 101010259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 101110259SAndrew.Bardsley@arm.com } 101210259SAndrew.Bardsley@arm.com return; 101310259SAndrew.Bardsley@arm.com } 101410259SAndrew.Bardsley@arm.com 101514105Sgabor.dozsa@arm.com if (request->inst->translationFault != NoFault) { 101610259SAndrew.Bardsley@arm.com if (request->inst->staticInst->isPrefetch()) { 101710259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n"); 101810259SAndrew.Bardsley@arm.com } 101910259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Moving faulting request into the transfers" 102010259SAndrew.Bardsley@arm.com " queue\n"); 102110259SAndrew.Bardsley@arm.com request->setState(LSQRequest::Complete); 102210259SAndrew.Bardsley@arm.com request->setSkipped(); 102310259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 102410259SAndrew.Bardsley@arm.com return; 102510259SAndrew.Bardsley@arm.com } 102610259SAndrew.Bardsley@arm.com 102710259SAndrew.Bardsley@arm.com bool is_load = request->isLoad; 102812749Sgiacomo.travaglini@arm.com bool is_llsc = request->request->isLLSC(); 102912749Sgiacomo.travaglini@arm.com bool is_swap = request->request->isSwap(); 103013652Sqtt2@cornell.edu bool is_atomic = request->request->isAtomic(); 103112749Sgiacomo.travaglini@arm.com bool bufferable = !(request->request->isStrictlyOrdered() || 103213652Sqtt2@cornell.edu is_llsc || is_swap || is_atomic); 103310259SAndrew.Bardsley@arm.com 103410259SAndrew.Bardsley@arm.com if (is_load) { 103510259SAndrew.Bardsley@arm.com if (numStoresInTransfers != 0) { 103610259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Load request with stores still in transfers" 103710259SAndrew.Bardsley@arm.com " queue, stalling\n"); 103810259SAndrew.Bardsley@arm.com return; 103910259SAndrew.Bardsley@arm.com } 104010259SAndrew.Bardsley@arm.com } else { 104110259SAndrew.Bardsley@arm.com /* Store. Can it be sent to the store buffer? */ 104212749Sgiacomo.travaglini@arm.com if (bufferable && !request->request->isMmappedIpr()) { 104310259SAndrew.Bardsley@arm.com request->setState(LSQRequest::StoreToStoreBuffer); 104410259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 104510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Moving store into transfers queue\n"); 104610259SAndrew.Bardsley@arm.com return; 104710259SAndrew.Bardsley@arm.com } 104810259SAndrew.Bardsley@arm.com } 104910259SAndrew.Bardsley@arm.com 105010259SAndrew.Bardsley@arm.com /* Check if this is the head instruction (and so must be executable as 105110259SAndrew.Bardsley@arm.com * its stream sequence number was checked above) for loads which must 105210259SAndrew.Bardsley@arm.com * not be speculatively issued and stores which must be issued here */ 105310259SAndrew.Bardsley@arm.com if (!bufferable) { 105410259SAndrew.Bardsley@arm.com if (!execute.instIsHeadInst(request->inst)) { 105510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Memory access not the head inst., can't be" 105610259SAndrew.Bardsley@arm.com " sure it can be performed, not issuing\n"); 105710259SAndrew.Bardsley@arm.com return; 105810259SAndrew.Bardsley@arm.com } 105910259SAndrew.Bardsley@arm.com 106010259SAndrew.Bardsley@arm.com unsigned int forwarding_slot = 0; 106110259SAndrew.Bardsley@arm.com 106210259SAndrew.Bardsley@arm.com if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) != 106310259SAndrew.Bardsley@arm.com NoAddrRangeCoverage) 106410259SAndrew.Bardsley@arm.com { 106513652Sqtt2@cornell.edu // There's at least another request that targets the same 106613652Sqtt2@cornell.edu // address and is staying in the storeBuffer. Since our 106713652Sqtt2@cornell.edu // request is non-bufferable (e.g., strictly ordered or atomic), 106813652Sqtt2@cornell.edu // we must wait for the other request in the storeBuffer to 106913652Sqtt2@cornell.edu // complete before we can issue this non-bufferable request. 107013652Sqtt2@cornell.edu // This is to make sure that the order they access the cache is 107113652Sqtt2@cornell.edu // correct. 107210259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Memory access can receive forwarded data" 107313652Sqtt2@cornell.edu " from the store buffer, but need to wait for store buffer" 107413652Sqtt2@cornell.edu " to drain\n"); 107510259SAndrew.Bardsley@arm.com return; 107610259SAndrew.Bardsley@arm.com } 107710259SAndrew.Bardsley@arm.com } 107810259SAndrew.Bardsley@arm.com 107910259SAndrew.Bardsley@arm.com /* True: submit this packet to the transfers queue to be sent to the 108010259SAndrew.Bardsley@arm.com * memory system. 108110259SAndrew.Bardsley@arm.com * False: skip the memory and push a packet for this request onto 108210259SAndrew.Bardsley@arm.com * requests */ 108310259SAndrew.Bardsley@arm.com bool do_access = true; 108410259SAndrew.Bardsley@arm.com 108510259SAndrew.Bardsley@arm.com if (!is_llsc) { 108610259SAndrew.Bardsley@arm.com /* Check for match in the store buffer */ 108710259SAndrew.Bardsley@arm.com if (is_load) { 108810259SAndrew.Bardsley@arm.com unsigned int forwarding_slot = 0; 108910259SAndrew.Bardsley@arm.com AddrRangeCoverage forwarding_result = 109010259SAndrew.Bardsley@arm.com storeBuffer.canForwardDataToLoad(request, 109110259SAndrew.Bardsley@arm.com forwarding_slot); 109210259SAndrew.Bardsley@arm.com 109310259SAndrew.Bardsley@arm.com switch (forwarding_result) { 109410259SAndrew.Bardsley@arm.com case FullAddrRangeCoverage: 109510259SAndrew.Bardsley@arm.com /* Forward data from the store buffer into this request and 109610259SAndrew.Bardsley@arm.com * repurpose this request's packet into a response packet */ 109710259SAndrew.Bardsley@arm.com storeBuffer.forwardStoreData(request, forwarding_slot); 109810259SAndrew.Bardsley@arm.com request->packet->makeResponse(); 109910259SAndrew.Bardsley@arm.com 110010259SAndrew.Bardsley@arm.com /* Just move between queues, no access */ 110110259SAndrew.Bardsley@arm.com do_access = false; 110210259SAndrew.Bardsley@arm.com break; 110310259SAndrew.Bardsley@arm.com case PartialAddrRangeCoverage: 110410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Load partly satisfied by store buffer" 110510259SAndrew.Bardsley@arm.com " data. Must wait for the store to complete\n"); 110610259SAndrew.Bardsley@arm.com return; 110710259SAndrew.Bardsley@arm.com break; 110810259SAndrew.Bardsley@arm.com case NoAddrRangeCoverage: 110910259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "No forwardable data from store buffer\n"); 111010259SAndrew.Bardsley@arm.com /* Fall through to try access */ 111110259SAndrew.Bardsley@arm.com break; 111210259SAndrew.Bardsley@arm.com } 111310259SAndrew.Bardsley@arm.com } 111410259SAndrew.Bardsley@arm.com } else { 111510259SAndrew.Bardsley@arm.com if (!canSendToMemorySystem()) { 111610259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 111710259SAndrew.Bardsley@arm.com return; 111810259SAndrew.Bardsley@arm.com } 111910259SAndrew.Bardsley@arm.com 112010259SAndrew.Bardsley@arm.com SimpleThread &thread = *cpu.threads[request->inst->id.threadId]; 112110259SAndrew.Bardsley@arm.com 112210259SAndrew.Bardsley@arm.com TheISA::PCState old_pc = thread.pcState(); 112310259SAndrew.Bardsley@arm.com ExecContext context(cpu, thread, execute, request->inst); 112410259SAndrew.Bardsley@arm.com 112510259SAndrew.Bardsley@arm.com /* Handle LLSC requests and tests */ 112610259SAndrew.Bardsley@arm.com if (is_load) { 112712749Sgiacomo.travaglini@arm.com TheISA::handleLockedRead(&context, request->request); 112810259SAndrew.Bardsley@arm.com } else { 112910259SAndrew.Bardsley@arm.com do_access = TheISA::handleLockedWrite(&context, 113012749Sgiacomo.travaglini@arm.com request->request, cacheBlockMask); 113110259SAndrew.Bardsley@arm.com 113210259SAndrew.Bardsley@arm.com if (!do_access) { 113310259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Not perfoming a memory " 113410259SAndrew.Bardsley@arm.com "access for store conditional\n"); 113510259SAndrew.Bardsley@arm.com } 113610259SAndrew.Bardsley@arm.com } 113710259SAndrew.Bardsley@arm.com thread.pcState(old_pc); 113810259SAndrew.Bardsley@arm.com } 113910259SAndrew.Bardsley@arm.com 114010259SAndrew.Bardsley@arm.com /* See the do_access comment above */ 114110259SAndrew.Bardsley@arm.com if (do_access) { 114210259SAndrew.Bardsley@arm.com if (!canSendToMemorySystem()) { 114310259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 114410259SAndrew.Bardsley@arm.com return; 114510259SAndrew.Bardsley@arm.com } 114610259SAndrew.Bardsley@arm.com 114710259SAndrew.Bardsley@arm.com /* Remember if this is an access which can't be idly 114810259SAndrew.Bardsley@arm.com * discarded by an interrupt */ 114910368SAndrew.Bardsley@arm.com if (!bufferable && !request->issuedToMemory) { 115010259SAndrew.Bardsley@arm.com numAccessesIssuedToMemory++; 115110259SAndrew.Bardsley@arm.com request->issuedToMemory = true; 115210259SAndrew.Bardsley@arm.com } 115310259SAndrew.Bardsley@arm.com 115411567Smitch.hayenga@arm.com if (tryToSend(request)) { 115510259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 115611567Smitch.hayenga@arm.com } 115710259SAndrew.Bardsley@arm.com } else { 115810259SAndrew.Bardsley@arm.com request->setState(LSQRequest::Complete); 115910259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(request); 116010259SAndrew.Bardsley@arm.com } 116110259SAndrew.Bardsley@arm.com} 116210259SAndrew.Bardsley@arm.com 116310259SAndrew.Bardsley@arm.combool 116410259SAndrew.Bardsley@arm.comLSQ::tryToSend(LSQRequestPtr request) 116510259SAndrew.Bardsley@arm.com{ 116610259SAndrew.Bardsley@arm.com bool ret = false; 116710259SAndrew.Bardsley@arm.com 116810259SAndrew.Bardsley@arm.com if (!canSendToMemorySystem()) { 116910259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n", 117010259SAndrew.Bardsley@arm.com *(request->inst)); 117110259SAndrew.Bardsley@arm.com } else { 117210259SAndrew.Bardsley@arm.com PacketPtr packet = request->getHeadPacket(); 117310259SAndrew.Bardsley@arm.com 117410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n", 117510259SAndrew.Bardsley@arm.com *(request->inst), packet->req->getVaddr()); 117610259SAndrew.Bardsley@arm.com 117710259SAndrew.Bardsley@arm.com /* The sender state of the packet *must* be an LSQRequest 117810259SAndrew.Bardsley@arm.com * so the response can be correctly handled */ 117910259SAndrew.Bardsley@arm.com assert(packet->findNextSenderState<LSQRequest>()); 118010259SAndrew.Bardsley@arm.com 118112749Sgiacomo.travaglini@arm.com if (request->request->isMmappedIpr()) { 118210259SAndrew.Bardsley@arm.com ThreadContext *thread = 118311435Smitch.hayenga@arm.com cpu.getContext(cpu.contextToThread( 118412749Sgiacomo.travaglini@arm.com request->request->contextId())); 118510259SAndrew.Bardsley@arm.com 118610259SAndrew.Bardsley@arm.com if (request->isLoad) { 118710259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst)); 118810259SAndrew.Bardsley@arm.com TheISA::handleIprRead(thread, packet); 118910259SAndrew.Bardsley@arm.com } else { 119010259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst)); 119110259SAndrew.Bardsley@arm.com TheISA::handleIprWrite(thread, packet); 119210259SAndrew.Bardsley@arm.com } 119310259SAndrew.Bardsley@arm.com 119410259SAndrew.Bardsley@arm.com request->stepToNextPacket(); 119510259SAndrew.Bardsley@arm.com ret = request->sentAllPackets(); 119610259SAndrew.Bardsley@arm.com 119710259SAndrew.Bardsley@arm.com if (!ret) { 119810259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "IPR access has another packet: %s\n", 119910259SAndrew.Bardsley@arm.com *(request->inst)); 120010259SAndrew.Bardsley@arm.com } 120110259SAndrew.Bardsley@arm.com 120210259SAndrew.Bardsley@arm.com if (ret) 120310259SAndrew.Bardsley@arm.com request->setState(LSQRequest::Complete); 120410259SAndrew.Bardsley@arm.com else 120510259SAndrew.Bardsley@arm.com request->setState(LSQRequest::RequestIssuing); 120610259SAndrew.Bardsley@arm.com } else if (dcachePort.sendTimingReq(packet)) { 120710259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Sent data memory request\n"); 120810259SAndrew.Bardsley@arm.com 120910259SAndrew.Bardsley@arm.com numAccessesInMemorySystem++; 121010259SAndrew.Bardsley@arm.com 121110259SAndrew.Bardsley@arm.com request->stepToNextPacket(); 121210259SAndrew.Bardsley@arm.com 121310259SAndrew.Bardsley@arm.com ret = request->sentAllPackets(); 121410259SAndrew.Bardsley@arm.com 121510259SAndrew.Bardsley@arm.com switch (request->state) { 121610259SAndrew.Bardsley@arm.com case LSQRequest::Translated: 121710259SAndrew.Bardsley@arm.com case LSQRequest::RequestIssuing: 121810259SAndrew.Bardsley@arm.com /* Fully or partially issued a request in the transfers 121910259SAndrew.Bardsley@arm.com * queue */ 122010259SAndrew.Bardsley@arm.com request->setState(LSQRequest::RequestIssuing); 122110259SAndrew.Bardsley@arm.com break; 122210259SAndrew.Bardsley@arm.com case LSQRequest::StoreInStoreBuffer: 122310259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferIssuing: 122410259SAndrew.Bardsley@arm.com /* Fully or partially issued a request in the store 122510259SAndrew.Bardsley@arm.com * buffer */ 122610259SAndrew.Bardsley@arm.com request->setState(LSQRequest::StoreBufferIssuing); 122710259SAndrew.Bardsley@arm.com break; 122810259SAndrew.Bardsley@arm.com default: 122913449Sgabeblack@google.com panic("Unrecognized LSQ request state %d.", request->state); 123010259SAndrew.Bardsley@arm.com } 123110259SAndrew.Bardsley@arm.com 123210259SAndrew.Bardsley@arm.com state = MemoryRunning; 123310259SAndrew.Bardsley@arm.com } else { 123410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, 123510259SAndrew.Bardsley@arm.com "Sending data memory request - needs retry\n"); 123610259SAndrew.Bardsley@arm.com 123710259SAndrew.Bardsley@arm.com /* Needs to be resent, wait for that */ 123810259SAndrew.Bardsley@arm.com state = MemoryNeedsRetry; 123910259SAndrew.Bardsley@arm.com retryRequest = request; 124010259SAndrew.Bardsley@arm.com 124110259SAndrew.Bardsley@arm.com switch (request->state) { 124210259SAndrew.Bardsley@arm.com case LSQRequest::Translated: 124310259SAndrew.Bardsley@arm.com case LSQRequest::RequestIssuing: 124410259SAndrew.Bardsley@arm.com request->setState(LSQRequest::RequestNeedsRetry); 124510259SAndrew.Bardsley@arm.com break; 124610259SAndrew.Bardsley@arm.com case LSQRequest::StoreInStoreBuffer: 124710259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferIssuing: 124810259SAndrew.Bardsley@arm.com request->setState(LSQRequest::StoreBufferNeedsRetry); 124910259SAndrew.Bardsley@arm.com break; 125010259SAndrew.Bardsley@arm.com default: 125113449Sgabeblack@google.com panic("Unrecognized LSQ request state %d.", request->state); 125210259SAndrew.Bardsley@arm.com } 125310259SAndrew.Bardsley@arm.com } 125410259SAndrew.Bardsley@arm.com } 125510259SAndrew.Bardsley@arm.com 125611567Smitch.hayenga@arm.com if (ret) 125711567Smitch.hayenga@arm.com threadSnoop(request); 125811567Smitch.hayenga@arm.com 125910259SAndrew.Bardsley@arm.com return ret; 126010259SAndrew.Bardsley@arm.com} 126110259SAndrew.Bardsley@arm.com 126210259SAndrew.Bardsley@arm.comvoid 126310259SAndrew.Bardsley@arm.comLSQ::moveFromRequestsToTransfers(LSQRequestPtr request) 126410259SAndrew.Bardsley@arm.com{ 126510259SAndrew.Bardsley@arm.com assert(!requests.empty() && requests.front() == request); 126610259SAndrew.Bardsley@arm.com assert(transfers.unreservedRemainingSpace() != 0); 126710259SAndrew.Bardsley@arm.com 126810259SAndrew.Bardsley@arm.com /* Need to count the number of stores in the transfers 126910259SAndrew.Bardsley@arm.com * queue so that loads know when their store buffer forwarding 127010259SAndrew.Bardsley@arm.com * results will be correct (only when all those stores 127110259SAndrew.Bardsley@arm.com * have reached the store buffer) */ 127210259SAndrew.Bardsley@arm.com if (!request->isLoad) 127310259SAndrew.Bardsley@arm.com numStoresInTransfers++; 127410259SAndrew.Bardsley@arm.com 127510259SAndrew.Bardsley@arm.com requests.pop(); 127610259SAndrew.Bardsley@arm.com transfers.push(request); 127710259SAndrew.Bardsley@arm.com} 127810259SAndrew.Bardsley@arm.com 127910259SAndrew.Bardsley@arm.combool 128010259SAndrew.Bardsley@arm.comLSQ::canSendToMemorySystem() 128110259SAndrew.Bardsley@arm.com{ 128210259SAndrew.Bardsley@arm.com return state == MemoryRunning && 128310259SAndrew.Bardsley@arm.com numAccessesInMemorySystem < inMemorySystemLimit; 128410259SAndrew.Bardsley@arm.com} 128510259SAndrew.Bardsley@arm.com 128610259SAndrew.Bardsley@arm.combool 128710259SAndrew.Bardsley@arm.comLSQ::recvTimingResp(PacketPtr response) 128810259SAndrew.Bardsley@arm.com{ 128910259SAndrew.Bardsley@arm.com LSQRequestPtr request = 129010259SAndrew.Bardsley@arm.com safe_cast<LSQRequestPtr>(response->popSenderState()); 129110259SAndrew.Bardsley@arm.com 129210259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Received response packet inst: %s" 129310259SAndrew.Bardsley@arm.com " addr: 0x%x cmd: %s\n", 129410259SAndrew.Bardsley@arm.com *(request->inst), response->getAddr(), 129510259SAndrew.Bardsley@arm.com response->cmd.toString()); 129610259SAndrew.Bardsley@arm.com 129710259SAndrew.Bardsley@arm.com numAccessesInMemorySystem--; 129810259SAndrew.Bardsley@arm.com 129910259SAndrew.Bardsley@arm.com if (response->isError()) { 130010259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Received error response packet: %s\n", 130110259SAndrew.Bardsley@arm.com *request->inst); 130210259SAndrew.Bardsley@arm.com } 130310259SAndrew.Bardsley@arm.com 130410259SAndrew.Bardsley@arm.com switch (request->state) { 130510259SAndrew.Bardsley@arm.com case LSQRequest::RequestIssuing: 130610259SAndrew.Bardsley@arm.com case LSQRequest::RequestNeedsRetry: 130710259SAndrew.Bardsley@arm.com /* Response to a request from the transfers queue */ 130810259SAndrew.Bardsley@arm.com request->retireResponse(response); 130910259SAndrew.Bardsley@arm.com 131010259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n", 131110259SAndrew.Bardsley@arm.com request->hasPacketsInMemSystem(), request->isComplete()); 131210259SAndrew.Bardsley@arm.com 131310259SAndrew.Bardsley@arm.com break; 131410259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferIssuing: 131510259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferNeedsRetry: 131610259SAndrew.Bardsley@arm.com /* Response to a request from the store buffer */ 131710259SAndrew.Bardsley@arm.com request->retireResponse(response); 131810259SAndrew.Bardsley@arm.com 131910581SAndrew.Bardsley@arm.com /* Remove completed requests unless they are barriers (which will 132010259SAndrew.Bardsley@arm.com * need to be removed in order */ 132110259SAndrew.Bardsley@arm.com if (request->isComplete()) { 132210259SAndrew.Bardsley@arm.com if (!request->isBarrier()) { 132310259SAndrew.Bardsley@arm.com storeBuffer.deleteRequest(request); 132410259SAndrew.Bardsley@arm.com } else { 132510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Completed transfer for barrier: %s" 132610259SAndrew.Bardsley@arm.com " leaving the request as it is also a barrier\n", 132710259SAndrew.Bardsley@arm.com *(request->inst)); 132810259SAndrew.Bardsley@arm.com } 132910259SAndrew.Bardsley@arm.com } 133010259SAndrew.Bardsley@arm.com break; 133110259SAndrew.Bardsley@arm.com default: 133213449Sgabeblack@google.com panic("Shouldn't be allowed to receive a response from another state"); 133310259SAndrew.Bardsley@arm.com } 133410259SAndrew.Bardsley@arm.com 133510259SAndrew.Bardsley@arm.com /* We go to idle even if there are more things in the requests queue 133610259SAndrew.Bardsley@arm.com * as it's the job of step to actually step us on to the next 133710259SAndrew.Bardsley@arm.com * transaction */ 133810259SAndrew.Bardsley@arm.com 133910259SAndrew.Bardsley@arm.com /* Let's try and wake up the processor for the next cycle */ 134010259SAndrew.Bardsley@arm.com cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 134110259SAndrew.Bardsley@arm.com 134210259SAndrew.Bardsley@arm.com /* Never busy */ 134310259SAndrew.Bardsley@arm.com return true; 134410259SAndrew.Bardsley@arm.com} 134510259SAndrew.Bardsley@arm.com 134610259SAndrew.Bardsley@arm.comvoid 134710713Sandreas.hansson@arm.comLSQ::recvReqRetry() 134810259SAndrew.Bardsley@arm.com{ 134910259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Received retry request\n"); 135010259SAndrew.Bardsley@arm.com 135110259SAndrew.Bardsley@arm.com assert(state == MemoryNeedsRetry); 135210259SAndrew.Bardsley@arm.com 135310259SAndrew.Bardsley@arm.com switch (retryRequest->state) { 135410259SAndrew.Bardsley@arm.com case LSQRequest::RequestNeedsRetry: 135510259SAndrew.Bardsley@arm.com /* Retry in the requests queue */ 135610259SAndrew.Bardsley@arm.com retryRequest->setState(LSQRequest::Translated); 135710259SAndrew.Bardsley@arm.com break; 135810259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferNeedsRetry: 135910259SAndrew.Bardsley@arm.com /* Retry in the store buffer */ 136010259SAndrew.Bardsley@arm.com retryRequest->setState(LSQRequest::StoreInStoreBuffer); 136110259SAndrew.Bardsley@arm.com break; 136210259SAndrew.Bardsley@arm.com default: 136313449Sgabeblack@google.com panic("Unrecognized retry request state %d.", retryRequest->state); 136410259SAndrew.Bardsley@arm.com } 136510259SAndrew.Bardsley@arm.com 136610259SAndrew.Bardsley@arm.com /* Set state back to MemoryRunning so that the following 136710259SAndrew.Bardsley@arm.com * tryToSend can actually send. Note that this won't 136810259SAndrew.Bardsley@arm.com * allow another transfer in as tryToSend should 136910259SAndrew.Bardsley@arm.com * issue a memory request and either succeed for this 137010259SAndrew.Bardsley@arm.com * request or return the LSQ back to MemoryNeedsRetry */ 137110259SAndrew.Bardsley@arm.com state = MemoryRunning; 137210259SAndrew.Bardsley@arm.com 137310259SAndrew.Bardsley@arm.com /* Try to resend the request */ 137410259SAndrew.Bardsley@arm.com if (tryToSend(retryRequest)) { 137510259SAndrew.Bardsley@arm.com /* Successfully sent, need to move the request */ 137610259SAndrew.Bardsley@arm.com switch (retryRequest->state) { 137710259SAndrew.Bardsley@arm.com case LSQRequest::RequestIssuing: 137810259SAndrew.Bardsley@arm.com /* In the requests queue */ 137910259SAndrew.Bardsley@arm.com moveFromRequestsToTransfers(retryRequest); 138010259SAndrew.Bardsley@arm.com break; 138110259SAndrew.Bardsley@arm.com case LSQRequest::StoreBufferIssuing: 138210259SAndrew.Bardsley@arm.com /* In the store buffer */ 138310581SAndrew.Bardsley@arm.com storeBuffer.countIssuedStore(retryRequest); 138410259SAndrew.Bardsley@arm.com break; 138510259SAndrew.Bardsley@arm.com default: 138613449Sgabeblack@google.com panic("Unrecognized retry request state %d.", retryRequest->state); 138710259SAndrew.Bardsley@arm.com } 138810647Sandreas.hansson@arm.com 138910647Sandreas.hansson@arm.com retryRequest = NULL; 139010259SAndrew.Bardsley@arm.com } 139110259SAndrew.Bardsley@arm.com} 139210259SAndrew.Bardsley@arm.com 139310259SAndrew.Bardsley@arm.comLSQ::LSQ(std::string name_, std::string dcache_port_name_, 139410259SAndrew.Bardsley@arm.com MinorCPU &cpu_, Execute &execute_, 139510259SAndrew.Bardsley@arm.com unsigned int in_memory_system_limit, unsigned int line_width, 139610259SAndrew.Bardsley@arm.com unsigned int requests_queue_size, unsigned int transfers_queue_size, 139710259SAndrew.Bardsley@arm.com unsigned int store_buffer_size, 139810259SAndrew.Bardsley@arm.com unsigned int store_buffer_cycle_store_limit) : 139910259SAndrew.Bardsley@arm.com Named(name_), 140010259SAndrew.Bardsley@arm.com cpu(cpu_), 140110259SAndrew.Bardsley@arm.com execute(execute_), 140210259SAndrew.Bardsley@arm.com dcachePort(dcache_port_name_, *this, cpu_), 140311567Smitch.hayenga@arm.com lastMemBarrier(cpu.numThreads, 0), 140410259SAndrew.Bardsley@arm.com state(MemoryRunning), 140510259SAndrew.Bardsley@arm.com inMemorySystemLimit(in_memory_system_limit), 140610259SAndrew.Bardsley@arm.com lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)), 140710259SAndrew.Bardsley@arm.com requests(name_ + ".requests", "addr", requests_queue_size), 140810259SAndrew.Bardsley@arm.com transfers(name_ + ".transfers", "addr", transfers_queue_size), 140910259SAndrew.Bardsley@arm.com storeBuffer(name_ + ".storeBuffer", 141010259SAndrew.Bardsley@arm.com *this, store_buffer_size, store_buffer_cycle_store_limit), 141110259SAndrew.Bardsley@arm.com numAccessesInMemorySystem(0), 141210259SAndrew.Bardsley@arm.com numAccessesInDTLB(0), 141310259SAndrew.Bardsley@arm.com numStoresInTransfers(0), 141410259SAndrew.Bardsley@arm.com numAccessesIssuedToMemory(0), 141510259SAndrew.Bardsley@arm.com retryRequest(NULL), 141610259SAndrew.Bardsley@arm.com cacheBlockMask(~(cpu_.cacheLineSize() - 1)) 141710259SAndrew.Bardsley@arm.com{ 141810259SAndrew.Bardsley@arm.com if (in_memory_system_limit < 1) { 141910259SAndrew.Bardsley@arm.com fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_, 142010259SAndrew.Bardsley@arm.com in_memory_system_limit); 142110259SAndrew.Bardsley@arm.com } 142210259SAndrew.Bardsley@arm.com 142310259SAndrew.Bardsley@arm.com if (store_buffer_cycle_store_limit < 1) { 142410259SAndrew.Bardsley@arm.com fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be" 142510259SAndrew.Bardsley@arm.com " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit); 142610259SAndrew.Bardsley@arm.com } 142710259SAndrew.Bardsley@arm.com 142810259SAndrew.Bardsley@arm.com if (requests_queue_size < 1) { 142910259SAndrew.Bardsley@arm.com fatal("%s: executeLSQRequestsQueueSize must be" 143010259SAndrew.Bardsley@arm.com " >= 1 (%d)\n", name_, requests_queue_size); 143110259SAndrew.Bardsley@arm.com } 143210259SAndrew.Bardsley@arm.com 143310259SAndrew.Bardsley@arm.com if (transfers_queue_size < 1) { 143410259SAndrew.Bardsley@arm.com fatal("%s: executeLSQTransfersQueueSize must be" 143510259SAndrew.Bardsley@arm.com " >= 1 (%d)\n", name_, transfers_queue_size); 143610259SAndrew.Bardsley@arm.com } 143710259SAndrew.Bardsley@arm.com 143810259SAndrew.Bardsley@arm.com if (store_buffer_size < 1) { 143910259SAndrew.Bardsley@arm.com fatal("%s: executeLSQStoreBufferSize must be" 144010259SAndrew.Bardsley@arm.com " >= 1 (%d)\n", name_, store_buffer_size); 144110259SAndrew.Bardsley@arm.com } 144210259SAndrew.Bardsley@arm.com 144310259SAndrew.Bardsley@arm.com if ((lineWidth & (lineWidth - 1)) != 0) { 144410259SAndrew.Bardsley@arm.com fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth); 144510259SAndrew.Bardsley@arm.com } 144610259SAndrew.Bardsley@arm.com} 144710259SAndrew.Bardsley@arm.com 144810259SAndrew.Bardsley@arm.comLSQ::~LSQ() 144910259SAndrew.Bardsley@arm.com{ } 145010259SAndrew.Bardsley@arm.com 145110259SAndrew.Bardsley@arm.comLSQ::LSQRequest::~LSQRequest() 145210259SAndrew.Bardsley@arm.com{ 145310259SAndrew.Bardsley@arm.com if (packet) 145410259SAndrew.Bardsley@arm.com delete packet; 145510259SAndrew.Bardsley@arm.com if (data) 145610259SAndrew.Bardsley@arm.com delete [] data; 145710259SAndrew.Bardsley@arm.com} 145810259SAndrew.Bardsley@arm.com 145910259SAndrew.Bardsley@arm.com/** 146010259SAndrew.Bardsley@arm.com * Step the memory access mechanism on to its next state. In reality, most 146110259SAndrew.Bardsley@arm.com * of the stepping is done by the callbacks on the LSQ but this 146210259SAndrew.Bardsley@arm.com * function is responsible for issuing memory requests lodged in the 146310259SAndrew.Bardsley@arm.com * requests queue. 146410259SAndrew.Bardsley@arm.com */ 146510259SAndrew.Bardsley@arm.comvoid 146610259SAndrew.Bardsley@arm.comLSQ::step() 146710259SAndrew.Bardsley@arm.com{ 146810259SAndrew.Bardsley@arm.com /* Try to move address-translated requests between queues and issue 146910259SAndrew.Bardsley@arm.com * them */ 147010259SAndrew.Bardsley@arm.com if (!requests.empty()) 147110259SAndrew.Bardsley@arm.com tryToSendToTransfers(requests.front()); 147210259SAndrew.Bardsley@arm.com 147310259SAndrew.Bardsley@arm.com storeBuffer.step(); 147410259SAndrew.Bardsley@arm.com} 147510259SAndrew.Bardsley@arm.com 147610259SAndrew.Bardsley@arm.comLSQ::LSQRequestPtr 147710259SAndrew.Bardsley@arm.comLSQ::findResponse(MinorDynInstPtr inst) 147810259SAndrew.Bardsley@arm.com{ 147910259SAndrew.Bardsley@arm.com LSQ::LSQRequestPtr ret = NULL; 148010259SAndrew.Bardsley@arm.com 148110259SAndrew.Bardsley@arm.com if (!transfers.empty()) { 148210259SAndrew.Bardsley@arm.com LSQRequestPtr request = transfers.front(); 148310259SAndrew.Bardsley@arm.com 148410259SAndrew.Bardsley@arm.com /* Same instruction and complete access or a store that's 148510259SAndrew.Bardsley@arm.com * capable of being moved to the store buffer */ 148610259SAndrew.Bardsley@arm.com if (request->inst->id == inst->id) { 148710504SAndrew.Bardsley@arm.com bool complete = request->isComplete(); 148810504SAndrew.Bardsley@arm.com bool can_store = storeBuffer.canInsert(); 148910504SAndrew.Bardsley@arm.com bool to_store_buffer = request->state == 149010504SAndrew.Bardsley@arm.com LSQRequest::StoreToStoreBuffer; 149110504SAndrew.Bardsley@arm.com 149210504SAndrew.Bardsley@arm.com if ((complete && !(request->isBarrier() && !can_store)) || 149310504SAndrew.Bardsley@arm.com (to_store_buffer && can_store)) 149410259SAndrew.Bardsley@arm.com { 149510259SAndrew.Bardsley@arm.com ret = request; 149610259SAndrew.Bardsley@arm.com } 149710259SAndrew.Bardsley@arm.com } 149810259SAndrew.Bardsley@arm.com } 149910259SAndrew.Bardsley@arm.com 150010259SAndrew.Bardsley@arm.com if (ret) { 150110259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Found matching memory response for inst: %s\n", 150210259SAndrew.Bardsley@arm.com *inst); 150310259SAndrew.Bardsley@arm.com } else { 150410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "No matching memory response for inst: %s\n", 150510259SAndrew.Bardsley@arm.com *inst); 150610259SAndrew.Bardsley@arm.com } 150710259SAndrew.Bardsley@arm.com 150810259SAndrew.Bardsley@arm.com return ret; 150910259SAndrew.Bardsley@arm.com} 151010259SAndrew.Bardsley@arm.com 151110259SAndrew.Bardsley@arm.comvoid 151210259SAndrew.Bardsley@arm.comLSQ::popResponse(LSQ::LSQRequestPtr response) 151310259SAndrew.Bardsley@arm.com{ 151410259SAndrew.Bardsley@arm.com assert(!transfers.empty() && transfers.front() == response); 151510259SAndrew.Bardsley@arm.com 151610259SAndrew.Bardsley@arm.com transfers.pop(); 151710259SAndrew.Bardsley@arm.com 151810259SAndrew.Bardsley@arm.com if (!response->isLoad) 151910259SAndrew.Bardsley@arm.com numStoresInTransfers--; 152010259SAndrew.Bardsley@arm.com 152110259SAndrew.Bardsley@arm.com if (response->issuedToMemory) 152210259SAndrew.Bardsley@arm.com numAccessesIssuedToMemory--; 152310259SAndrew.Bardsley@arm.com 152410259SAndrew.Bardsley@arm.com if (response->state != LSQRequest::StoreInStoreBuffer) { 152510259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Deleting %s request: %s\n", 152610259SAndrew.Bardsley@arm.com (response->isLoad ? "load" : "store"), 152710259SAndrew.Bardsley@arm.com *(response->inst)); 152810259SAndrew.Bardsley@arm.com 152910259SAndrew.Bardsley@arm.com delete response; 153010259SAndrew.Bardsley@arm.com } 153110259SAndrew.Bardsley@arm.com} 153210259SAndrew.Bardsley@arm.com 153310259SAndrew.Bardsley@arm.comvoid 153410259SAndrew.Bardsley@arm.comLSQ::sendStoreToStoreBuffer(LSQRequestPtr request) 153510259SAndrew.Bardsley@arm.com{ 153610259SAndrew.Bardsley@arm.com assert(request->state == LSQRequest::StoreToStoreBuffer); 153710259SAndrew.Bardsley@arm.com 153810259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Sending store: %s to store buffer\n", 153910259SAndrew.Bardsley@arm.com *(request->inst)); 154010259SAndrew.Bardsley@arm.com 154110259SAndrew.Bardsley@arm.com request->inst->inStoreBuffer = true; 154210259SAndrew.Bardsley@arm.com 154310259SAndrew.Bardsley@arm.com storeBuffer.insert(request); 154410259SAndrew.Bardsley@arm.com} 154510259SAndrew.Bardsley@arm.com 154610259SAndrew.Bardsley@arm.combool 154710259SAndrew.Bardsley@arm.comLSQ::isDrained() 154810259SAndrew.Bardsley@arm.com{ 154910259SAndrew.Bardsley@arm.com return requests.empty() && transfers.empty() && 155010259SAndrew.Bardsley@arm.com storeBuffer.isDrained(); 155110259SAndrew.Bardsley@arm.com} 155210259SAndrew.Bardsley@arm.com 155310259SAndrew.Bardsley@arm.combool 155410259SAndrew.Bardsley@arm.comLSQ::needsToTick() 155510259SAndrew.Bardsley@arm.com{ 155610259SAndrew.Bardsley@arm.com bool ret = false; 155710259SAndrew.Bardsley@arm.com 155810259SAndrew.Bardsley@arm.com if (canSendToMemorySystem()) { 155910259SAndrew.Bardsley@arm.com bool have_translated_requests = !requests.empty() && 156010259SAndrew.Bardsley@arm.com requests.front()->state != LSQRequest::InTranslation && 156110259SAndrew.Bardsley@arm.com transfers.unreservedRemainingSpace() != 0; 156210259SAndrew.Bardsley@arm.com 156310259SAndrew.Bardsley@arm.com ret = have_translated_requests || 156410259SAndrew.Bardsley@arm.com storeBuffer.numUnissuedStores() != 0; 156510259SAndrew.Bardsley@arm.com } 156610259SAndrew.Bardsley@arm.com 156710259SAndrew.Bardsley@arm.com if (ret) 156810259SAndrew.Bardsley@arm.com DPRINTF(Activity, "Need to tick\n"); 156910259SAndrew.Bardsley@arm.com 157010259SAndrew.Bardsley@arm.com return ret; 157110259SAndrew.Bardsley@arm.com} 157210259SAndrew.Bardsley@arm.com 157314105Sgabor.dozsa@arm.comFault 157410259SAndrew.Bardsley@arm.comLSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data, 157511608Snikos.nikoleris@arm.com unsigned int size, Addr addr, Request::Flags flags, 157614297Sjordi.vaquero@metempsy.com uint64_t *res, AtomicOpFunctorPtr amo_op, 157713954Sgiacomo.gabrielli@arm.com const std::vector<bool>& byteEnable) 157810259SAndrew.Bardsley@arm.com{ 157914105Sgabor.dozsa@arm.com assert(inst->translationFault == NoFault || inst->inLSQ); 158014105Sgabor.dozsa@arm.com 158114105Sgabor.dozsa@arm.com if (inst->inLSQ) { 158214105Sgabor.dozsa@arm.com return inst->translationFault; 158314105Sgabor.dozsa@arm.com } 158414105Sgabor.dozsa@arm.com 158510259SAndrew.Bardsley@arm.com bool needs_burst = transferNeedsBurst(addr, size, lineWidth); 158613652Sqtt2@cornell.edu 158713652Sqtt2@cornell.edu if (needs_burst && inst->staticInst->isAtomic()) { 158813652Sqtt2@cornell.edu // AMO requests that access across a cache line boundary are not 158913652Sqtt2@cornell.edu // allowed since the cache does not guarantee AMO ops to be executed 159013652Sqtt2@cornell.edu // atomically in two cache lines 159113652Sqtt2@cornell.edu // For ISAs such as x86 that requires AMO operations to work on 159213652Sqtt2@cornell.edu // accesses that cross cache-line boundaries, the cache needs to be 159313652Sqtt2@cornell.edu // modified to support locking both cache lines to guarantee the 159413652Sqtt2@cornell.edu // atomicity. 159513652Sqtt2@cornell.edu panic("Do not expect cross-cache-line atomic memory request\n"); 159613652Sqtt2@cornell.edu } 159713652Sqtt2@cornell.edu 159810259SAndrew.Bardsley@arm.com LSQRequestPtr request; 159910259SAndrew.Bardsley@arm.com 160010259SAndrew.Bardsley@arm.com /* Copy given data into the request. The request will pass this to the 160110259SAndrew.Bardsley@arm.com * packet and then it will own the data */ 160210259SAndrew.Bardsley@arm.com uint8_t *request_data = NULL; 160310259SAndrew.Bardsley@arm.com 160410259SAndrew.Bardsley@arm.com DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:" 160510259SAndrew.Bardsley@arm.com " 0x%x%s lineWidth : 0x%x\n", 160613652Sqtt2@cornell.edu (isLoad ? "load" : "store/atomic"), addr, size, flags, 160710259SAndrew.Bardsley@arm.com (needs_burst ? " (needs burst)" : ""), lineWidth); 160810259SAndrew.Bardsley@arm.com 160910259SAndrew.Bardsley@arm.com if (!isLoad) { 161013652Sqtt2@cornell.edu /* Request_data becomes the property of a ...DataRequest (see below) 161110259SAndrew.Bardsley@arm.com * and destroyed by its destructor */ 161210259SAndrew.Bardsley@arm.com request_data = new uint8_t[size]; 161313652Sqtt2@cornell.edu if (inst->staticInst->isAtomic() || 161413652Sqtt2@cornell.edu (flags & Request::STORE_NO_DATA)) { 161513652Sqtt2@cornell.edu /* For atomic or store-no-data, just use zeroed data */ 161610259SAndrew.Bardsley@arm.com std::memset(request_data, 0, size); 161710259SAndrew.Bardsley@arm.com } else { 161810259SAndrew.Bardsley@arm.com std::memcpy(request_data, data, size); 161910259SAndrew.Bardsley@arm.com } 162010259SAndrew.Bardsley@arm.com } 162110259SAndrew.Bardsley@arm.com 162210259SAndrew.Bardsley@arm.com if (needs_burst) { 162310259SAndrew.Bardsley@arm.com request = new SplitDataRequest( 162410259SAndrew.Bardsley@arm.com *this, inst, isLoad, request_data, res); 162510259SAndrew.Bardsley@arm.com } else { 162610259SAndrew.Bardsley@arm.com request = new SingleDataRequest( 162710259SAndrew.Bardsley@arm.com *this, inst, isLoad, request_data, res); 162810259SAndrew.Bardsley@arm.com } 162910259SAndrew.Bardsley@arm.com 163010259SAndrew.Bardsley@arm.com if (inst->traceData) 163110665SAli.Saidi@ARM.com inst->traceData->setMem(addr, size, flags); 163210259SAndrew.Bardsley@arm.com 163311148Smitch.hayenga@arm.com int cid = cpu.threads[inst->id.threadId]->getTC()->contextId(); 163412749Sgiacomo.travaglini@arm.com request->request->setContext(cid); 163512749Sgiacomo.travaglini@arm.com request->request->setVirt(0 /* asid */, 163610634Slukefahr@umich.edu addr, size, flags, cpu.dataMasterId(), 163710259SAndrew.Bardsley@arm.com /* I've no idea why we need the PC, but give it */ 163814297Sjordi.vaquero@metempsy.com inst->pc.instAddr(), std::move(amo_op)); 163914105Sgabor.dozsa@arm.com request->request->setByteEnable(byteEnable); 164010259SAndrew.Bardsley@arm.com 164110259SAndrew.Bardsley@arm.com requests.push(request); 164214105Sgabor.dozsa@arm.com inst->inLSQ = true; 164310259SAndrew.Bardsley@arm.com request->startAddrTranslation(); 164414105Sgabor.dozsa@arm.com 164514105Sgabor.dozsa@arm.com return inst->translationFault; 164610259SAndrew.Bardsley@arm.com} 164710259SAndrew.Bardsley@arm.com 164810259SAndrew.Bardsley@arm.comvoid 164910259SAndrew.Bardsley@arm.comLSQ::pushFailedRequest(MinorDynInstPtr inst) 165010259SAndrew.Bardsley@arm.com{ 165110259SAndrew.Bardsley@arm.com LSQRequestPtr request = new FailedDataRequest(*this, inst); 165210259SAndrew.Bardsley@arm.com requests.push(request); 165310259SAndrew.Bardsley@arm.com} 165410259SAndrew.Bardsley@arm.com 165510259SAndrew.Bardsley@arm.comvoid 165610259SAndrew.Bardsley@arm.comLSQ::minorTrace() const 165710259SAndrew.Bardsley@arm.com{ 165810259SAndrew.Bardsley@arm.com MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d" 165910259SAndrew.Bardsley@arm.com " lastMemBarrier=%d\n", 166010259SAndrew.Bardsley@arm.com state, numAccessesInDTLB, numAccessesInMemorySystem, 166111567Smitch.hayenga@arm.com numStoresInTransfers, lastMemBarrier[0]); 166210259SAndrew.Bardsley@arm.com requests.minorTrace(); 166310259SAndrew.Bardsley@arm.com transfers.minorTrace(); 166410259SAndrew.Bardsley@arm.com storeBuffer.minorTrace(); 166510259SAndrew.Bardsley@arm.com} 166610259SAndrew.Bardsley@arm.com 166710259SAndrew.Bardsley@arm.comLSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_, 166810259SAndrew.Bardsley@arm.com unsigned int store_buffer_size, 166910259SAndrew.Bardsley@arm.com unsigned int store_limit_per_cycle) : 167010259SAndrew.Bardsley@arm.com Named(name_), lsq(lsq_), 167110259SAndrew.Bardsley@arm.com numSlots(store_buffer_size), 167210259SAndrew.Bardsley@arm.com storeLimitPerCycle(store_limit_per_cycle), 167310259SAndrew.Bardsley@arm.com slots(), 167410259SAndrew.Bardsley@arm.com numUnissuedAccesses(0) 167510259SAndrew.Bardsley@arm.com{ 167610259SAndrew.Bardsley@arm.com} 167710259SAndrew.Bardsley@arm.com 167810259SAndrew.Bardsley@arm.comPacketPtr 167912749Sgiacomo.travaglini@arm.commakePacketForRequest(const RequestPtr &request, bool isLoad, 168010259SAndrew.Bardsley@arm.com Packet::SenderState *sender_state, PacketDataPtr data) 168110259SAndrew.Bardsley@arm.com{ 168212749Sgiacomo.travaglini@arm.com PacketPtr ret = isLoad ? Packet::createRead(request) 168312749Sgiacomo.travaglini@arm.com : Packet::createWrite(request); 168410259SAndrew.Bardsley@arm.com 168510259SAndrew.Bardsley@arm.com if (sender_state) 168610259SAndrew.Bardsley@arm.com ret->pushSenderState(sender_state); 168710259SAndrew.Bardsley@arm.com 168812355Snikos.nikoleris@arm.com if (isLoad) { 168910259SAndrew.Bardsley@arm.com ret->allocate(); 169012749Sgiacomo.travaglini@arm.com } else if (!request->isCacheMaintenance()) { 169112355Snikos.nikoleris@arm.com // CMOs are treated as stores but they don't have data. All 169212355Snikos.nikoleris@arm.com // stores otherwise need to allocate for data. 169310566Sandreas.hansson@arm.com ret->dataDynamic(data); 169412355Snikos.nikoleris@arm.com } 169510259SAndrew.Bardsley@arm.com 169610259SAndrew.Bardsley@arm.com return ret; 169710259SAndrew.Bardsley@arm.com} 169810259SAndrew.Bardsley@arm.com 169910259SAndrew.Bardsley@arm.comvoid 170010259SAndrew.Bardsley@arm.comLSQ::issuedMemBarrierInst(MinorDynInstPtr inst) 170110259SAndrew.Bardsley@arm.com{ 170210259SAndrew.Bardsley@arm.com assert(inst->isInst() && inst->staticInst->isMemBarrier()); 170311567Smitch.hayenga@arm.com assert(inst->id.execSeqNum > lastMemBarrier[inst->id.threadId]); 170410259SAndrew.Bardsley@arm.com 170510259SAndrew.Bardsley@arm.com /* Remember the barrier. We only have a notion of one 170610259SAndrew.Bardsley@arm.com * barrier so this may result in some mem refs being 170710259SAndrew.Bardsley@arm.com * delayed if they are between barriers */ 170811567Smitch.hayenga@arm.com lastMemBarrier[inst->id.threadId] = inst->id.execSeqNum; 170910259SAndrew.Bardsley@arm.com} 171010259SAndrew.Bardsley@arm.com 171110259SAndrew.Bardsley@arm.comvoid 171210259SAndrew.Bardsley@arm.comLSQ::LSQRequest::makePacket() 171310259SAndrew.Bardsley@arm.com{ 171414105Sgabor.dozsa@arm.com assert(inst->translationFault == NoFault); 171514105Sgabor.dozsa@arm.com 171610259SAndrew.Bardsley@arm.com /* Make the function idempotent */ 171710259SAndrew.Bardsley@arm.com if (packet) 171810259SAndrew.Bardsley@arm.com return; 171910259SAndrew.Bardsley@arm.com 172010259SAndrew.Bardsley@arm.com packet = makePacketForRequest(request, isLoad, this, data); 172110259SAndrew.Bardsley@arm.com /* Null the ret data so we know not to deallocate it when the 172210259SAndrew.Bardsley@arm.com * ret is destroyed. The data now belongs to the ret and 172310259SAndrew.Bardsley@arm.com * the ret is responsible for its destruction */ 172410259SAndrew.Bardsley@arm.com data = NULL; 172510259SAndrew.Bardsley@arm.com} 172610259SAndrew.Bardsley@arm.com 172710259SAndrew.Bardsley@arm.comstd::ostream & 172810259SAndrew.Bardsley@arm.comoperator <<(std::ostream &os, LSQ::MemoryState state) 172910259SAndrew.Bardsley@arm.com{ 173010259SAndrew.Bardsley@arm.com switch (state) { 173110259SAndrew.Bardsley@arm.com case LSQ::MemoryRunning: 173210259SAndrew.Bardsley@arm.com os << "MemoryRunning"; 173310259SAndrew.Bardsley@arm.com break; 173410259SAndrew.Bardsley@arm.com case LSQ::MemoryNeedsRetry: 173510259SAndrew.Bardsley@arm.com os << "MemoryNeedsRetry"; 173610259SAndrew.Bardsley@arm.com break; 173710259SAndrew.Bardsley@arm.com default: 173810259SAndrew.Bardsley@arm.com os << "MemoryState-" << static_cast<int>(state); 173910259SAndrew.Bardsley@arm.com break; 174010259SAndrew.Bardsley@arm.com } 174110259SAndrew.Bardsley@arm.com return os; 174210259SAndrew.Bardsley@arm.com} 174310259SAndrew.Bardsley@arm.com 174410259SAndrew.Bardsley@arm.comvoid 174510259SAndrew.Bardsley@arm.comLSQ::recvTimingSnoopReq(PacketPtr pkt) 174610259SAndrew.Bardsley@arm.com{ 174710259SAndrew.Bardsley@arm.com /* LLSC operations in Minor can't be speculative and are executed from 174810259SAndrew.Bardsley@arm.com * the head of the requests queue. We shouldn't need to do more than 174910259SAndrew.Bardsley@arm.com * this action on snoops. */ 175011567Smitch.hayenga@arm.com for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 175111567Smitch.hayenga@arm.com if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) { 175211567Smitch.hayenga@arm.com cpu.wakeup(tid); 175311567Smitch.hayenga@arm.com } 175411567Smitch.hayenga@arm.com } 175510259SAndrew.Bardsley@arm.com 175611356Skrinat01@arm.com if (pkt->isInvalidate() || pkt->isWrite()) { 175711567Smitch.hayenga@arm.com for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 175811567Smitch.hayenga@arm.com TheISA::handleLockedSnoop(cpu.getContext(tid), pkt, 175911567Smitch.hayenga@arm.com cacheBlockMask); 176011567Smitch.hayenga@arm.com } 176111567Smitch.hayenga@arm.com } 176211567Smitch.hayenga@arm.com} 176311567Smitch.hayenga@arm.com 176411567Smitch.hayenga@arm.comvoid 176511567Smitch.hayenga@arm.comLSQ::threadSnoop(LSQRequestPtr request) 176611567Smitch.hayenga@arm.com{ 176711567Smitch.hayenga@arm.com /* LLSC operations in Minor can't be speculative and are executed from 176811567Smitch.hayenga@arm.com * the head of the requests queue. We shouldn't need to do more than 176911567Smitch.hayenga@arm.com * this action on snoops. */ 177011567Smitch.hayenga@arm.com ThreadID req_tid = request->inst->id.threadId; 177111567Smitch.hayenga@arm.com PacketPtr pkt = request->packet; 177211567Smitch.hayenga@arm.com 177311567Smitch.hayenga@arm.com for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 177411567Smitch.hayenga@arm.com if (tid != req_tid) { 177511567Smitch.hayenga@arm.com if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) { 177611567Smitch.hayenga@arm.com cpu.wakeup(tid); 177711567Smitch.hayenga@arm.com } 177811567Smitch.hayenga@arm.com 177911567Smitch.hayenga@arm.com if (pkt->isInvalidate() || pkt->isWrite()) { 178011567Smitch.hayenga@arm.com TheISA::handleLockedSnoop(cpu.getContext(tid), pkt, 178111567Smitch.hayenga@arm.com cacheBlockMask); 178211567Smitch.hayenga@arm.com } 178311567Smitch.hayenga@arm.com } 178411356Skrinat01@arm.com } 178510259SAndrew.Bardsley@arm.com} 178610259SAndrew.Bardsley@arm.com 178710259SAndrew.Bardsley@arm.com} 1788