lsq.cc revision 10563
19243SN/A/* 210206Sandreas.hansson@arm.com * Copyright (c) 2013-2014 ARM Limited 39243SN/A * All rights reserved 49243SN/A * 59243SN/A * The license below extends only to copyright in the software and shall 69243SN/A * not be construed as granting a license to any other intellectual 79243SN/A * property including but not limited to intellectual property relating 89243SN/A * to a hardware implementation of the functionality of the software 99243SN/A * licensed hereunder. You may use the software subject to the license 109243SN/A * terms below provided that you ensure that this notice is replicated 119243SN/A * unmodified and in its entirety in all distributions of the software, 129243SN/A * modified or unmodified, in source code or in binary form. 139243SN/A * 149831SN/A * Redistribution and use in source and binary forms, with or without 159831SN/A * modification, are permitted provided that the following conditions are 169831SN/A * met: redistributions of source code must retain the above copyright 179243SN/A * notice, this list of conditions and the following disclaimer; 189243SN/A * redistributions in binary form must reproduce the above copyright 199243SN/A * notice, this list of conditions and the following disclaimer in the 209243SN/A * documentation and/or other materials provided with the distribution; 219243SN/A * neither the name of the copyright holders nor the names of its 229243SN/A * contributors may be used to endorse or promote products derived from 239243SN/A * this software without specific prior written permission. 249243SN/A * 259243SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 269243SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 279243SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 289243SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 299243SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 309243SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 319243SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 329243SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 339243SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 349243SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 359243SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 369243SN/A * 379243SN/A * Authors: Andrew Bardsley 389243SN/A */ 399243SN/A 409243SN/A#include <iomanip> 419243SN/A#include <sstream> 429967SN/A 439243SN/A#include "arch/locked_mem.hh" 449243SN/A#include "arch/mmapped_ipr.hh" 4510146Sandreas.hansson@arm.com#include "cpu/minor/cpu.hh" 469356SN/A#include "cpu/minor/exec_context.hh" 4710146Sandreas.hansson@arm.com#include "cpu/minor/execute.hh" 4810247Sandreas.hansson@arm.com#include "cpu/minor/lsq.hh" 4910208Sandreas.hansson@arm.com#include "cpu/minor/pipeline.hh" 509352SN/A#include "debug/Activity.hh" 5110146Sandreas.hansson@arm.com#include "debug/MinorMem.hh" 529814SN/A 539243SN/Anamespace Minor 549243SN/A{ 559243SN/A 5610146Sandreas.hansson@arm.com/** Returns the offset of addr into an aligned a block of size block_size */ 579243SN/Astatic Addr 589243SN/AaddrBlockOffset(Addr addr, unsigned int block_size) 599243SN/A{ 6010211Sandreas.hansson@arm.com return addr & (block_size - 1); 6110208Sandreas.hansson@arm.com} 6210208Sandreas.hansson@arm.com 6310208Sandreas.hansson@arm.com/** Returns true if the given [addr .. addr+size-1] transfer needs to be 649831SN/A * fragmented across a block size of block_size */ 659831SN/Astatic bool 669831SN/AtransferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size) 679831SN/A{ 689831SN/A return (addrBlockOffset(addr, block_size) + size) > block_size; 6910140SN/A} 7010286Sandreas.hansson@arm.com 719243SN/ALSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_, 7210394Swendy.elsasser@arm.com PacketDataPtr data_, uint64_t *res_) : 7310394Swendy.elsasser@arm.com SenderState(), 749566SN/A port(port_), 759243SN/A inst(inst_), 769243SN/A isLoad(isLoad_), 7710140SN/A data(data_), 7810140SN/A packet(NULL), 7910147Sandreas.hansson@arm.com request(), 8010147Sandreas.hansson@arm.com fault(NoFault), 8110393Swendy.elsasser@arm.com res(res_), 8210394Swendy.elsasser@arm.com skipped(false), 8310394Swendy.elsasser@arm.com issuedToMemory(false), 8410394Swendy.elsasser@arm.com state(NotIssued) 859243SN/A{ } 869243SN/A 8710141SN/ALSQ::AddrRangeCoverage 889726SN/ALSQ::LSQRequest::containsAddrRangeOf( 899726SN/A Addr req1_addr, unsigned int req1_size, 9010208Sandreas.hansson@arm.com Addr req2_addr, unsigned int req2_size) 9110208Sandreas.hansson@arm.com{ 9210393Swendy.elsasser@arm.com /* 'end' here means the address of the byte just past the request 9310393Swendy.elsasser@arm.com * blocks */ 949243SN/A Addr req2_end_addr = req2_addr + req2_size; 959243SN/A Addr req1_end_addr = req1_addr + req1_size; 969243SN/A 979243SN/A AddrRangeCoverage ret; 989969SN/A 999243SN/A if (req1_addr > req2_end_addr || req1_end_addr < req2_addr) 1009243SN/A ret = NoAddrRangeCoverage; 1019969SN/A else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr) 1029243SN/A ret = FullAddrRangeCoverage; 1039243SN/A else 10410246Sandreas.hansson@arm.com ret = PartialAddrRangeCoverage; 10510246Sandreas.hansson@arm.com 10610246Sandreas.hansson@arm.com return ret; 10710246Sandreas.hansson@arm.com} 10810246Sandreas.hansson@arm.com 10910394Swendy.elsasser@arm.comLSQ::AddrRangeCoverage 11010394Swendy.elsasser@arm.comLSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request) 11110394Swendy.elsasser@arm.com{ 11210394Swendy.elsasser@arm.com return containsAddrRangeOf(request.getPaddr(), request.getSize(), 11310394Swendy.elsasser@arm.com other_request->request.getPaddr(), other_request->request.getSize()); 11410394Swendy.elsasser@arm.com} 11510394Swendy.elsasser@arm.com 11610394Swendy.elsasser@arm.combool 11710394Swendy.elsasser@arm.comLSQ::LSQRequest::isBarrier() 11810394Swendy.elsasser@arm.com{ 11910394Swendy.elsasser@arm.com return inst->isInst() && inst->staticInst->isMemBarrier(); 12010394Swendy.elsasser@arm.com} 12110394Swendy.elsasser@arm.com 12210246Sandreas.hansson@arm.combool 12310246Sandreas.hansson@arm.comLSQ::LSQRequest::needsToBeSentToStoreBuffer() 12410246Sandreas.hansson@arm.com{ 12510140SN/A return state == StoreToStoreBuffer; 12610140SN/A} 12710140SN/A 12810140SN/Avoid 12910140SN/ALSQ::LSQRequest::setState(LSQRequestState new_state) 1309243SN/A{ 1319243SN/A DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:" 1329567SN/A " %s\n", state, new_state, *inst); 1339243SN/A state = new_state; 1349243SN/A} 1359243SN/A 1369831SN/Abool 1379831SN/ALSQ::LSQRequest::isComplete() const 1389831SN/A{ 1399831SN/A /* @todo, There is currently only one 'completed' state. This 1409831SN/A * may not be a good choice */ 1419243SN/A return state == Complete; 14210286Sandreas.hansson@arm.com} 1439566SN/A 1449566SN/Avoid 14510143SN/ALSQ::LSQRequest::reportData(std::ostream &os) const 1469566SN/A{ 1479566SN/A os << (isLoad ? 'R' : 'W') << ';'; 14810136SN/A inst->reportData(os); 1499831SN/A os << ';' << state; 15010286Sandreas.hansson@arm.com} 15110136SN/A 1529566SN/Astd::ostream & 15310286Sandreas.hansson@arm.comoperator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage) 15410286Sandreas.hansson@arm.com{ 15510286Sandreas.hansson@arm.com switch (coverage) { 15610286Sandreas.hansson@arm.com case LSQ::PartialAddrRangeCoverage: 15710286Sandreas.hansson@arm.com os << "PartialAddrRangeCoverage"; 15810286Sandreas.hansson@arm.com break; 15910286Sandreas.hansson@arm.com case LSQ::FullAddrRangeCoverage: 16010286Sandreas.hansson@arm.com os << "FullAddrRangeCoverage"; 16110286Sandreas.hansson@arm.com break; 16210286Sandreas.hansson@arm.com case LSQ::NoAddrRangeCoverage: 16310286Sandreas.hansson@arm.com os << "NoAddrRangeCoverage"; 16410286Sandreas.hansson@arm.com break; 16510286Sandreas.hansson@arm.com default: 16610286Sandreas.hansson@arm.com os << "AddrRangeCoverage-" << static_cast<int>(coverage); 16710286Sandreas.hansson@arm.com break; 16810286Sandreas.hansson@arm.com } 1699669SN/A return os; 17010286Sandreas.hansson@arm.com} 17110286Sandreas.hansson@arm.com 17210286Sandreas.hansson@arm.comstd::ostream & 17310286Sandreas.hansson@arm.comoperator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state) 17410286Sandreas.hansson@arm.com{ 17510286Sandreas.hansson@arm.com switch (state) { 17610286Sandreas.hansson@arm.com case LSQ::LSQRequest::NotIssued: 17710286Sandreas.hansson@arm.com os << "NotIssued"; 1789566SN/A break; 1799566SN/A case LSQ::LSQRequest::InTranslation: 18010207Sandreas.hansson@arm.com os << "InTranslation"; 18110207Sandreas.hansson@arm.com break; 18210207Sandreas.hansson@arm.com case LSQ::LSQRequest::Translated: 18310207Sandreas.hansson@arm.com os << "Translated"; 18410207Sandreas.hansson@arm.com break; 18510207Sandreas.hansson@arm.com case LSQ::LSQRequest::Failed: 18610394Swendy.elsasser@arm.com os << "Failed"; 18710394Swendy.elsasser@arm.com break; 18810394Swendy.elsasser@arm.com case LSQ::LSQRequest::RequestIssuing: 18910394Swendy.elsasser@arm.com os << "RequestIssuing"; 19010394Swendy.elsasser@arm.com break; 19110394Swendy.elsasser@arm.com case LSQ::LSQRequest::StoreToStoreBuffer: 19210394Swendy.elsasser@arm.com os << "StoreToStoreBuffer"; 19310394Swendy.elsasser@arm.com break; 19410394Swendy.elsasser@arm.com case LSQ::LSQRequest::StoreInStoreBuffer: 19510394Swendy.elsasser@arm.com os << "StoreInStoreBuffer"; 19610394Swendy.elsasser@arm.com break; 19710394Swendy.elsasser@arm.com case LSQ::LSQRequest::StoreBufferIssuing: 19810394Swendy.elsasser@arm.com os << "StoreBufferIssuing"; 19910394Swendy.elsasser@arm.com break; 20010394Swendy.elsasser@arm.com case LSQ::LSQRequest::RequestNeedsRetry: 20110394Swendy.elsasser@arm.com os << "RequestNeedsRetry"; 20210394Swendy.elsasser@arm.com break; 20310394Swendy.elsasser@arm.com case LSQ::LSQRequest::StoreBufferNeedsRetry: 20410394Swendy.elsasser@arm.com os << "StoreBufferNeedsRetry"; 20510394Swendy.elsasser@arm.com break; 20610394Swendy.elsasser@arm.com case LSQ::LSQRequest::Complete: 20710394Swendy.elsasser@arm.com os << "Complete"; 20810394Swendy.elsasser@arm.com break; 20910394Swendy.elsasser@arm.com default: 21010394Swendy.elsasser@arm.com os << "LSQRequestState-" << static_cast<int>(state); 21110394Swendy.elsasser@arm.com break; 21210394Swendy.elsasser@arm.com } 21310394Swendy.elsasser@arm.com return os; 21410394Swendy.elsasser@arm.com} 2159243SN/A 2169243SN/Avoid 2179243SN/ALSQ::clearMemBarrier(MinorDynInstPtr inst) 21810146Sandreas.hansson@arm.com{ 21910140SN/A bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier; 22010140SN/A 22110146Sandreas.hansson@arm.com DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n", 22210140SN/A (is_last_barrier ? "last" : "a"), *inst); 22310140SN/A 22410140SN/A if (is_last_barrier) 22510140SN/A lastMemBarrier = 0; 22610140SN/A} 22710140SN/A 22810146Sandreas.hansson@arm.comvoid 2299243SN/ALSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_, 23010143SN/A ThreadContext *tc, BaseTLB::Mode mode) 23110143SN/A{ 23210208Sandreas.hansson@arm.com fault = fault_; 23310143SN/A 23410206Sandreas.hansson@arm.com port.numAccessesInDTLB--; 23510206Sandreas.hansson@arm.com 23610206Sandreas.hansson@arm.com DPRINTFS(MinorMem, (&port), "Received translation response for" 23710206Sandreas.hansson@arm.com " request: %s\n", *inst); 23810206Sandreas.hansson@arm.com 23910206Sandreas.hansson@arm.com makePacket(); 24010207Sandreas.hansson@arm.com 24110207Sandreas.hansson@arm.com setState(Translated); 24210207Sandreas.hansson@arm.com port.tryToSendToTransfers(this); 2439243SN/A 2449243SN/A /* Let's try and wake up the processor for the next cycle */ 2459243SN/A port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 24610146Sandreas.hansson@arm.com} 2479243SN/A 2489243SN/Avoid 2499243SN/ALSQ::SingleDataRequest::startAddrTranslation() 2509243SN/A{ 2519243SN/A ThreadContext *thread = port.cpu.getContext( 2529243SN/A inst->id.threadId); 2539243SN/A 2549243SN/A port.numAccessesInDTLB++; 2559243SN/A 2569243SN/A setState(LSQ::LSQRequest::InTranslation); 2579243SN/A 2589243SN/A DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n"); 2599243SN/A /* Submit the translation request. The response will come through 2609243SN/A * finish/markDelayed on the LSQRequest as it bears the Translation 2619243SN/A * interface */ 2629243SN/A thread->getDTBPtr()->translateTiming( 26310146Sandreas.hansson@arm.com &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write)); 2649243SN/A} 2659831SN/A 2669831SN/Avoid 2679831SN/ALSQ::SingleDataRequest::retireResponse(PacketPtr packet_) 2689243SN/A{ 2699831SN/A DPRINTFS(MinorMem, (&port), "Retiring packet\n"); 2709831SN/A packet = packet_; 2719243SN/A packetInFlight = false; 2729243SN/A setState(Complete); 2739243SN/A} 27410146Sandreas.hansson@arm.com 2759243SN/Avoid 2769831SN/ALSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_, 2779831SN/A ThreadContext *tc, BaseTLB::Mode mode) 2789831SN/A{ 2799243SN/A fault = fault_; 2809243SN/A 28110146Sandreas.hansson@arm.com port.numAccessesInDTLB--; 28210146Sandreas.hansson@arm.com 28310143SN/A unsigned int M5_VAR_USED expected_fragment_index = 2849243SN/A numTranslatedFragments; 2859669SN/A 28610136SN/A numInTranslationFragments--; 28710136SN/A numTranslatedFragments++; 2889243SN/A 2899967SN/A DPRINTFS(MinorMem, (&port), "Received translation response for fragment" 29010245Sandreas.hansson@arm.com " %d of request: %s\n", expected_fragment_index, *inst); 29110245Sandreas.hansson@arm.com 29210245Sandreas.hansson@arm.com assert(request_ == fragmentRequests[expected_fragment_index]); 2939243SN/A 29410286Sandreas.hansson@arm.com /* Wake up next cycle to get things going again in case the 29510286Sandreas.hansson@arm.com * tryToSendToTransfers does take */ 2969831SN/A port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 2979243SN/A 2989491SN/A if (fault != NoFault) { 2999831SN/A /* tryToSendToTransfers will handle the fault */ 30010136SN/A 3019491SN/A DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:" 3029491SN/A " %d of request: %s\n", 3039831SN/A expected_fragment_index, *inst); 3049243SN/A 3059669SN/A setState(Translated); 3069566SN/A port.tryToSendToTransfers(this); 3079566SN/A } else if (numTranslatedFragments == numFragments) { 3089669SN/A makeFragmentPackets(); 3099669SN/A 3109669SN/A setState(Translated); 3119669SN/A port.tryToSendToTransfers(this); 3129669SN/A } else { 3139669SN/A /* Avoid calling translateTiming from within ::finish */ 3149669SN/A assert(!translationEvent.scheduled()); 3159669SN/A port.cpu.schedule(translationEvent, curTick()); 3169669SN/A } 3179669SN/A} 3189669SN/A 3199669SN/ALSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_, 3209669SN/A bool isLoad_, PacketDataPtr data_, uint64_t *res_) : 32110136SN/A LSQRequest(port_, inst_, isLoad_, data_, res_), 32210286Sandreas.hansson@arm.com translationEvent(*this), 32310286Sandreas.hansson@arm.com numFragments(0), 32410286Sandreas.hansson@arm.com numInTranslationFragments(0), 3259669SN/A numTranslatedFragments(0), 3269669SN/A numIssuedFragments(0), 3279669SN/A numRetiredFragments(0), 32810286Sandreas.hansson@arm.com fragmentRequests(), 32910286Sandreas.hansson@arm.com fragmentPackets() 3309669SN/A{ 3319669SN/A /* Don't know how many elements are needed until the request is 3329491SN/A * populated by the caller. */ 3339243SN/A} 3349243SN/A 3359243SN/ALSQ::SplitDataRequest::~SplitDataRequest() 3369491SN/A{ 3379491SN/A for (auto i = fragmentRequests.begin(); 3389243SN/A i != fragmentRequests.end(); i++) 3399243SN/A { 3409243SN/A delete *i; 3419491SN/A } 3429243SN/A 3439243SN/A for (auto i = fragmentPackets.begin(); 34410136SN/A i != fragmentPackets.end(); i++) 3459491SN/A { 3469491SN/A delete *i; 3479491SN/A } 34810286Sandreas.hansson@arm.com} 34910286Sandreas.hansson@arm.com 35010286Sandreas.hansson@arm.comvoid 3519566SN/ALSQ::SplitDataRequest::makeFragmentRequests() 3529566SN/A{ 3539566SN/A Addr base_addr = request.getVaddr(); 3549566SN/A unsigned int whole_size = request.getSize(); 3559566SN/A unsigned int line_width = port.lineWidth; 3569491SN/A 3579491SN/A unsigned int fragment_size; 3589243SN/A Addr fragment_addr; 3599243SN/A 3609243SN/A /* Assume that this transfer is across potentially many block snap 3619491SN/A * boundaries: 3629243SN/A * 3639243SN/A * | _|________|________|________|___ | 3649243SN/A * | |0| 1 | 2 | 3 | 4 | | 36510286Sandreas.hansson@arm.com * | |_|________|________|________|___| | 36610286Sandreas.hansson@arm.com * | | | | | | 3679243SN/A * 3689491SN/A * The first transfer (0) can be up to lineWidth in size. 3699243SN/A * All the middle transfers (1-3) are lineWidth in size 3709243SN/A * The last transfer (4) can be from zero to lineWidth - 1 in size 3719243SN/A */ 3729243SN/A unsigned int first_fragment_offset = 3739243SN/A addrBlockOffset(base_addr, line_width); 3749243SN/A unsigned int last_fragment_size = 3759243SN/A addrBlockOffset(base_addr + whole_size, line_width); 3769243SN/A unsigned int first_fragment_size = 37710245Sandreas.hansson@arm.com line_width - first_fragment_offset; 3789243SN/A 3799243SN/A unsigned int middle_fragments_total_size = 3809831SN/A whole_size - (first_fragment_size + last_fragment_size); 3819243SN/A 3829243SN/A assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0); 3839567SN/A 3849567SN/A unsigned int middle_fragment_count = 3859967SN/A middle_fragments_total_size / line_width; 3869967SN/A 3879967SN/A numFragments = 1 /* first */ + middle_fragment_count + 3889243SN/A (last_fragment_size == 0 ? 0 : 1); 3899243SN/A 3909243SN/A DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests." 39110146Sandreas.hansson@arm.com " First fragment size: %d Last fragment size: %d\n", 3929243SN/A numFragments, first_fragment_size, 3939243SN/A (last_fragment_size == 0 ? line_width : last_fragment_size)); 3949243SN/A 3959243SN/A assert(((middle_fragment_count * line_width) + 3969243SN/A first_fragment_size + last_fragment_size) == whole_size); 3979831SN/A 3989831SN/A fragment_addr = base_addr; 3999831SN/A fragment_size = first_fragment_size; 4009831SN/A 4019831SN/A /* Just past the last address in the request */ 4029831SN/A Addr end_addr = base_addr + whole_size; 4039831SN/A 4049831SN/A for (unsigned int fragment_index = 0; fragment_index < numFragments; 4059243SN/A fragment_index++) 4069831SN/A { 4079831SN/A bool M5_VAR_USED is_last_fragment = false; 4089831SN/A 4099831SN/A if (fragment_addr == base_addr) { 4109831SN/A /* First fragment */ 4119831SN/A fragment_size = first_fragment_size; 4129831SN/A } else { 4139243SN/A if ((fragment_addr + line_width) > end_addr) { 4149831SN/A /* Adjust size of last fragment */ 4159831SN/A fragment_size = end_addr - fragment_addr; 4169831SN/A is_last_fragment = true; 4179833SN/A } else { 4189832SN/A /* Middle fragments */ 4199832SN/A fragment_size = line_width; 4209832SN/A } 4219832SN/A } 4229831SN/A 4239831SN/A Request *fragment = new Request(); 4249831SN/A 4259831SN/A fragment->setThreadContext(request.contextId(), /* thread id */ 0); 4269831SN/A fragment->setVirt(0 /* asid */, 4279975SN/A fragment_addr, fragment_size, request.getFlags(), 4289831SN/A request.masterId(), 4299831SN/A request.getPC()); 4309243SN/A 4319831SN/A DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d" 4329831SN/A " (whole request addr: 0x%x size: %d) %s\n", 4339831SN/A fragment_addr, fragment_size, base_addr, whole_size, 4349831SN/A (is_last_fragment ? "last fragment" : "")); 4359831SN/A 4369831SN/A fragment_addr += fragment_size; 4379831SN/A 4389831SN/A fragmentRequests.push_back(fragment); 4399831SN/A } 4409831SN/A} 4419831SN/A 4429831SN/Avoid 4439966SN/ALSQ::SplitDataRequest::makeFragmentPackets() 4449831SN/A{ 4459831SN/A Addr base_addr = request.getVaddr(); 4469831SN/A 4479831SN/A DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst); 4489831SN/A 4499831SN/A for (unsigned int fragment_index = 0; fragment_index < numFragments; 4509831SN/A fragment_index++) 4519831SN/A { 4529831SN/A Request *fragment = fragmentRequests[fragment_index]; 4539831SN/A 4549831SN/A DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s" 4559831SN/A " (%d, 0x%x)\n", 4569831SN/A fragment_index, *inst, 4579831SN/A (fragment->hasPaddr() ? "has paddr" : "no paddr"), 4589831SN/A (fragment->hasPaddr() ? fragment->getPaddr() : 0)); 4599243SN/A 4609243SN/A Addr fragment_addr = fragment->getVaddr(); 4619831SN/A unsigned int fragment_size = fragment->getSize(); 4629831SN/A 4639831SN/A uint8_t *request_data = NULL; 4649831SN/A 4659831SN/A if (!isLoad) { 4669243SN/A /* Split data for Packets. Will become the property of the 4679831SN/A * outgoing Packets */ 4689831SN/A request_data = new uint8_t[fragment_size]; 4699831SN/A std::memcpy(request_data, data + (fragment_addr - base_addr), 4709243SN/A fragment_size); 47110206Sandreas.hansson@arm.com } 47210206Sandreas.hansson@arm.com 47310206Sandreas.hansson@arm.com assert(fragment->hasPaddr()); 4749567SN/A 4759567SN/A PacketPtr fragment_packet = 4769243SN/A makePacketForRequest(*fragment, isLoad, this, request_data); 4779243SN/A 4789243SN/A fragmentPackets.push_back(fragment_packet); 4799243SN/A /* Accumulate flags in parent request */ 48010146Sandreas.hansson@arm.com request.setFlags(fragment->getFlags()); 4819243SN/A } 4829243SN/A 4839243SN/A /* Might as well make the overall/response packet here */ 4849243SN/A /* Get the physical address for the whole request/packet from the first 4859243SN/A * fragment */ 4869831SN/A request.setPaddr(fragmentRequests[0]->getPaddr()); 4879831SN/A makePacket(); 4889831SN/A} 4899831SN/A 4909831SN/Avoid 4919831SN/ALSQ::SplitDataRequest::startAddrTranslation() 4929831SN/A{ 4939831SN/A setState(LSQ::LSQRequest::InTranslation); 4949243SN/A 4959832SN/A makeFragmentRequests(); 4969838SN/A 4979838SN/A numInTranslationFragments = 0; 4989838SN/A numTranslatedFragments = 0; 4999832SN/A 5009832SN/A /* @todo, just do these in sequence for now with 5019243SN/A * a loop of: 5029832SN/A * do { 5039832SN/A * sendNextFragmentToTranslation ; translateTiming ; finish 5049832SN/A * } while (numTranslatedFragments != numFragments); 5059832SN/A */ 5069838SN/A 5079838SN/A /* Do first translation */ 5089838SN/A sendNextFragmentToTranslation(); 5099832SN/A} 5109832SN/A 5119832SN/APacketPtr 5129832SN/ALSQ::SplitDataRequest::getHeadPacket() 5139832SN/A{ 5149832SN/A assert(numIssuedFragments < numFragments); 5159832SN/A 5169832SN/A return fragmentPackets[numIssuedFragments]; 5179832SN/A} 5189832SN/A 5199832SN/Avoid 5209832SN/ALSQ::SplitDataRequest::stepToNextPacket() 5219832SN/A{ 5229832SN/A assert(numIssuedFragments < numFragments); 5239832SN/A 5249832SN/A numIssuedFragments++; 5259832SN/A} 52610047SN/A 5279832SN/Avoid 5289832SN/ALSQ::SplitDataRequest::retireResponse(PacketPtr response) 5299832SN/A{ 5309838SN/A assert(numRetiredFragments < numFragments); 5319838SN/A 5329838SN/A DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d" 5339832SN/A " offset: 0x%x (retired fragment num: %d) %s\n", 5349832SN/A response->req->getVaddr(), response->req->getSize(), 5359832SN/A request.getVaddr() - response->req->getVaddr(), 5369832SN/A numRetiredFragments, 5379832SN/A (fault == NoFault ? "" : fault->name())); 5389832SN/A 5399832SN/A numRetiredFragments++; 5409832SN/A 5419832SN/A if (skipped) { 5429832SN/A /* Skip because we already knew the request had faulted or been 5439832SN/A * skipped */ 5449832SN/A DPRINTFS(MinorMem, (&port), "Skipping this fragment\n"); 5459832SN/A } else if (response->isError()) { 5469832SN/A /* Mark up the error and leave to execute to handle it */ 5479832SN/A DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n"); 5489832SN/A setSkipped(); 5499832SN/A packet->copyError(response); 5509832SN/A } else { 5519832SN/A if (isLoad) { 5529832SN/A if (!data) { 5539243SN/A /* For a split transfer, a Packet must be constructed 5549832SN/A * to contain all returning data. This is that packet's 5559832SN/A * data */ 5569832SN/A data = new uint8_t[request.getSize()]; 5579966SN/A } 5589243SN/A 5599832SN/A /* Populate the portion of the overall response data represented 5609832SN/A * by the response fragment */ 5619243SN/A std::memcpy( 5629832SN/A data + (response->req->getVaddr() - request.getVaddr()), 5639831SN/A response->getConstPtr<uint8_t>(), 5649832SN/A response->req->getSize()); 5659831SN/A } 5669832SN/A } 5679832SN/A 5689977SN/A /* Complete early if we're skipping are no more in-flight accesses */ 5699977SN/A if (skipped && !hasPacketsInMemSystem()) { 5709977SN/A DPRINTFS(MinorMem, (&port), "Completed skipped burst\n"); 5719977SN/A setState(Complete); 5729832SN/A if (packet->needsResponse()) 5739832SN/A packet->makeResponse(); 5749831SN/A } 5759831SN/A 5769831SN/A if (numRetiredFragments == numFragments) 5779243SN/A setState(Complete); 5789243SN/A 5799243SN/A if (!skipped && isComplete()) { 5809243SN/A DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL); 5819831SN/A 5829831SN/A DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d" 5839726SN/A " needsResponse: %d packetSize: %s requestSize: %s responseSize:" 5849243SN/A " %s\n", packet->isRead(), packet->isWrite(), 58510206Sandreas.hansson@arm.com packet->needsResponse(), packet->getSize(), request.getSize(), 58610206Sandreas.hansson@arm.com response->getSize()); 58710206Sandreas.hansson@arm.com 58810206Sandreas.hansson@arm.com /* A request can become complete by several paths, this is a sanity 58910206Sandreas.hansson@arm.com * check to make sure the packet's data is created */ 5909243SN/A if (!data) { 5919243SN/A data = new uint8_t[request.getSize()]; 5929243SN/A } 5939243SN/A 59410146Sandreas.hansson@arm.com if (isLoad) { 5959243SN/A DPRINTFS(MinorMem, (&port), "Copying read data\n"); 5969833SN/A std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize()); 5979243SN/A } 5989243SN/A packet->makeResponse(); 5999243SN/A } 6009833SN/A 6019243SN/A /* Packets are all deallocated together in ~SplitLSQRequest */ 6029243SN/A} 6039243SN/A 6049833SN/Avoid 6059243SN/ALSQ::SplitDataRequest::sendNextFragmentToTranslation() 6069243SN/A{ 6079243SN/A unsigned int fragment_index = numTranslatedFragments; 6089243SN/A 6099243SN/A ThreadContext *thread = port.cpu.getContext( 61010146Sandreas.hansson@arm.com inst->id.threadId); 6119243SN/A 6129349SN/A DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n", 6139349SN/A fragment_index); 6149349SN/A 6159349SN/A port.numAccessesInDTLB++; 6169349SN/A numInTranslationFragments++; 6179349SN/A 6189243SN/A thread->getDTBPtr()->translateTiming( 6199567SN/A fragmentRequests[fragment_index], thread, this, (isLoad ? 6209831SN/A BaseTLB::Read : BaseTLB::Write)); 6219243SN/A} 6229567SN/A 6239567SN/Abool 62410143SN/ALSQ::StoreBuffer::canInsert() const 6259567SN/A{ 6269567SN/A /* @todo, support store amalgamation */ 6279567SN/A return slots.size() < numSlots; 6289243SN/A} 6299243SN/A 6309243SN/Avoid 6319243SN/ALSQ::StoreBuffer::deleteRequest(LSQRequestPtr request) 6329243SN/A{ 6339243SN/A auto found = std::find(slots.begin(), slots.end(), request); 6349243SN/A 6359831SN/A if (found != slots.end()) { 6369831SN/A DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n", 6379831SN/A request, *found, *(request->inst)); 6389831SN/A slots.erase(found); 6399831SN/A 6409243SN/A delete request; 6419831SN/A } 6429831SN/A} 6439243SN/A 6449243SN/Avoid 6459243SN/ALSQ::StoreBuffer::insert(LSQRequestPtr request) 6469567SN/A{ 6479831SN/A if (!canInsert()) { 6489567SN/A warn("%s: store buffer insertion without space to insert from" 6499243SN/A " inst: %s\n", name(), *(request->inst)); 6509243SN/A } 6519243SN/A 6529243SN/A DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request); 6539243SN/A 6549831SN/A numUnissuedAccesses++; 6559243SN/A 6569977SN/A if (request->state != LSQRequest::Complete) 6579243SN/A request->setState(LSQRequest::StoreInStoreBuffer); 6589243SN/A 6599567SN/A slots.push_back(request); 6609831SN/A 6619567SN/A /* Let's try and wake up the processor for the next cycle to step 6629243SN/A * the store buffer */ 6639243SN/A lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 6649243SN/A} 6659243SN/A 6669243SN/ALSQ::AddrRangeCoverage 6679831SN/ALSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request, 6689243SN/A unsigned int &found_slot) 6699977SN/A{ 6709243SN/A unsigned int slot_index = slots.size() - 1; 6719243SN/A auto i = slots.rbegin(); 6729243SN/A AddrRangeCoverage ret = NoAddrRangeCoverage; 6739243SN/A 6749726SN/A /* Traverse the store buffer in reverse order (most to least recent) 6759243SN/A * and try to find a slot whose address range overlaps this request */ 6769243SN/A while (ret == NoAddrRangeCoverage && i != slots.rend()) { 6779243SN/A LSQRequestPtr slot = *i; 6789243SN/A 6799243SN/A if (slot->packet) { 6809243SN/A AddrRangeCoverage coverage = slot->containsAddrRangeOf(request); 68110146Sandreas.hansson@arm.com 6829243SN/A if (coverage != NoAddrRangeCoverage) { 6839243SN/A DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:" 6849243SN/A " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n", 6859243SN/A slot_index, coverage, 6869831SN/A request->request.getPaddr(), request->request.getSize(), 6879243SN/A slot->request.getPaddr(), slot->request.getSize()); 6889831SN/A 6899831SN/A found_slot = slot_index; 6909831SN/A ret = coverage; 6919831SN/A } 69210143SN/A } 6939831SN/A 6949831SN/A i++; 6959831SN/A slot_index--; 6969831SN/A } 6979831SN/A 6989831SN/A return ret; 6999831SN/A} 7009831SN/A 7019831SN/A/** Fill the given packet with appropriate date from slot slot_number */ 7029831SN/Avoid 7039831SN/ALSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load, 7049831SN/A unsigned int slot_number) 7059243SN/A{ 7069831SN/A assert(slot_number < slots.size()); 7079831SN/A assert(load->packet); 7089243SN/A assert(load->isLoad); 7099831SN/A 7109831SN/A LSQRequestPtr store = slots[slot_number]; 7119831SN/A 7129831SN/A assert(store->packet); 7139831SN/A assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage); 7149831SN/A 7159831SN/A Addr load_addr = load->request.getPaddr(); 7169831SN/A Addr store_addr = store->request.getPaddr(); 7179831SN/A Addr addr_offset = load_addr - store_addr; 7189831SN/A 7199831SN/A unsigned int load_size = load->request.getSize(); 7209831SN/A 7219567SN/A DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer" 7229831SN/A " slot: %d addr: 0x%x addressOffset: 0x%x\n", 7239831SN/A load_size, load_addr, slot_number, 7249831SN/A store_addr, addr_offset); 7259831SN/A 7269831SN/A void *load_packet_data = load->packet->getPtr<void>(); 7279831SN/A void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset; 7289243SN/A 7299243SN/A std::memcpy(load_packet_data, store_packet_data, load_size); 7309243SN/A} 73110393Swendy.elsasser@arm.com 7329243SN/Avoid 73310206Sandreas.hansson@arm.comLSQ::StoreBuffer::step() 73410206Sandreas.hansson@arm.com{ 73510206Sandreas.hansson@arm.com DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n", 73610206Sandreas.hansson@arm.com numUnissuedAccesses); 73710206Sandreas.hansson@arm.com 7389243SN/A if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) { 73910206Sandreas.hansson@arm.com /* Clear all the leading barriers */ 74010206Sandreas.hansson@arm.com while (!slots.empty() && 7419243SN/A slots.front()->isComplete() && slots.front()->isBarrier()) 7429243SN/A { 7439243SN/A LSQRequestPtr barrier = slots.front(); 7449243SN/A 7459243SN/A DPRINTF(MinorMem, "Clearing barrier for inst: %s\n", 7469243SN/A *(barrier->inst)); 74710393Swendy.elsasser@arm.com 7489243SN/A numUnissuedAccesses--; 7499243SN/A lsq.clearMemBarrier(barrier->inst); 7509243SN/A slots.pop_front(); 7519243SN/A 7529243SN/A delete barrier; 75310393Swendy.elsasser@arm.com } 7549974SN/A 7559974SN/A auto i = slots.begin(); 7569974SN/A bool issued = true; 7579974SN/A unsigned int issue_count = 0; 7589974SN/A 7599974SN/A /* Skip trying if the memory system is busy */ 7609974SN/A if (lsq.state == LSQ::MemoryNeedsRetry) 76110393Swendy.elsasser@arm.com issued = false; 7629974SN/A 7639974SN/A /* Try to issue all stores in order starting from the head 7649974SN/A * of the queue. Responses are allowed to be retired 7659974SN/A * out of order */ 7669974SN/A while (issued && 7679974SN/A issue_count < storeLimitPerCycle && 7689974SN/A lsq.canSendToMemorySystem() && 76910393Swendy.elsasser@arm.com i != slots.end()) 77010393Swendy.elsasser@arm.com { 77110393Swendy.elsasser@arm.com LSQRequestPtr request = *i; 77210393Swendy.elsasser@arm.com 77310393Swendy.elsasser@arm.com DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d" 77410393Swendy.elsasser@arm.com " state: %s\n", 77510393Swendy.elsasser@arm.com *(request->inst), request->sentAllPackets(), 77610393Swendy.elsasser@arm.com request->state); 77710393Swendy.elsasser@arm.com 77810393Swendy.elsasser@arm.com if (request->isBarrier() && request->isComplete()) { 77910393Swendy.elsasser@arm.com /* Give up at barriers */ 78010393Swendy.elsasser@arm.com issued = false; 78110393Swendy.elsasser@arm.com } else if (!(request->state == LSQRequest::StoreBufferIssuing && 78210393Swendy.elsasser@arm.com request->sentAllPackets())) 78310393Swendy.elsasser@arm.com { 78410393Swendy.elsasser@arm.com DPRINTF(MinorMem, "Trying to send request: %s to memory" 7859974SN/A " system\n", *(request->inst)); 78610393Swendy.elsasser@arm.com 78710393Swendy.elsasser@arm.com if (lsq.tryToSend(request)) { 78810393Swendy.elsasser@arm.com /* Barrier are accounted for as they are cleared from 78910393Swendy.elsasser@arm.com * the queue, not after their transfers are complete */ 79010211Sandreas.hansson@arm.com if (!request->isBarrier()) 79110393Swendy.elsasser@arm.com numUnissuedAccesses--; 79210393Swendy.elsasser@arm.com issue_count++; 7939974SN/A } else { 79410211Sandreas.hansson@arm.com /* Don't step on to the next store buffer entry if this 7959974SN/A * one hasn't issued all its packets as the store 7969974SN/A * buffer must still enforce ordering */ 7979974SN/A issued = false; 7989974SN/A } 7999974SN/A } 8009974SN/A i++; 8019974SN/A } 8029974SN/A } 8039974SN/A} 8049974SN/A 8059974SN/Avoid 8069974SN/ALSQ::completeMemBarrierInst(MinorDynInstPtr inst, 80710146Sandreas.hansson@arm.com bool committed) 8089243SN/A{ 8099243SN/A if (committed) { 8109243SN/A /* Not already sent to the store buffer as a store request? */ 8119243SN/A if (!inst->inStoreBuffer) { 8129243SN/A /* Insert an entry into the store buffer to tick off barriers 8139243SN/A * until there are none in flight */ 8149243SN/A storeBuffer.insert(new BarrierDataRequest(*this, inst)); 8159243SN/A } 8169243SN/A } else { 8179243SN/A /* Clear the barrier anyway if it wasn't actually committed */ 8189243SN/A clearMemBarrier(inst); 8199243SN/A } 8209243SN/A} 8219549SN/A 8229549SN/Avoid 8239549SN/ALSQ::StoreBuffer::minorTrace() const 8249726SN/A{ 8259726SN/A unsigned int size = slots.size(); 8269726SN/A unsigned int i = 0; 8279243SN/A std::ostringstream os; 8289587SN/A 8299587SN/A while (i < size) { 8309587SN/A LSQRequestPtr request = slots[i]; 8319243SN/A 8329243SN/A request->reportData(os); 8339243SN/A 8349243SN/A i++; 8359243SN/A if (i < numSlots) 8369243SN/A os << ','; 8379243SN/A } 8389243SN/A 83910246Sandreas.hansson@arm.com while (i < numSlots) { 8409488SN/A os << '-'; 84110246Sandreas.hansson@arm.com 84210246Sandreas.hansson@arm.com i++; 84310246Sandreas.hansson@arm.com if (i < numSlots) 8449969SN/A os << ','; 8459488SN/A } 8469488SN/A 8479488SN/A MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(), 84810207Sandreas.hansson@arm.com numUnissuedAccesses); 84910246Sandreas.hansson@arm.com} 85010246Sandreas.hansson@arm.com 85110207Sandreas.hansson@arm.comvoid 85210207Sandreas.hansson@arm.comLSQ::tryToSendToTransfers(LSQRequestPtr request) 85310207Sandreas.hansson@arm.com{ 85410207Sandreas.hansson@arm.com if (state == MemoryNeedsRetry) { 85510246Sandreas.hansson@arm.com DPRINTF(MinorMem, "Request needs retry, not issuing to" 85610246Sandreas.hansson@arm.com " memory until retry arrives\n"); 85710207Sandreas.hansson@arm.com return; 85810207Sandreas.hansson@arm.com } 85910207Sandreas.hansson@arm.com 86010207Sandreas.hansson@arm.com if (request->state == LSQRequest::InTranslation) { 86110247Sandreas.hansson@arm.com DPRINTF(MinorMem, "Request still in translation, not issuing to" 86210247Sandreas.hansson@arm.com " memory\n"); 86310247Sandreas.hansson@arm.com return; 86410247Sandreas.hansson@arm.com } 86510247Sandreas.hansson@arm.com 8669975SN/A assert(request->state == LSQRequest::Translated || 86710211Sandreas.hansson@arm.com request->state == LSQRequest::RequestIssuing || 86810246Sandreas.hansson@arm.com request->state == LSQRequest::Failed || 86910211Sandreas.hansson@arm.com request->state == LSQRequest::Complete); 87010211Sandreas.hansson@arm.com 87110394Swendy.elsasser@arm.com if (requests.empty() || requests.front() != request) { 87210211Sandreas.hansson@arm.com DPRINTF(MinorMem, "Request not at front of requests queue, can't" 8739971SN/A " issue to memory\n"); 8749971SN/A return; 87510210Sandreas.hansson@arm.com } 87610210Sandreas.hansson@arm.com 87710394Swendy.elsasser@arm.com if (transfers.unreservedRemainingSpace() == 0) { 87810394Swendy.elsasser@arm.com DPRINTF(MinorMem, "No space to insert request into transfers" 87910394Swendy.elsasser@arm.com " queue\n"); 88010394Swendy.elsasser@arm.com return; 88110394Swendy.elsasser@arm.com } 88210394Swendy.elsasser@arm.com 88310394Swendy.elsasser@arm.com if (request->isComplete() || request->state == LSQRequest::Failed) { 88410394Swendy.elsasser@arm.com DPRINTF(MinorMem, "Passing a %s transfer on to transfers" 88510394Swendy.elsasser@arm.com " queue\n", (request->isComplete() ? "completed" : "failed")); 88610394Swendy.elsasser@arm.com request->setState(LSQRequest::Complete); 88710394Swendy.elsasser@arm.com request->setSkipped(); 88810394Swendy.elsasser@arm.com moveFromRequestsToTransfers(request); 88910394Swendy.elsasser@arm.com return; 8909971SN/A } 89110208Sandreas.hansson@arm.com 8929971SN/A if (!execute.instIsRightStream(request->inst)) { 8939971SN/A /* Wrong stream, try to abort the transfer but only do so if 8949969SN/A * there are no packets in flight */ 8959824SN/A if (request->hasPacketsInMemSystem()) { 8969824SN/A DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 8979488SN/A " waiting for responses before aborting request\n"); 8989969SN/A } else { 89910210Sandreas.hansson@arm.com DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 90010210Sandreas.hansson@arm.com " aborting request\n"); 90110210Sandreas.hansson@arm.com request->setState(LSQRequest::Complete); 9029488SN/A request->setSkipped(); 9039488SN/A moveFromRequestsToTransfers(request); 9049488SN/A } 9059488SN/A return; 9069969SN/A } 9079488SN/A 9089488SN/A if (request->fault != NoFault) { 9099969SN/A if (request->inst->staticInst->isPrefetch()) { 9109488SN/A DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n"); 9119488SN/A } 9129488SN/A DPRINTF(MinorMem, "Moving faulting request into the transfers" 9139488SN/A " queue\n"); 9149969SN/A request->setState(LSQRequest::Complete); 9159488SN/A request->setSkipped(); 91610210Sandreas.hansson@arm.com moveFromRequestsToTransfers(request); 9179488SN/A return; 9189488SN/A } 91910210Sandreas.hansson@arm.com 92010210Sandreas.hansson@arm.com bool is_load = request->isLoad; 92110210Sandreas.hansson@arm.com bool is_llsc = request->request.isLLSC(); 9229488SN/A bool is_swap = request->request.isSwap(); 92310208Sandreas.hansson@arm.com bool bufferable = !(request->request.isUncacheable() || 92410208Sandreas.hansson@arm.com is_llsc || is_swap); 92510208Sandreas.hansson@arm.com 92610208Sandreas.hansson@arm.com if (is_load) { 92710208Sandreas.hansson@arm.com if (numStoresInTransfers != 0) { 92810208Sandreas.hansson@arm.com DPRINTF(MinorMem, "Load request with stores still in transfers" 92910208Sandreas.hansson@arm.com " queue, stalling\n"); 93010208Sandreas.hansson@arm.com return; 93110208Sandreas.hansson@arm.com } 93210208Sandreas.hansson@arm.com } else { 93310208Sandreas.hansson@arm.com /* Store. Can it be sent to the store buffer? */ 93410208Sandreas.hansson@arm.com if (bufferable && !request->request.isMmappedIpr()) { 93510208Sandreas.hansson@arm.com request->setState(LSQRequest::StoreToStoreBuffer); 93610208Sandreas.hansson@arm.com moveFromRequestsToTransfers(request); 93710208Sandreas.hansson@arm.com DPRINTF(MinorMem, "Moving store into transfers queue\n"); 93810208Sandreas.hansson@arm.com return; 93910208Sandreas.hansson@arm.com } 94010208Sandreas.hansson@arm.com } 9419488SN/A 9429488SN/A /* Check if this is the head instruction (and so must be executable as 9439488SN/A * its stream sequence number was checked above) for loads which must 94410247Sandreas.hansson@arm.com * not be speculatively issued and stores which must be issued here */ 94510207Sandreas.hansson@arm.com if (!bufferable) { 94610207Sandreas.hansson@arm.com if (!execute.instIsHeadInst(request->inst)) { 94710207Sandreas.hansson@arm.com DPRINTF(MinorMem, "Memory access not the head inst., can't be" 94810207Sandreas.hansson@arm.com " sure it can be performed, not issuing\n"); 94910207Sandreas.hansson@arm.com return; 95010207Sandreas.hansson@arm.com } 95110207Sandreas.hansson@arm.com 95210207Sandreas.hansson@arm.com unsigned int forwarding_slot = 0; 95310207Sandreas.hansson@arm.com 95410207Sandreas.hansson@arm.com if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) != 95510214Sandreas.hansson@arm.com NoAddrRangeCoverage) 95610214Sandreas.hansson@arm.com { 95710214Sandreas.hansson@arm.com DPRINTF(MinorMem, "Memory access can receive forwarded data" 95810211Sandreas.hansson@arm.com " from the store buffer, need to wait for store buffer to" 95910211Sandreas.hansson@arm.com " drain\n"); 96010211Sandreas.hansson@arm.com return; 96110207Sandreas.hansson@arm.com } 96210207Sandreas.hansson@arm.com } 96310207Sandreas.hansson@arm.com 96410207Sandreas.hansson@arm.com /* True: submit this packet to the transfers queue to be sent to the 96510247Sandreas.hansson@arm.com * memory system. 96610247Sandreas.hansson@arm.com * False: skip the memory and push a packet for this request onto 96710247Sandreas.hansson@arm.com * requests */ 96810247Sandreas.hansson@arm.com bool do_access = true; 96910247Sandreas.hansson@arm.com 97010247Sandreas.hansson@arm.com if (!is_llsc) { 97110207Sandreas.hansson@arm.com /* Check for match in the store buffer */ 97210208Sandreas.hansson@arm.com if (is_load) { 97310208Sandreas.hansson@arm.com unsigned int forwarding_slot = 0; 97410208Sandreas.hansson@arm.com AddrRangeCoverage forwarding_result = 97510208Sandreas.hansson@arm.com storeBuffer.canForwardDataToLoad(request, 97610208Sandreas.hansson@arm.com forwarding_slot); 97710208Sandreas.hansson@arm.com 97810208Sandreas.hansson@arm.com switch (forwarding_result) { 97910211Sandreas.hansson@arm.com case FullAddrRangeCoverage: 98010211Sandreas.hansson@arm.com /* Forward data from the store buffer into this request and 98110211Sandreas.hansson@arm.com * repurpose this request's packet into a response packet */ 98210208Sandreas.hansson@arm.com storeBuffer.forwardStoreData(request, forwarding_slot); 98310208Sandreas.hansson@arm.com request->packet->makeResponse(); 98410208Sandreas.hansson@arm.com 98510208Sandreas.hansson@arm.com /* Just move between queues, no access */ 98610208Sandreas.hansson@arm.com do_access = false; 98710207Sandreas.hansson@arm.com break; 98810207Sandreas.hansson@arm.com case PartialAddrRangeCoverage: 98910207Sandreas.hansson@arm.com DPRINTF(MinorMem, "Load partly satisfied by store buffer" 99010208Sandreas.hansson@arm.com " data. Must wait for the store to complete\n"); 99110208Sandreas.hansson@arm.com return; 99210208Sandreas.hansson@arm.com break; 99310207Sandreas.hansson@arm.com case NoAddrRangeCoverage: 99410207Sandreas.hansson@arm.com DPRINTF(MinorMem, "No forwardable data from store buffer\n"); 99510207Sandreas.hansson@arm.com /* Fall through to try access */ 99610207Sandreas.hansson@arm.com break; 99710146Sandreas.hansson@arm.com } 9989243SN/A } 9999243SN/A } else { 10009243SN/A if (!canSendToMemorySystem()) { 10019243SN/A DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 100210211Sandreas.hansson@arm.com return; 10039967SN/A } 10049243SN/A 100510211Sandreas.hansson@arm.com SimpleThread &thread = *cpu.threads[request->inst->id.threadId]; 100610211Sandreas.hansson@arm.com 100710211Sandreas.hansson@arm.com TheISA::PCState old_pc = thread.pcState(); 100810211Sandreas.hansson@arm.com ExecContext context(cpu, thread, execute, request->inst); 100910211Sandreas.hansson@arm.com 101010211Sandreas.hansson@arm.com /* Handle LLSC requests and tests */ 101110211Sandreas.hansson@arm.com if (is_load) { 101210211Sandreas.hansson@arm.com TheISA::handleLockedRead(&context, &request->request); 101310211Sandreas.hansson@arm.com } else { 101410209Sandreas.hansson@arm.com do_access = TheISA::handleLockedWrite(&context, 101510211Sandreas.hansson@arm.com &request->request, cacheBlockMask); 101610211Sandreas.hansson@arm.com 101710209Sandreas.hansson@arm.com if (!do_access) { 101810209Sandreas.hansson@arm.com DPRINTF(MinorMem, "Not perfoming a memory " 101910211Sandreas.hansson@arm.com "access for store conditional\n"); 10209488SN/A } 10219973SN/A } 102210211Sandreas.hansson@arm.com thread.pcState(old_pc); 102310211Sandreas.hansson@arm.com } 102410211Sandreas.hansson@arm.com 10259973SN/A /* See the do_access comment above */ 102610210Sandreas.hansson@arm.com if (do_access) { 102710210Sandreas.hansson@arm.com if (!canSendToMemorySystem()) { 102810246Sandreas.hansson@arm.com DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 102910210Sandreas.hansson@arm.com return; 103010211Sandreas.hansson@arm.com } 103110211Sandreas.hansson@arm.com 103210209Sandreas.hansson@arm.com /* Remember if this is an access which can't be idly 103310209Sandreas.hansson@arm.com * discarded by an interrupt */ 103410211Sandreas.hansson@arm.com if (!bufferable && !request->issuedToMemory) { 103510211Sandreas.hansson@arm.com numAccessesIssuedToMemory++; 103610211Sandreas.hansson@arm.com request->issuedToMemory = true; 103710211Sandreas.hansson@arm.com } 103810211Sandreas.hansson@arm.com 103910211Sandreas.hansson@arm.com if (tryToSend(request)) 104010211Sandreas.hansson@arm.com moveFromRequestsToTransfers(request); 104110211Sandreas.hansson@arm.com } else { 104210211Sandreas.hansson@arm.com request->setState(LSQRequest::Complete); 104310211Sandreas.hansson@arm.com moveFromRequestsToTransfers(request); 104410394Swendy.elsasser@arm.com } 104510394Swendy.elsasser@arm.com} 104610394Swendy.elsasser@arm.com 104710394Swendy.elsasser@arm.combool 104810394Swendy.elsasser@arm.comLSQ::tryToSend(LSQRequestPtr request) 104910394Swendy.elsasser@arm.com{ 105010394Swendy.elsasser@arm.com bool ret = false; 105110394Swendy.elsasser@arm.com 105210394Swendy.elsasser@arm.com if (!canSendToMemorySystem()) { 105310394Swendy.elsasser@arm.com DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n", 105410394Swendy.elsasser@arm.com *(request->inst)); 105510394Swendy.elsasser@arm.com } else { 105610394Swendy.elsasser@arm.com PacketPtr packet = request->getHeadPacket(); 105710394Swendy.elsasser@arm.com 105810394Swendy.elsasser@arm.com DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n", 105910394Swendy.elsasser@arm.com *(request->inst), packet->req->getVaddr()); 106010394Swendy.elsasser@arm.com 106110394Swendy.elsasser@arm.com /* The sender state of the packet *must* be an LSQRequest 106210394Swendy.elsasser@arm.com * so the response can be correctly handled */ 106310394Swendy.elsasser@arm.com assert(packet->findNextSenderState<LSQRequest>()); 106410394Swendy.elsasser@arm.com 106510394Swendy.elsasser@arm.com if (request->request.isMmappedIpr()) { 106610394Swendy.elsasser@arm.com ThreadContext *thread = 106710394Swendy.elsasser@arm.com cpu.getContext(request->request.threadId()); 106810394Swendy.elsasser@arm.com 106910394Swendy.elsasser@arm.com if (request->isLoad) { 107010394Swendy.elsasser@arm.com DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst)); 107110394Swendy.elsasser@arm.com TheISA::handleIprRead(thread, packet); 107210394Swendy.elsasser@arm.com } else { 107310394Swendy.elsasser@arm.com DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst)); 107410394Swendy.elsasser@arm.com TheISA::handleIprWrite(thread, packet); 107510394Swendy.elsasser@arm.com } 107610211Sandreas.hansson@arm.com 107710393Swendy.elsasser@arm.com request->stepToNextPacket(); 107810393Swendy.elsasser@arm.com ret = request->sentAllPackets(); 107910393Swendy.elsasser@arm.com 108010212Sandreas.hansson@arm.com if (!ret) { 108110212Sandreas.hansson@arm.com DPRINTF(MinorMem, "IPR access has another packet: %s\n", 108210212Sandreas.hansson@arm.com *(request->inst)); 108310212Sandreas.hansson@arm.com } 108410212Sandreas.hansson@arm.com 108510212Sandreas.hansson@arm.com if (ret) 108610210Sandreas.hansson@arm.com request->setState(LSQRequest::Complete); 108710209Sandreas.hansson@arm.com else 108810209Sandreas.hansson@arm.com request->setState(LSQRequest::RequestIssuing); 108910209Sandreas.hansson@arm.com } else if (dcachePort.sendTimingReq(packet)) { 109010209Sandreas.hansson@arm.com DPRINTF(MinorMem, "Sent data memory request\n"); 109110209Sandreas.hansson@arm.com 109210209Sandreas.hansson@arm.com numAccessesInMemorySystem++; 109310209Sandreas.hansson@arm.com 109410209Sandreas.hansson@arm.com request->stepToNextPacket(); 109510209Sandreas.hansson@arm.com 109610209Sandreas.hansson@arm.com ret = request->sentAllPackets(); 109710209Sandreas.hansson@arm.com 109810209Sandreas.hansson@arm.com switch (request->state) { 109910209Sandreas.hansson@arm.com case LSQRequest::Translated: 110010209Sandreas.hansson@arm.com case LSQRequest::RequestIssuing: 110110209Sandreas.hansson@arm.com /* Fully or partially issued a request in the transfers 110210209Sandreas.hansson@arm.com * queue */ 110310209Sandreas.hansson@arm.com request->setState(LSQRequest::RequestIssuing); 110410209Sandreas.hansson@arm.com break; 110510209Sandreas.hansson@arm.com case LSQRequest::StoreInStoreBuffer: 110610209Sandreas.hansson@arm.com case LSQRequest::StoreBufferIssuing: 110710209Sandreas.hansson@arm.com /* Fully or partially issued a request in the store 110810209Sandreas.hansson@arm.com * buffer */ 110910209Sandreas.hansson@arm.com request->setState(LSQRequest::StoreBufferIssuing); 111010209Sandreas.hansson@arm.com break; 111110209Sandreas.hansson@arm.com default: 111210209Sandreas.hansson@arm.com assert(false); 111310209Sandreas.hansson@arm.com break; 111410209Sandreas.hansson@arm.com } 111510209Sandreas.hansson@arm.com 111610209Sandreas.hansson@arm.com state = MemoryRunning; 111710209Sandreas.hansson@arm.com } else { 111810209Sandreas.hansson@arm.com DPRINTF(MinorMem, 111910209Sandreas.hansson@arm.com "Sending data memory request - needs retry\n"); 112010209Sandreas.hansson@arm.com 112110209Sandreas.hansson@arm.com /* Needs to be resent, wait for that */ 112210209Sandreas.hansson@arm.com state = MemoryNeedsRetry; 112310209Sandreas.hansson@arm.com retryRequest = request; 112410209Sandreas.hansson@arm.com 112510209Sandreas.hansson@arm.com switch (request->state) { 112610209Sandreas.hansson@arm.com case LSQRequest::Translated: 112710209Sandreas.hansson@arm.com case LSQRequest::RequestIssuing: 112810209Sandreas.hansson@arm.com request->setState(LSQRequest::RequestNeedsRetry); 11299973SN/A break; 113010141SN/A case LSQRequest::StoreInStoreBuffer: 113110141SN/A case LSQRequest::StoreBufferIssuing: 113210209Sandreas.hansson@arm.com request->setState(LSQRequest::StoreBufferNeedsRetry); 113310209Sandreas.hansson@arm.com break; 113410209Sandreas.hansson@arm.com default: 113510209Sandreas.hansson@arm.com assert(false); 113610209Sandreas.hansson@arm.com break; 113710209Sandreas.hansson@arm.com } 113810209Sandreas.hansson@arm.com } 113910142SN/A } 114010247Sandreas.hansson@arm.com 114110247Sandreas.hansson@arm.com return ret; 114210247Sandreas.hansson@arm.com} 114310209Sandreas.hansson@arm.com 114410209Sandreas.hansson@arm.comvoid 114510209Sandreas.hansson@arm.comLSQ::moveFromRequestsToTransfers(LSQRequestPtr request) 114610247Sandreas.hansson@arm.com{ 114710247Sandreas.hansson@arm.com assert(!requests.empty() && requests.front() == request); 114810247Sandreas.hansson@arm.com assert(transfers.unreservedRemainingSpace() != 0); 11499973SN/A 115010209Sandreas.hansson@arm.com /* Need to count the number of stores in the transfers 115110209Sandreas.hansson@arm.com * queue so that loads know when their store buffer forwarding 11529963SN/A * results will be correct (only when all those stores 11539243SN/A * have reached the store buffer) */ 11549243SN/A if (!request->isLoad) 11559243SN/A numStoresInTransfers++; 115610211Sandreas.hansson@arm.com 115710211Sandreas.hansson@arm.com requests.pop(); 11589243SN/A transfers.push(request); 115910247Sandreas.hansson@arm.com} 116010247Sandreas.hansson@arm.com 116110247Sandreas.hansson@arm.combool 116210206Sandreas.hansson@arm.comLSQ::canSendToMemorySystem() 116310206Sandreas.hansson@arm.com{ 116410206Sandreas.hansson@arm.com return state == MemoryRunning && 116510206Sandreas.hansson@arm.com numAccessesInMemorySystem < inMemorySystemLimit; 116610206Sandreas.hansson@arm.com} 11679972SN/A 116810206Sandreas.hansson@arm.combool 11699977SN/ALSQ::recvTimingResp(PacketPtr response) 117010147Sandreas.hansson@arm.com{ 117110211Sandreas.hansson@arm.com LSQRequestPtr request = 11729977SN/A safe_cast<LSQRequestPtr>(response->popSenderState()); 11739977SN/A 11749977SN/A DPRINTF(MinorMem, "Received response packet inst: %s" 117510206Sandreas.hansson@arm.com " addr: 0x%x cmd: %s\n", 117610206Sandreas.hansson@arm.com *(request->inst), response->getAddr(), 117710206Sandreas.hansson@arm.com response->cmd.toString()); 117810206Sandreas.hansson@arm.com 117910211Sandreas.hansson@arm.com numAccessesInMemorySystem--; 11809977SN/A 118110147Sandreas.hansson@arm.com if (response->isError()) { 118210211Sandreas.hansson@arm.com DPRINTF(MinorMem, "Received error response packet: %s\n", 11839977SN/A *request->inst); 11849977SN/A } 11859977SN/A 11869243SN/A switch (request->state) { 11879243SN/A case LSQRequest::RequestIssuing: 11889243SN/A case LSQRequest::RequestNeedsRetry: 11899243SN/A /* Response to a request from the transfers queue */ 119010206Sandreas.hansson@arm.com request->retireResponse(response); 11919243SN/A 119210393Swendy.elsasser@arm.com DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n", 119310393Swendy.elsasser@arm.com request->hasPacketsInMemSystem(), request->isComplete()); 119410393Swendy.elsasser@arm.com 119510206Sandreas.hansson@arm.com break; 119610206Sandreas.hansson@arm.com case LSQRequest::StoreBufferIssuing: 119710206Sandreas.hansson@arm.com case LSQRequest::StoreBufferNeedsRetry: 11989243SN/A /* Response to a request from the store buffer */ 119910206Sandreas.hansson@arm.com request->retireResponse(response); 120010206Sandreas.hansson@arm.com 120110206Sandreas.hansson@arm.com /* Remove completed requests unless they are barrier (which will 120210206Sandreas.hansson@arm.com * need to be removed in order */ 120310206Sandreas.hansson@arm.com if (request->isComplete()) { 120410206Sandreas.hansson@arm.com if (!request->isBarrier()) { 120510206Sandreas.hansson@arm.com storeBuffer.deleteRequest(request); 120610393Swendy.elsasser@arm.com } else { 120710206Sandreas.hansson@arm.com DPRINTF(MinorMem, "Completed transfer for barrier: %s" 120810206Sandreas.hansson@arm.com " leaving the request as it is also a barrier\n", 120910206Sandreas.hansson@arm.com *(request->inst)); 121010206Sandreas.hansson@arm.com } 121110206Sandreas.hansson@arm.com } 121210206Sandreas.hansson@arm.com break; 121310206Sandreas.hansson@arm.com default: 121410206Sandreas.hansson@arm.com /* Shouldn't be allowed to receive a response from another 121510393Swendy.elsasser@arm.com * state */ 121610206Sandreas.hansson@arm.com assert(false); 121710206Sandreas.hansson@arm.com break; 121810207Sandreas.hansson@arm.com } 121910207Sandreas.hansson@arm.com 122010207Sandreas.hansson@arm.com /* We go to idle even if there are more things in the requests queue 122110207Sandreas.hansson@arm.com * as it's the job of step to actually step us on to the next 122210207Sandreas.hansson@arm.com * transaction */ 122310207Sandreas.hansson@arm.com 122410207Sandreas.hansson@arm.com /* Let's try and wake up the processor for the next cycle */ 122510207Sandreas.hansson@arm.com cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 122610207Sandreas.hansson@arm.com 122710207Sandreas.hansson@arm.com /* Never busy */ 122810207Sandreas.hansson@arm.com return true; 122910207Sandreas.hansson@arm.com} 123010207Sandreas.hansson@arm.com 123110207Sandreas.hansson@arm.comvoid 123210207Sandreas.hansson@arm.comLSQ::recvRetry() 123310207Sandreas.hansson@arm.com{ 123410206Sandreas.hansson@arm.com DPRINTF(MinorMem, "Received retry request\n"); 123510206Sandreas.hansson@arm.com 123610206Sandreas.hansson@arm.com assert(state == MemoryNeedsRetry); 123710206Sandreas.hansson@arm.com 123810206Sandreas.hansson@arm.com switch (retryRequest->state) { 123910206Sandreas.hansson@arm.com case LSQRequest::RequestNeedsRetry: 124010206Sandreas.hansson@arm.com /* Retry in the requests queue */ 124110206Sandreas.hansson@arm.com retryRequest->setState(LSQRequest::Translated); 124210206Sandreas.hansson@arm.com break; 124310206Sandreas.hansson@arm.com case LSQRequest::StoreBufferNeedsRetry: 124410206Sandreas.hansson@arm.com /* Retry in the store buffer */ 124510206Sandreas.hansson@arm.com retryRequest->setState(LSQRequest::StoreInStoreBuffer); 124610206Sandreas.hansson@arm.com break; 124710206Sandreas.hansson@arm.com default: 124810206Sandreas.hansson@arm.com assert(false); 124910206Sandreas.hansson@arm.com } 125010206Sandreas.hansson@arm.com 125110206Sandreas.hansson@arm.com /* Set state back to MemoryRunning so that the following 125210206Sandreas.hansson@arm.com * tryToSend can actually send. Note that this won't 125310206Sandreas.hansson@arm.com * allow another transfer in as tryToSend should 125410206Sandreas.hansson@arm.com * issue a memory request and either succeed for this 125510206Sandreas.hansson@arm.com * request or return the LSQ back to MemoryNeedsRetry */ 125610206Sandreas.hansson@arm.com state = MemoryRunning; 125710206Sandreas.hansson@arm.com 125810206Sandreas.hansson@arm.com /* Try to resend the request */ 125910206Sandreas.hansson@arm.com if (tryToSend(retryRequest)) { 126010206Sandreas.hansson@arm.com /* Successfully sent, need to move the request */ 126110206Sandreas.hansson@arm.com switch (retryRequest->state) { 126210393Swendy.elsasser@arm.com case LSQRequest::RequestIssuing: 126310206Sandreas.hansson@arm.com /* In the requests queue */ 126410215Sandreas.hansson@arm.com moveFromRequestsToTransfers(retryRequest); 126510215Sandreas.hansson@arm.com break; 126610393Swendy.elsasser@arm.com case LSQRequest::StoreBufferIssuing: 126710393Swendy.elsasser@arm.com /* In the store buffer */ 126810393Swendy.elsasser@arm.com storeBuffer.numUnissuedAccesses--; 126910393Swendy.elsasser@arm.com break; 127010393Swendy.elsasser@arm.com default: 127110394Swendy.elsasser@arm.com assert(false); 127210394Swendy.elsasser@arm.com break; 127310393Swendy.elsasser@arm.com } 127410393Swendy.elsasser@arm.com } 127510215Sandreas.hansson@arm.com 127610206Sandreas.hansson@arm.com retryRequest = NULL; 127710206Sandreas.hansson@arm.com} 127810215Sandreas.hansson@arm.com 127910215Sandreas.hansson@arm.comLSQ::LSQ(std::string name_, std::string dcache_port_name_, 128010215Sandreas.hansson@arm.com MinorCPU &cpu_, Execute &execute_, 128110215Sandreas.hansson@arm.com unsigned int in_memory_system_limit, unsigned int line_width, 128210215Sandreas.hansson@arm.com unsigned int requests_queue_size, unsigned int transfers_queue_size, 128310215Sandreas.hansson@arm.com unsigned int store_buffer_size, 128410215Sandreas.hansson@arm.com unsigned int store_buffer_cycle_store_limit) : 128510215Sandreas.hansson@arm.com Named(name_), 128610215Sandreas.hansson@arm.com cpu(cpu_), 128710215Sandreas.hansson@arm.com execute(execute_), 128810215Sandreas.hansson@arm.com dcachePort(dcache_port_name_, *this, cpu_), 128910215Sandreas.hansson@arm.com lastMemBarrier(0), 129010215Sandreas.hansson@arm.com state(MemoryRunning), 129110215Sandreas.hansson@arm.com inMemorySystemLimit(in_memory_system_limit), 129210215Sandreas.hansson@arm.com lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)), 129310215Sandreas.hansson@arm.com requests(name_ + ".requests", "addr", requests_queue_size), 129410215Sandreas.hansson@arm.com transfers(name_ + ".transfers", "addr", transfers_queue_size), 129510206Sandreas.hansson@arm.com storeBuffer(name_ + ".storeBuffer", 129610206Sandreas.hansson@arm.com *this, store_buffer_size, store_buffer_cycle_store_limit), 129710206Sandreas.hansson@arm.com numAccessesInMemorySystem(0), 129810206Sandreas.hansson@arm.com numAccessesInDTLB(0), 129910206Sandreas.hansson@arm.com numStoresInTransfers(0), 130010206Sandreas.hansson@arm.com numAccessesIssuedToMemory(0), 130110206Sandreas.hansson@arm.com retryRequest(NULL), 130210206Sandreas.hansson@arm.com cacheBlockMask(~(cpu_.cacheLineSize() - 1)) 130310206Sandreas.hansson@arm.com{ 130410206Sandreas.hansson@arm.com if (in_memory_system_limit < 1) { 130510206Sandreas.hansson@arm.com fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_, 130610206Sandreas.hansson@arm.com in_memory_system_limit); 130710206Sandreas.hansson@arm.com } 130810206Sandreas.hansson@arm.com 13099352SN/A if (store_buffer_cycle_store_limit < 1) { 131010393Swendy.elsasser@arm.com fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be" 131110206Sandreas.hansson@arm.com " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit); 131210206Sandreas.hansson@arm.com } 131310206Sandreas.hansson@arm.com 131410393Swendy.elsasser@arm.com if (requests_queue_size < 1) { 131510394Swendy.elsasser@arm.com fatal("%s: executeLSQRequestsQueueSize must be" 131610394Swendy.elsasser@arm.com " >= 1 (%d)\n", name_, requests_queue_size); 131710394Swendy.elsasser@arm.com } 131810394Swendy.elsasser@arm.com 131910394Swendy.elsasser@arm.com if (transfers_queue_size < 1) { 132010394Swendy.elsasser@arm.com fatal("%s: executeLSQTransfersQueueSize must be" 132110393Swendy.elsasser@arm.com " >= 1 (%d)\n", name_, transfers_queue_size); 132210393Swendy.elsasser@arm.com } 132310206Sandreas.hansson@arm.com 132410206Sandreas.hansson@arm.com if (store_buffer_size < 1) { 132510206Sandreas.hansson@arm.com fatal("%s: executeLSQStoreBufferSize must be" 132610206Sandreas.hansson@arm.com " >= 1 (%d)\n", name_, store_buffer_size); 132710206Sandreas.hansson@arm.com } 132810206Sandreas.hansson@arm.com 132910206Sandreas.hansson@arm.com if ((lineWidth & (lineWidth - 1)) != 0) { 133010206Sandreas.hansson@arm.com fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth); 133110206Sandreas.hansson@arm.com } 133210206Sandreas.hansson@arm.com} 133310206Sandreas.hansson@arm.com 133410206Sandreas.hansson@arm.comLSQ::~LSQ() 133510206Sandreas.hansson@arm.com{ } 133610206Sandreas.hansson@arm.com 133710206Sandreas.hansson@arm.comLSQ::LSQRequest::~LSQRequest() 133810206Sandreas.hansson@arm.com{ 133910206Sandreas.hansson@arm.com if (packet) 134010206Sandreas.hansson@arm.com delete packet; 134110206Sandreas.hansson@arm.com if (data) 134210206Sandreas.hansson@arm.com delete [] data; 134310206Sandreas.hansson@arm.com} 134410206Sandreas.hansson@arm.com 134510206Sandreas.hansson@arm.com/** 134610206Sandreas.hansson@arm.com * Step the memory access mechanism on to its next state. In reality, most 134710206Sandreas.hansson@arm.com * of the stepping is done by the callbacks on the LSQ but this 134810206Sandreas.hansson@arm.com * function is responsible for issuing memory requests lodged in the 134910206Sandreas.hansson@arm.com * requests queue. 135010206Sandreas.hansson@arm.com */ 135110206Sandreas.hansson@arm.comvoid 135210206Sandreas.hansson@arm.comLSQ::step() 135310206Sandreas.hansson@arm.com{ 135410206Sandreas.hansson@arm.com /* Try to move address-translated requests between queues and issue 13559352SN/A * them */ 13569243SN/A if (!requests.empty()) 13579243SN/A tryToSendToTransfers(requests.front()); 13589967SN/A 135910393Swendy.elsasser@arm.com storeBuffer.step(); 136010393Swendy.elsasser@arm.com} 13619967SN/A 13629967SN/ALSQ::LSQRequestPtr 136310211Sandreas.hansson@arm.comLSQ::findResponse(MinorDynInstPtr inst) 13649967SN/A{ 136510393Swendy.elsasser@arm.com LSQ::LSQRequestPtr ret = NULL; 136610393Swendy.elsasser@arm.com 136710393Swendy.elsasser@arm.com if (!transfers.empty()) { 136810393Swendy.elsasser@arm.com LSQRequestPtr request = transfers.front(); 136910393Swendy.elsasser@arm.com 137010393Swendy.elsasser@arm.com /* Same instruction and complete access or a store that's 137110393Swendy.elsasser@arm.com * capable of being moved to the store buffer */ 13729967SN/A if (request->inst->id == inst->id) { 13739967SN/A bool complete = request->isComplete(); 13749967SN/A bool can_store = storeBuffer.canInsert(); 13759967SN/A bool to_store_buffer = request->state == 13769967SN/A LSQRequest::StoreToStoreBuffer; 13779967SN/A 13789967SN/A if ((complete && !(request->isBarrier() && !can_store)) || 13799967SN/A (to_store_buffer && can_store)) 138010211Sandreas.hansson@arm.com { 138110211Sandreas.hansson@arm.com ret = request; 13829967SN/A } 13839967SN/A } 138410211Sandreas.hansson@arm.com } 138510211Sandreas.hansson@arm.com 138610211Sandreas.hansson@arm.com if (ret) { 138710393Swendy.elsasser@arm.com DPRINTF(MinorMem, "Found matching memory response for inst: %s\n", 138810211Sandreas.hansson@arm.com *inst); 138910211Sandreas.hansson@arm.com } else { 139010211Sandreas.hansson@arm.com DPRINTF(MinorMem, "No matching memory response for inst: %s\n", 139110211Sandreas.hansson@arm.com *inst); 139210393Swendy.elsasser@arm.com } 139310393Swendy.elsasser@arm.com 139410393Swendy.elsasser@arm.com return ret; 139510393Swendy.elsasser@arm.com} 139610393Swendy.elsasser@arm.com 139710393Swendy.elsasser@arm.comvoid 139810393Swendy.elsasser@arm.comLSQ::popResponse(LSQ::LSQRequestPtr response) 139910393Swendy.elsasser@arm.com{ 140010393Swendy.elsasser@arm.com assert(!transfers.empty() && transfers.front() == response); 140110393Swendy.elsasser@arm.com 140210393Swendy.elsasser@arm.com transfers.pop(); 140310393Swendy.elsasser@arm.com 140410393Swendy.elsasser@arm.com if (!response->isLoad) 140510393Swendy.elsasser@arm.com numStoresInTransfers--; 140610393Swendy.elsasser@arm.com 140710393Swendy.elsasser@arm.com if (response->issuedToMemory) 140810393Swendy.elsasser@arm.com numAccessesIssuedToMemory--; 140910393Swendy.elsasser@arm.com 141010393Swendy.elsasser@arm.com if (response->state != LSQRequest::StoreInStoreBuffer) { 141110393Swendy.elsasser@arm.com DPRINTF(MinorMem, "Deleting %s request: %s\n", 141210393Swendy.elsasser@arm.com (response->isLoad ? "load" : "store"), 141310393Swendy.elsasser@arm.com *(response->inst)); 141410393Swendy.elsasser@arm.com 141510393Swendy.elsasser@arm.com delete response; 141610393Swendy.elsasser@arm.com } 141710393Swendy.elsasser@arm.com} 141810393Swendy.elsasser@arm.com 141910393Swendy.elsasser@arm.comvoid 142010393Swendy.elsasser@arm.comLSQ::sendStoreToStoreBuffer(LSQRequestPtr request) 142110393Swendy.elsasser@arm.com{ 142210211Sandreas.hansson@arm.com assert(request->state == LSQRequest::StoreToStoreBuffer); 14239967SN/A 14249967SN/A DPRINTF(MinorMem, "Sending store: %s to store buffer\n", 14259967SN/A *(request->inst)); 142610211Sandreas.hansson@arm.com 142710393Swendy.elsasser@arm.com request->inst->inStoreBuffer = true; 142810393Swendy.elsasser@arm.com 142910393Swendy.elsasser@arm.com storeBuffer.insert(request); 143010393Swendy.elsasser@arm.com} 143110393Swendy.elsasser@arm.com 143210393Swendy.elsasser@arm.combool 143310393Swendy.elsasser@arm.comLSQ::isDrained() 143410393Swendy.elsasser@arm.com{ 143510393Swendy.elsasser@arm.com return requests.empty() && transfers.empty() && 143610393Swendy.elsasser@arm.com storeBuffer.isDrained(); 143710393Swendy.elsasser@arm.com} 143810393Swendy.elsasser@arm.com 143910393Swendy.elsasser@arm.combool 144010393Swendy.elsasser@arm.comLSQ::needsToTick() 144110393Swendy.elsasser@arm.com{ 144210393Swendy.elsasser@arm.com bool ret = false; 144310393Swendy.elsasser@arm.com 144410393Swendy.elsasser@arm.com if (canSendToMemorySystem()) { 14459967SN/A bool have_translated_requests = !requests.empty() && 14469967SN/A requests.front()->state != LSQRequest::InTranslation && 14479967SN/A transfers.unreservedRemainingSpace() != 0; 14489243SN/A 144910146Sandreas.hansson@arm.com ret = have_translated_requests || 14509243SN/A storeBuffer.numUnissuedStores() != 0; 145110207Sandreas.hansson@arm.com } 145210207Sandreas.hansson@arm.com 145310207Sandreas.hansson@arm.com if (ret) 145410207Sandreas.hansson@arm.com DPRINTF(Activity, "Need to tick\n"); 14559243SN/A 145610207Sandreas.hansson@arm.com return ret; 145710207Sandreas.hansson@arm.com} 14589243SN/A 145910207Sandreas.hansson@arm.comvoid 146010207Sandreas.hansson@arm.comLSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data, 146110207Sandreas.hansson@arm.com unsigned int size, Addr addr, unsigned int flags, uint64_t *res) 146210207Sandreas.hansson@arm.com{ 146310207Sandreas.hansson@arm.com bool needs_burst = transferNeedsBurst(addr, size, lineWidth); 146410207Sandreas.hansson@arm.com LSQRequestPtr request; 146510207Sandreas.hansson@arm.com 146610207Sandreas.hansson@arm.com /* Copy given data into the request. The request will pass this to the 146710207Sandreas.hansson@arm.com * packet and then it will own the data */ 146810207Sandreas.hansson@arm.com uint8_t *request_data = NULL; 146910207Sandreas.hansson@arm.com 147010207Sandreas.hansson@arm.com DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:" 147110207Sandreas.hansson@arm.com " 0x%x%s lineWidth : 0x%x\n", 147210207Sandreas.hansson@arm.com (isLoad ? "load" : "store"), addr, size, flags, 147310207Sandreas.hansson@arm.com (needs_burst ? " (needs burst)" : ""), lineWidth); 147410207Sandreas.hansson@arm.com 147510207Sandreas.hansson@arm.com if (!isLoad) { 147610207Sandreas.hansson@arm.com /* request_data becomes the property of a ...DataRequest (see below) 147710207Sandreas.hansson@arm.com * and destroyed by its destructor */ 147810208Sandreas.hansson@arm.com request_data = new uint8_t[size]; 147910208Sandreas.hansson@arm.com if (flags & Request::CACHE_BLOCK_ZERO) { 148010208Sandreas.hansson@arm.com /* For cache zeroing, just use zeroed data */ 148110214Sandreas.hansson@arm.com std::memset(request_data, 0, size); 148210214Sandreas.hansson@arm.com } else { 148310208Sandreas.hansson@arm.com std::memcpy(request_data, data, size); 148410214Sandreas.hansson@arm.com } 148510214Sandreas.hansson@arm.com } 148610214Sandreas.hansson@arm.com 148710214Sandreas.hansson@arm.com if (needs_burst) { 148810214Sandreas.hansson@arm.com request = new SplitDataRequest( 148910214Sandreas.hansson@arm.com *this, inst, isLoad, request_data, res); 149010214Sandreas.hansson@arm.com } else { 149110214Sandreas.hansson@arm.com request = new SingleDataRequest( 149210214Sandreas.hansson@arm.com *this, inst, isLoad, request_data, res); 149310214Sandreas.hansson@arm.com } 149410214Sandreas.hansson@arm.com 149510214Sandreas.hansson@arm.com if (inst->traceData) 149610214Sandreas.hansson@arm.com inst->traceData->setAddr(addr); 149710214Sandreas.hansson@arm.com 149810214Sandreas.hansson@arm.com request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0); 149910214Sandreas.hansson@arm.com request->request.setVirt(0 /* asid */, 150010208Sandreas.hansson@arm.com addr, size, flags, cpu.instMasterId(), 150110208Sandreas.hansson@arm.com /* I've no idea why we need the PC, but give it */ 150210208Sandreas.hansson@arm.com inst->pc.instAddr()); 150310247Sandreas.hansson@arm.com 150410214Sandreas.hansson@arm.com requests.push(request); 150510214Sandreas.hansson@arm.com request->startAddrTranslation(); 150610214Sandreas.hansson@arm.com} 150710214Sandreas.hansson@arm.com 150810214Sandreas.hansson@arm.comvoid 150910208Sandreas.hansson@arm.comLSQ::pushFailedRequest(MinorDynInstPtr inst) 151010207Sandreas.hansson@arm.com{ 151110247Sandreas.hansson@arm.com LSQRequestPtr request = new FailedDataRequest(*this, inst); 151210247Sandreas.hansson@arm.com requests.push(request); 151310247Sandreas.hansson@arm.com} 151410247Sandreas.hansson@arm.com 151510207Sandreas.hansson@arm.comvoid 151610208Sandreas.hansson@arm.comLSQ::minorTrace() const 151710208Sandreas.hansson@arm.com{ 151810208Sandreas.hansson@arm.com MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d" 151910208Sandreas.hansson@arm.com " lastMemBarrier=%d\n", 152010208Sandreas.hansson@arm.com state, numAccessesInDTLB, numAccessesInMemorySystem, 152110208Sandreas.hansson@arm.com numStoresInTransfers, lastMemBarrier); 15229975SN/A requests.minorTrace(); 15239975SN/A transfers.minorTrace(); 152410208Sandreas.hansson@arm.com storeBuffer.minorTrace(); 152510208Sandreas.hansson@arm.com} 15269243SN/A 152710208Sandreas.hansson@arm.comLSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_, 152810208Sandreas.hansson@arm.com unsigned int store_buffer_size, 152910208Sandreas.hansson@arm.com unsigned int store_limit_per_cycle) : 153010208Sandreas.hansson@arm.com Named(name_), lsq(lsq_), 153110207Sandreas.hansson@arm.com numSlots(store_buffer_size), 153210207Sandreas.hansson@arm.com storeLimitPerCycle(store_limit_per_cycle), 153310207Sandreas.hansson@arm.com slots(), 153410207Sandreas.hansson@arm.com numUnissuedAccesses(0) 153510207Sandreas.hansson@arm.com{ 153610207Sandreas.hansson@arm.com} 153710207Sandreas.hansson@arm.com 153810208Sandreas.hansson@arm.comPacketPtr 153910207Sandreas.hansson@arm.commakePacketForRequest(Request &request, bool isLoad, 154010211Sandreas.hansson@arm.com Packet::SenderState *sender_state, PacketDataPtr data) 154110207Sandreas.hansson@arm.com{ 154210207Sandreas.hansson@arm.com MemCmd command; 154310207Sandreas.hansson@arm.com 154410211Sandreas.hansson@arm.com /* Make a ret with the right command type to match the request */ 154510207Sandreas.hansson@arm.com if (request.isLLSC()) { 154610247Sandreas.hansson@arm.com command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq); 154710247Sandreas.hansson@arm.com } else if (request.isSwap()) { 154810247Sandreas.hansson@arm.com command = MemCmd::SwapReq; 154910207Sandreas.hansson@arm.com } else { 155010207Sandreas.hansson@arm.com command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq); 155110207Sandreas.hansson@arm.com } 155210207Sandreas.hansson@arm.com 155310211Sandreas.hansson@arm.com PacketPtr ret = new Packet(&request, command); 155410207Sandreas.hansson@arm.com 155510207Sandreas.hansson@arm.com if (sender_state) 155610207Sandreas.hansson@arm.com ret->pushSenderState(sender_state); 155710207Sandreas.hansson@arm.com 155810207Sandreas.hansson@arm.com if (isLoad) 155910207Sandreas.hansson@arm.com ret->allocate(); 156010207Sandreas.hansson@arm.com else 156110208Sandreas.hansson@arm.com ret->dataDynamicArray(data); 156210207Sandreas.hansson@arm.com 156310208Sandreas.hansson@arm.com return ret; 156410208Sandreas.hansson@arm.com} 156510208Sandreas.hansson@arm.com 156610211Sandreas.hansson@arm.comvoid 156710207Sandreas.hansson@arm.comLSQ::issuedMemBarrierInst(MinorDynInstPtr inst) 156810208Sandreas.hansson@arm.com{ 156910211Sandreas.hansson@arm.com assert(inst->isInst() && inst->staticInst->isMemBarrier()); 157010208Sandreas.hansson@arm.com assert(inst->id.execSeqNum > lastMemBarrier); 157110208Sandreas.hansson@arm.com 157210208Sandreas.hansson@arm.com /* Remember the barrier. We only have a notion of one 157310208Sandreas.hansson@arm.com * barrier so this may result in some mem refs being 157410208Sandreas.hansson@arm.com * delayed if they are between barriers */ 157510208Sandreas.hansson@arm.com lastMemBarrier = inst->id.execSeqNum; 157610208Sandreas.hansson@arm.com} 157710208Sandreas.hansson@arm.com 157810208Sandreas.hansson@arm.comvoid 157910208Sandreas.hansson@arm.comLSQ::LSQRequest::makePacket() 158010208Sandreas.hansson@arm.com{ 158110208Sandreas.hansson@arm.com /* Make the function idempotent */ 158210208Sandreas.hansson@arm.com if (packet) 158310208Sandreas.hansson@arm.com return; 158410208Sandreas.hansson@arm.com 158510208Sandreas.hansson@arm.com packet = makePacketForRequest(request, isLoad, this, data); 158610208Sandreas.hansson@arm.com /* Null the ret data so we know not to deallocate it when the 158710208Sandreas.hansson@arm.com * ret is destroyed. The data now belongs to the ret and 158810208Sandreas.hansson@arm.com * the ret is responsible for its destruction */ 158910208Sandreas.hansson@arm.com data = NULL; 159010208Sandreas.hansson@arm.com} 159110208Sandreas.hansson@arm.com 159210208Sandreas.hansson@arm.comstd::ostream & 159310208Sandreas.hansson@arm.comoperator <<(std::ostream &os, LSQ::MemoryState state) 159410208Sandreas.hansson@arm.com{ 159510208Sandreas.hansson@arm.com switch (state) { 159610208Sandreas.hansson@arm.com case LSQ::MemoryRunning: 159710208Sandreas.hansson@arm.com os << "MemoryRunning"; 159810208Sandreas.hansson@arm.com break; 159910208Sandreas.hansson@arm.com case LSQ::MemoryNeedsRetry: 160010208Sandreas.hansson@arm.com os << "MemoryNeedsRetry"; 160110208Sandreas.hansson@arm.com break; 160210208Sandreas.hansson@arm.com default: 160310208Sandreas.hansson@arm.com os << "MemoryState-" << static_cast<int>(state); 160410208Sandreas.hansson@arm.com break; 160510208Sandreas.hansson@arm.com } 160610208Sandreas.hansson@arm.com return os; 160710208Sandreas.hansson@arm.com} 160810208Sandreas.hansson@arm.com 160910208Sandreas.hansson@arm.comvoid 161010208Sandreas.hansson@arm.comLSQ::recvTimingSnoopReq(PacketPtr pkt) 161110208Sandreas.hansson@arm.com{ 161210208Sandreas.hansson@arm.com /* LLSC operations in Minor can't be speculative and are executed from 161310208Sandreas.hansson@arm.com * the head of the requests queue. We shouldn't need to do more than 161410208Sandreas.hansson@arm.com * this action on snoops. */ 161510208Sandreas.hansson@arm.com 161610208Sandreas.hansson@arm.com /* THREAD */ 161710208Sandreas.hansson@arm.com TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask); 161810208Sandreas.hansson@arm.com} 161910208Sandreas.hansson@arm.com 162010208Sandreas.hansson@arm.com} 162110208Sandreas.hansson@arm.com