lsq.cc revision 13449:2f7efa89c58b
1/*
2 * Copyright (c) 2013-2014,2017 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include "cpu/minor/lsq.hh"
41
42#include <iomanip>
43#include <sstream>
44
45#include "arch/locked_mem.hh"
46#include "arch/mmapped_ipr.hh"
47#include "base/logging.hh"
48#include "cpu/minor/cpu.hh"
49#include "cpu/minor/exec_context.hh"
50#include "cpu/minor/execute.hh"
51#include "cpu/minor/pipeline.hh"
52#include "debug/Activity.hh"
53#include "debug/MinorMem.hh"
54
55namespace Minor
56{
57
58/** Returns the offset of addr into an aligned a block of size block_size */
59static Addr
60addrBlockOffset(Addr addr, unsigned int block_size)
61{
62    return addr & (block_size - 1);
63}
64
65/** Returns true if the given [addr .. addr+size-1] transfer needs to be
66 *  fragmented across a block size of block_size */
67static bool
68transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
69{
70    return (addrBlockOffset(addr, block_size) + size) > block_size;
71}
72
73LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
74    PacketDataPtr data_, uint64_t *res_) :
75    SenderState(),
76    port(port_),
77    inst(inst_),
78    isLoad(isLoad_),
79    data(data_),
80    packet(NULL),
81    request(),
82    fault(NoFault),
83    res(res_),
84    skipped(false),
85    issuedToMemory(false),
86    state(NotIssued)
87{
88    request = std::make_shared<Request>();
89}
90
91LSQ::AddrRangeCoverage
92LSQ::LSQRequest::containsAddrRangeOf(
93    Addr req1_addr, unsigned int req1_size,
94    Addr req2_addr, unsigned int req2_size)
95{
96    /* 'end' here means the address of the byte just past the request
97     *  blocks */
98    Addr req2_end_addr = req2_addr + req2_size;
99    Addr req1_end_addr = req1_addr + req1_size;
100
101    AddrRangeCoverage ret;
102
103    if (req1_addr >= req2_end_addr || req1_end_addr <= req2_addr)
104        ret = NoAddrRangeCoverage;
105    else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
106        ret = FullAddrRangeCoverage;
107    else
108        ret = PartialAddrRangeCoverage;
109
110    return ret;
111}
112
113LSQ::AddrRangeCoverage
114LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
115{
116    return containsAddrRangeOf(request->getPaddr(), request->getSize(),
117        other_request->request->getPaddr(), other_request->request->getSize());
118}
119
120bool
121LSQ::LSQRequest::isBarrier()
122{
123    return inst->isInst() && inst->staticInst->isMemBarrier();
124}
125
126bool
127LSQ::LSQRequest::needsToBeSentToStoreBuffer()
128{
129    return state == StoreToStoreBuffer;
130}
131
132void
133LSQ::LSQRequest::setState(LSQRequestState new_state)
134{
135    DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
136        " %s\n", state, new_state, *inst);
137    state = new_state;
138}
139
140bool
141LSQ::LSQRequest::isComplete() const
142{
143    /* @todo, There is currently only one 'completed' state.  This
144     *  may not be a good choice */
145    return state == Complete;
146}
147
148void
149LSQ::LSQRequest::reportData(std::ostream &os) const
150{
151    os << (isLoad ? 'R' : 'W') << ';';
152    inst->reportData(os);
153    os << ';' << state;
154}
155
156std::ostream &
157operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
158{
159    switch (coverage) {
160      case LSQ::PartialAddrRangeCoverage:
161        os << "PartialAddrRangeCoverage";
162        break;
163      case LSQ::FullAddrRangeCoverage:
164        os << "FullAddrRangeCoverage";
165        break;
166      case LSQ::NoAddrRangeCoverage:
167        os << "NoAddrRangeCoverage";
168        break;
169      default:
170        os << "AddrRangeCoverage-" << static_cast<int>(coverage);
171        break;
172    }
173    return os;
174}
175
176std::ostream &
177operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
178{
179    switch (state) {
180      case LSQ::LSQRequest::NotIssued:
181        os << "NotIssued";
182        break;
183      case LSQ::LSQRequest::InTranslation:
184        os << "InTranslation";
185        break;
186      case LSQ::LSQRequest::Translated:
187        os << "Translated";
188        break;
189      case LSQ::LSQRequest::Failed:
190        os << "Failed";
191        break;
192      case LSQ::LSQRequest::RequestIssuing:
193        os << "RequestIssuing";
194        break;
195      case LSQ::LSQRequest::StoreToStoreBuffer:
196        os << "StoreToStoreBuffer";
197        break;
198      case LSQ::LSQRequest::StoreInStoreBuffer:
199        os << "StoreInStoreBuffer";
200        break;
201      case LSQ::LSQRequest::StoreBufferIssuing:
202        os << "StoreBufferIssuing";
203        break;
204      case LSQ::LSQRequest::RequestNeedsRetry:
205        os << "RequestNeedsRetry";
206        break;
207      case LSQ::LSQRequest::StoreBufferNeedsRetry:
208        os << "StoreBufferNeedsRetry";
209        break;
210      case LSQ::LSQRequest::Complete:
211        os << "Complete";
212        break;
213      default:
214        os << "LSQRequestState-" << static_cast<int>(state);
215        break;
216    }
217    return os;
218}
219
220void
221LSQ::clearMemBarrier(MinorDynInstPtr inst)
222{
223    bool is_last_barrier =
224        inst->id.execSeqNum >= lastMemBarrier[inst->id.threadId];
225
226    DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
227        (is_last_barrier ? "last" : "a"), *inst);
228
229    if (is_last_barrier)
230        lastMemBarrier[inst->id.threadId] = 0;
231}
232
233void
234LSQ::SingleDataRequest::finish(const Fault &fault_, const RequestPtr &request_,
235                               ThreadContext *tc, BaseTLB::Mode mode)
236{
237    fault = fault_;
238
239    port.numAccessesInDTLB--;
240
241    DPRINTFS(MinorMem, (&port), "Received translation response for"
242        " request: %s\n", *inst);
243
244    makePacket();
245
246    setState(Translated);
247    port.tryToSendToTransfers(this);
248
249    /* Let's try and wake up the processor for the next cycle */
250    port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
251}
252
253void
254LSQ::SingleDataRequest::startAddrTranslation()
255{
256    ThreadContext *thread = port.cpu.getContext(
257        inst->id.threadId);
258
259    port.numAccessesInDTLB++;
260
261    setState(LSQ::LSQRequest::InTranslation);
262
263    DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
264    /* Submit the translation request.  The response will come through
265     *  finish/markDelayed on the LSQRequest as it bears the Translation
266     *  interface */
267    thread->getDTBPtr()->translateTiming(
268        request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
269}
270
271void
272LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
273{
274    DPRINTFS(MinorMem, (&port), "Retiring packet\n");
275    packet = packet_;
276    packetInFlight = false;
277    setState(Complete);
278}
279
280void
281LSQ::SplitDataRequest::finish(const Fault &fault_, const RequestPtr &request_,
282                              ThreadContext *tc, BaseTLB::Mode mode)
283{
284    fault = fault_;
285
286    port.numAccessesInDTLB--;
287
288    unsigned int M5_VAR_USED expected_fragment_index =
289        numTranslatedFragments;
290
291    numInTranslationFragments--;
292    numTranslatedFragments++;
293
294    DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
295        " %d of request: %s\n", expected_fragment_index, *inst);
296
297    assert(request_ == fragmentRequests[expected_fragment_index]);
298
299    /* Wake up next cycle to get things going again in case the
300     *  tryToSendToTransfers does take */
301    port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
302
303    if (fault != NoFault) {
304        /* tryToSendToTransfers will handle the fault */
305
306        DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
307            " %d of request: %s\n",
308            expected_fragment_index, *inst);
309
310        setState(Translated);
311        port.tryToSendToTransfers(this);
312    } else if (numTranslatedFragments == numFragments) {
313        makeFragmentPackets();
314
315        setState(Translated);
316        port.tryToSendToTransfers(this);
317    } else {
318        /* Avoid calling translateTiming from within ::finish */
319        assert(!translationEvent.scheduled());
320        port.cpu.schedule(translationEvent, curTick());
321    }
322}
323
324LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
325    bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
326    LSQRequest(port_, inst_, isLoad_, data_, res_),
327    translationEvent([this]{ sendNextFragmentToTranslation(); },
328                     "translationEvent"),
329    numFragments(0),
330    numInTranslationFragments(0),
331    numTranslatedFragments(0),
332    numIssuedFragments(0),
333    numRetiredFragments(0),
334    fragmentRequests(),
335    fragmentPackets()
336{
337    /* Don't know how many elements are needed until the request is
338     *  populated by the caller. */
339}
340
341LSQ::SplitDataRequest::~SplitDataRequest()
342{
343    for (auto i = fragmentPackets.begin();
344         i != fragmentPackets.end(); i++)
345    {
346        delete *i;
347    }
348}
349
350void
351LSQ::SplitDataRequest::makeFragmentRequests()
352{
353    Addr base_addr = request->getVaddr();
354    unsigned int whole_size = request->getSize();
355    unsigned int line_width = port.lineWidth;
356
357    unsigned int fragment_size;
358    Addr fragment_addr;
359
360    /* Assume that this transfer is across potentially many block snap
361     * boundaries:
362     *
363     * |      _|________|________|________|___     |
364     * |     |0| 1      | 2      | 3      | 4 |    |
365     * |     |_|________|________|________|___|    |
366     * |       |        |        |        |        |
367     *
368     *  The first transfer (0) can be up to lineWidth in size.
369     *  All the middle transfers (1-3) are lineWidth in size
370     *  The last transfer (4) can be from zero to lineWidth - 1 in size
371     */
372    unsigned int first_fragment_offset =
373        addrBlockOffset(base_addr, line_width);
374    unsigned int last_fragment_size =
375        addrBlockOffset(base_addr + whole_size, line_width);
376    unsigned int first_fragment_size =
377        line_width - first_fragment_offset;
378
379    unsigned int middle_fragments_total_size =
380        whole_size - (first_fragment_size + last_fragment_size);
381
382    assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
383
384    unsigned int middle_fragment_count =
385        middle_fragments_total_size / line_width;
386
387    numFragments = 1 /* first */ + middle_fragment_count +
388        (last_fragment_size == 0 ? 0 : 1);
389
390    DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
391        " First fragment size: %d Last fragment size: %d\n",
392        numFragments, first_fragment_size,
393        (last_fragment_size == 0 ? line_width : last_fragment_size));
394
395    assert(((middle_fragment_count * line_width) +
396        first_fragment_size + last_fragment_size) == whole_size);
397
398    fragment_addr = base_addr;
399    fragment_size = first_fragment_size;
400
401    /* Just past the last address in the request */
402    Addr end_addr = base_addr + whole_size;
403
404    for (unsigned int fragment_index = 0; fragment_index < numFragments;
405         fragment_index++)
406    {
407        bool M5_VAR_USED is_last_fragment = false;
408
409        if (fragment_addr == base_addr) {
410            /* First fragment */
411            fragment_size = first_fragment_size;
412        } else {
413            if ((fragment_addr + line_width) > end_addr) {
414                /* Adjust size of last fragment */
415                fragment_size = end_addr - fragment_addr;
416                is_last_fragment = true;
417            } else {
418                /* Middle fragments */
419                fragment_size = line_width;
420            }
421        }
422
423        RequestPtr fragment = std::make_shared<Request>();
424
425        fragment->setContext(request->contextId());
426        fragment->setVirt(0 /* asid */,
427            fragment_addr, fragment_size, request->getFlags(),
428            request->masterId(),
429            request->getPC());
430
431        DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
432            " (whole request addr: 0x%x size: %d) %s\n",
433            fragment_addr, fragment_size, base_addr, whole_size,
434            (is_last_fragment ? "last fragment" : ""));
435
436        fragment_addr += fragment_size;
437
438        fragmentRequests.push_back(fragment);
439    }
440}
441
442void
443LSQ::SplitDataRequest::makeFragmentPackets()
444{
445    Addr base_addr = request->getVaddr();
446
447    DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
448
449    for (unsigned int fragment_index = 0; fragment_index < numFragments;
450         fragment_index++)
451    {
452        RequestPtr fragment = fragmentRequests[fragment_index];
453
454        DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
455            " (%d, 0x%x)\n",
456            fragment_index, *inst,
457            (fragment->hasPaddr() ? "has paddr" : "no paddr"),
458            (fragment->hasPaddr() ? fragment->getPaddr() : 0));
459
460        Addr fragment_addr = fragment->getVaddr();
461        unsigned int fragment_size = fragment->getSize();
462
463        uint8_t *request_data = NULL;
464
465        if (!isLoad) {
466            /* Split data for Packets.  Will become the property of the
467             *  outgoing Packets */
468            request_data = new uint8_t[fragment_size];
469            std::memcpy(request_data, data + (fragment_addr - base_addr),
470                fragment_size);
471        }
472
473        assert(fragment->hasPaddr());
474
475        PacketPtr fragment_packet =
476            makePacketForRequest(fragment, isLoad, this, request_data);
477
478        fragmentPackets.push_back(fragment_packet);
479        /* Accumulate flags in parent request */
480        request->setFlags(fragment->getFlags());
481    }
482
483    /* Might as well make the overall/response packet here */
484    /* Get the physical address for the whole request/packet from the first
485     *  fragment */
486    request->setPaddr(fragmentRequests[0]->getPaddr());
487    makePacket();
488}
489
490void
491LSQ::SplitDataRequest::startAddrTranslation()
492{
493    setState(LSQ::LSQRequest::InTranslation);
494
495    makeFragmentRequests();
496
497    numInTranslationFragments = 0;
498    numTranslatedFragments = 0;
499
500    /* @todo, just do these in sequence for now with
501     * a loop of:
502     * do {
503     *  sendNextFragmentToTranslation ; translateTiming ; finish
504     * } while (numTranslatedFragments != numFragments);
505     */
506
507    /* Do first translation */
508    sendNextFragmentToTranslation();
509}
510
511PacketPtr
512LSQ::SplitDataRequest::getHeadPacket()
513{
514    assert(numIssuedFragments < numFragments);
515
516    return fragmentPackets[numIssuedFragments];
517}
518
519void
520LSQ::SplitDataRequest::stepToNextPacket()
521{
522    assert(numIssuedFragments < numFragments);
523
524    numIssuedFragments++;
525}
526
527void
528LSQ::SplitDataRequest::retireResponse(PacketPtr response)
529{
530    assert(numRetiredFragments < numFragments);
531
532    DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
533        " offset: 0x%x (retired fragment num: %d) %s\n",
534        response->req->getVaddr(), response->req->getSize(),
535        request->getVaddr() - response->req->getVaddr(),
536        numRetiredFragments,
537        (fault == NoFault ? "" : fault->name()));
538
539    numRetiredFragments++;
540
541    if (skipped) {
542        /* Skip because we already knew the request had faulted or been
543         *  skipped */
544        DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
545    } else if (response->isError()) {
546        /* Mark up the error and leave to execute to handle it */
547        DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
548        setSkipped();
549        packet->copyError(response);
550    } else {
551        if (isLoad) {
552            if (!data) {
553                /* For a split transfer, a Packet must be constructed
554                 *  to contain all returning data.  This is that packet's
555                 *  data */
556                data = new uint8_t[request->getSize()];
557            }
558
559            /* Populate the portion of the overall response data represented
560             *  by the response fragment */
561            std::memcpy(
562                data + (response->req->getVaddr() - request->getVaddr()),
563                response->getConstPtr<uint8_t>(),
564                response->req->getSize());
565        }
566    }
567
568    /* Complete early if we're skipping are no more in-flight accesses */
569    if (skipped && !hasPacketsInMemSystem()) {
570        DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
571        setState(Complete);
572        if (packet->needsResponse())
573            packet->makeResponse();
574    }
575
576    if (numRetiredFragments == numFragments)
577        setState(Complete);
578
579    if (!skipped && isComplete()) {
580        DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
581
582        DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
583             " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
584             " %s\n", packet->isRead(), packet->isWrite(),
585             packet->needsResponse(), packet->getSize(), request->getSize(),
586             response->getSize());
587
588        /* A request can become complete by several paths, this is a sanity
589         *  check to make sure the packet's data is created */
590        if (!data) {
591            data = new uint8_t[request->getSize()];
592        }
593
594        if (isLoad) {
595            DPRINTFS(MinorMem, (&port), "Copying read data\n");
596            std::memcpy(packet->getPtr<uint8_t>(), data, request->getSize());
597        }
598        packet->makeResponse();
599    }
600
601    /* Packets are all deallocated together in ~SplitLSQRequest */
602}
603
604void
605LSQ::SplitDataRequest::sendNextFragmentToTranslation()
606{
607    unsigned int fragment_index = numTranslatedFragments;
608
609    ThreadContext *thread = port.cpu.getContext(
610        inst->id.threadId);
611
612    DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
613        fragment_index);
614
615    port.numAccessesInDTLB++;
616    numInTranslationFragments++;
617
618    thread->getDTBPtr()->translateTiming(
619        fragmentRequests[fragment_index], thread, this, (isLoad ?
620        BaseTLB::Read : BaseTLB::Write));
621}
622
623bool
624LSQ::StoreBuffer::canInsert() const
625{
626    /* @todo, support store amalgamation */
627    return slots.size() < numSlots;
628}
629
630void
631LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
632{
633    auto found = std::find(slots.begin(), slots.end(), request);
634
635    if (found != slots.end()) {
636        DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
637            request, *found, *(request->inst));
638        slots.erase(found);
639
640        delete request;
641    }
642}
643
644void
645LSQ::StoreBuffer::insert(LSQRequestPtr request)
646{
647    if (!canInsert()) {
648        warn("%s: store buffer insertion without space to insert from"
649            " inst: %s\n", name(), *(request->inst));
650    }
651
652    DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
653
654    numUnissuedAccesses++;
655
656    if (request->state != LSQRequest::Complete)
657        request->setState(LSQRequest::StoreInStoreBuffer);
658
659    slots.push_back(request);
660
661    /* Let's try and wake up the processor for the next cycle to step
662     *  the store buffer */
663    lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
664}
665
666LSQ::AddrRangeCoverage
667LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
668    unsigned int &found_slot)
669{
670    unsigned int slot_index = slots.size() - 1;
671    auto i = slots.rbegin();
672    AddrRangeCoverage ret = NoAddrRangeCoverage;
673
674    /* Traverse the store buffer in reverse order (most to least recent)
675     *  and try to find a slot whose address range overlaps this request */
676    while (ret == NoAddrRangeCoverage && i != slots.rend()) {
677        LSQRequestPtr slot = *i;
678
679        /* Cache maintenance instructions go down via the store path *
680         * but they carry no data and they shouldn't be considered for
681         * forwarding */
682        if (slot->packet &&
683            slot->inst->id.threadId == request->inst->id.threadId &&
684            !slot->packet->req->isCacheMaintenance()) {
685            AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
686
687            if (coverage != NoAddrRangeCoverage) {
688                DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
689                    " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
690                    slot_index, coverage,
691                    request->request->getPaddr(), request->request->getSize(),
692                    slot->request->getPaddr(), slot->request->getSize());
693
694                found_slot = slot_index;
695                ret = coverage;
696            }
697        }
698
699        i++;
700        slot_index--;
701    }
702
703    return ret;
704}
705
706/** Fill the given packet with appropriate date from slot slot_number */
707void
708LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
709    unsigned int slot_number)
710{
711    assert(slot_number < slots.size());
712    assert(load->packet);
713    assert(load->isLoad);
714
715    LSQRequestPtr store = slots[slot_number];
716
717    assert(store->packet);
718    assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
719
720    Addr load_addr = load->request->getPaddr();
721    Addr store_addr = store->request->getPaddr();
722    Addr addr_offset = load_addr - store_addr;
723
724    unsigned int load_size = load->request->getSize();
725
726    DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
727        " slot: %d addr: 0x%x addressOffset: 0x%x\n",
728        load_size, load_addr, slot_number,
729        store_addr, addr_offset);
730
731    void *load_packet_data = load->packet->getPtr<void>();
732    void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
733
734    std::memcpy(load_packet_data, store_packet_data, load_size);
735}
736
737void
738LSQ::StoreBuffer::countIssuedStore(LSQRequestPtr request)
739{
740    /* Barriers are accounted for as they are cleared from
741     *  the queue, not after their transfers are complete */
742    if (!request->isBarrier())
743        numUnissuedAccesses--;
744}
745
746void
747LSQ::StoreBuffer::step()
748{
749    DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
750        numUnissuedAccesses);
751
752    if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
753        /* Clear all the leading barriers */
754        while (!slots.empty() &&
755            slots.front()->isComplete() && slots.front()->isBarrier())
756        {
757            LSQRequestPtr barrier = slots.front();
758
759            DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
760                *(barrier->inst));
761
762            numUnissuedAccesses--;
763            lsq.clearMemBarrier(barrier->inst);
764            slots.pop_front();
765
766            delete barrier;
767        }
768
769        auto i = slots.begin();
770        bool issued = true;
771        unsigned int issue_count = 0;
772
773        /* Skip trying if the memory system is busy */
774        if (lsq.state == LSQ::MemoryNeedsRetry)
775            issued = false;
776
777        /* Try to issue all stores in order starting from the head
778         *  of the queue.  Responses are allowed to be retired
779         *  out of order */
780        while (issued &&
781            issue_count < storeLimitPerCycle &&
782            lsq.canSendToMemorySystem() &&
783            i != slots.end())
784        {
785            LSQRequestPtr request = *i;
786
787            DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
788                " state: %s\n",
789                *(request->inst), request->sentAllPackets(),
790                request->state);
791
792            if (request->isBarrier() && request->isComplete()) {
793                /* Give up at barriers */
794                issued = false;
795            } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
796                request->sentAllPackets()))
797            {
798                DPRINTF(MinorMem, "Trying to send request: %s to memory"
799                    " system\n", *(request->inst));
800
801                if (lsq.tryToSend(request)) {
802                    countIssuedStore(request);
803                    issue_count++;
804                } else {
805                    /* Don't step on to the next store buffer entry if this
806                     *  one hasn't issued all its packets as the store
807                     *  buffer must still enforce ordering */
808                    issued = false;
809                }
810            }
811            i++;
812        }
813    }
814}
815
816void
817LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
818    bool committed)
819{
820    if (committed) {
821        /* Not already sent to the store buffer as a store request? */
822        if (!inst->inStoreBuffer) {
823            /* Insert an entry into the store buffer to tick off barriers
824             *  until there are none in flight */
825            storeBuffer.insert(new BarrierDataRequest(*this, inst));
826        }
827    } else {
828        /* Clear the barrier anyway if it wasn't actually committed */
829        clearMemBarrier(inst);
830    }
831}
832
833void
834LSQ::StoreBuffer::minorTrace() const
835{
836    unsigned int size = slots.size();
837    unsigned int i = 0;
838    std::ostringstream os;
839
840    while (i < size) {
841        LSQRequestPtr request = slots[i];
842
843        request->reportData(os);
844
845        i++;
846        if (i < numSlots)
847            os << ',';
848    }
849
850    while (i < numSlots) {
851        os << '-';
852
853        i++;
854        if (i < numSlots)
855            os << ',';
856    }
857
858    MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
859        numUnissuedAccesses);
860}
861
862void
863LSQ::tryToSendToTransfers(LSQRequestPtr request)
864{
865    if (state == MemoryNeedsRetry) {
866        DPRINTF(MinorMem, "Request needs retry, not issuing to"
867            " memory until retry arrives\n");
868        return;
869    }
870
871    if (request->state == LSQRequest::InTranslation) {
872        DPRINTF(MinorMem, "Request still in translation, not issuing to"
873            " memory\n");
874        return;
875    }
876
877    assert(request->state == LSQRequest::Translated ||
878        request->state == LSQRequest::RequestIssuing ||
879        request->state == LSQRequest::Failed ||
880        request->state == LSQRequest::Complete);
881
882    if (requests.empty() || requests.front() != request) {
883        DPRINTF(MinorMem, "Request not at front of requests queue, can't"
884            " issue to memory\n");
885        return;
886    }
887
888    if (transfers.unreservedRemainingSpace() == 0) {
889        DPRINTF(MinorMem, "No space to insert request into transfers"
890            " queue\n");
891        return;
892    }
893
894    if (request->isComplete() || request->state == LSQRequest::Failed) {
895        DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
896            " queue\n", (request->isComplete() ? "completed" : "failed"));
897        request->setState(LSQRequest::Complete);
898        request->setSkipped();
899        moveFromRequestsToTransfers(request);
900        return;
901    }
902
903    if (!execute.instIsRightStream(request->inst)) {
904        /* Wrong stream, try to abort the transfer but only do so if
905         *  there are no packets in flight */
906        if (request->hasPacketsInMemSystem()) {
907            DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
908                " waiting for responses before aborting request\n");
909        } else {
910            DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
911                " aborting request\n");
912            request->setState(LSQRequest::Complete);
913            request->setSkipped();
914            moveFromRequestsToTransfers(request);
915        }
916        return;
917    }
918
919    if (request->fault != NoFault) {
920        if (request->inst->staticInst->isPrefetch()) {
921            DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
922        }
923        DPRINTF(MinorMem, "Moving faulting request into the transfers"
924            " queue\n");
925        request->setState(LSQRequest::Complete);
926        request->setSkipped();
927        moveFromRequestsToTransfers(request);
928        return;
929    }
930
931    bool is_load = request->isLoad;
932    bool is_llsc = request->request->isLLSC();
933    bool is_swap = request->request->isSwap();
934    bool bufferable = !(request->request->isStrictlyOrdered() ||
935        is_llsc || is_swap);
936
937    if (is_load) {
938        if (numStoresInTransfers != 0) {
939            DPRINTF(MinorMem, "Load request with stores still in transfers"
940                " queue, stalling\n");
941            return;
942        }
943    } else {
944        /* Store.  Can it be sent to the store buffer? */
945        if (bufferable && !request->request->isMmappedIpr()) {
946            request->setState(LSQRequest::StoreToStoreBuffer);
947            moveFromRequestsToTransfers(request);
948            DPRINTF(MinorMem, "Moving store into transfers queue\n");
949            return;
950        }
951    }
952
953    /* Check if this is the head instruction (and so must be executable as
954     *  its stream sequence number was checked above) for loads which must
955     *  not be speculatively issued and stores which must be issued here */
956    if (!bufferable) {
957        if (!execute.instIsHeadInst(request->inst)) {
958            DPRINTF(MinorMem, "Memory access not the head inst., can't be"
959                " sure it can be performed, not issuing\n");
960            return;
961        }
962
963        unsigned int forwarding_slot = 0;
964
965        if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
966            NoAddrRangeCoverage)
967        {
968            DPRINTF(MinorMem, "Memory access can receive forwarded data"
969                " from the store buffer, need to wait for store buffer to"
970                " drain\n");
971            return;
972        }
973    }
974
975    /* True: submit this packet to the transfers queue to be sent to the
976     * memory system.
977     * False: skip the memory and push a packet for this request onto
978     * requests */
979    bool do_access = true;
980
981    if (!is_llsc) {
982        /* Check for match in the store buffer */
983        if (is_load) {
984            unsigned int forwarding_slot = 0;
985            AddrRangeCoverage forwarding_result =
986                storeBuffer.canForwardDataToLoad(request,
987                forwarding_slot);
988
989            switch (forwarding_result) {
990              case FullAddrRangeCoverage:
991                /* Forward data from the store buffer into this request and
992                 *  repurpose this request's packet into a response packet */
993                storeBuffer.forwardStoreData(request, forwarding_slot);
994                request->packet->makeResponse();
995
996                /* Just move between queues, no access */
997                do_access = false;
998                break;
999              case PartialAddrRangeCoverage:
1000                DPRINTF(MinorMem, "Load partly satisfied by store buffer"
1001                    " data. Must wait for the store to complete\n");
1002                return;
1003                break;
1004              case NoAddrRangeCoverage:
1005                DPRINTF(MinorMem, "No forwardable data from store buffer\n");
1006                /* Fall through to try access */
1007                break;
1008            }
1009        }
1010    } else {
1011        if (!canSendToMemorySystem()) {
1012            DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1013            return;
1014        }
1015
1016        SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1017
1018        TheISA::PCState old_pc = thread.pcState();
1019        ExecContext context(cpu, thread, execute, request->inst);
1020
1021        /* Handle LLSC requests and tests */
1022        if (is_load) {
1023            TheISA::handleLockedRead(&context, request->request);
1024        } else {
1025            do_access = TheISA::handleLockedWrite(&context,
1026                request->request, cacheBlockMask);
1027
1028            if (!do_access) {
1029                DPRINTF(MinorMem, "Not perfoming a memory "
1030                    "access for store conditional\n");
1031            }
1032        }
1033        thread.pcState(old_pc);
1034    }
1035
1036    /* See the do_access comment above */
1037    if (do_access) {
1038        if (!canSendToMemorySystem()) {
1039            DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1040            return;
1041        }
1042
1043        /* Remember if this is an access which can't be idly
1044         *  discarded by an interrupt */
1045        if (!bufferable && !request->issuedToMemory) {
1046            numAccessesIssuedToMemory++;
1047            request->issuedToMemory = true;
1048        }
1049
1050        if (tryToSend(request)) {
1051            moveFromRequestsToTransfers(request);
1052        }
1053    } else {
1054        request->setState(LSQRequest::Complete);
1055        moveFromRequestsToTransfers(request);
1056    }
1057}
1058
1059bool
1060LSQ::tryToSend(LSQRequestPtr request)
1061{
1062    bool ret = false;
1063
1064    if (!canSendToMemorySystem()) {
1065        DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1066            *(request->inst));
1067    } else {
1068        PacketPtr packet = request->getHeadPacket();
1069
1070        DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1071            *(request->inst), packet->req->getVaddr());
1072
1073        /* The sender state of the packet *must* be an LSQRequest
1074         *  so the response can be correctly handled */
1075        assert(packet->findNextSenderState<LSQRequest>());
1076
1077        if (request->request->isMmappedIpr()) {
1078            ThreadContext *thread =
1079                cpu.getContext(cpu.contextToThread(
1080                                request->request->contextId()));
1081
1082            if (request->isLoad) {
1083                DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1084                TheISA::handleIprRead(thread, packet);
1085            } else {
1086                DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1087                TheISA::handleIprWrite(thread, packet);
1088            }
1089
1090            request->stepToNextPacket();
1091            ret = request->sentAllPackets();
1092
1093            if (!ret) {
1094                DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1095                    *(request->inst));
1096            }
1097
1098            if (ret)
1099                request->setState(LSQRequest::Complete);
1100            else
1101                request->setState(LSQRequest::RequestIssuing);
1102        } else if (dcachePort.sendTimingReq(packet)) {
1103            DPRINTF(MinorMem, "Sent data memory request\n");
1104
1105            numAccessesInMemorySystem++;
1106
1107            request->stepToNextPacket();
1108
1109            ret = request->sentAllPackets();
1110
1111            switch (request->state) {
1112              case LSQRequest::Translated:
1113              case LSQRequest::RequestIssuing:
1114                /* Fully or partially issued a request in the transfers
1115                 *  queue */
1116                request->setState(LSQRequest::RequestIssuing);
1117                break;
1118              case LSQRequest::StoreInStoreBuffer:
1119              case LSQRequest::StoreBufferIssuing:
1120                /* Fully or partially issued a request in the store
1121                 *  buffer */
1122                request->setState(LSQRequest::StoreBufferIssuing);
1123                break;
1124              default:
1125                panic("Unrecognized LSQ request state %d.", request->state);
1126            }
1127
1128            state = MemoryRunning;
1129        } else {
1130            DPRINTF(MinorMem,
1131                "Sending data memory request - needs retry\n");
1132
1133            /* Needs to be resent, wait for that */
1134            state = MemoryNeedsRetry;
1135            retryRequest = request;
1136
1137            switch (request->state) {
1138              case LSQRequest::Translated:
1139              case LSQRequest::RequestIssuing:
1140                request->setState(LSQRequest::RequestNeedsRetry);
1141                break;
1142              case LSQRequest::StoreInStoreBuffer:
1143              case LSQRequest::StoreBufferIssuing:
1144                request->setState(LSQRequest::StoreBufferNeedsRetry);
1145                break;
1146              default:
1147                panic("Unrecognized LSQ request state %d.", request->state);
1148            }
1149        }
1150    }
1151
1152    if (ret)
1153        threadSnoop(request);
1154
1155    return ret;
1156}
1157
1158void
1159LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1160{
1161    assert(!requests.empty() && requests.front() == request);
1162    assert(transfers.unreservedRemainingSpace() != 0);
1163
1164    /* Need to count the number of stores in the transfers
1165     *  queue so that loads know when their store buffer forwarding
1166     *  results will be correct (only when all those stores
1167     *  have reached the store buffer) */
1168    if (!request->isLoad)
1169        numStoresInTransfers++;
1170
1171    requests.pop();
1172    transfers.push(request);
1173}
1174
1175bool
1176LSQ::canSendToMemorySystem()
1177{
1178    return state == MemoryRunning &&
1179        numAccessesInMemorySystem < inMemorySystemLimit;
1180}
1181
1182bool
1183LSQ::recvTimingResp(PacketPtr response)
1184{
1185    LSQRequestPtr request =
1186        safe_cast<LSQRequestPtr>(response->popSenderState());
1187
1188    DPRINTF(MinorMem, "Received response packet inst: %s"
1189        " addr: 0x%x cmd: %s\n",
1190        *(request->inst), response->getAddr(),
1191        response->cmd.toString());
1192
1193    numAccessesInMemorySystem--;
1194
1195    if (response->isError()) {
1196        DPRINTF(MinorMem, "Received error response packet: %s\n",
1197            *request->inst);
1198    }
1199
1200    switch (request->state) {
1201      case LSQRequest::RequestIssuing:
1202      case LSQRequest::RequestNeedsRetry:
1203        /* Response to a request from the transfers queue */
1204        request->retireResponse(response);
1205
1206        DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1207            request->hasPacketsInMemSystem(), request->isComplete());
1208
1209        break;
1210      case LSQRequest::StoreBufferIssuing:
1211      case LSQRequest::StoreBufferNeedsRetry:
1212        /* Response to a request from the store buffer */
1213        request->retireResponse(response);
1214
1215        /* Remove completed requests unless they are barriers (which will
1216         *  need to be removed in order */
1217        if (request->isComplete()) {
1218            if (!request->isBarrier()) {
1219                storeBuffer.deleteRequest(request);
1220            } else {
1221                DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1222                    " leaving the request as it is also a barrier\n",
1223                    *(request->inst));
1224            }
1225        }
1226        break;
1227      default:
1228        panic("Shouldn't be allowed to receive a response from another state");
1229    }
1230
1231    /* We go to idle even if there are more things in the requests queue
1232     * as it's the job of step to actually step us on to the next
1233     * transaction */
1234
1235    /* Let's try and wake up the processor for the next cycle */
1236    cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1237
1238    /* Never busy */
1239    return true;
1240}
1241
1242void
1243LSQ::recvReqRetry()
1244{
1245    DPRINTF(MinorMem, "Received retry request\n");
1246
1247    assert(state == MemoryNeedsRetry);
1248
1249    switch (retryRequest->state) {
1250      case LSQRequest::RequestNeedsRetry:
1251        /* Retry in the requests queue */
1252        retryRequest->setState(LSQRequest::Translated);
1253        break;
1254      case LSQRequest::StoreBufferNeedsRetry:
1255        /* Retry in the store buffer */
1256        retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1257        break;
1258      default:
1259        panic("Unrecognized retry request state %d.", retryRequest->state);
1260    }
1261
1262    /* Set state back to MemoryRunning so that the following
1263     *  tryToSend can actually send.  Note that this won't
1264     *  allow another transfer in as tryToSend should
1265     *  issue a memory request and either succeed for this
1266     *  request or return the LSQ back to MemoryNeedsRetry */
1267    state = MemoryRunning;
1268
1269    /* Try to resend the request */
1270    if (tryToSend(retryRequest)) {
1271        /* Successfully sent, need to move the request */
1272        switch (retryRequest->state) {
1273          case LSQRequest::RequestIssuing:
1274            /* In the requests queue */
1275            moveFromRequestsToTransfers(retryRequest);
1276            break;
1277          case LSQRequest::StoreBufferIssuing:
1278            /* In the store buffer */
1279            storeBuffer.countIssuedStore(retryRequest);
1280            break;
1281          default:
1282            panic("Unrecognized retry request state %d.", retryRequest->state);
1283        }
1284
1285        retryRequest = NULL;
1286    }
1287}
1288
1289LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1290    MinorCPU &cpu_, Execute &execute_,
1291    unsigned int in_memory_system_limit, unsigned int line_width,
1292    unsigned int requests_queue_size, unsigned int transfers_queue_size,
1293    unsigned int store_buffer_size,
1294    unsigned int store_buffer_cycle_store_limit) :
1295    Named(name_),
1296    cpu(cpu_),
1297    execute(execute_),
1298    dcachePort(dcache_port_name_, *this, cpu_),
1299    lastMemBarrier(cpu.numThreads, 0),
1300    state(MemoryRunning),
1301    inMemorySystemLimit(in_memory_system_limit),
1302    lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1303    requests(name_ + ".requests", "addr", requests_queue_size),
1304    transfers(name_ + ".transfers", "addr", transfers_queue_size),
1305    storeBuffer(name_ + ".storeBuffer",
1306        *this, store_buffer_size, store_buffer_cycle_store_limit),
1307    numAccessesInMemorySystem(0),
1308    numAccessesInDTLB(0),
1309    numStoresInTransfers(0),
1310    numAccessesIssuedToMemory(0),
1311    retryRequest(NULL),
1312    cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1313{
1314    if (in_memory_system_limit < 1) {
1315        fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1316            in_memory_system_limit);
1317    }
1318
1319    if (store_buffer_cycle_store_limit < 1) {
1320        fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1321            " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1322    }
1323
1324    if (requests_queue_size < 1) {
1325        fatal("%s: executeLSQRequestsQueueSize must be"
1326            " >= 1 (%d)\n", name_, requests_queue_size);
1327    }
1328
1329    if (transfers_queue_size < 1) {
1330        fatal("%s: executeLSQTransfersQueueSize must be"
1331            " >= 1 (%d)\n", name_, transfers_queue_size);
1332    }
1333
1334    if (store_buffer_size < 1) {
1335        fatal("%s: executeLSQStoreBufferSize must be"
1336            " >= 1 (%d)\n", name_, store_buffer_size);
1337    }
1338
1339    if ((lineWidth & (lineWidth - 1)) != 0) {
1340        fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1341    }
1342}
1343
1344LSQ::~LSQ()
1345{ }
1346
1347LSQ::LSQRequest::~LSQRequest()
1348{
1349    if (packet)
1350        delete packet;
1351    if (data)
1352        delete [] data;
1353}
1354
1355/**
1356 *  Step the memory access mechanism on to its next state.  In reality, most
1357 *  of the stepping is done by the callbacks on the LSQ but this
1358 *  function is responsible for issuing memory requests lodged in the
1359 *  requests queue.
1360 */
1361void
1362LSQ::step()
1363{
1364    /* Try to move address-translated requests between queues and issue
1365     *  them */
1366    if (!requests.empty())
1367        tryToSendToTransfers(requests.front());
1368
1369    storeBuffer.step();
1370}
1371
1372LSQ::LSQRequestPtr
1373LSQ::findResponse(MinorDynInstPtr inst)
1374{
1375    LSQ::LSQRequestPtr ret = NULL;
1376
1377    if (!transfers.empty()) {
1378        LSQRequestPtr request = transfers.front();
1379
1380        /* Same instruction and complete access or a store that's
1381         *  capable of being moved to the store buffer */
1382        if (request->inst->id == inst->id) {
1383            bool complete = request->isComplete();
1384            bool can_store = storeBuffer.canInsert();
1385            bool to_store_buffer = request->state ==
1386                LSQRequest::StoreToStoreBuffer;
1387
1388            if ((complete && !(request->isBarrier() && !can_store)) ||
1389                (to_store_buffer && can_store))
1390            {
1391                ret = request;
1392            }
1393        }
1394    }
1395
1396    if (ret) {
1397        DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1398            *inst);
1399    } else {
1400        DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1401            *inst);
1402    }
1403
1404    return ret;
1405}
1406
1407void
1408LSQ::popResponse(LSQ::LSQRequestPtr response)
1409{
1410    assert(!transfers.empty() && transfers.front() == response);
1411
1412    transfers.pop();
1413
1414    if (!response->isLoad)
1415        numStoresInTransfers--;
1416
1417    if (response->issuedToMemory)
1418        numAccessesIssuedToMemory--;
1419
1420    if (response->state != LSQRequest::StoreInStoreBuffer) {
1421        DPRINTF(MinorMem, "Deleting %s request: %s\n",
1422            (response->isLoad ? "load" : "store"),
1423            *(response->inst));
1424
1425        delete response;
1426    }
1427}
1428
1429void
1430LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1431{
1432    assert(request->state == LSQRequest::StoreToStoreBuffer);
1433
1434    DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1435        *(request->inst));
1436
1437    request->inst->inStoreBuffer = true;
1438
1439    storeBuffer.insert(request);
1440}
1441
1442bool
1443LSQ::isDrained()
1444{
1445    return requests.empty() && transfers.empty() &&
1446        storeBuffer.isDrained();
1447}
1448
1449bool
1450LSQ::needsToTick()
1451{
1452    bool ret = false;
1453
1454    if (canSendToMemorySystem()) {
1455        bool have_translated_requests = !requests.empty() &&
1456            requests.front()->state != LSQRequest::InTranslation &&
1457            transfers.unreservedRemainingSpace() != 0;
1458
1459        ret = have_translated_requests ||
1460            storeBuffer.numUnissuedStores() != 0;
1461    }
1462
1463    if (ret)
1464        DPRINTF(Activity, "Need to tick\n");
1465
1466    return ret;
1467}
1468
1469void
1470LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1471                 unsigned int size, Addr addr, Request::Flags flags,
1472                 uint64_t *res)
1473{
1474    bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1475    LSQRequestPtr request;
1476
1477    /* Copy given data into the request.  The request will pass this to the
1478     *  packet and then it will own the data */
1479    uint8_t *request_data = NULL;
1480
1481    DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1482        " 0x%x%s lineWidth : 0x%x\n",
1483        (isLoad ? "load" : "store"), addr, size, flags,
1484            (needs_burst ? " (needs burst)" : ""), lineWidth);
1485
1486    if (!isLoad) {
1487        /* request_data becomes the property of a ...DataRequest (see below)
1488         *  and destroyed by its destructor */
1489        request_data = new uint8_t[size];
1490        if (flags & Request::STORE_NO_DATA) {
1491            /* For cache zeroing, just use zeroed data */
1492            std::memset(request_data, 0, size);
1493        } else {
1494            std::memcpy(request_data, data, size);
1495        }
1496    }
1497
1498    if (needs_burst) {
1499        request = new SplitDataRequest(
1500            *this, inst, isLoad, request_data, res);
1501    } else {
1502        request = new SingleDataRequest(
1503            *this, inst, isLoad, request_data, res);
1504    }
1505
1506    if (inst->traceData)
1507        inst->traceData->setMem(addr, size, flags);
1508
1509    int cid = cpu.threads[inst->id.threadId]->getTC()->contextId();
1510    request->request->setContext(cid);
1511    request->request->setVirt(0 /* asid */,
1512        addr, size, flags, cpu.dataMasterId(),
1513        /* I've no idea why we need the PC, but give it */
1514        inst->pc.instAddr());
1515
1516    requests.push(request);
1517    request->startAddrTranslation();
1518}
1519
1520void
1521LSQ::pushFailedRequest(MinorDynInstPtr inst)
1522{
1523    LSQRequestPtr request = new FailedDataRequest(*this, inst);
1524    requests.push(request);
1525}
1526
1527void
1528LSQ::minorTrace() const
1529{
1530    MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1531        " lastMemBarrier=%d\n",
1532        state, numAccessesInDTLB, numAccessesInMemorySystem,
1533        numStoresInTransfers, lastMemBarrier[0]);
1534    requests.minorTrace();
1535    transfers.minorTrace();
1536    storeBuffer.minorTrace();
1537}
1538
1539LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1540    unsigned int store_buffer_size,
1541    unsigned int store_limit_per_cycle) :
1542    Named(name_), lsq(lsq_),
1543    numSlots(store_buffer_size),
1544    storeLimitPerCycle(store_limit_per_cycle),
1545    slots(),
1546    numUnissuedAccesses(0)
1547{
1548}
1549
1550PacketPtr
1551makePacketForRequest(const RequestPtr &request, bool isLoad,
1552    Packet::SenderState *sender_state, PacketDataPtr data)
1553{
1554    PacketPtr ret = isLoad ? Packet::createRead(request)
1555                           : Packet::createWrite(request);
1556
1557    if (sender_state)
1558        ret->pushSenderState(sender_state);
1559
1560    if (isLoad) {
1561        ret->allocate();
1562    } else if (!request->isCacheMaintenance()) {
1563        // CMOs are treated as stores but they don't have data. All
1564        // stores otherwise need to allocate for data.
1565        ret->dataDynamic(data);
1566    }
1567
1568    return ret;
1569}
1570
1571void
1572LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1573{
1574    assert(inst->isInst() && inst->staticInst->isMemBarrier());
1575    assert(inst->id.execSeqNum > lastMemBarrier[inst->id.threadId]);
1576
1577    /* Remember the barrier.  We only have a notion of one
1578     *  barrier so this may result in some mem refs being
1579     *  delayed if they are between barriers */
1580    lastMemBarrier[inst->id.threadId] = inst->id.execSeqNum;
1581}
1582
1583void
1584LSQ::LSQRequest::makePacket()
1585{
1586    /* Make the function idempotent */
1587    if (packet)
1588        return;
1589
1590    // if the translation faulted, do not create a packet
1591    if (fault != NoFault) {
1592        assert(packet == NULL);
1593        return;
1594    }
1595
1596    packet = makePacketForRequest(request, isLoad, this, data);
1597    /* Null the ret data so we know not to deallocate it when the
1598     * ret is destroyed.  The data now belongs to the ret and
1599     * the ret is responsible for its destruction */
1600    data = NULL;
1601}
1602
1603std::ostream &
1604operator <<(std::ostream &os, LSQ::MemoryState state)
1605{
1606    switch (state) {
1607      case LSQ::MemoryRunning:
1608        os << "MemoryRunning";
1609        break;
1610      case LSQ::MemoryNeedsRetry:
1611        os << "MemoryNeedsRetry";
1612        break;
1613      default:
1614        os << "MemoryState-" << static_cast<int>(state);
1615        break;
1616    }
1617    return os;
1618}
1619
1620void
1621LSQ::recvTimingSnoopReq(PacketPtr pkt)
1622{
1623    /* LLSC operations in Minor can't be speculative and are executed from
1624     * the head of the requests queue.  We shouldn't need to do more than
1625     * this action on snoops. */
1626    for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1627        if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1628            cpu.wakeup(tid);
1629        }
1630    }
1631
1632    if (pkt->isInvalidate() || pkt->isWrite()) {
1633        for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1634            TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1635                                      cacheBlockMask);
1636        }
1637    }
1638}
1639
1640void
1641LSQ::threadSnoop(LSQRequestPtr request)
1642{
1643    /* LLSC operations in Minor can't be speculative and are executed from
1644     * the head of the requests queue.  We shouldn't need to do more than
1645     * this action on snoops. */
1646    ThreadID req_tid = request->inst->id.threadId;
1647    PacketPtr pkt = request->packet;
1648
1649    for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1650        if (tid != req_tid) {
1651            if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1652                cpu.wakeup(tid);
1653            }
1654
1655            if (pkt->isInvalidate() || pkt->isWrite()) {
1656                TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1657                                          cacheBlockMask);
1658            }
1659        }
1660    }
1661}
1662
1663}
1664