lsq.cc revision 10566
1/*
2 * Copyright (c) 2013-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include <iomanip>
41#include <sstream>
42
43#include "arch/locked_mem.hh"
44#include "arch/mmapped_ipr.hh"
45#include "cpu/minor/cpu.hh"
46#include "cpu/minor/exec_context.hh"
47#include "cpu/minor/execute.hh"
48#include "cpu/minor/lsq.hh"
49#include "cpu/minor/pipeline.hh"
50#include "debug/Activity.hh"
51#include "debug/MinorMem.hh"
52
53namespace Minor
54{
55
56/** Returns the offset of addr into an aligned a block of size block_size */
57static Addr
58addrBlockOffset(Addr addr, unsigned int block_size)
59{
60    return addr & (block_size - 1);
61}
62
63/** Returns true if the given [addr .. addr+size-1] transfer needs to be
64 *  fragmented across a block size of block_size */
65static bool
66transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
67{
68    return (addrBlockOffset(addr, block_size) + size) > block_size;
69}
70
71LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
72    PacketDataPtr data_, uint64_t *res_) :
73    SenderState(),
74    port(port_),
75    inst(inst_),
76    isLoad(isLoad_),
77    data(data_),
78    packet(NULL),
79    request(),
80    fault(NoFault),
81    res(res_),
82    skipped(false),
83    issuedToMemory(false),
84    state(NotIssued)
85{ }
86
87LSQ::AddrRangeCoverage
88LSQ::LSQRequest::containsAddrRangeOf(
89    Addr req1_addr, unsigned int req1_size,
90    Addr req2_addr, unsigned int req2_size)
91{
92    /* 'end' here means the address of the byte just past the request
93     *  blocks */
94    Addr req2_end_addr = req2_addr + req2_size;
95    Addr req1_end_addr = req1_addr + req1_size;
96
97    AddrRangeCoverage ret;
98
99    if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
100        ret = NoAddrRangeCoverage;
101    else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
102        ret = FullAddrRangeCoverage;
103    else
104        ret = PartialAddrRangeCoverage;
105
106    return ret;
107}
108
109LSQ::AddrRangeCoverage
110LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
111{
112    return containsAddrRangeOf(request.getPaddr(), request.getSize(),
113        other_request->request.getPaddr(), other_request->request.getSize());
114}
115
116bool
117LSQ::LSQRequest::isBarrier()
118{
119    return inst->isInst() && inst->staticInst->isMemBarrier();
120}
121
122bool
123LSQ::LSQRequest::needsToBeSentToStoreBuffer()
124{
125    return state == StoreToStoreBuffer;
126}
127
128void
129LSQ::LSQRequest::setState(LSQRequestState new_state)
130{
131    DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
132        " %s\n", state, new_state, *inst);
133    state = new_state;
134}
135
136bool
137LSQ::LSQRequest::isComplete() const
138{
139    /* @todo, There is currently only one 'completed' state.  This
140     *  may not be a good choice */
141    return state == Complete;
142}
143
144void
145LSQ::LSQRequest::reportData(std::ostream &os) const
146{
147    os << (isLoad ? 'R' : 'W') << ';';
148    inst->reportData(os);
149    os << ';' << state;
150}
151
152std::ostream &
153operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
154{
155    switch (coverage) {
156      case LSQ::PartialAddrRangeCoverage:
157        os << "PartialAddrRangeCoverage";
158        break;
159      case LSQ::FullAddrRangeCoverage:
160        os << "FullAddrRangeCoverage";
161        break;
162      case LSQ::NoAddrRangeCoverage:
163        os << "NoAddrRangeCoverage";
164        break;
165      default:
166        os << "AddrRangeCoverage-" << static_cast<int>(coverage);
167        break;
168    }
169    return os;
170}
171
172std::ostream &
173operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
174{
175    switch (state) {
176      case LSQ::LSQRequest::NotIssued:
177        os << "NotIssued";
178        break;
179      case LSQ::LSQRequest::InTranslation:
180        os << "InTranslation";
181        break;
182      case LSQ::LSQRequest::Translated:
183        os << "Translated";
184        break;
185      case LSQ::LSQRequest::Failed:
186        os << "Failed";
187        break;
188      case LSQ::LSQRequest::RequestIssuing:
189        os << "RequestIssuing";
190        break;
191      case LSQ::LSQRequest::StoreToStoreBuffer:
192        os << "StoreToStoreBuffer";
193        break;
194      case LSQ::LSQRequest::StoreInStoreBuffer:
195        os << "StoreInStoreBuffer";
196        break;
197      case LSQ::LSQRequest::StoreBufferIssuing:
198        os << "StoreBufferIssuing";
199        break;
200      case LSQ::LSQRequest::RequestNeedsRetry:
201        os << "RequestNeedsRetry";
202        break;
203      case LSQ::LSQRequest::StoreBufferNeedsRetry:
204        os << "StoreBufferNeedsRetry";
205        break;
206      case LSQ::LSQRequest::Complete:
207        os << "Complete";
208        break;
209      default:
210        os << "LSQRequestState-" << static_cast<int>(state);
211        break;
212    }
213    return os;
214}
215
216void
217LSQ::clearMemBarrier(MinorDynInstPtr inst)
218{
219    bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier;
220
221    DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
222        (is_last_barrier ? "last" : "a"), *inst);
223
224    if (is_last_barrier)
225        lastMemBarrier = 0;
226}
227
228void
229LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_,
230                               ThreadContext *tc, BaseTLB::Mode mode)
231{
232    fault = fault_;
233
234    port.numAccessesInDTLB--;
235
236    DPRINTFS(MinorMem, (&port), "Received translation response for"
237        " request: %s\n", *inst);
238
239    makePacket();
240
241    setState(Translated);
242    port.tryToSendToTransfers(this);
243
244    /* Let's try and wake up the processor for the next cycle */
245    port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
246}
247
248void
249LSQ::SingleDataRequest::startAddrTranslation()
250{
251    ThreadContext *thread = port.cpu.getContext(
252        inst->id.threadId);
253
254    port.numAccessesInDTLB++;
255
256    setState(LSQ::LSQRequest::InTranslation);
257
258    DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
259    /* Submit the translation request.  The response will come through
260     *  finish/markDelayed on the LSQRequest as it bears the Translation
261     *  interface */
262    thread->getDTBPtr()->translateTiming(
263        &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
264}
265
266void
267LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
268{
269    DPRINTFS(MinorMem, (&port), "Retiring packet\n");
270    packet = packet_;
271    packetInFlight = false;
272    setState(Complete);
273}
274
275void
276LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_,
277                              ThreadContext *tc, BaseTLB::Mode mode)
278{
279    fault = fault_;
280
281    port.numAccessesInDTLB--;
282
283    unsigned int M5_VAR_USED expected_fragment_index =
284        numTranslatedFragments;
285
286    numInTranslationFragments--;
287    numTranslatedFragments++;
288
289    DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
290        " %d of request: %s\n", expected_fragment_index, *inst);
291
292    assert(request_ == fragmentRequests[expected_fragment_index]);
293
294    /* Wake up next cycle to get things going again in case the
295     *  tryToSendToTransfers does take */
296    port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
297
298    if (fault != NoFault) {
299        /* tryToSendToTransfers will handle the fault */
300
301        DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
302            " %d of request: %s\n",
303            expected_fragment_index, *inst);
304
305        setState(Translated);
306        port.tryToSendToTransfers(this);
307    } else if (numTranslatedFragments == numFragments) {
308        makeFragmentPackets();
309
310        setState(Translated);
311        port.tryToSendToTransfers(this);
312    } else {
313        /* Avoid calling translateTiming from within ::finish */
314        assert(!translationEvent.scheduled());
315        port.cpu.schedule(translationEvent, curTick());
316    }
317}
318
319LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
320    bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
321    LSQRequest(port_, inst_, isLoad_, data_, res_),
322    translationEvent(*this),
323    numFragments(0),
324    numInTranslationFragments(0),
325    numTranslatedFragments(0),
326    numIssuedFragments(0),
327    numRetiredFragments(0),
328    fragmentRequests(),
329    fragmentPackets()
330{
331    /* Don't know how many elements are needed until the request is
332     *  populated by the caller. */
333}
334
335LSQ::SplitDataRequest::~SplitDataRequest()
336{
337    for (auto i = fragmentRequests.begin();
338        i != fragmentRequests.end(); i++)
339    {
340        delete *i;
341    }
342
343    for (auto i = fragmentPackets.begin();
344         i != fragmentPackets.end(); i++)
345    {
346        delete *i;
347    }
348}
349
350void
351LSQ::SplitDataRequest::makeFragmentRequests()
352{
353    Addr base_addr = request.getVaddr();
354    unsigned int whole_size = request.getSize();
355    unsigned int line_width = port.lineWidth;
356
357    unsigned int fragment_size;
358    Addr fragment_addr;
359
360    /* Assume that this transfer is across potentially many block snap
361     * boundaries:
362     *
363     * |      _|________|________|________|___     |
364     * |     |0| 1      | 2      | 3      | 4 |    |
365     * |     |_|________|________|________|___|    |
366     * |       |        |        |        |        |
367     *
368     *  The first transfer (0) can be up to lineWidth in size.
369     *  All the middle transfers (1-3) are lineWidth in size
370     *  The last transfer (4) can be from zero to lineWidth - 1 in size
371     */
372    unsigned int first_fragment_offset =
373        addrBlockOffset(base_addr, line_width);
374    unsigned int last_fragment_size =
375        addrBlockOffset(base_addr + whole_size, line_width);
376    unsigned int first_fragment_size =
377        line_width - first_fragment_offset;
378
379    unsigned int middle_fragments_total_size =
380        whole_size - (first_fragment_size + last_fragment_size);
381
382    assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
383
384    unsigned int middle_fragment_count =
385        middle_fragments_total_size / line_width;
386
387    numFragments = 1 /* first */ + middle_fragment_count +
388        (last_fragment_size == 0 ? 0 : 1);
389
390    DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
391        " First fragment size: %d Last fragment size: %d\n",
392        numFragments, first_fragment_size,
393        (last_fragment_size == 0 ? line_width : last_fragment_size));
394
395    assert(((middle_fragment_count * line_width) +
396        first_fragment_size + last_fragment_size) == whole_size);
397
398    fragment_addr = base_addr;
399    fragment_size = first_fragment_size;
400
401    /* Just past the last address in the request */
402    Addr end_addr = base_addr + whole_size;
403
404    for (unsigned int fragment_index = 0; fragment_index < numFragments;
405         fragment_index++)
406    {
407        bool M5_VAR_USED is_last_fragment = false;
408
409        if (fragment_addr == base_addr) {
410            /* First fragment */
411            fragment_size = first_fragment_size;
412        } else {
413            if ((fragment_addr + line_width) > end_addr) {
414                /* Adjust size of last fragment */
415                fragment_size = end_addr - fragment_addr;
416                is_last_fragment = true;
417            } else {
418                /* Middle fragments */
419                fragment_size = line_width;
420            }
421        }
422
423        Request *fragment = new Request();
424
425        fragment->setThreadContext(request.contextId(), /* thread id */ 0);
426        fragment->setVirt(0 /* asid */,
427            fragment_addr, fragment_size, request.getFlags(),
428            request.masterId(),
429            request.getPC());
430
431        DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
432            " (whole request addr: 0x%x size: %d) %s\n",
433            fragment_addr, fragment_size, base_addr, whole_size,
434            (is_last_fragment ? "last fragment" : ""));
435
436        fragment_addr += fragment_size;
437
438        fragmentRequests.push_back(fragment);
439    }
440}
441
442void
443LSQ::SplitDataRequest::makeFragmentPackets()
444{
445    Addr base_addr = request.getVaddr();
446
447    DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
448
449    for (unsigned int fragment_index = 0; fragment_index < numFragments;
450         fragment_index++)
451    {
452        Request *fragment = fragmentRequests[fragment_index];
453
454        DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
455            " (%d, 0x%x)\n",
456            fragment_index, *inst,
457            (fragment->hasPaddr() ? "has paddr" : "no paddr"),
458            (fragment->hasPaddr() ? fragment->getPaddr() : 0));
459
460        Addr fragment_addr = fragment->getVaddr();
461        unsigned int fragment_size = fragment->getSize();
462
463        uint8_t *request_data = NULL;
464
465        if (!isLoad) {
466            /* Split data for Packets.  Will become the property of the
467             *  outgoing Packets */
468            request_data = new uint8_t[fragment_size];
469            std::memcpy(request_data, data + (fragment_addr - base_addr),
470                fragment_size);
471        }
472
473        assert(fragment->hasPaddr());
474
475        PacketPtr fragment_packet =
476            makePacketForRequest(*fragment, isLoad, this, request_data);
477
478        fragmentPackets.push_back(fragment_packet);
479        /* Accumulate flags in parent request */
480        request.setFlags(fragment->getFlags());
481    }
482
483    /* Might as well make the overall/response packet here */
484    /* Get the physical address for the whole request/packet from the first
485     *  fragment */
486    request.setPaddr(fragmentRequests[0]->getPaddr());
487    makePacket();
488}
489
490void
491LSQ::SplitDataRequest::startAddrTranslation()
492{
493    setState(LSQ::LSQRequest::InTranslation);
494
495    makeFragmentRequests();
496
497    numInTranslationFragments = 0;
498    numTranslatedFragments = 0;
499
500    /* @todo, just do these in sequence for now with
501     * a loop of:
502     * do {
503     *  sendNextFragmentToTranslation ; translateTiming ; finish
504     * } while (numTranslatedFragments != numFragments);
505     */
506
507    /* Do first translation */
508    sendNextFragmentToTranslation();
509}
510
511PacketPtr
512LSQ::SplitDataRequest::getHeadPacket()
513{
514    assert(numIssuedFragments < numFragments);
515
516    return fragmentPackets[numIssuedFragments];
517}
518
519void
520LSQ::SplitDataRequest::stepToNextPacket()
521{
522    assert(numIssuedFragments < numFragments);
523
524    numIssuedFragments++;
525}
526
527void
528LSQ::SplitDataRequest::retireResponse(PacketPtr response)
529{
530    assert(numRetiredFragments < numFragments);
531
532    DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
533        " offset: 0x%x (retired fragment num: %d) %s\n",
534        response->req->getVaddr(), response->req->getSize(),
535        request.getVaddr() - response->req->getVaddr(),
536        numRetiredFragments,
537        (fault == NoFault ? "" : fault->name()));
538
539    numRetiredFragments++;
540
541    if (skipped) {
542        /* Skip because we already knew the request had faulted or been
543         *  skipped */
544        DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
545    } else if (response->isError()) {
546        /* Mark up the error and leave to execute to handle it */
547        DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
548        setSkipped();
549        packet->copyError(response);
550    } else {
551        if (isLoad) {
552            if (!data) {
553                /* For a split transfer, a Packet must be constructed
554                 *  to contain all returning data.  This is that packet's
555                 *  data */
556                data = new uint8_t[request.getSize()];
557            }
558
559            /* Populate the portion of the overall response data represented
560             *  by the response fragment */
561            std::memcpy(
562                data + (response->req->getVaddr() - request.getVaddr()),
563                response->getConstPtr<uint8_t>(),
564                response->req->getSize());
565        }
566    }
567
568    /* Complete early if we're skipping are no more in-flight accesses */
569    if (skipped && !hasPacketsInMemSystem()) {
570        DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
571        setState(Complete);
572        if (packet->needsResponse())
573            packet->makeResponse();
574    }
575
576    if (numRetiredFragments == numFragments)
577        setState(Complete);
578
579    if (!skipped && isComplete()) {
580        DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
581
582        DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
583             " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
584             " %s\n", packet->isRead(), packet->isWrite(),
585             packet->needsResponse(), packet->getSize(), request.getSize(),
586             response->getSize());
587
588        /* A request can become complete by several paths, this is a sanity
589         *  check to make sure the packet's data is created */
590        if (!data) {
591            data = new uint8_t[request.getSize()];
592        }
593
594        if (isLoad) {
595            DPRINTFS(MinorMem, (&port), "Copying read data\n");
596            std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
597        }
598        packet->makeResponse();
599    }
600
601    /* Packets are all deallocated together in ~SplitLSQRequest */
602}
603
604void
605LSQ::SplitDataRequest::sendNextFragmentToTranslation()
606{
607    unsigned int fragment_index = numTranslatedFragments;
608
609    ThreadContext *thread = port.cpu.getContext(
610        inst->id.threadId);
611
612    DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
613        fragment_index);
614
615    port.numAccessesInDTLB++;
616    numInTranslationFragments++;
617
618    thread->getDTBPtr()->translateTiming(
619        fragmentRequests[fragment_index], thread, this, (isLoad ?
620        BaseTLB::Read : BaseTLB::Write));
621}
622
623bool
624LSQ::StoreBuffer::canInsert() const
625{
626    /* @todo, support store amalgamation */
627    return slots.size() < numSlots;
628}
629
630void
631LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
632{
633    auto found = std::find(slots.begin(), slots.end(), request);
634
635    if (found != slots.end()) {
636        DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
637            request, *found, *(request->inst));
638        slots.erase(found);
639
640        delete request;
641    }
642}
643
644void
645LSQ::StoreBuffer::insert(LSQRequestPtr request)
646{
647    if (!canInsert()) {
648        warn("%s: store buffer insertion without space to insert from"
649            " inst: %s\n", name(), *(request->inst));
650    }
651
652    DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
653
654    numUnissuedAccesses++;
655
656    if (request->state != LSQRequest::Complete)
657        request->setState(LSQRequest::StoreInStoreBuffer);
658
659    slots.push_back(request);
660
661    /* Let's try and wake up the processor for the next cycle to step
662     *  the store buffer */
663    lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
664}
665
666LSQ::AddrRangeCoverage
667LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
668    unsigned int &found_slot)
669{
670    unsigned int slot_index = slots.size() - 1;
671    auto i = slots.rbegin();
672    AddrRangeCoverage ret = NoAddrRangeCoverage;
673
674    /* Traverse the store buffer in reverse order (most to least recent)
675     *  and try to find a slot whose address range overlaps this request */
676    while (ret == NoAddrRangeCoverage && i != slots.rend()) {
677        LSQRequestPtr slot = *i;
678
679        if (slot->packet) {
680            AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
681
682            if (coverage != NoAddrRangeCoverage) {
683                DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
684                    " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
685                    slot_index, coverage,
686                    request->request.getPaddr(), request->request.getSize(),
687                    slot->request.getPaddr(), slot->request.getSize());
688
689                found_slot = slot_index;
690                ret = coverage;
691            }
692        }
693
694        i++;
695        slot_index--;
696    }
697
698    return ret;
699}
700
701/** Fill the given packet with appropriate date from slot slot_number */
702void
703LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
704    unsigned int slot_number)
705{
706    assert(slot_number < slots.size());
707    assert(load->packet);
708    assert(load->isLoad);
709
710    LSQRequestPtr store = slots[slot_number];
711
712    assert(store->packet);
713    assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
714
715    Addr load_addr = load->request.getPaddr();
716    Addr store_addr = store->request.getPaddr();
717    Addr addr_offset = load_addr - store_addr;
718
719    unsigned int load_size = load->request.getSize();
720
721    DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
722        " slot: %d addr: 0x%x addressOffset: 0x%x\n",
723        load_size, load_addr, slot_number,
724        store_addr, addr_offset);
725
726    void *load_packet_data = load->packet->getPtr<void>();
727    void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
728
729    std::memcpy(load_packet_data, store_packet_data, load_size);
730}
731
732void
733LSQ::StoreBuffer::step()
734{
735    DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
736        numUnissuedAccesses);
737
738    if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
739        /* Clear all the leading barriers */
740        while (!slots.empty() &&
741            slots.front()->isComplete() && slots.front()->isBarrier())
742        {
743            LSQRequestPtr barrier = slots.front();
744
745            DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
746                *(barrier->inst));
747
748            numUnissuedAccesses--;
749            lsq.clearMemBarrier(barrier->inst);
750            slots.pop_front();
751
752            delete barrier;
753        }
754
755        auto i = slots.begin();
756        bool issued = true;
757        unsigned int issue_count = 0;
758
759        /* Skip trying if the memory system is busy */
760        if (lsq.state == LSQ::MemoryNeedsRetry)
761            issued = false;
762
763        /* Try to issue all stores in order starting from the head
764         *  of the queue.  Responses are allowed to be retired
765         *  out of order */
766        while (issued &&
767            issue_count < storeLimitPerCycle &&
768            lsq.canSendToMemorySystem() &&
769            i != slots.end())
770        {
771            LSQRequestPtr request = *i;
772
773            DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
774                " state: %s\n",
775                *(request->inst), request->sentAllPackets(),
776                request->state);
777
778            if (request->isBarrier() && request->isComplete()) {
779                /* Give up at barriers */
780                issued = false;
781            } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
782                request->sentAllPackets()))
783            {
784                DPRINTF(MinorMem, "Trying to send request: %s to memory"
785                    " system\n", *(request->inst));
786
787                if (lsq.tryToSend(request)) {
788                    /* Barrier are accounted for as they are cleared from
789                     *  the queue, not after their transfers are complete */
790                    if (!request->isBarrier())
791                        numUnissuedAccesses--;
792                    issue_count++;
793                } else {
794                    /* Don't step on to the next store buffer entry if this
795                     *  one hasn't issued all its packets as the store
796                     *  buffer must still enforce ordering */
797                    issued = false;
798                }
799            }
800            i++;
801        }
802    }
803}
804
805void
806LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
807    bool committed)
808{
809    if (committed) {
810        /* Not already sent to the store buffer as a store request? */
811        if (!inst->inStoreBuffer) {
812            /* Insert an entry into the store buffer to tick off barriers
813             *  until there are none in flight */
814            storeBuffer.insert(new BarrierDataRequest(*this, inst));
815        }
816    } else {
817        /* Clear the barrier anyway if it wasn't actually committed */
818        clearMemBarrier(inst);
819    }
820}
821
822void
823LSQ::StoreBuffer::minorTrace() const
824{
825    unsigned int size = slots.size();
826    unsigned int i = 0;
827    std::ostringstream os;
828
829    while (i < size) {
830        LSQRequestPtr request = slots[i];
831
832        request->reportData(os);
833
834        i++;
835        if (i < numSlots)
836            os << ',';
837    }
838
839    while (i < numSlots) {
840        os << '-';
841
842        i++;
843        if (i < numSlots)
844            os << ',';
845    }
846
847    MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
848        numUnissuedAccesses);
849}
850
851void
852LSQ::tryToSendToTransfers(LSQRequestPtr request)
853{
854    if (state == MemoryNeedsRetry) {
855        DPRINTF(MinorMem, "Request needs retry, not issuing to"
856            " memory until retry arrives\n");
857        return;
858    }
859
860    if (request->state == LSQRequest::InTranslation) {
861        DPRINTF(MinorMem, "Request still in translation, not issuing to"
862            " memory\n");
863        return;
864    }
865
866    assert(request->state == LSQRequest::Translated ||
867        request->state == LSQRequest::RequestIssuing ||
868        request->state == LSQRequest::Failed ||
869        request->state == LSQRequest::Complete);
870
871    if (requests.empty() || requests.front() != request) {
872        DPRINTF(MinorMem, "Request not at front of requests queue, can't"
873            " issue to memory\n");
874        return;
875    }
876
877    if (transfers.unreservedRemainingSpace() == 0) {
878        DPRINTF(MinorMem, "No space to insert request into transfers"
879            " queue\n");
880        return;
881    }
882
883    if (request->isComplete() || request->state == LSQRequest::Failed) {
884        DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
885            " queue\n", (request->isComplete() ? "completed" : "failed"));
886        request->setState(LSQRequest::Complete);
887        request->setSkipped();
888        moveFromRequestsToTransfers(request);
889        return;
890    }
891
892    if (!execute.instIsRightStream(request->inst)) {
893        /* Wrong stream, try to abort the transfer but only do so if
894         *  there are no packets in flight */
895        if (request->hasPacketsInMemSystem()) {
896            DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
897                " waiting for responses before aborting request\n");
898        } else {
899            DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
900                " aborting request\n");
901            request->setState(LSQRequest::Complete);
902            request->setSkipped();
903            moveFromRequestsToTransfers(request);
904        }
905        return;
906    }
907
908    if (request->fault != NoFault) {
909        if (request->inst->staticInst->isPrefetch()) {
910            DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
911        }
912        DPRINTF(MinorMem, "Moving faulting request into the transfers"
913            " queue\n");
914        request->setState(LSQRequest::Complete);
915        request->setSkipped();
916        moveFromRequestsToTransfers(request);
917        return;
918    }
919
920    bool is_load = request->isLoad;
921    bool is_llsc = request->request.isLLSC();
922    bool is_swap = request->request.isSwap();
923    bool bufferable = !(request->request.isUncacheable() ||
924        is_llsc || is_swap);
925
926    if (is_load) {
927        if (numStoresInTransfers != 0) {
928            DPRINTF(MinorMem, "Load request with stores still in transfers"
929                " queue, stalling\n");
930            return;
931        }
932    } else {
933        /* Store.  Can it be sent to the store buffer? */
934        if (bufferable && !request->request.isMmappedIpr()) {
935            request->setState(LSQRequest::StoreToStoreBuffer);
936            moveFromRequestsToTransfers(request);
937            DPRINTF(MinorMem, "Moving store into transfers queue\n");
938            return;
939        }
940    }
941
942    /* Check if this is the head instruction (and so must be executable as
943     *  its stream sequence number was checked above) for loads which must
944     *  not be speculatively issued and stores which must be issued here */
945    if (!bufferable) {
946        if (!execute.instIsHeadInst(request->inst)) {
947            DPRINTF(MinorMem, "Memory access not the head inst., can't be"
948                " sure it can be performed, not issuing\n");
949            return;
950        }
951
952        unsigned int forwarding_slot = 0;
953
954        if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
955            NoAddrRangeCoverage)
956        {
957            DPRINTF(MinorMem, "Memory access can receive forwarded data"
958                " from the store buffer, need to wait for store buffer to"
959                " drain\n");
960            return;
961        }
962    }
963
964    /* True: submit this packet to the transfers queue to be sent to the
965     * memory system.
966     * False: skip the memory and push a packet for this request onto
967     * requests */
968    bool do_access = true;
969
970    if (!is_llsc) {
971        /* Check for match in the store buffer */
972        if (is_load) {
973            unsigned int forwarding_slot = 0;
974            AddrRangeCoverage forwarding_result =
975                storeBuffer.canForwardDataToLoad(request,
976                forwarding_slot);
977
978            switch (forwarding_result) {
979              case FullAddrRangeCoverage:
980                /* Forward data from the store buffer into this request and
981                 *  repurpose this request's packet into a response packet */
982                storeBuffer.forwardStoreData(request, forwarding_slot);
983                request->packet->makeResponse();
984
985                /* Just move between queues, no access */
986                do_access = false;
987                break;
988              case PartialAddrRangeCoverage:
989                DPRINTF(MinorMem, "Load partly satisfied by store buffer"
990                    " data. Must wait for the store to complete\n");
991                return;
992                break;
993              case NoAddrRangeCoverage:
994                DPRINTF(MinorMem, "No forwardable data from store buffer\n");
995                /* Fall through to try access */
996                break;
997            }
998        }
999    } else {
1000        if (!canSendToMemorySystem()) {
1001            DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1002            return;
1003        }
1004
1005        SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1006
1007        TheISA::PCState old_pc = thread.pcState();
1008        ExecContext context(cpu, thread, execute, request->inst);
1009
1010        /* Handle LLSC requests and tests */
1011        if (is_load) {
1012            TheISA::handleLockedRead(&context, &request->request);
1013        } else {
1014            do_access = TheISA::handleLockedWrite(&context,
1015                &request->request, cacheBlockMask);
1016
1017            if (!do_access) {
1018                DPRINTF(MinorMem, "Not perfoming a memory "
1019                    "access for store conditional\n");
1020            }
1021        }
1022        thread.pcState(old_pc);
1023    }
1024
1025    /* See the do_access comment above */
1026    if (do_access) {
1027        if (!canSendToMemorySystem()) {
1028            DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1029            return;
1030        }
1031
1032        /* Remember if this is an access which can't be idly
1033         *  discarded by an interrupt */
1034        if (!bufferable && !request->issuedToMemory) {
1035            numAccessesIssuedToMemory++;
1036            request->issuedToMemory = true;
1037        }
1038
1039        if (tryToSend(request))
1040            moveFromRequestsToTransfers(request);
1041    } else {
1042        request->setState(LSQRequest::Complete);
1043        moveFromRequestsToTransfers(request);
1044    }
1045}
1046
1047bool
1048LSQ::tryToSend(LSQRequestPtr request)
1049{
1050    bool ret = false;
1051
1052    if (!canSendToMemorySystem()) {
1053        DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1054            *(request->inst));
1055    } else {
1056        PacketPtr packet = request->getHeadPacket();
1057
1058        DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1059            *(request->inst), packet->req->getVaddr());
1060
1061        /* The sender state of the packet *must* be an LSQRequest
1062         *  so the response can be correctly handled */
1063        assert(packet->findNextSenderState<LSQRequest>());
1064
1065        if (request->request.isMmappedIpr()) {
1066            ThreadContext *thread =
1067                cpu.getContext(request->request.threadId());
1068
1069            if (request->isLoad) {
1070                DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1071                TheISA::handleIprRead(thread, packet);
1072            } else {
1073                DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1074                TheISA::handleIprWrite(thread, packet);
1075            }
1076
1077            request->stepToNextPacket();
1078            ret = request->sentAllPackets();
1079
1080            if (!ret) {
1081                DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1082                    *(request->inst));
1083            }
1084
1085            if (ret)
1086                request->setState(LSQRequest::Complete);
1087            else
1088                request->setState(LSQRequest::RequestIssuing);
1089        } else if (dcachePort.sendTimingReq(packet)) {
1090            DPRINTF(MinorMem, "Sent data memory request\n");
1091
1092            numAccessesInMemorySystem++;
1093
1094            request->stepToNextPacket();
1095
1096            ret = request->sentAllPackets();
1097
1098            switch (request->state) {
1099              case LSQRequest::Translated:
1100              case LSQRequest::RequestIssuing:
1101                /* Fully or partially issued a request in the transfers
1102                 *  queue */
1103                request->setState(LSQRequest::RequestIssuing);
1104                break;
1105              case LSQRequest::StoreInStoreBuffer:
1106              case LSQRequest::StoreBufferIssuing:
1107                /* Fully or partially issued a request in the store
1108                 *  buffer */
1109                request->setState(LSQRequest::StoreBufferIssuing);
1110                break;
1111              default:
1112                assert(false);
1113                break;
1114            }
1115
1116            state = MemoryRunning;
1117        } else {
1118            DPRINTF(MinorMem,
1119                "Sending data memory request - needs retry\n");
1120
1121            /* Needs to be resent, wait for that */
1122            state = MemoryNeedsRetry;
1123            retryRequest = request;
1124
1125            switch (request->state) {
1126              case LSQRequest::Translated:
1127              case LSQRequest::RequestIssuing:
1128                request->setState(LSQRequest::RequestNeedsRetry);
1129                break;
1130              case LSQRequest::StoreInStoreBuffer:
1131              case LSQRequest::StoreBufferIssuing:
1132                request->setState(LSQRequest::StoreBufferNeedsRetry);
1133                break;
1134              default:
1135                assert(false);
1136                break;
1137            }
1138        }
1139    }
1140
1141    return ret;
1142}
1143
1144void
1145LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1146{
1147    assert(!requests.empty() && requests.front() == request);
1148    assert(transfers.unreservedRemainingSpace() != 0);
1149
1150    /* Need to count the number of stores in the transfers
1151     *  queue so that loads know when their store buffer forwarding
1152     *  results will be correct (only when all those stores
1153     *  have reached the store buffer) */
1154    if (!request->isLoad)
1155        numStoresInTransfers++;
1156
1157    requests.pop();
1158    transfers.push(request);
1159}
1160
1161bool
1162LSQ::canSendToMemorySystem()
1163{
1164    return state == MemoryRunning &&
1165        numAccessesInMemorySystem < inMemorySystemLimit;
1166}
1167
1168bool
1169LSQ::recvTimingResp(PacketPtr response)
1170{
1171    LSQRequestPtr request =
1172        safe_cast<LSQRequestPtr>(response->popSenderState());
1173
1174    DPRINTF(MinorMem, "Received response packet inst: %s"
1175        " addr: 0x%x cmd: %s\n",
1176        *(request->inst), response->getAddr(),
1177        response->cmd.toString());
1178
1179    numAccessesInMemorySystem--;
1180
1181    if (response->isError()) {
1182        DPRINTF(MinorMem, "Received error response packet: %s\n",
1183            *request->inst);
1184    }
1185
1186    switch (request->state) {
1187      case LSQRequest::RequestIssuing:
1188      case LSQRequest::RequestNeedsRetry:
1189        /* Response to a request from the transfers queue */
1190        request->retireResponse(response);
1191
1192        DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1193            request->hasPacketsInMemSystem(), request->isComplete());
1194
1195        break;
1196      case LSQRequest::StoreBufferIssuing:
1197      case LSQRequest::StoreBufferNeedsRetry:
1198        /* Response to a request from the store buffer */
1199        request->retireResponse(response);
1200
1201        /* Remove completed requests unless they are barrier (which will
1202         *  need to be removed in order */
1203        if (request->isComplete()) {
1204            if (!request->isBarrier()) {
1205                storeBuffer.deleteRequest(request);
1206            } else {
1207                DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1208                    " leaving the request as it is also a barrier\n",
1209                    *(request->inst));
1210            }
1211        }
1212        break;
1213      default:
1214        /* Shouldn't be allowed to receive a response from another
1215         *  state */
1216        assert(false);
1217        break;
1218    }
1219
1220    /* We go to idle even if there are more things in the requests queue
1221     * as it's the job of step to actually step us on to the next
1222     * transaction */
1223
1224    /* Let's try and wake up the processor for the next cycle */
1225    cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1226
1227    /* Never busy */
1228    return true;
1229}
1230
1231void
1232LSQ::recvRetry()
1233{
1234    DPRINTF(MinorMem, "Received retry request\n");
1235
1236    assert(state == MemoryNeedsRetry);
1237
1238    switch (retryRequest->state) {
1239      case LSQRequest::RequestNeedsRetry:
1240        /* Retry in the requests queue */
1241        retryRequest->setState(LSQRequest::Translated);
1242        break;
1243      case LSQRequest::StoreBufferNeedsRetry:
1244        /* Retry in the store buffer */
1245        retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1246        break;
1247      default:
1248        assert(false);
1249    }
1250
1251    /* Set state back to MemoryRunning so that the following
1252     *  tryToSend can actually send.  Note that this won't
1253     *  allow another transfer in as tryToSend should
1254     *  issue a memory request and either succeed for this
1255     *  request or return the LSQ back to MemoryNeedsRetry */
1256    state = MemoryRunning;
1257
1258    /* Try to resend the request */
1259    if (tryToSend(retryRequest)) {
1260        /* Successfully sent, need to move the request */
1261        switch (retryRequest->state) {
1262          case LSQRequest::RequestIssuing:
1263            /* In the requests queue */
1264            moveFromRequestsToTransfers(retryRequest);
1265            break;
1266          case LSQRequest::StoreBufferIssuing:
1267            /* In the store buffer */
1268            storeBuffer.numUnissuedAccesses--;
1269            break;
1270          default:
1271            assert(false);
1272            break;
1273        }
1274    }
1275
1276    retryRequest = NULL;
1277}
1278
1279LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1280    MinorCPU &cpu_, Execute &execute_,
1281    unsigned int in_memory_system_limit, unsigned int line_width,
1282    unsigned int requests_queue_size, unsigned int transfers_queue_size,
1283    unsigned int store_buffer_size,
1284    unsigned int store_buffer_cycle_store_limit) :
1285    Named(name_),
1286    cpu(cpu_),
1287    execute(execute_),
1288    dcachePort(dcache_port_name_, *this, cpu_),
1289    lastMemBarrier(0),
1290    state(MemoryRunning),
1291    inMemorySystemLimit(in_memory_system_limit),
1292    lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1293    requests(name_ + ".requests", "addr", requests_queue_size),
1294    transfers(name_ + ".transfers", "addr", transfers_queue_size),
1295    storeBuffer(name_ + ".storeBuffer",
1296        *this, store_buffer_size, store_buffer_cycle_store_limit),
1297    numAccessesInMemorySystem(0),
1298    numAccessesInDTLB(0),
1299    numStoresInTransfers(0),
1300    numAccessesIssuedToMemory(0),
1301    retryRequest(NULL),
1302    cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1303{
1304    if (in_memory_system_limit < 1) {
1305        fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1306            in_memory_system_limit);
1307    }
1308
1309    if (store_buffer_cycle_store_limit < 1) {
1310        fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1311            " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1312    }
1313
1314    if (requests_queue_size < 1) {
1315        fatal("%s: executeLSQRequestsQueueSize must be"
1316            " >= 1 (%d)\n", name_, requests_queue_size);
1317    }
1318
1319    if (transfers_queue_size < 1) {
1320        fatal("%s: executeLSQTransfersQueueSize must be"
1321            " >= 1 (%d)\n", name_, transfers_queue_size);
1322    }
1323
1324    if (store_buffer_size < 1) {
1325        fatal("%s: executeLSQStoreBufferSize must be"
1326            " >= 1 (%d)\n", name_, store_buffer_size);
1327    }
1328
1329    if ((lineWidth & (lineWidth - 1)) != 0) {
1330        fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1331    }
1332}
1333
1334LSQ::~LSQ()
1335{ }
1336
1337LSQ::LSQRequest::~LSQRequest()
1338{
1339    if (packet)
1340        delete packet;
1341    if (data)
1342        delete [] data;
1343}
1344
1345/**
1346 *  Step the memory access mechanism on to its next state.  In reality, most
1347 *  of the stepping is done by the callbacks on the LSQ but this
1348 *  function is responsible for issuing memory requests lodged in the
1349 *  requests queue.
1350 */
1351void
1352LSQ::step()
1353{
1354    /* Try to move address-translated requests between queues and issue
1355     *  them */
1356    if (!requests.empty())
1357        tryToSendToTransfers(requests.front());
1358
1359    storeBuffer.step();
1360}
1361
1362LSQ::LSQRequestPtr
1363LSQ::findResponse(MinorDynInstPtr inst)
1364{
1365    LSQ::LSQRequestPtr ret = NULL;
1366
1367    if (!transfers.empty()) {
1368        LSQRequestPtr request = transfers.front();
1369
1370        /* Same instruction and complete access or a store that's
1371         *  capable of being moved to the store buffer */
1372        if (request->inst->id == inst->id) {
1373            bool complete = request->isComplete();
1374            bool can_store = storeBuffer.canInsert();
1375            bool to_store_buffer = request->state ==
1376                LSQRequest::StoreToStoreBuffer;
1377
1378            if ((complete && !(request->isBarrier() && !can_store)) ||
1379                (to_store_buffer && can_store))
1380            {
1381                ret = request;
1382            }
1383        }
1384    }
1385
1386    if (ret) {
1387        DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1388            *inst);
1389    } else {
1390        DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1391            *inst);
1392    }
1393
1394    return ret;
1395}
1396
1397void
1398LSQ::popResponse(LSQ::LSQRequestPtr response)
1399{
1400    assert(!transfers.empty() && transfers.front() == response);
1401
1402    transfers.pop();
1403
1404    if (!response->isLoad)
1405        numStoresInTransfers--;
1406
1407    if (response->issuedToMemory)
1408        numAccessesIssuedToMemory--;
1409
1410    if (response->state != LSQRequest::StoreInStoreBuffer) {
1411        DPRINTF(MinorMem, "Deleting %s request: %s\n",
1412            (response->isLoad ? "load" : "store"),
1413            *(response->inst));
1414
1415        delete response;
1416    }
1417}
1418
1419void
1420LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1421{
1422    assert(request->state == LSQRequest::StoreToStoreBuffer);
1423
1424    DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1425        *(request->inst));
1426
1427    request->inst->inStoreBuffer = true;
1428
1429    storeBuffer.insert(request);
1430}
1431
1432bool
1433LSQ::isDrained()
1434{
1435    return requests.empty() && transfers.empty() &&
1436        storeBuffer.isDrained();
1437}
1438
1439bool
1440LSQ::needsToTick()
1441{
1442    bool ret = false;
1443
1444    if (canSendToMemorySystem()) {
1445        bool have_translated_requests = !requests.empty() &&
1446            requests.front()->state != LSQRequest::InTranslation &&
1447            transfers.unreservedRemainingSpace() != 0;
1448
1449        ret = have_translated_requests ||
1450            storeBuffer.numUnissuedStores() != 0;
1451    }
1452
1453    if (ret)
1454        DPRINTF(Activity, "Need to tick\n");
1455
1456    return ret;
1457}
1458
1459void
1460LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1461    unsigned int size, Addr addr, unsigned int flags, uint64_t *res)
1462{
1463    bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1464    LSQRequestPtr request;
1465
1466    /* Copy given data into the request.  The request will pass this to the
1467     *  packet and then it will own the data */
1468    uint8_t *request_data = NULL;
1469
1470    DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1471        " 0x%x%s lineWidth : 0x%x\n",
1472        (isLoad ? "load" : "store"), addr, size, flags,
1473            (needs_burst ? " (needs burst)" : ""), lineWidth);
1474
1475    if (!isLoad) {
1476        /* request_data becomes the property of a ...DataRequest (see below)
1477         *  and destroyed by its destructor */
1478        request_data = new uint8_t[size];
1479        if (flags & Request::CACHE_BLOCK_ZERO) {
1480            /* For cache zeroing, just use zeroed data */
1481            std::memset(request_data, 0, size);
1482        } else {
1483            std::memcpy(request_data, data, size);
1484        }
1485    }
1486
1487    if (needs_burst) {
1488        request = new SplitDataRequest(
1489            *this, inst, isLoad, request_data, res);
1490    } else {
1491        request = new SingleDataRequest(
1492            *this, inst, isLoad, request_data, res);
1493    }
1494
1495    if (inst->traceData)
1496        inst->traceData->setAddr(addr);
1497
1498    request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0);
1499    request->request.setVirt(0 /* asid */,
1500        addr, size, flags, cpu.instMasterId(),
1501        /* I've no idea why we need the PC, but give it */
1502        inst->pc.instAddr());
1503
1504    requests.push(request);
1505    request->startAddrTranslation();
1506}
1507
1508void
1509LSQ::pushFailedRequest(MinorDynInstPtr inst)
1510{
1511    LSQRequestPtr request = new FailedDataRequest(*this, inst);
1512    requests.push(request);
1513}
1514
1515void
1516LSQ::minorTrace() const
1517{
1518    MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1519        " lastMemBarrier=%d\n",
1520        state, numAccessesInDTLB, numAccessesInMemorySystem,
1521        numStoresInTransfers, lastMemBarrier);
1522    requests.minorTrace();
1523    transfers.minorTrace();
1524    storeBuffer.minorTrace();
1525}
1526
1527LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1528    unsigned int store_buffer_size,
1529    unsigned int store_limit_per_cycle) :
1530    Named(name_), lsq(lsq_),
1531    numSlots(store_buffer_size),
1532    storeLimitPerCycle(store_limit_per_cycle),
1533    slots(),
1534    numUnissuedAccesses(0)
1535{
1536}
1537
1538PacketPtr
1539makePacketForRequest(Request &request, bool isLoad,
1540    Packet::SenderState *sender_state, PacketDataPtr data)
1541{
1542    MemCmd command;
1543
1544    /* Make a ret with the right command type to match the request */
1545    if (request.isLLSC()) {
1546        command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq);
1547    } else if (request.isSwap()) {
1548        command = MemCmd::SwapReq;
1549    } else {
1550        command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq);
1551    }
1552
1553    PacketPtr ret = new Packet(&request, command);
1554
1555    if (sender_state)
1556        ret->pushSenderState(sender_state);
1557
1558    if (isLoad)
1559        ret->allocate();
1560    else
1561        ret->dataDynamic(data);
1562
1563    return ret;
1564}
1565
1566void
1567LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1568{
1569    assert(inst->isInst() && inst->staticInst->isMemBarrier());
1570    assert(inst->id.execSeqNum > lastMemBarrier);
1571
1572    /* Remember the barrier.  We only have a notion of one
1573     *  barrier so this may result in some mem refs being
1574     *  delayed if they are between barriers */
1575    lastMemBarrier = inst->id.execSeqNum;
1576}
1577
1578void
1579LSQ::LSQRequest::makePacket()
1580{
1581    /* Make the function idempotent */
1582    if (packet)
1583        return;
1584
1585    packet = makePacketForRequest(request, isLoad, this, data);
1586    /* Null the ret data so we know not to deallocate it when the
1587     * ret is destroyed.  The data now belongs to the ret and
1588     * the ret is responsible for its destruction */
1589    data = NULL;
1590}
1591
1592std::ostream &
1593operator <<(std::ostream &os, LSQ::MemoryState state)
1594{
1595    switch (state) {
1596      case LSQ::MemoryRunning:
1597        os << "MemoryRunning";
1598        break;
1599      case LSQ::MemoryNeedsRetry:
1600        os << "MemoryNeedsRetry";
1601        break;
1602      default:
1603        os << "MemoryState-" << static_cast<int>(state);
1604        break;
1605    }
1606    return os;
1607}
1608
1609void
1610LSQ::recvTimingSnoopReq(PacketPtr pkt)
1611{
1612    /* LLSC operations in Minor can't be speculative and are executed from
1613     * the head of the requests queue.  We shouldn't need to do more than
1614     * this action on snoops. */
1615
1616    /* THREAD */
1617    TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask);
1618}
1619
1620}
1621