lsq_impl.hh revision 14297 
1/*
2 * Copyright (c) 2011-2012, 2014, 2017-2018 ARM Limited
3 * Copyright (c) 2013 Advanced Micro Devices, Inc.
4 * All rights reserved
5 *
6 * The license below extends only to copyright in the software and shall
7 * not be construed as granting a license to any other intellectual
8 * property including but not limited to intellectual property relating
9 * to a hardware implementation of the functionality of the software
10 * licensed hereunder.  You may use the software subject to the license
11 * terms below provided that you ensure that this notice is replicated
12 * unmodified and in its entirety in all distributions of the software,
13 * modified or unmodified, in source code or in binary form.
14 *
15 * Copyright (c) 2005-2006 The Regents of The University of Michigan
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Korey Sewell
42 */
43
44#ifndef __CPU_O3_LSQ_IMPL_HH__
45#define __CPU_O3_LSQ_IMPL_HH__
46
47#include <algorithm>
48#include <list>
49#include <string>
50
51#include "base/logging.hh"
52#include "cpu/o3/cpu.hh"
53#include "cpu/o3/lsq.hh"
54#include "debug/Drain.hh"
55#include "debug/Fetch.hh"
56#include "debug/LSQ.hh"
57#include "debug/Writeback.hh"
58#include "params/DerivO3CPU.hh"
59
60using namespace std;
61
62template <class Impl>
63LSQ<Impl>::LSQ(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params)
64    : cpu(cpu_ptr), iewStage(iew_ptr),
65      _cacheBlocked(false),
66      cacheStorePorts(params->cacheStorePorts), usedStorePorts(0),
67      cacheLoadPorts(params->cacheLoadPorts), usedLoadPorts(0),
68      lsqPolicy(params->smtLSQPolicy),
69      LQEntries(params->LQEntries),
70      SQEntries(params->SQEntries),
71      maxLQEntries(maxLSQAllocation(lsqPolicy, LQEntries, params->numThreads,
72                  params->smtLSQThreshold)),
73      maxSQEntries(maxLSQAllocation(lsqPolicy, SQEntries, params->numThreads,
74                  params->smtLSQThreshold)),
75      dcachePort(this, cpu_ptr),
76      numThreads(params->numThreads)
77{
78    assert(numThreads > 0 && numThreads <= Impl::MaxThreads);
79
80    //**********************************************/
81    //************ Handle SMT Parameters ***********/
82    //**********************************************/
83
84    /* Run SMT olicy checks. */
85        if (lsqPolicy == SMTQueuePolicy::Dynamic) {
86        DPRINTF(LSQ, "LSQ sharing policy set to Dynamic\n");
87    } else if (lsqPolicy == SMTQueuePolicy::Partitioned) {
88        DPRINTF(Fetch, "LSQ sharing policy set to Partitioned: "
89                "%i entries per LQ | %i entries per SQ\n",
90                maxLQEntries,maxSQEntries);
91    } else if (lsqPolicy == SMTQueuePolicy::Threshold) {
92
93        assert(params->smtLSQThreshold > params->LQEntries);
94        assert(params->smtLSQThreshold > params->SQEntries);
95
96        DPRINTF(LSQ, "LSQ sharing policy set to Threshold: "
97                "%i entries per LQ | %i entries per SQ\n",
98                maxLQEntries,maxSQEntries);
99    } else {
100        panic("Invalid LSQ sharing policy. Options are: Dynamic, "
101                    "Partitioned, Threshold");
102    }
103
104    thread.reserve(numThreads);
105    for (ThreadID tid = 0; tid < numThreads; tid++) {
106        thread.emplace_back(maxLQEntries, maxSQEntries);
107        thread[tid].init(cpu, iew_ptr, params, this, tid);
108        thread[tid].setDcachePort(&dcachePort);
109    }
110}
111
112
113template<class Impl>
114std::string
115LSQ<Impl>::name() const
116{
117    return iewStage->name() + ".lsq";
118}
119
120template<class Impl>
121void
122LSQ<Impl>::regStats()
123{
124    //Initialize LSQs
125    for (ThreadID tid = 0; tid < numThreads; tid++) {
126        thread[tid].regStats();
127    }
128}
129
130template<class Impl>
131void
132LSQ<Impl>::setActiveThreads(list<ThreadID> *at_ptr)
133{
134    activeThreads = at_ptr;
135    assert(activeThreads != 0);
136}
137
138template <class Impl>
139void
140LSQ<Impl>::drainSanityCheck() const
141{
142    assert(isDrained());
143
144    for (ThreadID tid = 0; tid < numThreads; tid++)
145        thread[tid].drainSanityCheck();
146}
147
148template <class Impl>
149bool
150LSQ<Impl>::isDrained() const
151{
152    bool drained(true);
153
154    if (!lqEmpty()) {
155        DPRINTF(Drain, "Not drained, LQ not empty.\n");
156        drained = false;
157    }
158
159    if (!sqEmpty()) {
160        DPRINTF(Drain, "Not drained, SQ not empty.\n");
161        drained = false;
162    }
163
164    return drained;
165}
166
167template <class Impl>
168void
169LSQ<Impl>::takeOverFrom()
170{
171    usedStorePorts = 0;
172    _cacheBlocked = false;
173
174    for (ThreadID tid = 0; tid < numThreads; tid++) {
175        thread[tid].takeOverFrom();
176    }
177}
178
179template <class Impl>
180void
181LSQ<Impl>::tick()
182{
183    // Re-issue loads which got blocked on the per-cycle load ports limit.
184    if (usedLoadPorts == cacheLoadPorts && !_cacheBlocked)
185        iewStage->cacheUnblocked();
186
187    usedLoadPorts = 0;
188    usedStorePorts = 0;
189}
190
191template<class Impl>
192bool
193LSQ<Impl>::cacheBlocked() const
194{
195    return _cacheBlocked;
196}
197
198template<class Impl>
199void
200LSQ<Impl>::cacheBlocked(bool v)
201{
202    _cacheBlocked = v;
203}
204
205template<class Impl>
206bool
207LSQ<Impl>::cachePortAvailable(bool is_load) const
208{
209    bool ret;
210    if (is_load) {
211        ret  = usedLoadPorts < cacheLoadPorts;
212    } else {
213        ret  = usedStorePorts < cacheStorePorts;
214    }
215    return ret;
216}
217
218template<class Impl>
219void
220LSQ<Impl>::cachePortBusy(bool is_load)
221{
222    assert(cachePortAvailable(is_load));
223    if (is_load) {
224        usedLoadPorts++;
225    } else {
226        usedStorePorts++;
227    }
228}
229
230template<class Impl>
231void
232LSQ<Impl>::insertLoad(const DynInstPtr &load_inst)
233{
234    ThreadID tid = load_inst->threadNumber;
235
236    thread[tid].insertLoad(load_inst);
237}
238
239template<class Impl>
240void
241LSQ<Impl>::insertStore(const DynInstPtr &store_inst)
242{
243    ThreadID tid = store_inst->threadNumber;
244
245    thread[tid].insertStore(store_inst);
246}
247
248template<class Impl>
249Fault
250LSQ<Impl>::executeLoad(const DynInstPtr &inst)
251{
252    ThreadID tid = inst->threadNumber;
253
254    return thread[tid].executeLoad(inst);
255}
256
257template<class Impl>
258Fault
259LSQ<Impl>::executeStore(const DynInstPtr &inst)
260{
261    ThreadID tid = inst->threadNumber;
262
263    return thread[tid].executeStore(inst);
264}
265
266template<class Impl>
267void
268LSQ<Impl>::writebackStores()
269{
270    list<ThreadID>::iterator threads = activeThreads->begin();
271    list<ThreadID>::iterator end = activeThreads->end();
272
273    while (threads != end) {
274        ThreadID tid = *threads++;
275
276        if (numStoresToWB(tid) > 0) {
277            DPRINTF(Writeback,"[tid:%i] Writing back stores. %i stores "
278                "available for Writeback.\n", tid, numStoresToWB(tid));
279        }
280
281        thread[tid].writebackStores();
282    }
283}
284
285template<class Impl>
286bool
287LSQ<Impl>::violation()
288{
289    /* Answers: Does Anybody Have a Violation?*/
290    list<ThreadID>::iterator threads = activeThreads->begin();
291    list<ThreadID>::iterator end = activeThreads->end();
292
293    while (threads != end) {
294        ThreadID tid = *threads++;
295
296        if (thread[tid].violation())
297            return true;
298    }
299
300    return false;
301}
302
303template <class Impl>
304void
305LSQ<Impl>::recvReqRetry()
306{
307    iewStage->cacheUnblocked();
308    cacheBlocked(false);
309
310    for (ThreadID tid : *activeThreads) {
311        thread[tid].recvRetry();
312    }
313}
314
315template <class Impl>
316void
317LSQ<Impl>::completeDataAccess(PacketPtr pkt)
318{
319    auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
320    thread[cpu->contextToThread(senderState->contextId())]
321        .completeDataAccess(pkt);
322}
323
324template <class Impl>
325bool
326LSQ<Impl>::recvTimingResp(PacketPtr pkt)
327{
328    if (pkt->isError())
329        DPRINTF(LSQ, "Got error packet back for address: %#X\n",
330                pkt->getAddr());
331
332    auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
333    panic_if(!senderState, "Got packet back with unknown sender state\n");
334
335    thread[cpu->contextToThread(senderState->contextId())].recvTimingResp(pkt);
336
337    if (pkt->isInvalidate()) {
338        // This response also contains an invalidate; e.g. this can be the case
339        // if cmd is ReadRespWithInvalidate.
340        //
341        // The calling order between completeDataAccess and checkSnoop matters.
342        // By calling checkSnoop after completeDataAccess, we ensure that the
343        // fault set by checkSnoop is not lost. Calling writeback (more
344        // specifically inst->completeAcc) in completeDataAccess overwrites
345        // fault, and in case this instruction requires squashing (as
346        // determined by checkSnoop), the ReExec fault set by checkSnoop would
347        // be lost otherwise.
348
349        DPRINTF(LSQ, "received invalidation with response for addr:%#x\n",
350                pkt->getAddr());
351
352        for (ThreadID tid = 0; tid < numThreads; tid++) {
353            thread[tid].checkSnoop(pkt);
354        }
355    }
356    // Update the LSQRequest state (this may delete the request)
357    senderState->request()->packetReplied();
358
359    return true;
360}
361
362template <class Impl>
363void
364LSQ<Impl>::recvTimingSnoopReq(PacketPtr pkt)
365{
366    DPRINTF(LSQ, "received pkt for addr:%#x %s\n", pkt->getAddr(),
367            pkt->cmdString());
368
369    // must be a snoop
370    if (pkt->isInvalidate()) {
371        DPRINTF(LSQ, "received invalidation for addr:%#x\n",
372                pkt->getAddr());
373        for (ThreadID tid = 0; tid < numThreads; tid++) {
374            thread[tid].checkSnoop(pkt);
375        }
376    }
377}
378
379template<class Impl>
380int
381LSQ<Impl>::getCount()
382{
383    unsigned total = 0;
384
385    list<ThreadID>::iterator threads = activeThreads->begin();
386    list<ThreadID>::iterator end = activeThreads->end();
387
388    while (threads != end) {
389        ThreadID tid = *threads++;
390
391        total += getCount(tid);
392    }
393
394    return total;
395}
396
397template<class Impl>
398int
399LSQ<Impl>::numLoads()
400{
401    unsigned total = 0;
402
403    list<ThreadID>::iterator threads = activeThreads->begin();
404    list<ThreadID>::iterator end = activeThreads->end();
405
406    while (threads != end) {
407        ThreadID tid = *threads++;
408
409        total += numLoads(tid);
410    }
411
412    return total;
413}
414
415template<class Impl>
416int
417LSQ<Impl>::numStores()
418{
419    unsigned total = 0;
420
421    list<ThreadID>::iterator threads = activeThreads->begin();
422    list<ThreadID>::iterator end = activeThreads->end();
423
424    while (threads != end) {
425        ThreadID tid = *threads++;
426
427        total += thread[tid].numStores();
428    }
429
430    return total;
431}
432
433template<class Impl>
434unsigned
435LSQ<Impl>::numFreeLoadEntries()
436{
437    unsigned total = 0;
438
439    list<ThreadID>::iterator threads = activeThreads->begin();
440    list<ThreadID>::iterator end = activeThreads->end();
441
442    while (threads != end) {
443        ThreadID tid = *threads++;
444
445        total += thread[tid].numFreeLoadEntries();
446    }
447
448    return total;
449}
450
451template<class Impl>
452unsigned
453LSQ<Impl>::numFreeStoreEntries()
454{
455    unsigned total = 0;
456
457    list<ThreadID>::iterator threads = activeThreads->begin();
458    list<ThreadID>::iterator end = activeThreads->end();
459
460    while (threads != end) {
461        ThreadID tid = *threads++;
462
463        total += thread[tid].numFreeStoreEntries();
464    }
465
466    return total;
467}
468
469template<class Impl>
470unsigned
471LSQ<Impl>::numFreeLoadEntries(ThreadID tid)
472{
473        return thread[tid].numFreeLoadEntries();
474}
475
476template<class Impl>
477unsigned
478LSQ<Impl>::numFreeStoreEntries(ThreadID tid)
479{
480        return thread[tid].numFreeStoreEntries();
481}
482
483template<class Impl>
484bool
485LSQ<Impl>::isFull()
486{
487    list<ThreadID>::iterator threads = activeThreads->begin();
488    list<ThreadID>::iterator end = activeThreads->end();
489
490    while (threads != end) {
491        ThreadID tid = *threads++;
492
493        if (!(thread[tid].lqFull() || thread[tid].sqFull()))
494            return false;
495    }
496
497    return true;
498}
499
500template<class Impl>
501bool
502LSQ<Impl>::isFull(ThreadID tid)
503{
504    //@todo: Change to Calculate All Entries for
505    //Dynamic Policy
506    if (lsqPolicy == SMTQueuePolicy::Dynamic)
507        return isFull();
508    else
509        return thread[tid].lqFull() || thread[tid].sqFull();
510}
511
512template<class Impl>
513bool
514LSQ<Impl>::isEmpty() const
515{
516    return lqEmpty() && sqEmpty();
517}
518
519template<class Impl>
520bool
521LSQ<Impl>::lqEmpty() const
522{
523    list<ThreadID>::const_iterator threads = activeThreads->begin();
524    list<ThreadID>::const_iterator end = activeThreads->end();
525
526    while (threads != end) {
527        ThreadID tid = *threads++;
528
529        if (!thread[tid].lqEmpty())
530            return false;
531    }
532
533    return true;
534}
535
536template<class Impl>
537bool
538LSQ<Impl>::sqEmpty() const
539{
540    list<ThreadID>::const_iterator threads = activeThreads->begin();
541    list<ThreadID>::const_iterator end = activeThreads->end();
542
543    while (threads != end) {
544        ThreadID tid = *threads++;
545
546        if (!thread[tid].sqEmpty())
547            return false;
548    }
549
550    return true;
551}
552
553template<class Impl>
554bool
555LSQ<Impl>::lqFull()
556{
557    list<ThreadID>::iterator threads = activeThreads->begin();
558    list<ThreadID>::iterator end = activeThreads->end();
559
560    while (threads != end) {
561        ThreadID tid = *threads++;
562
563        if (!thread[tid].lqFull())
564            return false;
565    }
566
567    return true;
568}
569
570template<class Impl>
571bool
572LSQ<Impl>::lqFull(ThreadID tid)
573{
574    //@todo: Change to Calculate All Entries for
575    //Dynamic Policy
576    if (lsqPolicy == SMTQueuePolicy::Dynamic)
577        return lqFull();
578    else
579        return thread[tid].lqFull();
580}
581
582template<class Impl>
583bool
584LSQ<Impl>::sqFull()
585{
586    list<ThreadID>::iterator threads = activeThreads->begin();
587    list<ThreadID>::iterator end = activeThreads->end();
588
589    while (threads != end) {
590        ThreadID tid = *threads++;
591
592        if (!sqFull(tid))
593            return false;
594    }
595
596    return true;
597}
598
599template<class Impl>
600bool
601LSQ<Impl>::sqFull(ThreadID tid)
602{
603     //@todo: Change to Calculate All Entries for
604    //Dynamic Policy
605    if (lsqPolicy == SMTQueuePolicy::Dynamic)
606        return sqFull();
607    else
608        return thread[tid].sqFull();
609}
610
611template<class Impl>
612bool
613LSQ<Impl>::isStalled()
614{
615    list<ThreadID>::iterator threads = activeThreads->begin();
616    list<ThreadID>::iterator end = activeThreads->end();
617
618    while (threads != end) {
619        ThreadID tid = *threads++;
620
621        if (!thread[tid].isStalled())
622            return false;
623    }
624
625    return true;
626}
627
628template<class Impl>
629bool
630LSQ<Impl>::isStalled(ThreadID tid)
631{
632    if (lsqPolicy == SMTQueuePolicy::Dynamic)
633        return isStalled();
634    else
635        return thread[tid].isStalled();
636}
637
638template<class Impl>
639bool
640LSQ<Impl>::hasStoresToWB()
641{
642    list<ThreadID>::iterator threads = activeThreads->begin();
643    list<ThreadID>::iterator end = activeThreads->end();
644
645    while (threads != end) {
646        ThreadID tid = *threads++;
647
648        if (hasStoresToWB(tid))
649            return true;
650    }
651
652    return false;
653}
654
655template<class Impl>
656bool
657LSQ<Impl>::willWB()
658{
659    list<ThreadID>::iterator threads = activeThreads->begin();
660    list<ThreadID>::iterator end = activeThreads->end();
661
662    while (threads != end) {
663        ThreadID tid = *threads++;
664
665        if (willWB(tid))
666            return true;
667    }
668
669    return false;
670}
671
672template<class Impl>
673void
674LSQ<Impl>::dumpInsts() const
675{
676    list<ThreadID>::const_iterator threads = activeThreads->begin();
677    list<ThreadID>::const_iterator end = activeThreads->end();
678
679    while (threads != end) {
680        ThreadID tid = *threads++;
681
682        thread[tid].dumpInsts();
683    }
684}
685
686template<class Impl>
687Fault
688LSQ<Impl>::pushRequest(const DynInstPtr& inst, bool isLoad, uint8_t *data,
689                       unsigned int size, Addr addr, Request::Flags flags,
690                       uint64_t *res, AtomicOpFunctorPtr amo_op,
691                       const std::vector<bool>& byteEnable)
692{
693    // This comming request can be either load, store or atomic.
694    // Atomic request has a corresponding pointer to its atomic memory
695    // operation
696    bool isAtomic M5_VAR_USED = !isLoad && amo_op;
697
698    ThreadID tid = cpu->contextToThread(inst->contextId());
699    auto cacheLineSize = cpu->cacheLineSize();
700    bool needs_burst = transferNeedsBurst(addr, size, cacheLineSize);
701    LSQRequest* req = nullptr;
702
703    // Atomic requests that access data across cache line boundary are
704    // currently not allowed since the cache does not guarantee corresponding
705    // atomic memory operations to be executed atomically across a cache line.
706    // For ISAs such as x86 that supports cross-cache-line atomic instructions,
707    // the cache needs to be modified to perform atomic update to both cache
708    // lines. For now, such cross-line update is not supported.
709    assert(!isAtomic || (isAtomic && !needs_burst));
710
711    if (inst->translationStarted()) {
712        req = inst->savedReq;
713        assert(req);
714    } else {
715        if (needs_burst) {
716            req = new SplitDataRequest(&thread[tid], inst, isLoad, addr,
717                    size, flags, data, res);
718        } else {
719            req = new SingleDataRequest(&thread[tid], inst, isLoad, addr,
720                    size, flags, data, res, std::move(amo_op));
721        }
722        assert(req);
723        if (!byteEnable.empty()) {
724            req->_byteEnable = byteEnable;
725        }
726        inst->setRequest();
727        req->taskId(cpu->taskId());
728
729        // There might be fault from a previous execution attempt if this is
730        // a strictly ordered load
731        inst->getFault() = NoFault;
732
733        req->initiateTranslation();
734    }
735
736    /* This is the place were instructions get the effAddr. */
737    if (req->isTranslationComplete()) {
738        if (req->isMemAccessRequired()) {
739            inst->effAddr = req->getVaddr();
740            inst->effSize = size;
741            inst->effAddrValid(true);
742
743            if (cpu->checker) {
744                inst->reqToVerify = std::make_shared<Request>(*req->request());
745            }
746            Fault fault;
747            if (isLoad)
748                fault = cpu->read(req, inst->lqIdx);
749            else
750                fault = cpu->write(req, data, inst->sqIdx);
751            // inst->getFault() may have the first-fault of a
752            // multi-access split request at this point.
753            // Overwrite that only if we got another type of fault
754            // (e.g. re-exec).
755            if (fault != NoFault)
756                inst->getFault() = fault;
757        } else if (isLoad) {
758            inst->setMemAccPredicate(false);
759            // Commit will have to clean up whatever happened.  Set this
760            // instruction as executed.
761            inst->setExecuted();
762        }
763    }
764
765    if (inst->traceData)
766        inst->traceData->setMem(addr, size, flags);
767
768    return inst->getFault();
769}
770
771template<class Impl>
772void
773LSQ<Impl>::SingleDataRequest::finish(const Fault &fault, const RequestPtr &req,
774        ThreadContext* tc, BaseTLB::Mode mode)
775{
776    _fault.push_back(fault);
777    numInTranslationFragments = 0;
778    numTranslatedFragments = 1;
779    /* If the instruction has been squahsed, let the request know
780     * as it may have to self-destruct. */
781    if (_inst->isSquashed()) {
782        this->squashTranslation();
783    } else {
784        _inst->strictlyOrdered(req->isStrictlyOrdered());
785
786        flags.set(Flag::TranslationFinished);
787        if (fault == NoFault) {
788            _inst->physEffAddr = req->getPaddr();
789            _inst->memReqFlags = req->getFlags();
790            if (req->isCondSwap()) {
791                assert(_res);
792                req->setExtraData(*_res);
793            }
794            setState(State::Request);
795        } else {
796            setState(State::Fault);
797        }
798
799        LSQRequest::_inst->fault = fault;
800        LSQRequest::_inst->translationCompleted(true);
801    }
802}
803
804template<class Impl>
805void
806LSQ<Impl>::SplitDataRequest::finish(const Fault &fault, const RequestPtr &req,
807        ThreadContext* tc, BaseTLB::Mode mode)
808{
809    int i;
810    for (i = 0; i < _requests.size() && _requests[i] != req; i++);
811    assert(i < _requests.size());
812    _fault[i] = fault;
813
814    numInTranslationFragments--;
815    numTranslatedFragments++;
816
817    if (fault == NoFault)
818        mainReq->setFlags(req->getFlags());
819
820    if (numTranslatedFragments == _requests.size()) {
821        if (_inst->isSquashed()) {
822            this->squashTranslation();
823        } else {
824            _inst->strictlyOrdered(mainReq->isStrictlyOrdered());
825            flags.set(Flag::TranslationFinished);
826            _inst->translationCompleted(true);
827
828            for (i = 0; i < _fault.size() && _fault[i] == NoFault; i++);
829            if (i > 0) {
830                _inst->physEffAddr = request(0)->getPaddr();
831                _inst->memReqFlags = mainReq->getFlags();
832                if (mainReq->isCondSwap()) {
833                    assert (i == _fault.size());
834                    assert(_res);
835                    mainReq->setExtraData(*_res);
836                }
837                if (i == _fault.size()) {
838                    _inst->fault = NoFault;
839                    setState(State::Request);
840                } else {
841                  _inst->fault = _fault[i];
842                  setState(State::PartialFault);
843                }
844            } else {
845                _inst->fault = _fault[0];
846                setState(State::Fault);
847            }
848        }
849
850    }
851}
852
853template<class Impl>
854void
855LSQ<Impl>::SingleDataRequest::initiateTranslation()
856{
857    assert(_requests.size() == 0);
858
859    this->addRequest(_addr, _size, _byteEnable);
860
861    if (_requests.size() > 0) {
862        _requests.back()->setReqInstSeqNum(_inst->seqNum);
863        _requests.back()->taskId(_taskId);
864        _inst->translationStarted(true);
865        setState(State::Translation);
866        flags.set(Flag::TranslationStarted);
867
868        _inst->savedReq = this;
869        sendFragmentToTranslation(0);
870    } else {
871        _inst->setMemAccPredicate(false);
872    }
873}
874
875template<class Impl>
876PacketPtr
877LSQ<Impl>::SplitDataRequest::mainPacket()
878{
879    return _mainPacket;
880}
881
882template<class Impl>
883RequestPtr
884LSQ<Impl>::SplitDataRequest::mainRequest()
885{
886    return mainReq;
887}
888
889template<class Impl>
890void
891LSQ<Impl>::SplitDataRequest::initiateTranslation()
892{
893    auto cacheLineSize = _port.cacheLineSize();
894    Addr base_addr = _addr;
895    Addr next_addr = addrBlockAlign(_addr + cacheLineSize, cacheLineSize);
896    Addr final_addr = addrBlockAlign(_addr + _size, cacheLineSize);
897    uint32_t size_so_far = 0;
898
899    mainReq = std::make_shared<Request>(_inst->getASID(), base_addr,
900                _size, _flags, _inst->masterId(),
901                _inst->instAddr(), _inst->contextId());
902    if (!_byteEnable.empty()) {
903        mainReq->setByteEnable(_byteEnable);
904    }
905
906    // Paddr is not used in mainReq. However, we will accumulate the flags
907    // from the sub requests into mainReq by calling setFlags() in finish().
908    // setFlags() assumes that paddr is set so flip the paddr valid bit here to
909    // avoid a potential assert in setFlags() when we call it from  finish().
910    mainReq->setPaddr(0);
911
912    /* Get the pre-fix, possibly unaligned. */
913    if (_byteEnable.empty()) {
914        this->addRequest(base_addr, next_addr - base_addr, _byteEnable);
915    } else {
916        auto it_start = _byteEnable.begin();
917        auto it_end = _byteEnable.begin() + (next_addr - base_addr);
918        this->addRequest(base_addr, next_addr - base_addr,
919                         std::vector<bool>(it_start, it_end));
920    }
921    size_so_far = next_addr - base_addr;
922
923    /* We are block aligned now, reading whole blocks. */
924    base_addr = next_addr;
925    while (base_addr != final_addr) {
926        if (_byteEnable.empty()) {
927            this->addRequest(base_addr, cacheLineSize, _byteEnable);
928        } else {
929            auto it_start = _byteEnable.begin() + size_so_far;
930            auto it_end = _byteEnable.begin() + size_so_far + cacheLineSize;
931            this->addRequest(base_addr, cacheLineSize,
932                             std::vector<bool>(it_start, it_end));
933        }
934        size_so_far += cacheLineSize;
935        base_addr += cacheLineSize;
936    }
937
938    /* Deal with the tail. */
939    if (size_so_far < _size) {
940        if (_byteEnable.empty()) {
941            this->addRequest(base_addr, _size - size_so_far, _byteEnable);
942        } else {
943            auto it_start = _byteEnable.begin() + size_so_far;
944            auto it_end = _byteEnable.end();
945            this->addRequest(base_addr, _size - size_so_far,
946                             std::vector<bool>(it_start, it_end));
947        }
948    }
949
950    if (_requests.size() > 0) {
951        /* Setup the requests and send them to translation. */
952        for (auto& r: _requests) {
953            r->setReqInstSeqNum(_inst->seqNum);
954            r->taskId(_taskId);
955        }
956
957        _inst->translationStarted(true);
958        setState(State::Translation);
959        flags.set(Flag::TranslationStarted);
960        this->_inst->savedReq = this;
961        numInTranslationFragments = 0;
962        numTranslatedFragments = 0;
963        _fault.resize(_requests.size());
964
965        for (uint32_t i = 0; i < _requests.size(); i++) {
966            sendFragmentToTranslation(i);
967        }
968    } else {
969        _inst->setMemAccPredicate(false);
970    }
971}
972
973template<class Impl>
974void
975LSQ<Impl>::LSQRequest::sendFragmentToTranslation(int i)
976{
977    numInTranslationFragments++;
978    _port.dTLB()->translateTiming(
979            this->request(i),
980            this->_inst->thread->getTC(), this,
981            this->isLoad() ? BaseTLB::Read : BaseTLB::Write);
982}
983
984template<class Impl>
985bool
986LSQ<Impl>::SingleDataRequest::recvTimingResp(PacketPtr pkt)
987{
988    assert(_numOutstandingPackets == 1);
989    auto state = dynamic_cast<LSQSenderState*>(pkt->senderState);
990    flags.set(Flag::Complete);
991    state->outstanding--;
992    assert(pkt == _packets.front());
993    _port.completeDataAccess(pkt);
994    return true;
995}
996
997template<class Impl>
998bool
999LSQ<Impl>::SplitDataRequest::recvTimingResp(PacketPtr pkt)
1000{
1001    auto state = dynamic_cast<LSQSenderState*>(pkt->senderState);
1002    uint32_t pktIdx = 0;
1003    while (pktIdx < _packets.size() && pkt != _packets[pktIdx])
1004        pktIdx++;
1005    assert(pktIdx < _packets.size());
1006    numReceivedPackets++;
1007    state->outstanding--;
1008    if (numReceivedPackets == _packets.size()) {
1009        flags.set(Flag::Complete);
1010        /* Assemble packets. */
1011        PacketPtr resp = isLoad()
1012            ? Packet::createRead(mainReq)
1013            : Packet::createWrite(mainReq);
1014        if (isLoad())
1015            resp->dataStatic(_inst->memData);
1016        else
1017            resp->dataStatic(_data);
1018        resp->senderState = _senderState;
1019        _port.completeDataAccess(resp);
1020        delete resp;
1021    }
1022    return true;
1023}
1024
1025template<class Impl>
1026void
1027LSQ<Impl>::SingleDataRequest::buildPackets()
1028{
1029    assert(_senderState);
1030    /* Retries do not create new packets. */
1031    if (_packets.size() == 0) {
1032        _packets.push_back(
1033                isLoad()
1034                    ?  Packet::createRead(request())
1035                    :  Packet::createWrite(request()));
1036        _packets.back()->dataStatic(_inst->memData);
1037        _packets.back()->senderState = _senderState;
1038    }
1039    assert(_packets.size() == 1);
1040}
1041
1042template<class Impl>
1043void
1044LSQ<Impl>::SplitDataRequest::buildPackets()
1045{
1046    /* Extra data?? */
1047    Addr base_address = _addr;
1048
1049    if (_packets.size() == 0) {
1050        /* New stuff */
1051        if (isLoad()) {
1052            _mainPacket = Packet::createRead(mainReq);
1053            _mainPacket->dataStatic(_inst->memData);
1054        }
1055        for (int i = 0; i < _requests.size() && _fault[i] == NoFault; i++) {
1056            RequestPtr r = _requests[i];
1057            PacketPtr pkt = isLoad() ? Packet::createRead(r)
1058                                     : Packet::createWrite(r);
1059            ptrdiff_t offset = r->getVaddr() - base_address;
1060            if (isLoad()) {
1061                pkt->dataStatic(_inst->memData + offset);
1062            } else {
1063                uint8_t* req_data = new uint8_t[r->getSize()];
1064                std::memcpy(req_data,
1065                        _inst->memData + offset,
1066                        r->getSize());
1067                pkt->dataDynamic(req_data);
1068            }
1069            pkt->senderState = _senderState;
1070            _packets.push_back(pkt);
1071        }
1072    }
1073    assert(_packets.size() > 0);
1074}
1075
1076template<class Impl>
1077void
1078LSQ<Impl>::SingleDataRequest::sendPacketToCache()
1079{
1080    assert(_numOutstandingPackets == 0);
1081    if (lsqUnit()->trySendPacket(isLoad(), _packets.at(0)))
1082        _numOutstandingPackets = 1;
1083}
1084
1085template<class Impl>
1086void
1087LSQ<Impl>::SplitDataRequest::sendPacketToCache()
1088{
1089    /* Try to send the packets. */
1090    while (numReceivedPackets + _numOutstandingPackets < _packets.size() &&
1091            lsqUnit()->trySendPacket(isLoad(),
1092                _packets.at(numReceivedPackets + _numOutstandingPackets))) {
1093        _numOutstandingPackets++;
1094    }
1095}
1096
1097template<class Impl>
1098void
1099LSQ<Impl>::SingleDataRequest::handleIprWrite(ThreadContext *thread,
1100                                             PacketPtr pkt)
1101{
1102    TheISA::handleIprWrite(thread, pkt);
1103}
1104
1105template<class Impl>
1106void
1107LSQ<Impl>::SplitDataRequest::handleIprWrite(ThreadContext *thread,
1108                                            PacketPtr mainPkt)
1109{
1110    unsigned offset = 0;
1111    for (auto r: _requests) {
1112        PacketPtr pkt = new Packet(r, MemCmd::WriteReq);
1113        pkt->dataStatic(mainPkt->getPtr<uint8_t>() + offset);
1114        TheISA::handleIprWrite(thread, pkt);
1115        offset += r->getSize();
1116        delete pkt;
1117    }
1118}
1119
1120template<class Impl>
1121Cycles
1122LSQ<Impl>::SingleDataRequest::handleIprRead(ThreadContext *thread,
1123                                            PacketPtr pkt)
1124{
1125    return TheISA::handleIprRead(thread, pkt);
1126}
1127
1128template<class Impl>
1129Cycles
1130LSQ<Impl>::SplitDataRequest::handleIprRead(ThreadContext *thread,
1131                                           PacketPtr mainPkt)
1132{
1133    Cycles delay(0);
1134    unsigned offset = 0;
1135
1136    for (auto r: _requests) {
1137        PacketPtr pkt = new Packet(r, MemCmd::ReadReq);
1138        pkt->dataStatic(mainPkt->getPtr<uint8_t>() + offset);
1139        Cycles d = TheISA::handleIprRead(thread, pkt);
1140        if (d > delay)
1141            delay = d;
1142        offset += r->getSize();
1143        delete pkt;
1144    }
1145    return delay;
1146}
1147
1148template<class Impl>
1149bool
1150LSQ<Impl>::SingleDataRequest::isCacheBlockHit(Addr blockAddr, Addr blockMask)
1151{
1152    return ( (LSQRequest::_requests[0]->getPaddr() & blockMask) == blockAddr);
1153}
1154
1155template<class Impl>
1156bool
1157LSQ<Impl>::SplitDataRequest::isCacheBlockHit(Addr blockAddr, Addr blockMask)
1158{
1159    bool is_hit = false;
1160    for (auto &r: _requests) {
1161        if ((r->getPaddr() & blockMask) == blockAddr) {
1162            is_hit = true;
1163            break;
1164        }
1165    }
1166    return is_hit;
1167}
1168
1169template <class Impl>
1170bool
1171LSQ<Impl>::DcachePort::recvTimingResp(PacketPtr pkt)
1172{
1173    return lsq->recvTimingResp(pkt);
1174}
1175
1176template <class Impl>
1177void
1178LSQ<Impl>::DcachePort::recvTimingSnoopReq(PacketPtr pkt)
1179{
1180    for (ThreadID tid = 0; tid < cpu->numThreads; tid++) {
1181        if (cpu->getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1182            cpu->wakeup(tid);
1183        }
1184    }
1185    lsq->recvTimingSnoopReq(pkt);
1186}
1187
1188template <class Impl>
1189void
1190LSQ<Impl>::DcachePort::recvReqRetry()
1191{
1192    lsq->recvReqRetry();
1193}
1194
1195#endif//__CPU_O3_LSQ_IMPL_HH__
1196