lsq_impl.hh revision 13590 
1/*
2 * Copyright (c) 2011-2012, 2014, 2017-2018 ARM Limited
3 * Copyright (c) 2013 Advanced Micro Devices, Inc.
4 * All rights reserved
5 *
6 * The license below extends only to copyright in the software and shall
7 * not be construed as granting a license to any other intellectual
8 * property including but not limited to intellectual property relating
9 * to a hardware implementation of the functionality of the software
10 * licensed hereunder.  You may use the software subject to the license
11 * terms below provided that you ensure that this notice is replicated
12 * unmodified and in its entirety in all distributions of the software,
13 * modified or unmodified, in source code or in binary form.
14 *
15 * Copyright (c) 2005-2006 The Regents of The University of Michigan
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Korey Sewell
42 */
43
44#ifndef __CPU_O3_LSQ_IMPL_HH__
45#define __CPU_O3_LSQ_IMPL_HH__
46
47#include <algorithm>
48#include <list>
49#include <string>
50
51#include "base/logging.hh"
52#include "cpu/o3/lsq.hh"
53#include "debug/Drain.hh"
54#include "debug/Fetch.hh"
55#include "debug/LSQ.hh"
56#include "debug/Writeback.hh"
57#include "params/DerivO3CPU.hh"
58
59using namespace std;
60
61template <class Impl>
62LSQ<Impl>::LSQ(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params)
63    : cpu(cpu_ptr), iewStage(iew_ptr),
64      _cacheBlocked(false),
65      cacheStorePorts(params->cacheStorePorts), usedStorePorts(0),
66      lsqPolicy(params->smtLSQPolicy),
67      LQEntries(params->LQEntries),
68      SQEntries(params->SQEntries),
69      maxLQEntries(maxLSQAllocation(lsqPolicy, LQEntries, params->numThreads,
70                  params->smtLSQThreshold)),
71      maxSQEntries(maxLSQAllocation(lsqPolicy, SQEntries, params->numThreads,
72                  params->smtLSQThreshold)),
73      numThreads(params->numThreads)
74{
75    assert(numThreads > 0 && numThreads <= Impl::MaxThreads);
76
77    //**********************************************/
78    //************ Handle SMT Parameters ***********/
79    //**********************************************/
80
81    /* Run SMT olicy checks. */
82        if (lsqPolicy == SMTQueuePolicy::Dynamic) {
83        DPRINTF(LSQ, "LSQ sharing policy set to Dynamic\n");
84    } else if (lsqPolicy == SMTQueuePolicy::Partitioned) {
85        DPRINTF(Fetch, "LSQ sharing policy set to Partitioned: "
86                "%i entries per LQ | %i entries per SQ\n",
87                maxLQEntries,maxSQEntries);
88    } else if (lsqPolicy == SMTQueuePolicy::Threshold) {
89
90        assert(params->smtLSQThreshold > params->LQEntries);
91        assert(params->smtLSQThreshold > params->SQEntries);
92
93        DPRINTF(LSQ, "LSQ sharing policy set to Threshold: "
94                "%i entries per LQ | %i entries per SQ\n",
95                maxLQEntries,maxSQEntries);
96    } else {
97        panic("Invalid LSQ sharing policy. Options are: Dynamic, "
98                    "Partitioned, Threshold");
99    }
100
101    thread.reserve(numThreads);
102    for (ThreadID tid = 0; tid < numThreads; tid++) {
103        thread.emplace_back(maxLQEntries, maxSQEntries);
104        thread[tid].init(cpu, iew_ptr, params, this, tid);
105        thread[tid].setDcachePort(&cpu_ptr->getDataPort());
106    }
107}
108
109
110template<class Impl>
111std::string
112LSQ<Impl>::name() const
113{
114    return iewStage->name() + ".lsq";
115}
116
117template<class Impl>
118void
119LSQ<Impl>::regStats()
120{
121    //Initialize LSQs
122    for (ThreadID tid = 0; tid < numThreads; tid++) {
123        thread[tid].regStats();
124    }
125}
126
127template<class Impl>
128void
129LSQ<Impl>::setActiveThreads(list<ThreadID> *at_ptr)
130{
131    activeThreads = at_ptr;
132    assert(activeThreads != 0);
133}
134
135template <class Impl>
136void
137LSQ<Impl>::drainSanityCheck() const
138{
139    assert(isDrained());
140
141    for (ThreadID tid = 0; tid < numThreads; tid++)
142        thread[tid].drainSanityCheck();
143}
144
145template <class Impl>
146bool
147LSQ<Impl>::isDrained() const
148{
149    bool drained(true);
150
151    if (!lqEmpty()) {
152        DPRINTF(Drain, "Not drained, LQ not empty.\n");
153        drained = false;
154    }
155
156    if (!sqEmpty()) {
157        DPRINTF(Drain, "Not drained, SQ not empty.\n");
158        drained = false;
159    }
160
161    return drained;
162}
163
164template <class Impl>
165void
166LSQ<Impl>::takeOverFrom()
167{
168    usedStorePorts = 0;
169    _cacheBlocked = false;
170
171    for (ThreadID tid = 0; tid < numThreads; tid++) {
172        thread[tid].takeOverFrom();
173    }
174}
175
176template<class Impl>
177bool
178LSQ<Impl>::cacheBlocked() const
179{
180    return _cacheBlocked;
181}
182
183template<class Impl>
184void
185LSQ<Impl>::cacheBlocked(bool v)
186{
187    _cacheBlocked = v;
188}
189
190template<class Impl>
191bool
192LSQ<Impl>::storePortAvailable() const
193{
194    return usedStorePorts < cacheStorePorts;
195}
196
197template<class Impl>
198void
199LSQ<Impl>::storePortBusy()
200{
201    usedStorePorts++;
202    assert(usedStorePorts <= cacheStorePorts);
203}
204
205template<class Impl>
206void
207LSQ<Impl>::insertLoad(const DynInstPtr &load_inst)
208{
209    ThreadID tid = load_inst->threadNumber;
210
211    thread[tid].insertLoad(load_inst);
212}
213
214template<class Impl>
215void
216LSQ<Impl>::insertStore(const DynInstPtr &store_inst)
217{
218    ThreadID tid = store_inst->threadNumber;
219
220    thread[tid].insertStore(store_inst);
221}
222
223template<class Impl>
224Fault
225LSQ<Impl>::executeLoad(const DynInstPtr &inst)
226{
227    ThreadID tid = inst->threadNumber;
228
229    return thread[tid].executeLoad(inst);
230}
231
232template<class Impl>
233Fault
234LSQ<Impl>::executeStore(const DynInstPtr &inst)
235{
236    ThreadID tid = inst->threadNumber;
237
238    return thread[tid].executeStore(inst);
239}
240
241template<class Impl>
242void
243LSQ<Impl>::writebackStores()
244{
245    list<ThreadID>::iterator threads = activeThreads->begin();
246    list<ThreadID>::iterator end = activeThreads->end();
247
248    while (threads != end) {
249        ThreadID tid = *threads++;
250
251        if (numStoresToWB(tid) > 0) {
252            DPRINTF(Writeback,"[tid:%i] Writing back stores. %i stores "
253                "available for Writeback.\n", tid, numStoresToWB(tid));
254        }
255
256        thread[tid].writebackStores();
257    }
258}
259
260template<class Impl>
261bool
262LSQ<Impl>::violation()
263{
264    /* Answers: Does Anybody Have a Violation?*/
265    list<ThreadID>::iterator threads = activeThreads->begin();
266    list<ThreadID>::iterator end = activeThreads->end();
267
268    while (threads != end) {
269        ThreadID tid = *threads++;
270
271        if (thread[tid].violation())
272            return true;
273    }
274
275    return false;
276}
277
278template <class Impl>
279void
280LSQ<Impl>::recvReqRetry()
281{
282    iewStage->cacheUnblocked();
283    cacheBlocked(false);
284
285    for (ThreadID tid : *activeThreads) {
286        thread[tid].recvRetry();
287    }
288}
289
290template <class Impl>
291void
292LSQ<Impl>::completeDataAccess(PacketPtr pkt)
293{
294    auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
295    thread[cpu->contextToThread(senderState->contextId())]
296        .completeDataAccess(pkt);
297}
298
299template <class Impl>
300bool
301LSQ<Impl>::recvTimingResp(PacketPtr pkt)
302{
303    if (pkt->isError())
304        DPRINTF(LSQ, "Got error packet back for address: %#X\n",
305                pkt->getAddr());
306
307    auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
308    panic_if(!senderState, "Got packet back with unknown sender state\n");
309
310    thread[cpu->contextToThread(senderState->contextId())].recvTimingResp(pkt);
311
312    if (pkt->isInvalidate()) {
313        // This response also contains an invalidate; e.g. this can be the case
314        // if cmd is ReadRespWithInvalidate.
315        //
316        // The calling order between completeDataAccess and checkSnoop matters.
317        // By calling checkSnoop after completeDataAccess, we ensure that the
318        // fault set by checkSnoop is not lost. Calling writeback (more
319        // specifically inst->completeAcc) in completeDataAccess overwrites
320        // fault, and in case this instruction requires squashing (as
321        // determined by checkSnoop), the ReExec fault set by checkSnoop would
322        // be lost otherwise.
323
324        DPRINTF(LSQ, "received invalidation with response for addr:%#x\n",
325                pkt->getAddr());
326
327        for (ThreadID tid = 0; tid < numThreads; tid++) {
328            thread[tid].checkSnoop(pkt);
329        }
330    }
331    // Update the LSQRequest state (this may delete the request)
332    senderState->request()->packetReplied();
333
334    return true;
335}
336
337template <class Impl>
338void
339LSQ<Impl>::recvTimingSnoopReq(PacketPtr pkt)
340{
341    DPRINTF(LSQ, "received pkt for addr:%#x %s\n", pkt->getAddr(),
342            pkt->cmdString());
343
344    // must be a snoop
345    if (pkt->isInvalidate()) {
346        DPRINTF(LSQ, "received invalidation for addr:%#x\n",
347                pkt->getAddr());
348        for (ThreadID tid = 0; tid < numThreads; tid++) {
349            thread[tid].checkSnoop(pkt);
350        }
351    }
352}
353
354template<class Impl>
355int
356LSQ<Impl>::getCount()
357{
358    unsigned total = 0;
359
360    list<ThreadID>::iterator threads = activeThreads->begin();
361    list<ThreadID>::iterator end = activeThreads->end();
362
363    while (threads != end) {
364        ThreadID tid = *threads++;
365
366        total += getCount(tid);
367    }
368
369    return total;
370}
371
372template<class Impl>
373int
374LSQ<Impl>::numLoads()
375{
376    unsigned total = 0;
377
378    list<ThreadID>::iterator threads = activeThreads->begin();
379    list<ThreadID>::iterator end = activeThreads->end();
380
381    while (threads != end) {
382        ThreadID tid = *threads++;
383
384        total += numLoads(tid);
385    }
386
387    return total;
388}
389
390template<class Impl>
391int
392LSQ<Impl>::numStores()
393{
394    unsigned total = 0;
395
396    list<ThreadID>::iterator threads = activeThreads->begin();
397    list<ThreadID>::iterator end = activeThreads->end();
398
399    while (threads != end) {
400        ThreadID tid = *threads++;
401
402        total += thread[tid].numStores();
403    }
404
405    return total;
406}
407
408template<class Impl>
409unsigned
410LSQ<Impl>::numFreeLoadEntries()
411{
412    unsigned total = 0;
413
414    list<ThreadID>::iterator threads = activeThreads->begin();
415    list<ThreadID>::iterator end = activeThreads->end();
416
417    while (threads != end) {
418        ThreadID tid = *threads++;
419
420        total += thread[tid].numFreeLoadEntries();
421    }
422
423    return total;
424}
425
426template<class Impl>
427unsigned
428LSQ<Impl>::numFreeStoreEntries()
429{
430    unsigned total = 0;
431
432    list<ThreadID>::iterator threads = activeThreads->begin();
433    list<ThreadID>::iterator end = activeThreads->end();
434
435    while (threads != end) {
436        ThreadID tid = *threads++;
437
438        total += thread[tid].numFreeStoreEntries();
439    }
440
441    return total;
442}
443
444template<class Impl>
445unsigned
446LSQ<Impl>::numFreeLoadEntries(ThreadID tid)
447{
448        return thread[tid].numFreeLoadEntries();
449}
450
451template<class Impl>
452unsigned
453LSQ<Impl>::numFreeStoreEntries(ThreadID tid)
454{
455        return thread[tid].numFreeStoreEntries();
456}
457
458template<class Impl>
459bool
460LSQ<Impl>::isFull()
461{
462    list<ThreadID>::iterator threads = activeThreads->begin();
463    list<ThreadID>::iterator end = activeThreads->end();
464
465    while (threads != end) {
466        ThreadID tid = *threads++;
467
468        if (!(thread[tid].lqFull() || thread[tid].sqFull()))
469            return false;
470    }
471
472    return true;
473}
474
475template<class Impl>
476bool
477LSQ<Impl>::isFull(ThreadID tid)
478{
479    //@todo: Change to Calculate All Entries for
480    //Dynamic Policy
481    if (lsqPolicy == SMTQueuePolicy::Dynamic)
482        return isFull();
483    else
484        return thread[tid].lqFull() || thread[tid].sqFull();
485}
486
487template<class Impl>
488bool
489LSQ<Impl>::isEmpty() const
490{
491    return lqEmpty() && sqEmpty();
492}
493
494template<class Impl>
495bool
496LSQ<Impl>::lqEmpty() const
497{
498    list<ThreadID>::const_iterator threads = activeThreads->begin();
499    list<ThreadID>::const_iterator end = activeThreads->end();
500
501    while (threads != end) {
502        ThreadID tid = *threads++;
503
504        if (!thread[tid].lqEmpty())
505            return false;
506    }
507
508    return true;
509}
510
511template<class Impl>
512bool
513LSQ<Impl>::sqEmpty() const
514{
515    list<ThreadID>::const_iterator threads = activeThreads->begin();
516    list<ThreadID>::const_iterator end = activeThreads->end();
517
518    while (threads != end) {
519        ThreadID tid = *threads++;
520
521        if (!thread[tid].sqEmpty())
522            return false;
523    }
524
525    return true;
526}
527
528template<class Impl>
529bool
530LSQ<Impl>::lqFull()
531{
532    list<ThreadID>::iterator threads = activeThreads->begin();
533    list<ThreadID>::iterator end = activeThreads->end();
534
535    while (threads != end) {
536        ThreadID tid = *threads++;
537
538        if (!thread[tid].lqFull())
539            return false;
540    }
541
542    return true;
543}
544
545template<class Impl>
546bool
547LSQ<Impl>::lqFull(ThreadID tid)
548{
549    //@todo: Change to Calculate All Entries for
550    //Dynamic Policy
551    if (lsqPolicy == SMTQueuePolicy::Dynamic)
552        return lqFull();
553    else
554        return thread[tid].lqFull();
555}
556
557template<class Impl>
558bool
559LSQ<Impl>::sqFull()
560{
561    list<ThreadID>::iterator threads = activeThreads->begin();
562    list<ThreadID>::iterator end = activeThreads->end();
563
564    while (threads != end) {
565        ThreadID tid = *threads++;
566
567        if (!sqFull(tid))
568            return false;
569    }
570
571    return true;
572}
573
574template<class Impl>
575bool
576LSQ<Impl>::sqFull(ThreadID tid)
577{
578     //@todo: Change to Calculate All Entries for
579    //Dynamic Policy
580    if (lsqPolicy == SMTQueuePolicy::Dynamic)
581        return sqFull();
582    else
583        return thread[tid].sqFull();
584}
585
586template<class Impl>
587bool
588LSQ<Impl>::isStalled()
589{
590    list<ThreadID>::iterator threads = activeThreads->begin();
591    list<ThreadID>::iterator end = activeThreads->end();
592
593    while (threads != end) {
594        ThreadID tid = *threads++;
595
596        if (!thread[tid].isStalled())
597            return false;
598    }
599
600    return true;
601}
602
603template<class Impl>
604bool
605LSQ<Impl>::isStalled(ThreadID tid)
606{
607    if (lsqPolicy == SMTQueuePolicy::Dynamic)
608        return isStalled();
609    else
610        return thread[tid].isStalled();
611}
612
613template<class Impl>
614bool
615LSQ<Impl>::hasStoresToWB()
616{
617    list<ThreadID>::iterator threads = activeThreads->begin();
618    list<ThreadID>::iterator end = activeThreads->end();
619
620    while (threads != end) {
621        ThreadID tid = *threads++;
622
623        if (hasStoresToWB(tid))
624            return true;
625    }
626
627    return false;
628}
629
630template<class Impl>
631bool
632LSQ<Impl>::willWB()
633{
634    list<ThreadID>::iterator threads = activeThreads->begin();
635    list<ThreadID>::iterator end = activeThreads->end();
636
637    while (threads != end) {
638        ThreadID tid = *threads++;
639
640        if (willWB(tid))
641            return true;
642    }
643
644    return false;
645}
646
647template<class Impl>
648void
649LSQ<Impl>::dumpInsts() const
650{
651    list<ThreadID>::const_iterator threads = activeThreads->begin();
652    list<ThreadID>::const_iterator end = activeThreads->end();
653
654    while (threads != end) {
655        ThreadID tid = *threads++;
656
657        thread[tid].dumpInsts();
658    }
659}
660
661static Addr
662addrBlockOffset(Addr addr, unsigned int block_size)
663{
664    return addr & (block_size - 1);
665}
666
667static Addr
668addrBlockAlign(Addr addr, uint64_t block_size)
669{
670    return addr & ~(block_size - 1);
671}
672
673static bool
674transferNeedsBurst(Addr addr, uint64_t size, uint64_t block_size)
675{
676    return (addrBlockOffset(addr, block_size) + size) > block_size;
677}
678
679template<class Impl>
680Fault
681LSQ<Impl>::pushRequest(const DynInstPtr& inst, bool isLoad, uint8_t *data,
682                       unsigned int size, Addr addr, Request::Flags flags,
683                       uint64_t *res)
684{
685    ThreadID tid = cpu->contextToThread(inst->contextId());
686    auto cacheLineSize = cpu->cacheLineSize();
687    bool needs_burst = transferNeedsBurst(addr, size, cacheLineSize);
688    LSQRequest* req = nullptr;
689
690    if (inst->translationStarted()) {
691        req = inst->savedReq;
692        assert(req);
693    } else {
694        if (needs_burst) {
695            req = new SplitDataRequest(&thread[tid], inst, isLoad, addr,
696                    size, flags, data, res);
697        } else {
698            req = new SingleDataRequest(&thread[tid], inst, isLoad, addr,
699                    size, flags, data, res);
700        }
701        assert(req);
702        inst->setRequest();
703        req->taskId(cpu->taskId());
704
705        req->initiateTranslation();
706    }
707
708    /* This is the place were instructions get the effAddr. */
709    if (req->isTranslationComplete()) {
710        if (inst->getFault() == NoFault) {
711            inst->effAddr = req->getVaddr();
712            inst->effSize = size;
713            inst->effAddrValid(true);
714
715            if (cpu->checker) {
716                inst->reqToVerify = std::make_shared<Request>(*req->request());
717            }
718            if (isLoad)
719                inst->getFault() = cpu->read(req, inst->lqIdx);
720            else
721                inst->getFault() = cpu->write(req, data, inst->sqIdx);
722        } else if (isLoad) {
723            // Commit will have to clean up whatever happened.  Set this
724            // instruction as executed.
725            inst->setExecuted();
726        }
727    }
728
729    if (inst->traceData)
730        inst->traceData->setMem(addr, size, flags);
731
732    return inst->getFault();
733}
734
735template<class Impl>
736void
737LSQ<Impl>::SingleDataRequest::finish(const Fault &fault, const RequestPtr &req,
738        ThreadContext* tc, BaseTLB::Mode mode)
739{
740    _fault.push_back(fault);
741    numInTranslationFragments = 0;
742    numTranslatedFragments = 1;
743    /* If the instruction has been squahsed, let the request know
744     * as it may have to self-destruct. */
745    if (_inst->isSquashed()) {
746        this->squashTranslation();
747    } else {
748        _inst->strictlyOrdered(req->isStrictlyOrdered());
749
750        flags.set(Flag::TranslationFinished);
751        if (fault == NoFault) {
752            _inst->physEffAddr = req->getPaddr();
753            _inst->memReqFlags = req->getFlags();
754            if (req->isCondSwap()) {
755                assert(_res);
756                req->setExtraData(*_res);
757            }
758            setState(State::Request);
759        } else {
760            setState(State::Fault);
761        }
762
763        LSQRequest::_inst->fault = fault;
764        LSQRequest::_inst->translationCompleted(true);
765    }
766}
767
768template<class Impl>
769void
770LSQ<Impl>::SplitDataRequest::finish(const Fault &fault, const RequestPtr &req,
771        ThreadContext* tc, BaseTLB::Mode mode)
772{
773    _fault.push_back(fault);
774    assert(req == _requests[numTranslatedFragments] || this->isDelayed());
775
776    numInTranslationFragments--;
777    numTranslatedFragments++;
778
779    mainReq->setFlags(req->getFlags());
780
781    if (numTranslatedFragments == _requests.size()) {
782        if (_inst->isSquashed()) {
783            this->squashTranslation();
784        } else {
785            _inst->strictlyOrdered(mainReq->isStrictlyOrdered());
786            flags.set(Flag::TranslationFinished);
787            auto fault_it = _fault.begin();
788            /* Ffwd to the first NoFault. */
789            while (fault_it != _fault.end() && *fault_it == NoFault)
790                fault_it++;
791            /* If none of the fragments faulted: */
792            if (fault_it == _fault.end()) {
793                _inst->physEffAddr = request(0)->getPaddr();
794
795                _inst->memReqFlags = mainReq->getFlags();
796                if (mainReq->isCondSwap()) {
797                    assert(_res);
798                    mainReq->setExtraData(*_res);
799                }
800                setState(State::Request);
801                _inst->fault = NoFault;
802            } else {
803                setState(State::Fault);
804                _inst->fault = *fault_it;
805            }
806            _inst->translationCompleted(true);
807        }
808    }
809}
810
811template<class Impl>
812void
813LSQ<Impl>::SingleDataRequest::initiateTranslation()
814{
815    _inst->translationStarted(true);
816    setState(State::Translation);
817    flags.set(Flag::TranslationStarted);
818
819    _inst->savedReq = this;
820    sendFragmentToTranslation(0);
821
822    if (isTranslationComplete()) {
823    }
824}
825
826template<class Impl>
827PacketPtr
828LSQ<Impl>::SplitDataRequest::mainPacket()
829{
830    return _mainPacket;
831}
832
833template<class Impl>
834RequestPtr
835LSQ<Impl>::SplitDataRequest::mainRequest()
836{
837    return mainReq;
838}
839
840template<class Impl>
841void
842LSQ<Impl>::SplitDataRequest::initiateTranslation()
843{
844    _inst->translationStarted(true);
845    setState(State::Translation);
846    flags.set(Flag::TranslationStarted);
847
848    unsigned int cacheLineSize = _port.cacheLineSize();
849    Addr base_addr = _addr;
850    Addr next_addr = addrBlockAlign(_addr + cacheLineSize, cacheLineSize);
851    Addr final_addr = addrBlockAlign(_addr + _size, cacheLineSize);
852    uint32_t size_so_far = 0;
853
854    mainReq = std::make_shared<Request>(_inst->getASID(), base_addr,
855                _size, _flags, _inst->masterId(),
856                _inst->instAddr(), _inst->contextId());
857
858    // Paddr is not used in mainReq. However, we will accumulate the flags
859    // from the sub requests into mainReq by calling setFlags() in finish().
860    // setFlags() assumes that paddr is set so flip the paddr valid bit here to
861    // avoid a potential assert in setFlags() when we call it from  finish().
862    mainReq->setPaddr(0);
863
864    /* Get the pre-fix, possibly unaligned. */
865    _requests.push_back(std::make_shared<Request>(_inst->getASID(), base_addr,
866                next_addr - base_addr, _flags, _inst->masterId(),
867                _inst->instAddr(), _inst->contextId()));
868    size_so_far = next_addr - base_addr;
869
870    /* We are block aligned now, reading whole blocks. */
871    base_addr = next_addr;
872    while (base_addr != final_addr) {
873        _requests.push_back(std::make_shared<Request>(_inst->getASID(),
874                    base_addr, cacheLineSize, _flags, _inst->masterId(),
875                    _inst->instAddr(), _inst->contextId()));
876        size_so_far += cacheLineSize;
877        base_addr += cacheLineSize;
878    }
879
880    /* Deal with the tail. */
881    if (size_so_far < _size) {
882        _requests.push_back(std::make_shared<Request>(_inst->getASID(),
883                    base_addr, _size - size_so_far, _flags, _inst->masterId(),
884                    _inst->instAddr(), _inst->contextId()));
885    }
886
887    /* Setup the requests and send them to translation. */
888    for (auto& r: _requests) {
889        r->setReqInstSeqNum(_inst->seqNum);
890        r->taskId(_taskId);
891    }
892    this->_inst->savedReq = this;
893    numInTranslationFragments = 0;
894    numTranslatedFragments = 0;
895
896    for (uint32_t i = 0; i < _requests.size(); i++) {
897        sendFragmentToTranslation(i);
898    }
899}
900
901template<class Impl>
902void
903LSQ<Impl>::LSQRequest::sendFragmentToTranslation(int i)
904{
905    numInTranslationFragments++;
906    _port.dTLB()->translateTiming(
907            this->request(i),
908            this->_inst->thread->getTC(), this,
909            this->isLoad() ? BaseTLB::Read : BaseTLB::Write);
910}
911
912template<class Impl>
913bool
914LSQ<Impl>::SingleDataRequest::recvTimingResp(PacketPtr pkt)
915{
916    assert(_numOutstandingPackets == 1);
917    auto state = dynamic_cast<LSQSenderState*>(pkt->senderState);
918    setState(State::Complete);
919    flags.set(Flag::Complete);
920    state->outstanding--;
921    assert(pkt == _packets.front());
922    _port.completeDataAccess(pkt);
923    return true;
924}
925
926template<class Impl>
927bool
928LSQ<Impl>::SplitDataRequest::recvTimingResp(PacketPtr pkt)
929{
930    auto state = dynamic_cast<LSQSenderState*>(pkt->senderState);
931    uint32_t pktIdx = 0;
932    while (pktIdx < _packets.size() && pkt != _packets[pktIdx])
933        pktIdx++;
934    assert(pktIdx < _packets.size());
935    assert(pkt->req == _requests[pktIdx]);
936    assert(pkt == _packets[pktIdx]);
937    numReceivedPackets++;
938    state->outstanding--;
939    if (numReceivedPackets == _packets.size()) {
940        setState(State::Complete);
941        flags.set(Flag::Complete);
942        /* Assemble packets. */
943        PacketPtr resp = isLoad()
944            ? Packet::createRead(mainReq)
945            : Packet::createWrite(mainReq);
946        if (isLoad())
947            resp->dataStatic(_inst->memData);
948        else
949            resp->dataStatic(_data);
950        resp->senderState = _senderState;
951        _port.completeDataAccess(resp);
952        delete resp;
953    }
954    return true;
955}
956
957template<class Impl>
958void
959LSQ<Impl>::SingleDataRequest::buildPackets()
960{
961    assert(_senderState);
962    /* Retries do not create new packets. */
963    if (_packets.size() == 0) {
964        _packets.push_back(
965                isLoad()
966                    ?  Packet::createRead(request())
967                    :  Packet::createWrite(request()));
968        _packets.back()->dataStatic(_inst->memData);
969        _packets.back()->senderState = _senderState;
970    }
971    assert(_packets.size() == 1);
972}
973
974template<class Impl>
975void
976LSQ<Impl>::SplitDataRequest::buildPackets()
977{
978    /* Extra data?? */
979    ptrdiff_t offset = 0;
980    if (_packets.size() == 0) {
981        /* New stuff */
982        if (isLoad()) {
983            _mainPacket = Packet::createRead(mainReq);
984            _mainPacket->dataStatic(_inst->memData);
985        }
986        for (auto& r: _requests) {
987            PacketPtr pkt = isLoad() ? Packet::createRead(r)
988                                    : Packet::createWrite(r);
989            if (isLoad()) {
990                pkt->dataStatic(_inst->memData + offset);
991            } else {
992                uint8_t* req_data = new uint8_t[r->getSize()];
993                std::memcpy(req_data,
994                        _inst->memData + offset,
995                        r->getSize());
996                pkt->dataDynamic(req_data);
997            }
998            offset += r->getSize();
999            pkt->senderState = _senderState;
1000            _packets.push_back(pkt);
1001        }
1002    }
1003    assert(_packets.size() == _requests.size());
1004}
1005
1006template<class Impl>
1007void
1008LSQ<Impl>::SingleDataRequest::sendPacketToCache()
1009{
1010    assert(_numOutstandingPackets == 0);
1011    if (lsqUnit()->trySendPacket(isLoad(), _packets.at(0)))
1012        _numOutstandingPackets = 1;
1013}
1014
1015template<class Impl>
1016void
1017LSQ<Impl>::SplitDataRequest::sendPacketToCache()
1018{
1019    /* Try to send the packets. */
1020    while (numReceivedPackets + _numOutstandingPackets < _packets.size() &&
1021            lsqUnit()->trySendPacket(isLoad(),
1022                _packets.at(numReceivedPackets + _numOutstandingPackets))) {
1023        _numOutstandingPackets++;
1024    }
1025}
1026
1027template<class Impl>
1028void
1029LSQ<Impl>::SingleDataRequest::handleIprWrite(ThreadContext *thread,
1030                                             PacketPtr pkt)
1031{
1032    TheISA::handleIprWrite(thread, pkt);
1033}
1034
1035template<class Impl>
1036void
1037LSQ<Impl>::SplitDataRequest::handleIprWrite(ThreadContext *thread,
1038                                            PacketPtr mainPkt)
1039{
1040    unsigned offset = 0;
1041    for (auto r: _requests) {
1042        PacketPtr pkt = new Packet(r, MemCmd::WriteReq);
1043        pkt->dataStatic(mainPkt->getPtr<uint8_t>() + offset);
1044        TheISA::handleIprWrite(thread, pkt);
1045        offset += r->getSize();
1046        delete pkt;
1047    }
1048}
1049
1050template<class Impl>
1051Cycles
1052LSQ<Impl>::SingleDataRequest::handleIprRead(ThreadContext *thread,
1053                                            PacketPtr pkt)
1054{
1055    return TheISA::handleIprRead(thread, pkt);
1056}
1057
1058template<class Impl>
1059Cycles
1060LSQ<Impl>::SplitDataRequest::handleIprRead(ThreadContext *thread,
1061                                           PacketPtr mainPkt)
1062{
1063    Cycles delay(0);
1064    unsigned offset = 0;
1065
1066    for (auto r: _requests) {
1067        PacketPtr pkt = new Packet(r, MemCmd::ReadReq);
1068        pkt->dataStatic(mainPkt->getPtr<uint8_t>() + offset);
1069        Cycles d = TheISA::handleIprRead(thread, pkt);
1070        if (d > delay)
1071            delay = d;
1072        offset += r->getSize();
1073        delete pkt;
1074    }
1075    return delay;
1076}
1077
1078template<class Impl>
1079bool
1080LSQ<Impl>::SingleDataRequest::isCacheBlockHit(Addr blockAddr, Addr blockMask)
1081{
1082    return ( (LSQRequest::_requests[0]->getPaddr() & blockMask) == blockAddr);
1083}
1084
1085template<class Impl>
1086bool
1087LSQ<Impl>::SplitDataRequest::isCacheBlockHit(Addr blockAddr, Addr blockMask)
1088{
1089    bool is_hit = false;
1090    for (auto &r: _requests) {
1091        if ((r->getPaddr() & blockMask) == blockAddr) {
1092            is_hit = true;
1093            break;
1094        }
1095    }
1096    return is_hit;
1097}
1098
1099#endif//__CPU_O3_LSQ_IMPL_HH__
1100