1/*
2 * Copyright (c) 2011-2012, 2014 ARM Limited
3 * Copyright (c) 2013 Advanced Micro Devices, Inc.
4 * All rights reserved
5 *
6 * The license below extends only to copyright in the software and shall
7 * not be construed as granting a license to any other intellectual
8 * property including but not limited to intellectual property relating
9 * to a hardware implementation of the functionality of the software
10 * licensed hereunder. You may use the software subject to the license
11 * terms below provided that you ensure that this notice is replicated
12 * unmodified and in its entirety in all distributions of the software,
13 * modified or unmodified, in source code or in binary form.
14 *
15 * Copyright (c) 2005-2006 The Regents of The University of Michigan
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Korey Sewell
42 */
43
44#ifndef __CPU_O3_LSQ_IMPL_HH__
45#define __CPU_O3_LSQ_IMPL_HH__
46
47#include <algorithm>
48#include <list>
49#include <string>
50
51#include "base/logging.hh"
52#include "cpu/o3/lsq.hh"
53#include "debug/Drain.hh"
54#include "debug/Fetch.hh"
55#include "debug/LSQ.hh"
56#include "debug/Writeback.hh"
57#include "params/DerivO3CPU.hh"
58
59using namespace std;
60
61template <class Impl>
62LSQ<Impl>::LSQ(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params)
63 : cpu(cpu_ptr), iewStage(iew_ptr),
64 lsqPolicy(params->smtLSQPolicy),
65 LQEntries(params->LQEntries),
66 SQEntries(params->SQEntries),
67 maxLQEntries(maxLSQAllocation(lsqPolicy, LQEntries, params->numThreads,
68 params->smtLSQThreshold)),
69 maxSQEntries(maxLSQAllocation(lsqPolicy, SQEntries, params->numThreads,
70 params->smtLSQThreshold)),
71 numThreads(params->numThreads)
72{
73 assert(numThreads > 0 && numThreads <= Impl::MaxThreads);
74
75 //**********************************************/
76 //************ Handle SMT Parameters ***********/
77 //**********************************************/
78
79 //Figure out fetch policy
80 if (lsqPolicy == SMTQueuePolicy::Dynamic) {
81 DPRINTF(LSQ, "LSQ sharing policy set to Dynamic\n");
82 } else if (lsqPolicy == SMTQueuePolicy::Partitioned) {
83 DPRINTF(Fetch, "LSQ sharing policy set to Partitioned: "
84 "%i entries per LQ | %i entries per SQ\n",
85 maxLQEntries,maxSQEntries);
86 } else if (lsqPolicy == SMTQueuePolicy::Threshold) {
87
88 assert(params->smtLSQThreshold > LQEntries);
89 assert(params->smtLSQThreshold > SQEntries);
90
91 DPRINTF(LSQ, "LSQ sharing policy set to Threshold: "
92 "%i entries per LQ | %i entries per SQ\n",
93 maxLQEntries,maxSQEntries);
94 } else {
95 panic("Invalid LSQ sharing policy. Options are: Dynamic, "
96 "Partitioned, Threshold");
97 }
98
99 thread.reserve(numThreads);
100 for (ThreadID tid = 0; tid < numThreads; tid++) {
101 thread.emplace_back(maxLQEntries, maxSQEntries);
102 thread[tid].init(cpu, iew_ptr, params, this, tid);
103 thread[tid].setDcachePort(&cpu_ptr->getDataPort());
104 }
105}
106
107
108template<class Impl>
109std::string
110LSQ<Impl>::name() const
111{
112 return iewStage->name() + ".lsq";
113}
114
115template<class Impl>
116void
117LSQ<Impl>::regStats()
118{
119 //Initialize LSQs
120 for (ThreadID tid = 0; tid < numThreads; tid++) {
121 thread[tid].regStats();
122 }
123}
124
125template<class Impl>
126void
127LSQ<Impl>::setActiveThreads(list<ThreadID> *at_ptr)
128{
129 activeThreads = at_ptr;
130 assert(activeThreads != 0);
131}
132
133template <class Impl>
134void
135LSQ<Impl>::drainSanityCheck() const
136{
137 assert(isDrained());
138
139 for (ThreadID tid = 0; tid < numThreads; tid++)
140 thread[tid].drainSanityCheck();
141}
142
143template <class Impl>
144bool
145LSQ<Impl>::isDrained() const
146{
147 bool drained(true);
148
149 if (!lqEmpty()) {
150 DPRINTF(Drain, "Not drained, LQ not empty.\n");
151 drained = false;
152 }
153
154 if (!sqEmpty()) {
155 DPRINTF(Drain, "Not drained, SQ not empty.\n");
156 drained = false;
157 }
158
159 return drained;
160}
161
162template <class Impl>
163void
164LSQ<Impl>::takeOverFrom()
165{
166 for (ThreadID tid = 0; tid < numThreads; tid++) {
167 thread[tid].takeOverFrom();
168 }
169}
170
171template <class Impl>
172int
173LSQ<Impl>::entryAmount(ThreadID num_threads)
174{
175 if (lsqPolicy == SMTQueuePolicy::Partitioned) {
176 return LQEntries / num_threads;
177 } else {
178 return 0;
179 }
180}
181
182template <class Impl>
183void
184LSQ<Impl>::resetEntries()
185{
186 if (lsqPolicy != SMTQueuePolicy::Dynamic || numThreads > 1) {
187 int active_threads = activeThreads->size();
188
189 int maxEntries;
190
191 if (lsqPolicy == SMTQueuePolicy::Partitioned) {
192 maxEntries = LQEntries / active_threads;
193 } else if (lsqPolicy == SMTQueuePolicy::Threshold &&
194 active_threads == 1) {
195 maxEntries = LQEntries;
196 } else {
197 maxEntries = LQEntries;
198 }
199
200 list<ThreadID>::iterator threads = activeThreads->begin();
201 list<ThreadID>::iterator end = activeThreads->end();
202
203 while (threads != end) {
204 ThreadID tid = *threads++;
205
206 resizeEntries(maxEntries, tid);
207 }
208 }
209}
210
211template<class Impl>
212void
213LSQ<Impl>::removeEntries(ThreadID tid)
214{
215 thread[tid].clearLQ();
216 thread[tid].clearSQ();
217}
218
219template<class Impl>
220void
221LSQ<Impl>::resizeEntries(unsigned size, ThreadID tid)
222{
223 thread[tid].resizeLQ(size);
224 thread[tid].resizeSQ(size);
225}
226
227template<class Impl>
228void
229LSQ<Impl>::tick()
230{
231 list<ThreadID>::iterator threads = activeThreads->begin();
232 list<ThreadID>::iterator end = activeThreads->end();
233
234 while (threads != end) {
235 ThreadID tid = *threads++;
236
237 thread[tid].tick();
238 }
239}
240
241template<class Impl>
242void
243LSQ<Impl>::insertLoad(const DynInstPtr &load_inst)
244{
245 ThreadID tid = load_inst->threadNumber;
246
247 thread[tid].insertLoad(load_inst);
248}
249
250template<class Impl>
251void
252LSQ<Impl>::insertStore(const DynInstPtr &store_inst)
253{
254 ThreadID tid = store_inst->threadNumber;
255
256 thread[tid].insertStore(store_inst);
257}
258
259template<class Impl>
260Fault
261LSQ<Impl>::executeLoad(const DynInstPtr &inst)
262{
263 ThreadID tid = inst->threadNumber;
264
265 return thread[tid].executeLoad(inst);
266}
267
268template<class Impl>
269Fault
270LSQ<Impl>::executeStore(const DynInstPtr &inst)
271{
272 ThreadID tid = inst->threadNumber;
273
274 return thread[tid].executeStore(inst);
275}
276
277template<class Impl>
278void
279LSQ<Impl>::writebackStores()
280{
281 list<ThreadID>::iterator threads = activeThreads->begin();
282 list<ThreadID>::iterator end = activeThreads->end();
283
284 while (threads != end) {
285 ThreadID tid = *threads++;
286
287 if (numStoresToWB(tid) > 0) {
288 DPRINTF(Writeback,"[tid:%i] Writing back stores. %i stores "
289 "available for Writeback.\n", tid, numStoresToWB(tid));
290 }
291
292 thread[tid].writebackStores();
293 }
294}
295
296template<class Impl>
297bool
298LSQ<Impl>::violation()
299{
300 /* Answers: Does Anybody Have a Violation?*/
301 list<ThreadID>::iterator threads = activeThreads->begin();
302 list<ThreadID>::iterator end = activeThreads->end();
303
304 while (threads != end) {
305 ThreadID tid = *threads++;
306
307 if (thread[tid].violation())
308 return true;
309 }
310
311 return false;
312}
313
314template <class Impl>
315void
316LSQ<Impl>::recvReqRetry()
317{
318 iewStage->cacheUnblocked();
319
320 for (ThreadID tid : *activeThreads) {
321 thread[tid].recvRetry();
322 }
323}
324
325template <class Impl>
326bool
327LSQ<Impl>::recvTimingResp(PacketPtr pkt)
328{
329 if (pkt->isError())
330 DPRINTF(LSQ, "Got error packet back for address: %#X\n",
331 pkt->getAddr());
332
333 thread[cpu->contextToThread(pkt->req->contextId())]
334 .completeDataAccess(pkt);
335
336 if (pkt->isInvalidate()) {
337 // This response also contains an invalidate; e.g. this can be the case
338 // if cmd is ReadRespWithInvalidate.
339 //
340 // The calling order between completeDataAccess and checkSnoop matters.
341 // By calling checkSnoop after completeDataAccess, we ensure that the
342 // fault set by checkSnoop is not lost. Calling writeback (more
343 // specifically inst->completeAcc) in completeDataAccess overwrites
344 // fault, and in case this instruction requires squashing (as
345 // determined by checkSnoop), the ReExec fault set by checkSnoop would
346 // be lost otherwise.
347
348 DPRINTF(LSQ, "received invalidation with response for addr:%#x\n",
349 pkt->getAddr());
350
351 for (ThreadID tid = 0; tid < numThreads; tid++) {
352 thread[tid].checkSnoop(pkt);
353 }
354 }
355
356 delete pkt;
357 return true;
358}
359
360template <class Impl>
361void
362LSQ<Impl>::recvTimingSnoopReq(PacketPtr pkt)
363{
364 DPRINTF(LSQ, "received pkt for addr:%#x %s\n", pkt->getAddr(),
365 pkt->cmdString());
366
367 // must be a snoop
368 if (pkt->isInvalidate()) {
369 DPRINTF(LSQ, "received invalidation for addr:%#x\n",
370 pkt->getAddr());
371 for (ThreadID tid = 0; tid < numThreads; tid++) {
372 thread[tid].checkSnoop(pkt);
373 }
374 }
375}
376
377template<class Impl>
378int
379LSQ<Impl>::getCount()
380{
381 unsigned total = 0;
382
383 list<ThreadID>::iterator threads = activeThreads->begin();
384 list<ThreadID>::iterator end = activeThreads->end();
385
386 while (threads != end) {
387 ThreadID tid = *threads++;
388
389 total += getCount(tid);
390 }
391
392 return total;
393}
394
395template<class Impl>
396int
397LSQ<Impl>::numLoads()
398{
399 unsigned total = 0;
400
401 list<ThreadID>::iterator threads = activeThreads->begin();
402 list<ThreadID>::iterator end = activeThreads->end();
403
404 while (threads != end) {
405 ThreadID tid = *threads++;
406
407 total += numLoads(tid);
408 }
409
410 return total;
411}
412
413template<class Impl>
414int
415LSQ<Impl>::numStores()
416{
417 unsigned total = 0;
418
419 list<ThreadID>::iterator threads = activeThreads->begin();
420 list<ThreadID>::iterator end = activeThreads->end();
421
422 while (threads != end) {
423 ThreadID tid = *threads++;
424
425 total += thread[tid].numStores();
426 }
427
428 return total;
429}
430
431template<class Impl>
432unsigned
433LSQ<Impl>::numFreeLoadEntries()
434{
435 unsigned total = 0;
436
437 list<ThreadID>::iterator threads = activeThreads->begin();
438 list<ThreadID>::iterator end = activeThreads->end();
439
440 while (threads != end) {
441 ThreadID tid = *threads++;
442
443 total += thread[tid].numFreeLoadEntries();
444 }
445
446 return total;
447}
448
449template<class Impl>
450unsigned
451LSQ<Impl>::numFreeStoreEntries()
452{
453 unsigned total = 0;
454
455 list<ThreadID>::iterator threads = activeThreads->begin();
456 list<ThreadID>::iterator end = activeThreads->end();
457
458 while (threads != end) {
459 ThreadID tid = *threads++;
460
461 total += thread[tid].numFreeStoreEntries();
462 }
463
464 return total;
465}
466
467template<class Impl>
468unsigned
469LSQ<Impl>::numFreeLoadEntries(ThreadID tid)
470{
471 return thread[tid].numFreeLoadEntries();
472}
473
474template<class Impl>
475unsigned
476LSQ<Impl>::numFreeStoreEntries(ThreadID tid)
477{
478 return thread[tid].numFreeStoreEntries();
479}
480
481template<class Impl>
482bool
483LSQ<Impl>::isFull()
484{
485 list<ThreadID>::iterator threads = activeThreads->begin();
486 list<ThreadID>::iterator end = activeThreads->end();
487
488 while (threads != end) {
489 ThreadID tid = *threads++;
490
491 if (!(thread[tid].lqFull() || thread[tid].sqFull()))
492 return false;
493 }
494
495 return true;
496}
497
498template<class Impl>
499bool
500LSQ<Impl>::isFull(ThreadID tid)
501{
502 //@todo: Change to Calculate All Entries for
503 //Dynamic Policy
504 if (lsqPolicy == SMTQueuePolicy::Dynamic)
505 return isFull();
506 else
507 return thread[tid].lqFull() || thread[tid].sqFull();
508}
509
510template<class Impl>
511bool
512LSQ<Impl>::isEmpty() const
513{
514 return lqEmpty() && sqEmpty();
515}
516
517template<class Impl>
518bool
519LSQ<Impl>::lqEmpty() const
520{
521 list<ThreadID>::const_iterator threads = activeThreads->begin();
522 list<ThreadID>::const_iterator end = activeThreads->end();
523
524 while (threads != end) {
525 ThreadID tid = *threads++;
526
527 if (!thread[tid].lqEmpty())
528 return false;
529 }
530
531 return true;
532}
533
534template<class Impl>
535bool
536LSQ<Impl>::sqEmpty() const
537{
538 list<ThreadID>::const_iterator threads = activeThreads->begin();
539 list<ThreadID>::const_iterator end = activeThreads->end();
540
541 while (threads != end) {
542 ThreadID tid = *threads++;
543
544 if (!thread[tid].sqEmpty())
545 return false;
546 }
547
548 return true;
549}
550
551template<class Impl>
552bool
553LSQ<Impl>::lqFull()
554{
555 list<ThreadID>::iterator threads = activeThreads->begin();
556 list<ThreadID>::iterator end = activeThreads->end();
557
558 while (threads != end) {
559 ThreadID tid = *threads++;
560
561 if (!thread[tid].lqFull())
562 return false;
563 }
564
565 return true;
566}
567
568template<class Impl>
569bool
570LSQ<Impl>::lqFull(ThreadID tid)
571{
572 //@todo: Change to Calculate All Entries for
573 //Dynamic Policy
574 if (lsqPolicy == SMTQueuePolicy::Dynamic)
575 return lqFull();
576 else
577 return thread[tid].lqFull();
578}
579
580template<class Impl>
581bool
582LSQ<Impl>::sqFull()
583{
584 list<ThreadID>::iterator threads = activeThreads->begin();
585 list<ThreadID>::iterator end = activeThreads->end();
586
587 while (threads != end) {
588 ThreadID tid = *threads++;
589
590 if (!sqFull(tid))
591 return false;
592 }
593
594 return true;
595}
596
597template<class Impl>
598bool
599LSQ<Impl>::sqFull(ThreadID tid)
600{
601 //@todo: Change to Calculate All Entries for
602 //Dynamic Policy
603 if (lsqPolicy == SMTQueuePolicy::Dynamic)
604 return sqFull();
605 else
606 return thread[tid].sqFull();
607}
608
609template<class Impl>
610bool
611LSQ<Impl>::isStalled()
612{
613 list<ThreadID>::iterator threads = activeThreads->begin();
614 list<ThreadID>::iterator end = activeThreads->end();
615
616 while (threads != end) {
617 ThreadID tid = *threads++;
618
619 if (!thread[tid].isStalled())
620 return false;
621 }
622
623 return true;
624}
625
626template<class Impl>
627bool
628LSQ<Impl>::isStalled(ThreadID tid)
629{
630 if (lsqPolicy == SMTQueuePolicy::Dynamic)
631 return isStalled();
632 else
633 return thread[tid].isStalled();
634}
635
636template<class Impl>
637bool
638LSQ<Impl>::hasStoresToWB()
639{
640 list<ThreadID>::iterator threads = activeThreads->begin();
641 list<ThreadID>::iterator end = activeThreads->end();
642
643 while (threads != end) {
644 ThreadID tid = *threads++;
645
646 if (hasStoresToWB(tid))
647 return true;
648 }
649
650 return false;
651}
652
653template<class Impl>
654bool
655LSQ<Impl>::willWB()
656{
657 list<ThreadID>::iterator threads = activeThreads->begin();
658 list<ThreadID>::iterator end = activeThreads->end();
659
660 while (threads != end) {
661 ThreadID tid = *threads++;
662
663 if (willWB(tid))
664 return true;
665 }
666
667 return false;
668}
669
670template<class Impl>
671void
672LSQ<Impl>::dumpInsts() const
673{
674 list<ThreadID>::const_iterator threads = activeThreads->begin();
675 list<ThreadID>::const_iterator end = activeThreads->end();
676
677 while (threads != end) {
678 ThreadID tid = *threads++;
679
680 thread[tid].dumpInsts();
681 }
682}
683
684#endif//__CPU_O3_LSQ_IMPL_HH__