15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 * Korey Sewell
42 */
43
44#ifndef __CPU_O3_LSQ_UNIT_HH__
45#define __CPU_O3_LSQ_UNIT_HH__
46
47#include <algorithm>
48#include <cstring>
49#include <map>
50#include <queue>
51
52#include "arch/generic/debugfaults.hh"
53#include "arch/isa_traits.hh"
54#include "arch/locked_mem.hh"
55#include "arch/mmapped_ipr.hh"
56#include "base/hashmap.hh"
57#include "config/the_isa.hh"
58#include "cpu/inst_seq.hh"
59#include "cpu/timebuf.hh"
60#include "debug/LSQUnit.hh"
61#include "mem/packet.hh"
62#include "mem/port.hh"
63#include "sim/fault_fwd.hh"
64
65struct DerivO3CPUParams;
66
67/**
68 * Class that implements the actual LQ and SQ for each specific
69 * thread. Both are circular queues; load entries are freed upon
70 * committing, while store entries are freed once they writeback. The
71 * LSQUnit tracks if there are memory ordering violations, and also
72 * detects partial load to store forwarding cases (a store only has
73 * part of a load's data) that requires the load to wait until the
74 * store writes back. In the former case it holds onto the instruction
75 * until the dependence unit looks at it, and in the latter it stalls
76 * the LSQ until the store writes back. At that point the load is
77 * replayed.
78 */
79template <class Impl>
80class LSQUnit {
81 public:
82 typedef typename Impl::O3CPU O3CPU;
83 typedef typename Impl::DynInstPtr DynInstPtr;
84 typedef typename Impl::CPUPol::IEW IEW;
85 typedef typename Impl::CPUPol::LSQ LSQ;
86 typedef typename Impl::CPUPol::IssueStruct IssueStruct;
87
88 public:
89 /** Constructs an LSQ unit. init() must be called prior to use. */
90 LSQUnit();
91
92 /** Initializes the LSQ unit with the specified number of entries. */
93 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
94 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
95 unsigned id);
96
97 /** Returns the name of the LSQ unit. */
98 std::string name() const;
99
100 /** Registers statistics. */
101 void regStats();
102
103 /** Sets the pointer to the dcache port. */
104 void setDcachePort(MasterPort *dcache_port);
105
106 /** Perform sanity checks after a drain. */
107 void drainSanityCheck() const;
108
109 /** Takes over from another CPU's thread. */
110 void takeOverFrom();
111
112 /** Ticks the LSQ unit, which in this case only resets the number of
113 * used cache ports.
114 * @todo: Move the number of used ports up to the LSQ level so it can
115 * be shared by all LSQ units.
116 */
117 void tick() { usedPorts = 0; }
118
119 /** Inserts an instruction. */
120 void insert(DynInstPtr &inst);
121 /** Inserts a load instruction. */
122 void insertLoad(DynInstPtr &load_inst);
123 /** Inserts a store instruction. */
124 void insertStore(DynInstPtr &store_inst);
125
126 /** Check for ordering violations in the LSQ. For a store squash if we
127 * ever find a conflicting load. For a load, only squash if we
128 * an external snoop invalidate has been seen for that load address
129 * @param load_idx index to start checking at
130 * @param inst the instruction to check
131 */
132 Fault checkViolations(int load_idx, DynInstPtr &inst);
133
134 /** Check if an incoming invalidate hits in the lsq on a load
135 * that might have issued out of order wrt another load beacuse
136 * of the intermediate invalidate.
137 */
138 void checkSnoop(PacketPtr pkt);
139
140 /** Executes a load instruction. */
141 Fault executeLoad(DynInstPtr &inst);
142
143 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; }
144 /** Executes a store instruction. */
145 Fault executeStore(DynInstPtr &inst);
146
147 /** Commits the head load. */
148 void commitLoad();
149 /** Commits loads older than a specific sequence number. */
150 void commitLoads(InstSeqNum &youngest_inst);
151
152 /** Commits stores older than a specific sequence number. */
153 void commitStores(InstSeqNum &youngest_inst);
154
155 /** Writes back stores. */
156 void writebackStores();
157
158 /** Completes the data access that has been returned from the
159 * memory system. */
160 void completeDataAccess(PacketPtr pkt);
161
162 /** Clears all the entries in the LQ. */
163 void clearLQ();
164
165 /** Clears all the entries in the SQ. */
166 void clearSQ();
167
168 /** Resizes the LQ to a given size. */
169 void resizeLQ(unsigned size);
170
171 /** Resizes the SQ to a given size. */
172 void resizeSQ(unsigned size);
173
174 /** Squashes all instructions younger than a specific sequence number. */
175 void squash(const InstSeqNum &squashed_num);
176
177 /** Returns if there is a memory ordering violation. Value is reset upon
178 * call to getMemDepViolator().
179 */
180 bool violation() { return memDepViolator; }
181
182 /** Returns the memory ordering violator. */
183 DynInstPtr getMemDepViolator();
184
185 /** Returns if a load became blocked due to the memory system. */
186 bool loadBlocked()
187 { return isLoadBlocked; }
188
189 /** Clears the signal that a load became blocked. */
190 void clearLoadBlocked()
191 { isLoadBlocked = false; }
192
193 /** Returns if the blocked load was handled. */
194 bool isLoadBlockedHandled()
195 { return loadBlockedHandled; }
196
197 /** Records the blocked load as being handled. */
198 void setLoadBlockedHandled()
199 { loadBlockedHandled = true; }
200
| 16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Kevin Lim
42 * Korey Sewell
43 */
44
45#ifndef __CPU_O3_LSQ_UNIT_HH__
46#define __CPU_O3_LSQ_UNIT_HH__
47
48#include <algorithm>
49#include <cstring>
50#include <map>
51#include <queue>
52
53#include "arch/generic/debugfaults.hh"
54#include "arch/isa_traits.hh"
55#include "arch/locked_mem.hh"
56#include "arch/mmapped_ipr.hh"
57#include "base/hashmap.hh"
58#include "config/the_isa.hh"
59#include "cpu/inst_seq.hh"
60#include "cpu/timebuf.hh"
61#include "debug/LSQUnit.hh"
62#include "mem/packet.hh"
63#include "mem/port.hh"
64#include "sim/fault_fwd.hh"
65
66struct DerivO3CPUParams;
67
68/**
69 * Class that implements the actual LQ and SQ for each specific
70 * thread. Both are circular queues; load entries are freed upon
71 * committing, while store entries are freed once they writeback. The
72 * LSQUnit tracks if there are memory ordering violations, and also
73 * detects partial load to store forwarding cases (a store only has
74 * part of a load's data) that requires the load to wait until the
75 * store writes back. In the former case it holds onto the instruction
76 * until the dependence unit looks at it, and in the latter it stalls
77 * the LSQ until the store writes back. At that point the load is
78 * replayed.
79 */
80template <class Impl>
81class LSQUnit {
82 public:
83 typedef typename Impl::O3CPU O3CPU;
84 typedef typename Impl::DynInstPtr DynInstPtr;
85 typedef typename Impl::CPUPol::IEW IEW;
86 typedef typename Impl::CPUPol::LSQ LSQ;
87 typedef typename Impl::CPUPol::IssueStruct IssueStruct;
88
89 public:
90 /** Constructs an LSQ unit. init() must be called prior to use. */
91 LSQUnit();
92
93 /** Initializes the LSQ unit with the specified number of entries. */
94 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
95 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
96 unsigned id);
97
98 /** Returns the name of the LSQ unit. */
99 std::string name() const;
100
101 /** Registers statistics. */
102 void regStats();
103
104 /** Sets the pointer to the dcache port. */
105 void setDcachePort(MasterPort *dcache_port);
106
107 /** Perform sanity checks after a drain. */
108 void drainSanityCheck() const;
109
110 /** Takes over from another CPU's thread. */
111 void takeOverFrom();
112
113 /** Ticks the LSQ unit, which in this case only resets the number of
114 * used cache ports.
115 * @todo: Move the number of used ports up to the LSQ level so it can
116 * be shared by all LSQ units.
117 */
118 void tick() { usedPorts = 0; }
119
120 /** Inserts an instruction. */
121 void insert(DynInstPtr &inst);
122 /** Inserts a load instruction. */
123 void insertLoad(DynInstPtr &load_inst);
124 /** Inserts a store instruction. */
125 void insertStore(DynInstPtr &store_inst);
126
127 /** Check for ordering violations in the LSQ. For a store squash if we
128 * ever find a conflicting load. For a load, only squash if we
129 * an external snoop invalidate has been seen for that load address
130 * @param load_idx index to start checking at
131 * @param inst the instruction to check
132 */
133 Fault checkViolations(int load_idx, DynInstPtr &inst);
134
135 /** Check if an incoming invalidate hits in the lsq on a load
136 * that might have issued out of order wrt another load beacuse
137 * of the intermediate invalidate.
138 */
139 void checkSnoop(PacketPtr pkt);
140
141 /** Executes a load instruction. */
142 Fault executeLoad(DynInstPtr &inst);
143
144 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; }
145 /** Executes a store instruction. */
146 Fault executeStore(DynInstPtr &inst);
147
148 /** Commits the head load. */
149 void commitLoad();
150 /** Commits loads older than a specific sequence number. */
151 void commitLoads(InstSeqNum &youngest_inst);
152
153 /** Commits stores older than a specific sequence number. */
154 void commitStores(InstSeqNum &youngest_inst);
155
156 /** Writes back stores. */
157 void writebackStores();
158
159 /** Completes the data access that has been returned from the
160 * memory system. */
161 void completeDataAccess(PacketPtr pkt);
162
163 /** Clears all the entries in the LQ. */
164 void clearLQ();
165
166 /** Clears all the entries in the SQ. */
167 void clearSQ();
168
169 /** Resizes the LQ to a given size. */
170 void resizeLQ(unsigned size);
171
172 /** Resizes the SQ to a given size. */
173 void resizeSQ(unsigned size);
174
175 /** Squashes all instructions younger than a specific sequence number. */
176 void squash(const InstSeqNum &squashed_num);
177
178 /** Returns if there is a memory ordering violation. Value is reset upon
179 * call to getMemDepViolator().
180 */
181 bool violation() { return memDepViolator; }
182
183 /** Returns the memory ordering violator. */
184 DynInstPtr getMemDepViolator();
185
186 /** Returns if a load became blocked due to the memory system. */
187 bool loadBlocked()
188 { return isLoadBlocked; }
189
190 /** Clears the signal that a load became blocked. */
191 void clearLoadBlocked()
192 { isLoadBlocked = false; }
193
194 /** Returns if the blocked load was handled. */
195 bool isLoadBlockedHandled()
196 { return loadBlockedHandled; }
197
198 /** Records the blocked load as being handled. */
199 void setLoadBlockedHandled()
200 { loadBlockedHandled = true; }
201
|
204 /** Returns the number of loads in the LQ. */
205 int numLoads() { return loads; }
206
207 /** Returns the number of stores in the SQ. */
208 int numStores() { return stores; }
209
210 /** Returns if either the LQ or SQ is full. */
211 bool isFull() { return lqFull() || sqFull(); }
212
213 /** Returns if both the LQ and SQ are empty. */
214 bool isEmpty() const { return lqEmpty() && sqEmpty(); }
215
216 /** Returns if the LQ is full. */
217 bool lqFull() { return loads >= (LQEntries - 1); }
218
219 /** Returns if the SQ is full. */
220 bool sqFull() { return stores >= (SQEntries - 1); }
221
222 /** Returns if the LQ is empty. */
223 bool lqEmpty() const { return loads == 0; }
224
225 /** Returns if the SQ is empty. */
226 bool sqEmpty() const { return stores == 0; }
227
228 /** Returns the number of instructions in the LSQ. */
229 unsigned getCount() { return loads + stores; }
230
231 /** Returns if there are any stores to writeback. */
232 bool hasStoresToWB() { return storesToWB; }
233
234 /** Returns the number of stores to writeback. */
235 int numStoresToWB() { return storesToWB; }
236
237 /** Returns if the LSQ unit will writeback on this cycle. */
238 bool willWB() { return storeQueue[storeWBIdx].canWB &&
239 !storeQueue[storeWBIdx].completed &&
240 !isStoreBlocked; }
241
242 /** Handles doing the retry. */
243 void recvRetry();
244
245 private:
246 /** Reset the LSQ state */
247 void resetState();
248
249 /** Writes back the instruction, sending it to IEW. */
250 void writeback(DynInstPtr &inst, PacketPtr pkt);
251
252 /** Writes back a store that couldn't be completed the previous cycle. */
253 void writebackPendingStore();
254
255 /** Handles completing the send of a store to memory. */
256 void storePostSend(PacketPtr pkt);
257
258 /** Completes the store at the specified index. */
259 void completeStore(int store_idx);
260
261 /** Attempts to send a store to the cache. */
262 bool sendStore(PacketPtr data_pkt);
263
264 /** Increments the given store index (circular queue). */
265 inline void incrStIdx(int &store_idx) const;
266 /** Decrements the given store index (circular queue). */
267 inline void decrStIdx(int &store_idx) const;
268 /** Increments the given load index (circular queue). */
269 inline void incrLdIdx(int &load_idx) const;
270 /** Decrements the given load index (circular queue). */
271 inline void decrLdIdx(int &load_idx) const;
272
273 public:
274 /** Debugging function to dump instructions in the LSQ. */
275 void dumpInsts() const;
276
277 private:
278 /** Pointer to the CPU. */
279 O3CPU *cpu;
280
281 /** Pointer to the IEW stage. */
282 IEW *iewStage;
283
284 /** Pointer to the LSQ. */
285 LSQ *lsq;
286
287 /** Pointer to the dcache port. Used only for sending. */
288 MasterPort *dcachePort;
289
290 /** Derived class to hold any sender state the LSQ needs. */
291 class LSQSenderState : public Packet::SenderState
292 {
293 public:
294 /** Default constructor. */
295 LSQSenderState()
296 : mainPkt(NULL), pendingPacket(NULL), outstanding(1),
297 noWB(false), isSplit(false), pktToSend(false)
298 { }
299
300 /** Instruction who initiated the access to memory. */
301 DynInstPtr inst;
302 /** The main packet from a split load, used during writeback. */
303 PacketPtr mainPkt;
304 /** A second packet from a split store that needs sending. */
305 PacketPtr pendingPacket;
306 /** The LQ/SQ index of the instruction. */
307 uint8_t idx;
308 /** Number of outstanding packets to complete. */
309 uint8_t outstanding;
310 /** Whether or not it is a load. */
311 bool isLoad;
312 /** Whether or not the instruction will need to writeback. */
313 bool noWB;
314 /** Whether or not this access is split in two. */
315 bool isSplit;
316 /** Whether or not there is a packet that needs sending. */
317 bool pktToSend;
318
319 /** Completes a packet and returns whether the access is finished. */
320 inline bool complete() { return --outstanding == 0; }
321 };
322
323 /** Writeback event, specifically for when stores forward data to loads. */
324 class WritebackEvent : public Event {
325 public:
326 /** Constructs a writeback event. */
327 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr);
328
329 /** Processes the writeback event. */
330 void process();
331
332 /** Returns the description of this event. */
333 const char *description() const;
334
335 private:
336 /** Instruction whose results are being written back. */
337 DynInstPtr inst;
338
339 /** The packet that would have been sent to memory. */
340 PacketPtr pkt;
341
342 /** The pointer to the LSQ unit that issued the store. */
343 LSQUnit<Impl> *lsqPtr;
344 };
345
346 public:
347 struct SQEntry {
348 /** Constructs an empty store queue entry. */
349 SQEntry()
350 : inst(NULL), req(NULL), size(0),
351 canWB(0), committed(0), completed(0)
352 {
353 std::memset(data, 0, sizeof(data));
354 }
355
356 ~SQEntry()
357 {
358 inst = NULL;
359 }
360
361 /** Constructs a store queue entry for a given instruction. */
362 SQEntry(DynInstPtr &_inst)
363 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0),
364 isSplit(0), canWB(0), committed(0), completed(0), isAllZeros(0)
365 {
366 std::memset(data, 0, sizeof(data));
367 }
368 /** The store data. */
369 char data[16];
370 /** The store instruction. */
371 DynInstPtr inst;
372 /** The request for the store. */
373 RequestPtr req;
374 /** The split requests for the store. */
375 RequestPtr sreqLow;
376 RequestPtr sreqHigh;
377 /** The size of the store. */
378 uint8_t size;
379 /** Whether or not the store is split into two requests. */
380 bool isSplit;
381 /** Whether or not the store can writeback. */
382 bool canWB;
383 /** Whether or not the store is committed. */
384 bool committed;
385 /** Whether or not the store is completed. */
386 bool completed;
387 /** Does this request write all zeros and thus doesn't
388 * have any data attached to it. Used for cache block zero
389 * style instructs (ARM DC ZVA; ALPHA WH64)
390 */
391 bool isAllZeros;
392 };
393
394 private:
395 /** The LSQUnit thread id. */
396 ThreadID lsqID;
397
398 /** The store queue. */
399 std::vector<SQEntry> storeQueue;
400
401 /** The load queue. */
402 std::vector<DynInstPtr> loadQueue;
403
404 /** The number of LQ entries, plus a sentinel entry (circular queue).
405 * @todo: Consider having var that records the true number of LQ entries.
406 */
407 unsigned LQEntries;
408 /** The number of SQ entries, plus a sentinel entry (circular queue).
409 * @todo: Consider having var that records the true number of SQ entries.
410 */
411 unsigned SQEntries;
412
413 /** The number of places to shift addresses in the LSQ before checking
414 * for dependency violations
415 */
416 unsigned depCheckShift;
417
418 /** Should loads be checked for dependency issues */
419 bool checkLoads;
420
421 /** The number of load instructions in the LQ. */
422 int loads;
423 /** The number of store instructions in the SQ. */
424 int stores;
425 /** The number of store instructions in the SQ waiting to writeback. */
426 int storesToWB;
427
428 /** The index of the head instruction in the LQ. */
429 int loadHead;
430 /** The index of the tail instruction in the LQ. */
431 int loadTail;
432
433 /** The index of the head instruction in the SQ. */
434 int storeHead;
435 /** The index of the first instruction that may be ready to be
436 * written back, and has not yet been written back.
437 */
438 int storeWBIdx;
439 /** The index of the tail instruction in the SQ. */
440 int storeTail;
441
442 /// @todo Consider moving to a more advanced model with write vs read ports
443 /** The number of cache ports available each cycle. */
444 int cachePorts;
445
446 /** The number of used cache ports in this cycle. */
447 int usedPorts;
448
449 //list<InstSeqNum> mshrSeqNums;
450
451 /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */
452 Addr cacheBlockMask;
453
454 /** Wire to read information from the issue stage time queue. */
455 typename TimeBuffer<IssueStruct>::wire fromIssue;
456
457 /** Whether or not the LSQ is stalled. */
458 bool stalled;
459 /** The store that causes the stall due to partial store to load
460 * forwarding.
461 */
462 InstSeqNum stallingStoreIsn;
463 /** The index of the above store. */
464 int stallingLoadIdx;
465
466 /** The packet that needs to be retried. */
467 PacketPtr retryPkt;
468
469 /** Whehter or not a store is blocked due to the memory system. */
470 bool isStoreBlocked;
471
472 /** Whether or not a load is blocked due to the memory system. */
473 bool isLoadBlocked;
474
475 /** Has the blocked load been handled. */
476 bool loadBlockedHandled;
477
478 /** Whether or not a store is in flight. */
479 bool storeInFlight;
480
481 /** The sequence number of the blocked load. */
482 InstSeqNum blockedLoadSeqNum;
483
484 /** The oldest load that caused a memory ordering violation. */
485 DynInstPtr memDepViolator;
486
487 /** Whether or not there is a packet that couldn't be sent because of
488 * a lack of cache ports. */
489 bool hasPendingPkt;
490
491 /** The packet that is pending free cache ports. */
492 PacketPtr pendingPkt;
493
494 /** Flag for memory model. */
495 bool needsTSO;
496
497 // Will also need how many read/write ports the Dcache has. Or keep track
498 // of that in stage that is one level up, and only call executeLoad/Store
499 // the appropriate number of times.
500 /** Total number of loads forwaded from LSQ stores. */
501 Stats::Scalar lsqForwLoads;
502
503 /** Total number of loads ignored due to invalid addresses. */
504 Stats::Scalar invAddrLoads;
505
506 /** Total number of squashed loads. */
507 Stats::Scalar lsqSquashedLoads;
508
509 /** Total number of responses from the memory system that are
510 * ignored due to the instruction already being squashed. */
511 Stats::Scalar lsqIgnoredResponses;
512
513 /** Tota number of memory ordering violations. */
514 Stats::Scalar lsqMemOrderViolation;
515
516 /** Total number of squashed stores. */
517 Stats::Scalar lsqSquashedStores;
518
519 /** Total number of software prefetches ignored due to invalid addresses. */
520 Stats::Scalar invAddrSwpfs;
521
522 /** Ready loads blocked due to partial store-forwarding. */
523 Stats::Scalar lsqBlockedLoads;
524
525 /** Number of loads that were rescheduled. */
526 Stats::Scalar lsqRescheduledLoads;
527
528 /** Number of times the LSQ is blocked due to the cache. */
529 Stats::Scalar lsqCacheBlocked;
530
531 public:
532 /** Executes the load at the given index. */
533 Fault read(Request *req, Request *sreqLow, Request *sreqHigh,
534 uint8_t *data, int load_idx);
535
536 /** Executes the store at the given index. */
537 Fault write(Request *req, Request *sreqLow, Request *sreqHigh,
538 uint8_t *data, int store_idx);
539
540 /** Returns the index of the head load instruction. */
541 int getLoadHead() { return loadHead; }
542 /** Returns the sequence number of the head load instruction. */
543 InstSeqNum getLoadHeadSeqNum()
544 {
545 if (loadQueue[loadHead]) {
546 return loadQueue[loadHead]->seqNum;
547 } else {
548 return 0;
549 }
550
551 }
552
553 /** Returns the index of the head store instruction. */
554 int getStoreHead() { return storeHead; }
555 /** Returns the sequence number of the head store instruction. */
556 InstSeqNum getStoreHeadSeqNum()
557 {
558 if (storeQueue[storeHead].inst) {
559 return storeQueue[storeHead].inst->seqNum;
560 } else {
561 return 0;
562 }
563
564 }
565
566 /** Returns whether or not the LSQ unit is stalled. */
567 bool isStalled() { return stalled; }
568};
569
570template <class Impl>
571Fault
572LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh,
573 uint8_t *data, int load_idx)
574{
575 DynInstPtr load_inst = loadQueue[load_idx];
576
577 assert(load_inst);
578
579 assert(!load_inst->isExecuted());
580
581 // Make sure this isn't an uncacheable access
582 // A bit of a hackish way to get uncached accesses to work only if they're
583 // at the head of the LSQ and are ready to commit (at the head of the ROB
584 // too).
585 if (req->isUncacheable() &&
586 (load_idx != loadHead || !load_inst->isAtCommit())) {
587 iewStage->rescheduleMemInst(load_inst);
588 ++lsqRescheduledLoads;
589 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n",
590 load_inst->seqNum, load_inst->pcState());
591
592 // Must delete request now that it wasn't handed off to
593 // memory. This is quite ugly. @todo: Figure out the proper
594 // place to really handle request deletes.
595 delete req;
596 if (TheISA::HasUnalignedMemAcc && sreqLow) {
597 delete sreqLow;
598 delete sreqHigh;
599 }
600 return new GenericISA::M5PanicFault(
601 "Uncachable load [sn:%llx] PC %s\n",
602 load_inst->seqNum, load_inst->pcState());
603 }
604
605 // Check the SQ for any previous stores that might lead to forwarding
606 int store_idx = load_inst->sqIdx;
607
608 int store_size = 0;
609
610 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, "
611 "storeHead: %i addr: %#x%s\n",
612 load_idx, store_idx, storeHead, req->getPaddr(),
613 sreqLow ? " split" : "");
614
615 if (req->isLLSC()) {
616 assert(!sreqLow);
617 // Disable recording the result temporarily. Writing to misc
618 // regs normally updates the result, but this is not the
619 // desired behavior when handling store conditionals.
620 load_inst->recordResult(false);
621 TheISA::handleLockedRead(load_inst.get(), req);
622 load_inst->recordResult(true);
623 }
624
625 if (req->isMmappedIpr()) {
626 assert(!load_inst->memData);
627 load_inst->memData = new uint8_t[64];
628
629 ThreadContext *thread = cpu->tcBase(lsqID);
630 Cycles delay(0);
631 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq);
632
633 if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
634 data_pkt->dataStatic(load_inst->memData);
635 delay = TheISA::handleIprRead(thread, data_pkt);
636 } else {
637 assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr());
638 PacketPtr fst_data_pkt = new Packet(sreqLow, MemCmd::ReadReq);
639 PacketPtr snd_data_pkt = new Packet(sreqHigh, MemCmd::ReadReq);
640
641 fst_data_pkt->dataStatic(load_inst->memData);
642 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
643
644 delay = TheISA::handleIprRead(thread, fst_data_pkt);
645 Cycles delay2 = TheISA::handleIprRead(thread, snd_data_pkt);
646 if (delay2 > delay)
647 delay = delay2;
648
649 delete sreqLow;
650 delete sreqHigh;
651 delete fst_data_pkt;
652 delete snd_data_pkt;
653 }
654 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
655 cpu->schedule(wb, cpu->clockEdge(delay));
656 return NoFault;
657 }
658
659 while (store_idx != -1) {
660 // End once we've reached the top of the LSQ
661 if (store_idx == storeWBIdx) {
662 break;
663 }
664
665 // Move the index to one younger
666 if (--store_idx < 0)
667 store_idx += SQEntries;
668
669 assert(storeQueue[store_idx].inst);
670
671 store_size = storeQueue[store_idx].size;
672
673 if (store_size == 0)
674 continue;
675 else if (storeQueue[store_idx].inst->uncacheable())
676 continue;
677
678 assert(storeQueue[store_idx].inst->effAddrValid());
679
680 // Check if the store data is within the lower and upper bounds of
681 // addresses that the request needs.
682 bool store_has_lower_limit =
683 req->getVaddr() >= storeQueue[store_idx].inst->effAddr;
684 bool store_has_upper_limit =
685 (req->getVaddr() + req->getSize()) <=
686 (storeQueue[store_idx].inst->effAddr + store_size);
687 bool lower_load_has_store_part =
688 req->getVaddr() < (storeQueue[store_idx].inst->effAddr +
689 store_size);
690 bool upper_load_has_store_part =
691 (req->getVaddr() + req->getSize()) >
692 storeQueue[store_idx].inst->effAddr;
693
694 // If the store's data has all of the data needed, we can forward.
695 if ((store_has_lower_limit && store_has_upper_limit)) {
696 // Get shift amount for offset into the store's data.
697 int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr;
698
699 if (storeQueue[store_idx].isAllZeros)
700 memset(data, 0, req->getSize());
701 else
702 memcpy(data, storeQueue[store_idx].data + shift_amt,
703 req->getSize());
704
705 assert(!load_inst->memData);
706 load_inst->memData = new uint8_t[req->getSize()];
707 if (storeQueue[store_idx].isAllZeros)
708 memset(load_inst->memData, 0, req->getSize());
709 else
710 memcpy(load_inst->memData,
711 storeQueue[store_idx].data + shift_amt, req->getSize());
712
713 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to "
714 "addr %#x\n", store_idx, req->getVaddr());
715
716 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq);
717 data_pkt->dataStatic(load_inst->memData);
718
719 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
720
721 // We'll say this has a 1 cycle load-store forwarding latency
722 // for now.
723 // @todo: Need to make this a parameter.
724 cpu->schedule(wb, curTick());
725
726 // Don't need to do anything special for split loads.
727 if (TheISA::HasUnalignedMemAcc && sreqLow) {
728 delete sreqLow;
729 delete sreqHigh;
730 }
731
732 ++lsqForwLoads;
733 return NoFault;
734 } else if ((store_has_lower_limit && lower_load_has_store_part) ||
735 (store_has_upper_limit && upper_load_has_store_part) ||
736 (lower_load_has_store_part && upper_load_has_store_part)) {
737 // This is the partial store-load forwarding case where a store
738 // has only part of the load's data.
739
740 // If it's already been written back, then don't worry about
741 // stalling on it.
742 if (storeQueue[store_idx].completed) {
743 panic("Should not check one of these");
744 continue;
745 }
746
747 // Must stall load and force it to retry, so long as it's the oldest
748 // load that needs to do so.
749 if (!stalled ||
750 (stalled &&
751 load_inst->seqNum <
752 loadQueue[stallingLoadIdx]->seqNum)) {
753 stalled = true;
754 stallingStoreIsn = storeQueue[store_idx].inst->seqNum;
755 stallingLoadIdx = load_idx;
756 }
757
758 // Tell IQ/mem dep unit that this instruction will need to be
759 // rescheduled eventually
760 iewStage->rescheduleMemInst(load_inst);
761 iewStage->decrWb(load_inst->seqNum);
762 load_inst->clearIssued();
763 ++lsqRescheduledLoads;
764
765 // Do not generate a writeback event as this instruction is not
766 // complete.
767 DPRINTF(LSQUnit, "Load-store forwarding mis-match. "
768 "Store idx %i to load addr %#x\n",
769 store_idx, req->getVaddr());
770
771 // Must delete request now that it wasn't handed off to
772 // memory. This is quite ugly. @todo: Figure out the
773 // proper place to really handle request deletes.
774 delete req;
775 if (TheISA::HasUnalignedMemAcc && sreqLow) {
776 delete sreqLow;
777 delete sreqHigh;
778 }
779
780 return NoFault;
781 }
782 }
783
784 // If there's no forwarding case, then go access memory
785 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n",
786 load_inst->seqNum, load_inst->pcState());
787
788 assert(!load_inst->memData);
789 load_inst->memData = new uint8_t[req->getSize()];
790
791 ++usedPorts;
792
793 // if we the cache is not blocked, do cache access
794 bool completedFirst = false;
795 if (!lsq->cacheBlocked()) {
796 MemCmd command =
797 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq;
798 PacketPtr data_pkt = new Packet(req, command);
799 PacketPtr fst_data_pkt = NULL;
800 PacketPtr snd_data_pkt = NULL;
801
802 data_pkt->dataStatic(load_inst->memData);
803
804 LSQSenderState *state = new LSQSenderState;
805 state->isLoad = true;
806 state->idx = load_idx;
807 state->inst = load_inst;
808 data_pkt->senderState = state;
809
810 if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
811
812 // Point the first packet at the main data packet.
813 fst_data_pkt = data_pkt;
814 } else {
815
816 // Create the split packets.
817 fst_data_pkt = new Packet(sreqLow, command);
818 snd_data_pkt = new Packet(sreqHigh, command);
819
820 fst_data_pkt->dataStatic(load_inst->memData);
821 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
822
823 fst_data_pkt->senderState = state;
824 snd_data_pkt->senderState = state;
825
826 state->isSplit = true;
827 state->outstanding = 2;
828 state->mainPkt = data_pkt;
829 }
830
831 if (!dcachePort->sendTimingReq(fst_data_pkt)) {
832 // Delete state and data packet because a load retry
833 // initiates a pipeline restart; it does not retry.
834 delete state;
835 delete data_pkt->req;
836 delete data_pkt;
837 if (TheISA::HasUnalignedMemAcc && sreqLow) {
838 delete fst_data_pkt->req;
839 delete fst_data_pkt;
840 delete snd_data_pkt->req;
841 delete snd_data_pkt;
842 sreqLow = NULL;
843 sreqHigh = NULL;
844 }
845
846 req = NULL;
847
848 // If the access didn't succeed, tell the LSQ by setting
849 // the retry thread id.
850 lsq->setRetryTid(lsqID);
851 } else if (TheISA::HasUnalignedMemAcc && sreqLow) {
852 completedFirst = true;
853
854 // The first packet was sent without problems, so send this one
855 // too. If there is a problem with this packet then the whole
856 // load will be squashed, so indicate this to the state object.
857 // The first packet will return in completeDataAccess and be
858 // handled there.
859 ++usedPorts;
860 if (!dcachePort->sendTimingReq(snd_data_pkt)) {
861
862 // The main packet will be deleted in completeDataAccess.
863 delete snd_data_pkt->req;
864 delete snd_data_pkt;
865
866 state->complete();
867
868 req = NULL;
869 sreqHigh = NULL;
870
871 lsq->setRetryTid(lsqID);
872 }
873 }
874 }
875
876 // If the cache was blocked, or has become blocked due to the access,
877 // handle it.
878 if (lsq->cacheBlocked()) {
879 if (req)
880 delete req;
881 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) {
882 delete sreqLow;
883 delete sreqHigh;
884 }
885
886 ++lsqCacheBlocked;
887
888 // If the first part of a split access succeeds, then let the LSQ
889 // handle the decrWb when completeDataAccess is called upon return
890 // of the requested first part of data
891 if (!completedFirst)
892 iewStage->decrWb(load_inst->seqNum);
893
894 // There's an older load that's already going to squash.
895 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum)
896 return NoFault;
897
898 // Record that the load was blocked due to memory. This
899 // load will squash all instructions after it, be
900 // refetched, and re-executed.
901 isLoadBlocked = true;
902 loadBlockedHandled = false;
903 blockedLoadSeqNum = load_inst->seqNum;
904 // No fault occurred, even though the interface is blocked.
905 return NoFault;
906 }
907
908 return NoFault;
909}
910
911template <class Impl>
912Fault
913LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh,
914 uint8_t *data, int store_idx)
915{
916 assert(storeQueue[store_idx].inst);
917
918 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x"
919 " | storeHead:%i [sn:%i]\n",
920 store_idx, req->getPaddr(), storeHead,
921 storeQueue[store_idx].inst->seqNum);
922
923 storeQueue[store_idx].req = req;
924 storeQueue[store_idx].sreqLow = sreqLow;
925 storeQueue[store_idx].sreqHigh = sreqHigh;
926 unsigned size = req->getSize();
927 storeQueue[store_idx].size = size;
928 storeQueue[store_idx].isAllZeros = req->getFlags() & Request::CACHE_BLOCK_ZERO;
929 assert(size <= sizeof(storeQueue[store_idx].data) ||
930 (req->getFlags() & Request::CACHE_BLOCK_ZERO));
931
932 // Split stores can only occur in ISAs with unaligned memory accesses. If
933 // a store request has been split, sreqLow and sreqHigh will be non-null.
934 if (TheISA::HasUnalignedMemAcc && sreqLow) {
935 storeQueue[store_idx].isSplit = true;
936 }
937
938 if (!(req->getFlags() & Request::CACHE_BLOCK_ZERO))
939 memcpy(storeQueue[store_idx].data, data, size);
940
941 // This function only writes the data to the store queue, so no fault
942 // can happen here.
943 return NoFault;
944}
945
946#endif // __CPU_O3_LSQ_UNIT_HH__
| 208 /** Returns the number of loads in the LQ. */
209 int numLoads() { return loads; }
210
211 /** Returns the number of stores in the SQ. */
212 int numStores() { return stores; }
213
214 /** Returns if either the LQ or SQ is full. */
215 bool isFull() { return lqFull() || sqFull(); }
216
217 /** Returns if both the LQ and SQ are empty. */
218 bool isEmpty() const { return lqEmpty() && sqEmpty(); }
219
220 /** Returns if the LQ is full. */
221 bool lqFull() { return loads >= (LQEntries - 1); }
222
223 /** Returns if the SQ is full. */
224 bool sqFull() { return stores >= (SQEntries - 1); }
225
226 /** Returns if the LQ is empty. */
227 bool lqEmpty() const { return loads == 0; }
228
229 /** Returns if the SQ is empty. */
230 bool sqEmpty() const { return stores == 0; }
231
232 /** Returns the number of instructions in the LSQ. */
233 unsigned getCount() { return loads + stores; }
234
235 /** Returns if there are any stores to writeback. */
236 bool hasStoresToWB() { return storesToWB; }
237
238 /** Returns the number of stores to writeback. */
239 int numStoresToWB() { return storesToWB; }
240
241 /** Returns if the LSQ unit will writeback on this cycle. */
242 bool willWB() { return storeQueue[storeWBIdx].canWB &&
243 !storeQueue[storeWBIdx].completed &&
244 !isStoreBlocked; }
245
246 /** Handles doing the retry. */
247 void recvRetry();
248
249 private:
250 /** Reset the LSQ state */
251 void resetState();
252
253 /** Writes back the instruction, sending it to IEW. */
254 void writeback(DynInstPtr &inst, PacketPtr pkt);
255
256 /** Writes back a store that couldn't be completed the previous cycle. */
257 void writebackPendingStore();
258
259 /** Handles completing the send of a store to memory. */
260 void storePostSend(PacketPtr pkt);
261
262 /** Completes the store at the specified index. */
263 void completeStore(int store_idx);
264
265 /** Attempts to send a store to the cache. */
266 bool sendStore(PacketPtr data_pkt);
267
268 /** Increments the given store index (circular queue). */
269 inline void incrStIdx(int &store_idx) const;
270 /** Decrements the given store index (circular queue). */
271 inline void decrStIdx(int &store_idx) const;
272 /** Increments the given load index (circular queue). */
273 inline void incrLdIdx(int &load_idx) const;
274 /** Decrements the given load index (circular queue). */
275 inline void decrLdIdx(int &load_idx) const;
276
277 public:
278 /** Debugging function to dump instructions in the LSQ. */
279 void dumpInsts() const;
280
281 private:
282 /** Pointer to the CPU. */
283 O3CPU *cpu;
284
285 /** Pointer to the IEW stage. */
286 IEW *iewStage;
287
288 /** Pointer to the LSQ. */
289 LSQ *lsq;
290
291 /** Pointer to the dcache port. Used only for sending. */
292 MasterPort *dcachePort;
293
294 /** Derived class to hold any sender state the LSQ needs. */
295 class LSQSenderState : public Packet::SenderState
296 {
297 public:
298 /** Default constructor. */
299 LSQSenderState()
300 : mainPkt(NULL), pendingPacket(NULL), outstanding(1),
301 noWB(false), isSplit(false), pktToSend(false)
302 { }
303
304 /** Instruction who initiated the access to memory. */
305 DynInstPtr inst;
306 /** The main packet from a split load, used during writeback. */
307 PacketPtr mainPkt;
308 /** A second packet from a split store that needs sending. */
309 PacketPtr pendingPacket;
310 /** The LQ/SQ index of the instruction. */
311 uint8_t idx;
312 /** Number of outstanding packets to complete. */
313 uint8_t outstanding;
314 /** Whether or not it is a load. */
315 bool isLoad;
316 /** Whether or not the instruction will need to writeback. */
317 bool noWB;
318 /** Whether or not this access is split in two. */
319 bool isSplit;
320 /** Whether or not there is a packet that needs sending. */
321 bool pktToSend;
322
323 /** Completes a packet and returns whether the access is finished. */
324 inline bool complete() { return --outstanding == 0; }
325 };
326
327 /** Writeback event, specifically for when stores forward data to loads. */
328 class WritebackEvent : public Event {
329 public:
330 /** Constructs a writeback event. */
331 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr);
332
333 /** Processes the writeback event. */
334 void process();
335
336 /** Returns the description of this event. */
337 const char *description() const;
338
339 private:
340 /** Instruction whose results are being written back. */
341 DynInstPtr inst;
342
343 /** The packet that would have been sent to memory. */
344 PacketPtr pkt;
345
346 /** The pointer to the LSQ unit that issued the store. */
347 LSQUnit<Impl> *lsqPtr;
348 };
349
350 public:
351 struct SQEntry {
352 /** Constructs an empty store queue entry. */
353 SQEntry()
354 : inst(NULL), req(NULL), size(0),
355 canWB(0), committed(0), completed(0)
356 {
357 std::memset(data, 0, sizeof(data));
358 }
359
360 ~SQEntry()
361 {
362 inst = NULL;
363 }
364
365 /** Constructs a store queue entry for a given instruction. */
366 SQEntry(DynInstPtr &_inst)
367 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0),
368 isSplit(0), canWB(0), committed(0), completed(0), isAllZeros(0)
369 {
370 std::memset(data, 0, sizeof(data));
371 }
372 /** The store data. */
373 char data[16];
374 /** The store instruction. */
375 DynInstPtr inst;
376 /** The request for the store. */
377 RequestPtr req;
378 /** The split requests for the store. */
379 RequestPtr sreqLow;
380 RequestPtr sreqHigh;
381 /** The size of the store. */
382 uint8_t size;
383 /** Whether or not the store is split into two requests. */
384 bool isSplit;
385 /** Whether or not the store can writeback. */
386 bool canWB;
387 /** Whether or not the store is committed. */
388 bool committed;
389 /** Whether or not the store is completed. */
390 bool completed;
391 /** Does this request write all zeros and thus doesn't
392 * have any data attached to it. Used for cache block zero
393 * style instructs (ARM DC ZVA; ALPHA WH64)
394 */
395 bool isAllZeros;
396 };
397
398 private:
399 /** The LSQUnit thread id. */
400 ThreadID lsqID;
401
402 /** The store queue. */
403 std::vector<SQEntry> storeQueue;
404
405 /** The load queue. */
406 std::vector<DynInstPtr> loadQueue;
407
408 /** The number of LQ entries, plus a sentinel entry (circular queue).
409 * @todo: Consider having var that records the true number of LQ entries.
410 */
411 unsigned LQEntries;
412 /** The number of SQ entries, plus a sentinel entry (circular queue).
413 * @todo: Consider having var that records the true number of SQ entries.
414 */
415 unsigned SQEntries;
416
417 /** The number of places to shift addresses in the LSQ before checking
418 * for dependency violations
419 */
420 unsigned depCheckShift;
421
422 /** Should loads be checked for dependency issues */
423 bool checkLoads;
424
425 /** The number of load instructions in the LQ. */
426 int loads;
427 /** The number of store instructions in the SQ. */
428 int stores;
429 /** The number of store instructions in the SQ waiting to writeback. */
430 int storesToWB;
431
432 /** The index of the head instruction in the LQ. */
433 int loadHead;
434 /** The index of the tail instruction in the LQ. */
435 int loadTail;
436
437 /** The index of the head instruction in the SQ. */
438 int storeHead;
439 /** The index of the first instruction that may be ready to be
440 * written back, and has not yet been written back.
441 */
442 int storeWBIdx;
443 /** The index of the tail instruction in the SQ. */
444 int storeTail;
445
446 /// @todo Consider moving to a more advanced model with write vs read ports
447 /** The number of cache ports available each cycle. */
448 int cachePorts;
449
450 /** The number of used cache ports in this cycle. */
451 int usedPorts;
452
453 //list<InstSeqNum> mshrSeqNums;
454
455 /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */
456 Addr cacheBlockMask;
457
458 /** Wire to read information from the issue stage time queue. */
459 typename TimeBuffer<IssueStruct>::wire fromIssue;
460
461 /** Whether or not the LSQ is stalled. */
462 bool stalled;
463 /** The store that causes the stall due to partial store to load
464 * forwarding.
465 */
466 InstSeqNum stallingStoreIsn;
467 /** The index of the above store. */
468 int stallingLoadIdx;
469
470 /** The packet that needs to be retried. */
471 PacketPtr retryPkt;
472
473 /** Whehter or not a store is blocked due to the memory system. */
474 bool isStoreBlocked;
475
476 /** Whether or not a load is blocked due to the memory system. */
477 bool isLoadBlocked;
478
479 /** Has the blocked load been handled. */
480 bool loadBlockedHandled;
481
482 /** Whether or not a store is in flight. */
483 bool storeInFlight;
484
485 /** The sequence number of the blocked load. */
486 InstSeqNum blockedLoadSeqNum;
487
488 /** The oldest load that caused a memory ordering violation. */
489 DynInstPtr memDepViolator;
490
491 /** Whether or not there is a packet that couldn't be sent because of
492 * a lack of cache ports. */
493 bool hasPendingPkt;
494
495 /** The packet that is pending free cache ports. */
496 PacketPtr pendingPkt;
497
498 /** Flag for memory model. */
499 bool needsTSO;
500
501 // Will also need how many read/write ports the Dcache has. Or keep track
502 // of that in stage that is one level up, and only call executeLoad/Store
503 // the appropriate number of times.
504 /** Total number of loads forwaded from LSQ stores. */
505 Stats::Scalar lsqForwLoads;
506
507 /** Total number of loads ignored due to invalid addresses. */
508 Stats::Scalar invAddrLoads;
509
510 /** Total number of squashed loads. */
511 Stats::Scalar lsqSquashedLoads;
512
513 /** Total number of responses from the memory system that are
514 * ignored due to the instruction already being squashed. */
515 Stats::Scalar lsqIgnoredResponses;
516
517 /** Tota number of memory ordering violations. */
518 Stats::Scalar lsqMemOrderViolation;
519
520 /** Total number of squashed stores. */
521 Stats::Scalar lsqSquashedStores;
522
523 /** Total number of software prefetches ignored due to invalid addresses. */
524 Stats::Scalar invAddrSwpfs;
525
526 /** Ready loads blocked due to partial store-forwarding. */
527 Stats::Scalar lsqBlockedLoads;
528
529 /** Number of loads that were rescheduled. */
530 Stats::Scalar lsqRescheduledLoads;
531
532 /** Number of times the LSQ is blocked due to the cache. */
533 Stats::Scalar lsqCacheBlocked;
534
535 public:
536 /** Executes the load at the given index. */
537 Fault read(Request *req, Request *sreqLow, Request *sreqHigh,
538 uint8_t *data, int load_idx);
539
540 /** Executes the store at the given index. */
541 Fault write(Request *req, Request *sreqLow, Request *sreqHigh,
542 uint8_t *data, int store_idx);
543
544 /** Returns the index of the head load instruction. */
545 int getLoadHead() { return loadHead; }
546 /** Returns the sequence number of the head load instruction. */
547 InstSeqNum getLoadHeadSeqNum()
548 {
549 if (loadQueue[loadHead]) {
550 return loadQueue[loadHead]->seqNum;
551 } else {
552 return 0;
553 }
554
555 }
556
557 /** Returns the index of the head store instruction. */
558 int getStoreHead() { return storeHead; }
559 /** Returns the sequence number of the head store instruction. */
560 InstSeqNum getStoreHeadSeqNum()
561 {
562 if (storeQueue[storeHead].inst) {
563 return storeQueue[storeHead].inst->seqNum;
564 } else {
565 return 0;
566 }
567
568 }
569
570 /** Returns whether or not the LSQ unit is stalled. */
571 bool isStalled() { return stalled; }
572};
573
574template <class Impl>
575Fault
576LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh,
577 uint8_t *data, int load_idx)
578{
579 DynInstPtr load_inst = loadQueue[load_idx];
580
581 assert(load_inst);
582
583 assert(!load_inst->isExecuted());
584
585 // Make sure this isn't an uncacheable access
586 // A bit of a hackish way to get uncached accesses to work only if they're
587 // at the head of the LSQ and are ready to commit (at the head of the ROB
588 // too).
589 if (req->isUncacheable() &&
590 (load_idx != loadHead || !load_inst->isAtCommit())) {
591 iewStage->rescheduleMemInst(load_inst);
592 ++lsqRescheduledLoads;
593 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n",
594 load_inst->seqNum, load_inst->pcState());
595
596 // Must delete request now that it wasn't handed off to
597 // memory. This is quite ugly. @todo: Figure out the proper
598 // place to really handle request deletes.
599 delete req;
600 if (TheISA::HasUnalignedMemAcc && sreqLow) {
601 delete sreqLow;
602 delete sreqHigh;
603 }
604 return new GenericISA::M5PanicFault(
605 "Uncachable load [sn:%llx] PC %s\n",
606 load_inst->seqNum, load_inst->pcState());
607 }
608
609 // Check the SQ for any previous stores that might lead to forwarding
610 int store_idx = load_inst->sqIdx;
611
612 int store_size = 0;
613
614 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, "
615 "storeHead: %i addr: %#x%s\n",
616 load_idx, store_idx, storeHead, req->getPaddr(),
617 sreqLow ? " split" : "");
618
619 if (req->isLLSC()) {
620 assert(!sreqLow);
621 // Disable recording the result temporarily. Writing to misc
622 // regs normally updates the result, but this is not the
623 // desired behavior when handling store conditionals.
624 load_inst->recordResult(false);
625 TheISA::handleLockedRead(load_inst.get(), req);
626 load_inst->recordResult(true);
627 }
628
629 if (req->isMmappedIpr()) {
630 assert(!load_inst->memData);
631 load_inst->memData = new uint8_t[64];
632
633 ThreadContext *thread = cpu->tcBase(lsqID);
634 Cycles delay(0);
635 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq);
636
637 if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
638 data_pkt->dataStatic(load_inst->memData);
639 delay = TheISA::handleIprRead(thread, data_pkt);
640 } else {
641 assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr());
642 PacketPtr fst_data_pkt = new Packet(sreqLow, MemCmd::ReadReq);
643 PacketPtr snd_data_pkt = new Packet(sreqHigh, MemCmd::ReadReq);
644
645 fst_data_pkt->dataStatic(load_inst->memData);
646 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
647
648 delay = TheISA::handleIprRead(thread, fst_data_pkt);
649 Cycles delay2 = TheISA::handleIprRead(thread, snd_data_pkt);
650 if (delay2 > delay)
651 delay = delay2;
652
653 delete sreqLow;
654 delete sreqHigh;
655 delete fst_data_pkt;
656 delete snd_data_pkt;
657 }
658 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
659 cpu->schedule(wb, cpu->clockEdge(delay));
660 return NoFault;
661 }
662
663 while (store_idx != -1) {
664 // End once we've reached the top of the LSQ
665 if (store_idx == storeWBIdx) {
666 break;
667 }
668
669 // Move the index to one younger
670 if (--store_idx < 0)
671 store_idx += SQEntries;
672
673 assert(storeQueue[store_idx].inst);
674
675 store_size = storeQueue[store_idx].size;
676
677 if (store_size == 0)
678 continue;
679 else if (storeQueue[store_idx].inst->uncacheable())
680 continue;
681
682 assert(storeQueue[store_idx].inst->effAddrValid());
683
684 // Check if the store data is within the lower and upper bounds of
685 // addresses that the request needs.
686 bool store_has_lower_limit =
687 req->getVaddr() >= storeQueue[store_idx].inst->effAddr;
688 bool store_has_upper_limit =
689 (req->getVaddr() + req->getSize()) <=
690 (storeQueue[store_idx].inst->effAddr + store_size);
691 bool lower_load_has_store_part =
692 req->getVaddr() < (storeQueue[store_idx].inst->effAddr +
693 store_size);
694 bool upper_load_has_store_part =
695 (req->getVaddr() + req->getSize()) >
696 storeQueue[store_idx].inst->effAddr;
697
698 // If the store's data has all of the data needed, we can forward.
699 if ((store_has_lower_limit && store_has_upper_limit)) {
700 // Get shift amount for offset into the store's data.
701 int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr;
702
703 if (storeQueue[store_idx].isAllZeros)
704 memset(data, 0, req->getSize());
705 else
706 memcpy(data, storeQueue[store_idx].data + shift_amt,
707 req->getSize());
708
709 assert(!load_inst->memData);
710 load_inst->memData = new uint8_t[req->getSize()];
711 if (storeQueue[store_idx].isAllZeros)
712 memset(load_inst->memData, 0, req->getSize());
713 else
714 memcpy(load_inst->memData,
715 storeQueue[store_idx].data + shift_amt, req->getSize());
716
717 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to "
718 "addr %#x\n", store_idx, req->getVaddr());
719
720 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq);
721 data_pkt->dataStatic(load_inst->memData);
722
723 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
724
725 // We'll say this has a 1 cycle load-store forwarding latency
726 // for now.
727 // @todo: Need to make this a parameter.
728 cpu->schedule(wb, curTick());
729
730 // Don't need to do anything special for split loads.
731 if (TheISA::HasUnalignedMemAcc && sreqLow) {
732 delete sreqLow;
733 delete sreqHigh;
734 }
735
736 ++lsqForwLoads;
737 return NoFault;
738 } else if ((store_has_lower_limit && lower_load_has_store_part) ||
739 (store_has_upper_limit && upper_load_has_store_part) ||
740 (lower_load_has_store_part && upper_load_has_store_part)) {
741 // This is the partial store-load forwarding case where a store
742 // has only part of the load's data.
743
744 // If it's already been written back, then don't worry about
745 // stalling on it.
746 if (storeQueue[store_idx].completed) {
747 panic("Should not check one of these");
748 continue;
749 }
750
751 // Must stall load and force it to retry, so long as it's the oldest
752 // load that needs to do so.
753 if (!stalled ||
754 (stalled &&
755 load_inst->seqNum <
756 loadQueue[stallingLoadIdx]->seqNum)) {
757 stalled = true;
758 stallingStoreIsn = storeQueue[store_idx].inst->seqNum;
759 stallingLoadIdx = load_idx;
760 }
761
762 // Tell IQ/mem dep unit that this instruction will need to be
763 // rescheduled eventually
764 iewStage->rescheduleMemInst(load_inst);
765 iewStage->decrWb(load_inst->seqNum);
766 load_inst->clearIssued();
767 ++lsqRescheduledLoads;
768
769 // Do not generate a writeback event as this instruction is not
770 // complete.
771 DPRINTF(LSQUnit, "Load-store forwarding mis-match. "
772 "Store idx %i to load addr %#x\n",
773 store_idx, req->getVaddr());
774
775 // Must delete request now that it wasn't handed off to
776 // memory. This is quite ugly. @todo: Figure out the
777 // proper place to really handle request deletes.
778 delete req;
779 if (TheISA::HasUnalignedMemAcc && sreqLow) {
780 delete sreqLow;
781 delete sreqHigh;
782 }
783
784 return NoFault;
785 }
786 }
787
788 // If there's no forwarding case, then go access memory
789 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n",
790 load_inst->seqNum, load_inst->pcState());
791
792 assert(!load_inst->memData);
793 load_inst->memData = new uint8_t[req->getSize()];
794
795 ++usedPorts;
796
797 // if we the cache is not blocked, do cache access
798 bool completedFirst = false;
799 if (!lsq->cacheBlocked()) {
800 MemCmd command =
801 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq;
802 PacketPtr data_pkt = new Packet(req, command);
803 PacketPtr fst_data_pkt = NULL;
804 PacketPtr snd_data_pkt = NULL;
805
806 data_pkt->dataStatic(load_inst->memData);
807
808 LSQSenderState *state = new LSQSenderState;
809 state->isLoad = true;
810 state->idx = load_idx;
811 state->inst = load_inst;
812 data_pkt->senderState = state;
813
814 if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
815
816 // Point the first packet at the main data packet.
817 fst_data_pkt = data_pkt;
818 } else {
819
820 // Create the split packets.
821 fst_data_pkt = new Packet(sreqLow, command);
822 snd_data_pkt = new Packet(sreqHigh, command);
823
824 fst_data_pkt->dataStatic(load_inst->memData);
825 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
826
827 fst_data_pkt->senderState = state;
828 snd_data_pkt->senderState = state;
829
830 state->isSplit = true;
831 state->outstanding = 2;
832 state->mainPkt = data_pkt;
833 }
834
835 if (!dcachePort->sendTimingReq(fst_data_pkt)) {
836 // Delete state and data packet because a load retry
837 // initiates a pipeline restart; it does not retry.
838 delete state;
839 delete data_pkt->req;
840 delete data_pkt;
841 if (TheISA::HasUnalignedMemAcc && sreqLow) {
842 delete fst_data_pkt->req;
843 delete fst_data_pkt;
844 delete snd_data_pkt->req;
845 delete snd_data_pkt;
846 sreqLow = NULL;
847 sreqHigh = NULL;
848 }
849
850 req = NULL;
851
852 // If the access didn't succeed, tell the LSQ by setting
853 // the retry thread id.
854 lsq->setRetryTid(lsqID);
855 } else if (TheISA::HasUnalignedMemAcc && sreqLow) {
856 completedFirst = true;
857
858 // The first packet was sent without problems, so send this one
859 // too. If there is a problem with this packet then the whole
860 // load will be squashed, so indicate this to the state object.
861 // The first packet will return in completeDataAccess and be
862 // handled there.
863 ++usedPorts;
864 if (!dcachePort->sendTimingReq(snd_data_pkt)) {
865
866 // The main packet will be deleted in completeDataAccess.
867 delete snd_data_pkt->req;
868 delete snd_data_pkt;
869
870 state->complete();
871
872 req = NULL;
873 sreqHigh = NULL;
874
875 lsq->setRetryTid(lsqID);
876 }
877 }
878 }
879
880 // If the cache was blocked, or has become blocked due to the access,
881 // handle it.
882 if (lsq->cacheBlocked()) {
883 if (req)
884 delete req;
885 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) {
886 delete sreqLow;
887 delete sreqHigh;
888 }
889
890 ++lsqCacheBlocked;
891
892 // If the first part of a split access succeeds, then let the LSQ
893 // handle the decrWb when completeDataAccess is called upon return
894 // of the requested first part of data
895 if (!completedFirst)
896 iewStage->decrWb(load_inst->seqNum);
897
898 // There's an older load that's already going to squash.
899 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum)
900 return NoFault;
901
902 // Record that the load was blocked due to memory. This
903 // load will squash all instructions after it, be
904 // refetched, and re-executed.
905 isLoadBlocked = true;
906 loadBlockedHandled = false;
907 blockedLoadSeqNum = load_inst->seqNum;
908 // No fault occurred, even though the interface is blocked.
909 return NoFault;
910 }
911
912 return NoFault;
913}
914
915template <class Impl>
916Fault
917LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh,
918 uint8_t *data, int store_idx)
919{
920 assert(storeQueue[store_idx].inst);
921
922 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x"
923 " | storeHead:%i [sn:%i]\n",
924 store_idx, req->getPaddr(), storeHead,
925 storeQueue[store_idx].inst->seqNum);
926
927 storeQueue[store_idx].req = req;
928 storeQueue[store_idx].sreqLow = sreqLow;
929 storeQueue[store_idx].sreqHigh = sreqHigh;
930 unsigned size = req->getSize();
931 storeQueue[store_idx].size = size;
932 storeQueue[store_idx].isAllZeros = req->getFlags() & Request::CACHE_BLOCK_ZERO;
933 assert(size <= sizeof(storeQueue[store_idx].data) ||
934 (req->getFlags() & Request::CACHE_BLOCK_ZERO));
935
936 // Split stores can only occur in ISAs with unaligned memory accesses. If
937 // a store request has been split, sreqLow and sreqHigh will be non-null.
938 if (TheISA::HasUnalignedMemAcc && sreqLow) {
939 storeQueue[store_idx].isSplit = true;
940 }
941
942 if (!(req->getFlags() & Request::CACHE_BLOCK_ZERO))
943 memcpy(storeQueue[store_idx].data, data, size);
944
945 // This function only writes the data to the store queue, so no fault
946 // can happen here.
947 return NoFault;
948}
949
950#endif // __CPU_O3_LSQ_UNIT_HH__
|