1/*
2 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include "arch/x86/ldstflags.hh"
30#include "base/misc.hh"
31#include "base/str.hh"
32#include "cpu/testers/rubytest/RubyTester.hh"
33#include "debug/MemoryAccess.hh"
34#include "debug/ProtocolTrace.hh"
35#include "debug/RubySequencer.hh"
36#include "debug/RubyStats.hh"
37#include "mem/protocol/PrefetchBit.hh"
38#include "mem/protocol/RubyAccessMode.hh"
39#include "mem/ruby/profiler/Profiler.hh"
40#include "mem/ruby/slicc_interface/RubyRequest.hh"
41#include "mem/ruby/system/RubySystem.hh"
42#include "mem/ruby/system/Sequencer.hh"
43#include "mem/packet.hh"
44#include "sim/system.hh"
45
46using namespace std;
47
48Sequencer *
49RubySequencerParams::create()
50{
51 return new Sequencer(this);
52}
53
54Sequencer::Sequencer(const Params *p)
55 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this)
56{
57 m_outstanding_count = 0;
58
59 m_instCache_ptr = p->icache;
60 m_dataCache_ptr = p->dcache;
61 m_data_cache_hit_latency = p->dcache_hit_latency;
62 m_inst_cache_hit_latency = p->icache_hit_latency;
63 m_max_outstanding_requests = p->max_outstanding_requests;
64 m_deadlock_threshold = p->deadlock_threshold;
65
66 assert(m_max_outstanding_requests > 0);
67 assert(m_deadlock_threshold > 0);
68 assert(m_instCache_ptr != NULL);
69 assert(m_dataCache_ptr != NULL);
70 assert(m_data_cache_hit_latency > 0);
71 assert(m_inst_cache_hit_latency > 0);
72
73 m_usingNetworkTester = p->using_network_tester;
74}
75
76Sequencer::~Sequencer()
77{
78}
79
80void
81Sequencer::wakeup()
82{
83 assert(drainState() != DrainState::Draining);
84
85 // Check for deadlock of any of the requests
86 Cycles current_time = curCycle();
87
88 // Check across all outstanding requests
89 int total_outstanding = 0;
90
91 RequestTable::iterator read = m_readRequestTable.begin();
92 RequestTable::iterator read_end = m_readRequestTable.end();
93 for (; read != read_end; ++read) {
94 SequencerRequest* request = read->second;
95 if (current_time - request->issue_time < m_deadlock_threshold)
96 continue;
97
98 panic("Possible Deadlock detected. Aborting!\n"
99 "version: %d request.paddr: 0x%x m_readRequestTable: %d "
100 "current time: %u issue_time: %d difference: %d\n", m_version,
101 request->pkt->getAddr(), m_readRequestTable.size(),
102 current_time * clockPeriod(), request->issue_time * clockPeriod(),
103 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
104 }
105
106 RequestTable::iterator write = m_writeRequestTable.begin();
107 RequestTable::iterator write_end = m_writeRequestTable.end();
108 for (; write != write_end; ++write) {
109 SequencerRequest* request = write->second;
110 if (current_time - request->issue_time < m_deadlock_threshold)
111 continue;
112
113 panic("Possible Deadlock detected. Aborting!\n"
114 "version: %d request.paddr: 0x%x m_writeRequestTable: %d "
115 "current time: %u issue_time: %d difference: %d\n", m_version,
116 request->pkt->getAddr(), m_writeRequestTable.size(),
117 current_time * clockPeriod(), request->issue_time * clockPeriod(),
118 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
119 }
120
121 total_outstanding += m_writeRequestTable.size();
122 total_outstanding += m_readRequestTable.size();
123
124 assert(m_outstanding_count == total_outstanding);
125
126 if (m_outstanding_count > 0) {
127 // If there are still outstanding requests, keep checking
128 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
129 }
130}
131
132void Sequencer::resetStats()
133{
134 m_latencyHist.reset();
135 m_hitLatencyHist.reset();
136 m_missLatencyHist.reset();
137 for (int i = 0; i < RubyRequestType_NUM; i++) {
138 m_typeLatencyHist[i]->reset();
139 m_hitTypeLatencyHist[i]->reset();
140 m_missTypeLatencyHist[i]->reset();
141 for (int j = 0; j < MachineType_NUM; j++) {
142 m_hitTypeMachLatencyHist[i][j]->reset();
143 m_missTypeMachLatencyHist[i][j]->reset();
144 }
145 }
146
147 for (int i = 0; i < MachineType_NUM; i++) {
148 m_missMachLatencyHist[i]->reset();
149 m_hitMachLatencyHist[i]->reset();
150
151 m_IssueToInitialDelayHist[i]->reset();
152 m_InitialToForwardDelayHist[i]->reset();
153 m_ForwardToFirstResponseDelayHist[i]->reset();
154 m_FirstResponseToCompletionDelayHist[i]->reset();
155
156 m_IncompleteTimes[i] = 0;
157 }
158}
159
160// Insert the request on the correct request table. Return true if
161// the entry was already present.
162RequestStatus
163Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type)
164{
165 assert(m_outstanding_count ==
166 (m_writeRequestTable.size() + m_readRequestTable.size()));
167
168 // See if we should schedule a deadlock check
169 if (!deadlockCheckEvent.scheduled() &&
170 drainState() != DrainState::Draining) {
171 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
172 }
173
174 Addr line_addr = makeLineAddress(pkt->getAddr());
175 // Create a default entry, mapping the address to NULL, the cast is
176 // there to make gcc 4.4 happy
177 RequestTable::value_type default_entry(line_addr,
178 (SequencerRequest*) NULL);
179
180 if ((request_type == RubyRequestType_ST) ||
181 (request_type == RubyRequestType_RMW_Read) ||
182 (request_type == RubyRequestType_RMW_Write) ||
183 (request_type == RubyRequestType_Load_Linked) ||
184 (request_type == RubyRequestType_Store_Conditional) ||
185 (request_type == RubyRequestType_Locked_RMW_Read) ||
186 (request_type == RubyRequestType_Locked_RMW_Write) ||
187 (request_type == RubyRequestType_FLUSH)) {
188
189 // Check if there is any outstanding read request for the same
190 // cache line.
191 if (m_readRequestTable.count(line_addr) > 0) {
192 m_store_waiting_on_load++;
193 return RequestStatus_Aliased;
194 }
195
196 pair<RequestTable::iterator, bool> r =
197 m_writeRequestTable.insert(default_entry);
198 if (r.second) {
199 RequestTable::iterator i = r.first;
200 i->second = new SequencerRequest(pkt, request_type, curCycle());
201 m_outstanding_count++;
202 } else {
203 // There is an outstanding write request for the cache line
204 m_store_waiting_on_store++;
205 return RequestStatus_Aliased;
206 }
207 } else {
208 // Check if there is any outstanding write request for the same
209 // cache line.
210 if (m_writeRequestTable.count(line_addr) > 0) {
211 m_load_waiting_on_store++;
212 return RequestStatus_Aliased;
213 }
214
215 pair<RequestTable::iterator, bool> r =
216 m_readRequestTable.insert(default_entry);
217
218 if (r.second) {
219 RequestTable::iterator i = r.first;
220 i->second = new SequencerRequest(pkt, request_type, curCycle());
221 m_outstanding_count++;
222 } else {
223 // There is an outstanding read request for the cache line
224 m_load_waiting_on_load++;
225 return RequestStatus_Aliased;
226 }
227 }
228
229 m_outstandReqHist.sample(m_outstanding_count);
230 assert(m_outstanding_count ==
231 (m_writeRequestTable.size() + m_readRequestTable.size()));
232
233 return RequestStatus_Ready;
234}
235
236void
237Sequencer::markRemoved()
238{
239 m_outstanding_count--;
240 assert(m_outstanding_count ==
241 m_writeRequestTable.size() + m_readRequestTable.size());
242}
243
244void
245Sequencer::invalidateSC(Addr address)
246{
247 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
248 // The controller has lost the coherence permissions, hence the lock
249 // on the cache line maintained by the cache should be cleared.
250 if (e && e->isLocked(m_version)) {
251 e->clearLocked();
252 }
253}
254
255bool
256Sequencer::handleLlsc(Addr address, SequencerRequest* request)
257{
258 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
259 if (!e)
260 return true;
261
262 // The success flag indicates whether the LLSC operation was successful.
263 // LL ops will always succeed, but SC may fail if the cache line is no
264 // longer locked.
265 bool success = true;
266 if (request->m_type == RubyRequestType_Store_Conditional) {
267 if (!e->isLocked(m_version)) {
268 //
269 // For failed SC requests, indicate the failure to the cpu by
270 // setting the extra data to zero.
271 //
272 request->pkt->req->setExtraData(0);
273 success = false;
274 } else {
275 //
276 // For successful SC requests, indicate the success to the cpu by
277 // setting the extra data to one.
278 //
279 request->pkt->req->setExtraData(1);
280 }
281 //
282 // Independent of success, all SC operations must clear the lock
283 //
284 e->clearLocked();
285 } else if (request->m_type == RubyRequestType_Load_Linked) {
286 //
287 // Note: To fully follow Alpha LLSC semantics, should the LL clear any
288 // previously locked cache lines?
289 //
290 e->setLocked(m_version);
291 } else if (e->isLocked(m_version)) {
292 //
293 // Normal writes should clear the locked address
294 //
295 e->clearLocked();
296 }
297 return success;
298}
299
300void
301Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type,
302 const MachineType respondingMach,
303 bool isExternalHit, Cycles issuedTime,
304 Cycles initialRequestTime,
305 Cycles forwardRequestTime,
306 Cycles firstResponseTime, Cycles completionTime)
307{
308 m_latencyHist.sample(cycles);
309 m_typeLatencyHist[type]->sample(cycles);
310
311 if (isExternalHit) {
312 m_missLatencyHist.sample(cycles);
313 m_missTypeLatencyHist[type]->sample(cycles);
314
315 if (respondingMach != MachineType_NUM) {
316 m_missMachLatencyHist[respondingMach]->sample(cycles);
317 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles);
318
319 if ((issuedTime <= initialRequestTime) &&
320 (initialRequestTime <= forwardRequestTime) &&
321 (forwardRequestTime <= firstResponseTime) &&
322 (firstResponseTime <= completionTime)) {
323
324 m_IssueToInitialDelayHist[respondingMach]->sample(
325 initialRequestTime - issuedTime);
326 m_InitialToForwardDelayHist[respondingMach]->sample(
327 forwardRequestTime - initialRequestTime);
328 m_ForwardToFirstResponseDelayHist[respondingMach]->sample(
329 firstResponseTime - forwardRequestTime);
330 m_FirstResponseToCompletionDelayHist[respondingMach]->sample(
331 completionTime - firstResponseTime);
332 } else {
333 m_IncompleteTimes[respondingMach]++;
334 }
335 }
336 } else {
337 m_hitLatencyHist.sample(cycles);
338 m_hitTypeLatencyHist[type]->sample(cycles);
339
340 if (respondingMach != MachineType_NUM) {
341 m_hitMachLatencyHist[respondingMach]->sample(cycles);
342 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles);
343 }
344 }
345}
346
347void
348Sequencer::writeCallback(Addr address, DataBlock& data,
349 const bool externalHit, const MachineType mach,
350 const Cycles initialRequestTime,
351 const Cycles forwardRequestTime,
352 const Cycles firstResponseTime)
353{
354 assert(address == makeLineAddress(address));
355 assert(m_writeRequestTable.count(makeLineAddress(address)));
356
357 RequestTable::iterator i = m_writeRequestTable.find(address);
358 assert(i != m_writeRequestTable.end());
359 SequencerRequest* request = i->second;
360
361 m_writeRequestTable.erase(i);
362 markRemoved();
363
364 assert((request->m_type == RubyRequestType_ST) ||
365 (request->m_type == RubyRequestType_ATOMIC) ||
366 (request->m_type == RubyRequestType_RMW_Read) ||
367 (request->m_type == RubyRequestType_RMW_Write) ||
368 (request->m_type == RubyRequestType_Load_Linked) ||
369 (request->m_type == RubyRequestType_Store_Conditional) ||
370 (request->m_type == RubyRequestType_Locked_RMW_Read) ||
371 (request->m_type == RubyRequestType_Locked_RMW_Write) ||
372 (request->m_type == RubyRequestType_FLUSH));
373
374 //
375 // For Alpha, properly handle LL, SC, and write requests with respect to
376 // locked cache blocks.
377 //
378 // Not valid for Network_test protocl
379 //
380 bool success = true;
381 if(!m_usingNetworkTester)
382 success = handleLlsc(address, request);
383
384 if (request->m_type == RubyRequestType_Locked_RMW_Read) {
385 m_controller->blockOnQueue(address, m_mandatory_q_ptr);
386 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) {
387 m_controller->unblock(address);
388 }
389
390 hitCallback(request, data, success, mach, externalHit,
391 initialRequestTime, forwardRequestTime, firstResponseTime);
392}
393
394void
395Sequencer::readCallback(Addr address, DataBlock& data,
396 bool externalHit, const MachineType mach,
397 Cycles initialRequestTime,
398 Cycles forwardRequestTime,
399 Cycles firstResponseTime)
400{
401 assert(address == makeLineAddress(address));
402 assert(m_readRequestTable.count(makeLineAddress(address)));
403
404 RequestTable::iterator i = m_readRequestTable.find(address);
405 assert(i != m_readRequestTable.end());
406 SequencerRequest* request = i->second;
407
408 m_readRequestTable.erase(i);
409 markRemoved();
410
411 assert((request->m_type == RubyRequestType_LD) ||
412 (request->m_type == RubyRequestType_IFETCH));
413
414 hitCallback(request, data, true, mach, externalHit,
415 initialRequestTime, forwardRequestTime, firstResponseTime);
416}
417
418void
419Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data,
420 bool llscSuccess,
421 const MachineType mach, const bool externalHit,
422 const Cycles initialRequestTime,
423 const Cycles forwardRequestTime,
424 const Cycles firstResponseTime)
425{
426 warn_once("Replacement policy updates recently became the responsibility "
427 "of SLICC state machines. Make sure to setMRU() near callbacks "
428 "in .sm files!");
429
430 PacketPtr pkt = srequest->pkt;
431 Addr request_address(pkt->getAddr());
432 RubyRequestType type = srequest->m_type;
433 Cycles issued_time = srequest->issue_time;
434
435 assert(curCycle() >= issued_time);
436 Cycles total_latency = curCycle() - issued_time;
437
438 // Profile the latency for all demand accesses.
439 recordMissLatency(total_latency, type, mach, externalHit, issued_time,
440 initialRequestTime, forwardRequestTime,
441 firstResponseTime, curCycle());
442
443 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n",
444 curTick(), m_version, "Seq",
445 llscSuccess ? "Done" : "SC_Failed", "", "",
446 request_address, total_latency);
447
448 // update the data unless it is a non-data-carrying flush
449 if (RubySystem::getWarmupEnabled()) {
450 data.setData(pkt->getConstPtr<uint8_t>(),
451 getOffset(request_address), pkt->getSize());
452 } else if (!pkt->isFlush()) {
453 if ((type == RubyRequestType_LD) ||
454 (type == RubyRequestType_IFETCH) ||
455 (type == RubyRequestType_RMW_Read) ||
456 (type == RubyRequestType_Locked_RMW_Read) ||
457 (type == RubyRequestType_Load_Linked)) {
458 memcpy(pkt->getPtr<uint8_t>(),
459 data.getData(getOffset(request_address), pkt->getSize()),
460 pkt->getSize());
461 DPRINTF(RubySequencer, "read data %s\n", data);
462 } else {
463 data.setData(pkt->getConstPtr<uint8_t>(),
464 getOffset(request_address), pkt->getSize());
465 DPRINTF(RubySequencer, "set data %s\n", data);
466 }
467 }
468
469 // If using the RubyTester, update the RubyTester sender state's
470 // subBlock with the recieved data. The tester will later access
471 // this state.
472 if (m_usingRubyTester) {
473 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n",
474 pkt->cmdString(), pkt->getAddr());
475 RubyTester::SenderState* testerSenderState =
476 pkt->findNextSenderState<RubyTester::SenderState>();
477 assert(testerSenderState);
478 testerSenderState->subBlock.mergeFrom(data);
479 }
480
481 delete srequest;
482
483 RubySystem *rs = m_ruby_system;
484 if (RubySystem::getWarmupEnabled()) {
485 assert(pkt->req);
486 delete pkt->req;
487 delete pkt;
488 rs->m_cache_recorder->enqueueNextFetchRequest();
489 } else if (RubySystem::getCooldownEnabled()) {
490 delete pkt;
491 rs->m_cache_recorder->enqueueNextFlushRequest();
492 } else {
493 ruby_hit_callback(pkt);
494 }
495}
496
497bool
498Sequencer::empty() const
499{
500 return m_writeRequestTable.empty() && m_readRequestTable.empty();
501}
502
503RequestStatus
504Sequencer::makeRequest(PacketPtr pkt)
505{
506 if (m_outstanding_count >= m_max_outstanding_requests) {
507 return RequestStatus_BufferFull;
508 }
509
510 RubyRequestType primary_type = RubyRequestType_NULL;
511 RubyRequestType secondary_type = RubyRequestType_NULL;
512
513 if (pkt->isLLSC()) {
514 //
515 // Alpha LL/SC instructions need to be handled carefully by the cache
516 // coherence protocol to ensure they follow the proper semantics. In
517 // particular, by identifying the operations as atomic, the protocol
518 // should understand that migratory sharing optimizations should not
519 // be performed (i.e. a load between the LL and SC should not steal
520 // away exclusive permission).
521 //
522 if (pkt->isWrite()) {
523 DPRINTF(RubySequencer, "Issuing SC\n");
524 primary_type = RubyRequestType_Store_Conditional;
525 } else {
526 DPRINTF(RubySequencer, "Issuing LL\n");
527 assert(pkt->isRead());
528 primary_type = RubyRequestType_Load_Linked;
529 }
530 secondary_type = RubyRequestType_ATOMIC;
531 } else if (pkt->req->isLockedRMW()) {
532 //
533 // x86 locked instructions are translated to store cache coherence
534 // requests because these requests should always be treated as read
535 // exclusive operations and should leverage any migratory sharing
536 // optimization built into the protocol.
537 //
538 if (pkt->isWrite()) {
539 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n");
540 primary_type = RubyRequestType_Locked_RMW_Write;
541 } else {
542 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n");
543 assert(pkt->isRead());
544 primary_type = RubyRequestType_Locked_RMW_Read;
545 }
546 secondary_type = RubyRequestType_ST;
547 } else {
548 if (pkt->isRead()) {
549 if (pkt->req->isInstFetch()) {
550 primary_type = secondary_type = RubyRequestType_IFETCH;
551 } else {
552 bool storeCheck = false;
553 // only X86 need the store check
554 if (system->getArch() == Arch::X86ISA) {
555 uint32_t flags = pkt->req->getFlags();
556 storeCheck = flags &
557 (X86ISA::StoreCheck << X86ISA::FlagShift);
558 }
559 if (storeCheck) {
560 primary_type = RubyRequestType_RMW_Read;
561 secondary_type = RubyRequestType_ST;
562 } else {
563 primary_type = secondary_type = RubyRequestType_LD;
564 }
565 }
566 } else if (pkt->isWrite()) {
567 //
568 // Note: M5 packets do not differentiate ST from RMW_Write
569 //
570 primary_type = secondary_type = RubyRequestType_ST;
571 } else if (pkt->isFlush()) {
572 primary_type = secondary_type = RubyRequestType_FLUSH;
573 } else {
574 panic("Unsupported ruby packet type\n");
575 }
576 }
577
578 RequestStatus status = insertRequest(pkt, primary_type);
579 if (status != RequestStatus_Ready)
580 return status;
581
582 issueRequest(pkt, secondary_type);
583
584 // TODO: issue hardware prefetches here
585 return RequestStatus_Issued;
586}
587
588void
589Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type)
590{
591 assert(pkt != NULL);
592 ContextID proc_id = pkt->req->hasContextId() ?
593 pkt->req->contextId() : InvalidContextID;
594
595 // If valid, copy the pc to the ruby request
596 Addr pc = 0;
597 if (pkt->req->hasPC()) {
598 pc = pkt->req->getPC();
599 }
600
601 // check if the packet has data as for example prefetch and flush
602 // requests do not
603 std::shared_ptr<RubyRequest> msg =
604 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(),
605 pkt->isFlush() ?
606 nullptr : pkt->getPtr<uint8_t>(),
607 pkt->getSize(), pc, secondary_type,
608 RubyAccessMode_Supervisor, pkt,
609 PrefetchBit_No, proc_id);
610
611 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n",
612 curTick(), m_version, "Seq", "Begin", "", "",
613 msg->getPhysicalAddress(),
614 RubyRequestType_to_string(secondary_type));
615
616 // The Sequencer currently assesses instruction and data cache hit latency
617 // for the top-level caches at the beginning of a memory access.
618 // TODO: Eventually, this latency should be moved to represent the actual
619 // cache access latency portion of the memory access. This will require
620 // changing cache controller protocol files to assess the latency on the
621 // access response path.
622 Cycles latency(0); // Initialize to zero to catch misconfigured latency
623 if (secondary_type == RubyRequestType_IFETCH)
624 latency = m_inst_cache_hit_latency;
625 else
626 latency = m_data_cache_hit_latency;
627
628 // Send the message to the cache controller
629 assert(latency > 0);
630
631 assert(m_mandatory_q_ptr != NULL);
| 1/*
2 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include "arch/x86/ldstflags.hh"
30#include "base/misc.hh"
31#include "base/str.hh"
32#include "cpu/testers/rubytest/RubyTester.hh"
33#include "debug/MemoryAccess.hh"
34#include "debug/ProtocolTrace.hh"
35#include "debug/RubySequencer.hh"
36#include "debug/RubyStats.hh"
37#include "mem/protocol/PrefetchBit.hh"
38#include "mem/protocol/RubyAccessMode.hh"
39#include "mem/ruby/profiler/Profiler.hh"
40#include "mem/ruby/slicc_interface/RubyRequest.hh"
41#include "mem/ruby/system/RubySystem.hh"
42#include "mem/ruby/system/Sequencer.hh"
43#include "mem/packet.hh"
44#include "sim/system.hh"
45
46using namespace std;
47
48Sequencer *
49RubySequencerParams::create()
50{
51 return new Sequencer(this);
52}
53
54Sequencer::Sequencer(const Params *p)
55 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this)
56{
57 m_outstanding_count = 0;
58
59 m_instCache_ptr = p->icache;
60 m_dataCache_ptr = p->dcache;
61 m_data_cache_hit_latency = p->dcache_hit_latency;
62 m_inst_cache_hit_latency = p->icache_hit_latency;
63 m_max_outstanding_requests = p->max_outstanding_requests;
64 m_deadlock_threshold = p->deadlock_threshold;
65
66 assert(m_max_outstanding_requests > 0);
67 assert(m_deadlock_threshold > 0);
68 assert(m_instCache_ptr != NULL);
69 assert(m_dataCache_ptr != NULL);
70 assert(m_data_cache_hit_latency > 0);
71 assert(m_inst_cache_hit_latency > 0);
72
73 m_usingNetworkTester = p->using_network_tester;
74}
75
76Sequencer::~Sequencer()
77{
78}
79
80void
81Sequencer::wakeup()
82{
83 assert(drainState() != DrainState::Draining);
84
85 // Check for deadlock of any of the requests
86 Cycles current_time = curCycle();
87
88 // Check across all outstanding requests
89 int total_outstanding = 0;
90
91 RequestTable::iterator read = m_readRequestTable.begin();
92 RequestTable::iterator read_end = m_readRequestTable.end();
93 for (; read != read_end; ++read) {
94 SequencerRequest* request = read->second;
95 if (current_time - request->issue_time < m_deadlock_threshold)
96 continue;
97
98 panic("Possible Deadlock detected. Aborting!\n"
99 "version: %d request.paddr: 0x%x m_readRequestTable: %d "
100 "current time: %u issue_time: %d difference: %d\n", m_version,
101 request->pkt->getAddr(), m_readRequestTable.size(),
102 current_time * clockPeriod(), request->issue_time * clockPeriod(),
103 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
104 }
105
106 RequestTable::iterator write = m_writeRequestTable.begin();
107 RequestTable::iterator write_end = m_writeRequestTable.end();
108 for (; write != write_end; ++write) {
109 SequencerRequest* request = write->second;
110 if (current_time - request->issue_time < m_deadlock_threshold)
111 continue;
112
113 panic("Possible Deadlock detected. Aborting!\n"
114 "version: %d request.paddr: 0x%x m_writeRequestTable: %d "
115 "current time: %u issue_time: %d difference: %d\n", m_version,
116 request->pkt->getAddr(), m_writeRequestTable.size(),
117 current_time * clockPeriod(), request->issue_time * clockPeriod(),
118 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
119 }
120
121 total_outstanding += m_writeRequestTable.size();
122 total_outstanding += m_readRequestTable.size();
123
124 assert(m_outstanding_count == total_outstanding);
125
126 if (m_outstanding_count > 0) {
127 // If there are still outstanding requests, keep checking
128 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
129 }
130}
131
132void Sequencer::resetStats()
133{
134 m_latencyHist.reset();
135 m_hitLatencyHist.reset();
136 m_missLatencyHist.reset();
137 for (int i = 0; i < RubyRequestType_NUM; i++) {
138 m_typeLatencyHist[i]->reset();
139 m_hitTypeLatencyHist[i]->reset();
140 m_missTypeLatencyHist[i]->reset();
141 for (int j = 0; j < MachineType_NUM; j++) {
142 m_hitTypeMachLatencyHist[i][j]->reset();
143 m_missTypeMachLatencyHist[i][j]->reset();
144 }
145 }
146
147 for (int i = 0; i < MachineType_NUM; i++) {
148 m_missMachLatencyHist[i]->reset();
149 m_hitMachLatencyHist[i]->reset();
150
151 m_IssueToInitialDelayHist[i]->reset();
152 m_InitialToForwardDelayHist[i]->reset();
153 m_ForwardToFirstResponseDelayHist[i]->reset();
154 m_FirstResponseToCompletionDelayHist[i]->reset();
155
156 m_IncompleteTimes[i] = 0;
157 }
158}
159
160// Insert the request on the correct request table. Return true if
161// the entry was already present.
162RequestStatus
163Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type)
164{
165 assert(m_outstanding_count ==
166 (m_writeRequestTable.size() + m_readRequestTable.size()));
167
168 // See if we should schedule a deadlock check
169 if (!deadlockCheckEvent.scheduled() &&
170 drainState() != DrainState::Draining) {
171 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
172 }
173
174 Addr line_addr = makeLineAddress(pkt->getAddr());
175 // Create a default entry, mapping the address to NULL, the cast is
176 // there to make gcc 4.4 happy
177 RequestTable::value_type default_entry(line_addr,
178 (SequencerRequest*) NULL);
179
180 if ((request_type == RubyRequestType_ST) ||
181 (request_type == RubyRequestType_RMW_Read) ||
182 (request_type == RubyRequestType_RMW_Write) ||
183 (request_type == RubyRequestType_Load_Linked) ||
184 (request_type == RubyRequestType_Store_Conditional) ||
185 (request_type == RubyRequestType_Locked_RMW_Read) ||
186 (request_type == RubyRequestType_Locked_RMW_Write) ||
187 (request_type == RubyRequestType_FLUSH)) {
188
189 // Check if there is any outstanding read request for the same
190 // cache line.
191 if (m_readRequestTable.count(line_addr) > 0) {
192 m_store_waiting_on_load++;
193 return RequestStatus_Aliased;
194 }
195
196 pair<RequestTable::iterator, bool> r =
197 m_writeRequestTable.insert(default_entry);
198 if (r.second) {
199 RequestTable::iterator i = r.first;
200 i->second = new SequencerRequest(pkt, request_type, curCycle());
201 m_outstanding_count++;
202 } else {
203 // There is an outstanding write request for the cache line
204 m_store_waiting_on_store++;
205 return RequestStatus_Aliased;
206 }
207 } else {
208 // Check if there is any outstanding write request for the same
209 // cache line.
210 if (m_writeRequestTable.count(line_addr) > 0) {
211 m_load_waiting_on_store++;
212 return RequestStatus_Aliased;
213 }
214
215 pair<RequestTable::iterator, bool> r =
216 m_readRequestTable.insert(default_entry);
217
218 if (r.second) {
219 RequestTable::iterator i = r.first;
220 i->second = new SequencerRequest(pkt, request_type, curCycle());
221 m_outstanding_count++;
222 } else {
223 // There is an outstanding read request for the cache line
224 m_load_waiting_on_load++;
225 return RequestStatus_Aliased;
226 }
227 }
228
229 m_outstandReqHist.sample(m_outstanding_count);
230 assert(m_outstanding_count ==
231 (m_writeRequestTable.size() + m_readRequestTable.size()));
232
233 return RequestStatus_Ready;
234}
235
236void
237Sequencer::markRemoved()
238{
239 m_outstanding_count--;
240 assert(m_outstanding_count ==
241 m_writeRequestTable.size() + m_readRequestTable.size());
242}
243
244void
245Sequencer::invalidateSC(Addr address)
246{
247 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
248 // The controller has lost the coherence permissions, hence the lock
249 // on the cache line maintained by the cache should be cleared.
250 if (e && e->isLocked(m_version)) {
251 e->clearLocked();
252 }
253}
254
255bool
256Sequencer::handleLlsc(Addr address, SequencerRequest* request)
257{
258 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
259 if (!e)
260 return true;
261
262 // The success flag indicates whether the LLSC operation was successful.
263 // LL ops will always succeed, but SC may fail if the cache line is no
264 // longer locked.
265 bool success = true;
266 if (request->m_type == RubyRequestType_Store_Conditional) {
267 if (!e->isLocked(m_version)) {
268 //
269 // For failed SC requests, indicate the failure to the cpu by
270 // setting the extra data to zero.
271 //
272 request->pkt->req->setExtraData(0);
273 success = false;
274 } else {
275 //
276 // For successful SC requests, indicate the success to the cpu by
277 // setting the extra data to one.
278 //
279 request->pkt->req->setExtraData(1);
280 }
281 //
282 // Independent of success, all SC operations must clear the lock
283 //
284 e->clearLocked();
285 } else if (request->m_type == RubyRequestType_Load_Linked) {
286 //
287 // Note: To fully follow Alpha LLSC semantics, should the LL clear any
288 // previously locked cache lines?
289 //
290 e->setLocked(m_version);
291 } else if (e->isLocked(m_version)) {
292 //
293 // Normal writes should clear the locked address
294 //
295 e->clearLocked();
296 }
297 return success;
298}
299
300void
301Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type,
302 const MachineType respondingMach,
303 bool isExternalHit, Cycles issuedTime,
304 Cycles initialRequestTime,
305 Cycles forwardRequestTime,
306 Cycles firstResponseTime, Cycles completionTime)
307{
308 m_latencyHist.sample(cycles);
309 m_typeLatencyHist[type]->sample(cycles);
310
311 if (isExternalHit) {
312 m_missLatencyHist.sample(cycles);
313 m_missTypeLatencyHist[type]->sample(cycles);
314
315 if (respondingMach != MachineType_NUM) {
316 m_missMachLatencyHist[respondingMach]->sample(cycles);
317 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles);
318
319 if ((issuedTime <= initialRequestTime) &&
320 (initialRequestTime <= forwardRequestTime) &&
321 (forwardRequestTime <= firstResponseTime) &&
322 (firstResponseTime <= completionTime)) {
323
324 m_IssueToInitialDelayHist[respondingMach]->sample(
325 initialRequestTime - issuedTime);
326 m_InitialToForwardDelayHist[respondingMach]->sample(
327 forwardRequestTime - initialRequestTime);
328 m_ForwardToFirstResponseDelayHist[respondingMach]->sample(
329 firstResponseTime - forwardRequestTime);
330 m_FirstResponseToCompletionDelayHist[respondingMach]->sample(
331 completionTime - firstResponseTime);
332 } else {
333 m_IncompleteTimes[respondingMach]++;
334 }
335 }
336 } else {
337 m_hitLatencyHist.sample(cycles);
338 m_hitTypeLatencyHist[type]->sample(cycles);
339
340 if (respondingMach != MachineType_NUM) {
341 m_hitMachLatencyHist[respondingMach]->sample(cycles);
342 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles);
343 }
344 }
345}
346
347void
348Sequencer::writeCallback(Addr address, DataBlock& data,
349 const bool externalHit, const MachineType mach,
350 const Cycles initialRequestTime,
351 const Cycles forwardRequestTime,
352 const Cycles firstResponseTime)
353{
354 assert(address == makeLineAddress(address));
355 assert(m_writeRequestTable.count(makeLineAddress(address)));
356
357 RequestTable::iterator i = m_writeRequestTable.find(address);
358 assert(i != m_writeRequestTable.end());
359 SequencerRequest* request = i->second;
360
361 m_writeRequestTable.erase(i);
362 markRemoved();
363
364 assert((request->m_type == RubyRequestType_ST) ||
365 (request->m_type == RubyRequestType_ATOMIC) ||
366 (request->m_type == RubyRequestType_RMW_Read) ||
367 (request->m_type == RubyRequestType_RMW_Write) ||
368 (request->m_type == RubyRequestType_Load_Linked) ||
369 (request->m_type == RubyRequestType_Store_Conditional) ||
370 (request->m_type == RubyRequestType_Locked_RMW_Read) ||
371 (request->m_type == RubyRequestType_Locked_RMW_Write) ||
372 (request->m_type == RubyRequestType_FLUSH));
373
374 //
375 // For Alpha, properly handle LL, SC, and write requests with respect to
376 // locked cache blocks.
377 //
378 // Not valid for Network_test protocl
379 //
380 bool success = true;
381 if(!m_usingNetworkTester)
382 success = handleLlsc(address, request);
383
384 if (request->m_type == RubyRequestType_Locked_RMW_Read) {
385 m_controller->blockOnQueue(address, m_mandatory_q_ptr);
386 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) {
387 m_controller->unblock(address);
388 }
389
390 hitCallback(request, data, success, mach, externalHit,
391 initialRequestTime, forwardRequestTime, firstResponseTime);
392}
393
394void
395Sequencer::readCallback(Addr address, DataBlock& data,
396 bool externalHit, const MachineType mach,
397 Cycles initialRequestTime,
398 Cycles forwardRequestTime,
399 Cycles firstResponseTime)
400{
401 assert(address == makeLineAddress(address));
402 assert(m_readRequestTable.count(makeLineAddress(address)));
403
404 RequestTable::iterator i = m_readRequestTable.find(address);
405 assert(i != m_readRequestTable.end());
406 SequencerRequest* request = i->second;
407
408 m_readRequestTable.erase(i);
409 markRemoved();
410
411 assert((request->m_type == RubyRequestType_LD) ||
412 (request->m_type == RubyRequestType_IFETCH));
413
414 hitCallback(request, data, true, mach, externalHit,
415 initialRequestTime, forwardRequestTime, firstResponseTime);
416}
417
418void
419Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data,
420 bool llscSuccess,
421 const MachineType mach, const bool externalHit,
422 const Cycles initialRequestTime,
423 const Cycles forwardRequestTime,
424 const Cycles firstResponseTime)
425{
426 warn_once("Replacement policy updates recently became the responsibility "
427 "of SLICC state machines. Make sure to setMRU() near callbacks "
428 "in .sm files!");
429
430 PacketPtr pkt = srequest->pkt;
431 Addr request_address(pkt->getAddr());
432 RubyRequestType type = srequest->m_type;
433 Cycles issued_time = srequest->issue_time;
434
435 assert(curCycle() >= issued_time);
436 Cycles total_latency = curCycle() - issued_time;
437
438 // Profile the latency for all demand accesses.
439 recordMissLatency(total_latency, type, mach, externalHit, issued_time,
440 initialRequestTime, forwardRequestTime,
441 firstResponseTime, curCycle());
442
443 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n",
444 curTick(), m_version, "Seq",
445 llscSuccess ? "Done" : "SC_Failed", "", "",
446 request_address, total_latency);
447
448 // update the data unless it is a non-data-carrying flush
449 if (RubySystem::getWarmupEnabled()) {
450 data.setData(pkt->getConstPtr<uint8_t>(),
451 getOffset(request_address), pkt->getSize());
452 } else if (!pkt->isFlush()) {
453 if ((type == RubyRequestType_LD) ||
454 (type == RubyRequestType_IFETCH) ||
455 (type == RubyRequestType_RMW_Read) ||
456 (type == RubyRequestType_Locked_RMW_Read) ||
457 (type == RubyRequestType_Load_Linked)) {
458 memcpy(pkt->getPtr<uint8_t>(),
459 data.getData(getOffset(request_address), pkt->getSize()),
460 pkt->getSize());
461 DPRINTF(RubySequencer, "read data %s\n", data);
462 } else {
463 data.setData(pkt->getConstPtr<uint8_t>(),
464 getOffset(request_address), pkt->getSize());
465 DPRINTF(RubySequencer, "set data %s\n", data);
466 }
467 }
468
469 // If using the RubyTester, update the RubyTester sender state's
470 // subBlock with the recieved data. The tester will later access
471 // this state.
472 if (m_usingRubyTester) {
473 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n",
474 pkt->cmdString(), pkt->getAddr());
475 RubyTester::SenderState* testerSenderState =
476 pkt->findNextSenderState<RubyTester::SenderState>();
477 assert(testerSenderState);
478 testerSenderState->subBlock.mergeFrom(data);
479 }
480
481 delete srequest;
482
483 RubySystem *rs = m_ruby_system;
484 if (RubySystem::getWarmupEnabled()) {
485 assert(pkt->req);
486 delete pkt->req;
487 delete pkt;
488 rs->m_cache_recorder->enqueueNextFetchRequest();
489 } else if (RubySystem::getCooldownEnabled()) {
490 delete pkt;
491 rs->m_cache_recorder->enqueueNextFlushRequest();
492 } else {
493 ruby_hit_callback(pkt);
494 }
495}
496
497bool
498Sequencer::empty() const
499{
500 return m_writeRequestTable.empty() && m_readRequestTable.empty();
501}
502
503RequestStatus
504Sequencer::makeRequest(PacketPtr pkt)
505{
506 if (m_outstanding_count >= m_max_outstanding_requests) {
507 return RequestStatus_BufferFull;
508 }
509
510 RubyRequestType primary_type = RubyRequestType_NULL;
511 RubyRequestType secondary_type = RubyRequestType_NULL;
512
513 if (pkt->isLLSC()) {
514 //
515 // Alpha LL/SC instructions need to be handled carefully by the cache
516 // coherence protocol to ensure they follow the proper semantics. In
517 // particular, by identifying the operations as atomic, the protocol
518 // should understand that migratory sharing optimizations should not
519 // be performed (i.e. a load between the LL and SC should not steal
520 // away exclusive permission).
521 //
522 if (pkt->isWrite()) {
523 DPRINTF(RubySequencer, "Issuing SC\n");
524 primary_type = RubyRequestType_Store_Conditional;
525 } else {
526 DPRINTF(RubySequencer, "Issuing LL\n");
527 assert(pkt->isRead());
528 primary_type = RubyRequestType_Load_Linked;
529 }
530 secondary_type = RubyRequestType_ATOMIC;
531 } else if (pkt->req->isLockedRMW()) {
532 //
533 // x86 locked instructions are translated to store cache coherence
534 // requests because these requests should always be treated as read
535 // exclusive operations and should leverage any migratory sharing
536 // optimization built into the protocol.
537 //
538 if (pkt->isWrite()) {
539 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n");
540 primary_type = RubyRequestType_Locked_RMW_Write;
541 } else {
542 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n");
543 assert(pkt->isRead());
544 primary_type = RubyRequestType_Locked_RMW_Read;
545 }
546 secondary_type = RubyRequestType_ST;
547 } else {
548 if (pkt->isRead()) {
549 if (pkt->req->isInstFetch()) {
550 primary_type = secondary_type = RubyRequestType_IFETCH;
551 } else {
552 bool storeCheck = false;
553 // only X86 need the store check
554 if (system->getArch() == Arch::X86ISA) {
555 uint32_t flags = pkt->req->getFlags();
556 storeCheck = flags &
557 (X86ISA::StoreCheck << X86ISA::FlagShift);
558 }
559 if (storeCheck) {
560 primary_type = RubyRequestType_RMW_Read;
561 secondary_type = RubyRequestType_ST;
562 } else {
563 primary_type = secondary_type = RubyRequestType_LD;
564 }
565 }
566 } else if (pkt->isWrite()) {
567 //
568 // Note: M5 packets do not differentiate ST from RMW_Write
569 //
570 primary_type = secondary_type = RubyRequestType_ST;
571 } else if (pkt->isFlush()) {
572 primary_type = secondary_type = RubyRequestType_FLUSH;
573 } else {
574 panic("Unsupported ruby packet type\n");
575 }
576 }
577
578 RequestStatus status = insertRequest(pkt, primary_type);
579 if (status != RequestStatus_Ready)
580 return status;
581
582 issueRequest(pkt, secondary_type);
583
584 // TODO: issue hardware prefetches here
585 return RequestStatus_Issued;
586}
587
588void
589Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type)
590{
591 assert(pkt != NULL);
592 ContextID proc_id = pkt->req->hasContextId() ?
593 pkt->req->contextId() : InvalidContextID;
594
595 // If valid, copy the pc to the ruby request
596 Addr pc = 0;
597 if (pkt->req->hasPC()) {
598 pc = pkt->req->getPC();
599 }
600
601 // check if the packet has data as for example prefetch and flush
602 // requests do not
603 std::shared_ptr<RubyRequest> msg =
604 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(),
605 pkt->isFlush() ?
606 nullptr : pkt->getPtr<uint8_t>(),
607 pkt->getSize(), pc, secondary_type,
608 RubyAccessMode_Supervisor, pkt,
609 PrefetchBit_No, proc_id);
610
611 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n",
612 curTick(), m_version, "Seq", "Begin", "", "",
613 msg->getPhysicalAddress(),
614 RubyRequestType_to_string(secondary_type));
615
616 // The Sequencer currently assesses instruction and data cache hit latency
617 // for the top-level caches at the beginning of a memory access.
618 // TODO: Eventually, this latency should be moved to represent the actual
619 // cache access latency portion of the memory access. This will require
620 // changing cache controller protocol files to assess the latency on the
621 // access response path.
622 Cycles latency(0); // Initialize to zero to catch misconfigured latency
623 if (secondary_type == RubyRequestType_IFETCH)
624 latency = m_inst_cache_hit_latency;
625 else
626 latency = m_data_cache_hit_latency;
627
628 // Send the message to the cache controller
629 assert(latency > 0);
630
631 assert(m_mandatory_q_ptr != NULL);
|