1/*
2 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include "mem/ruby/system/Sequencer.hh"
30
31#include "arch/x86/ldstflags.hh"
32#include "base/misc.hh"
33#include "base/str.hh"
34#include "cpu/testers/rubytest/RubyTester.hh"
35#include "debug/MemoryAccess.hh"
36#include "debug/ProtocolTrace.hh"
37#include "debug/RubySequencer.hh"
38#include "debug/RubyStats.hh"
39#include "mem/packet.hh"
40#include "mem/protocol/PrefetchBit.hh"
41#include "mem/protocol/RubyAccessMode.hh"
42#include "mem/ruby/profiler/Profiler.hh"
43#include "mem/ruby/slicc_interface/RubyRequest.hh"
44#include "mem/ruby/system/RubySystem.hh"
45#include "sim/system.hh"
46
47using namespace std;
48
49Sequencer *
50RubySequencerParams::create()
51{
52 return new Sequencer(this);
53}
54
55Sequencer::Sequencer(const Params *p)
56 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this)
57{
58 m_outstanding_count = 0;
59
60 m_instCache_ptr = p->icache;
61 m_dataCache_ptr = p->dcache;
62 m_data_cache_hit_latency = p->dcache_hit_latency;
63 m_inst_cache_hit_latency = p->icache_hit_latency;
64 m_max_outstanding_requests = p->max_outstanding_requests;
65 m_deadlock_threshold = p->deadlock_threshold;
66
67 m_coreId = p->coreid; // for tracking the two CorePair sequencers
68 assert(m_max_outstanding_requests > 0);
69 assert(m_deadlock_threshold > 0);
70 assert(m_instCache_ptr != NULL);
71 assert(m_dataCache_ptr != NULL);
72 assert(m_data_cache_hit_latency > 0);
73 assert(m_inst_cache_hit_latency > 0);
74
75 m_runningGarnetStandalone = p->garnet_standalone;
76}
77
78Sequencer::~Sequencer()
79{
80}
81
82void
83Sequencer::wakeup()
84{
85 assert(drainState() != DrainState::Draining);
86
87 // Check for deadlock of any of the requests
88 Cycles current_time = curCycle();
89
90 // Check across all outstanding requests
91 int total_outstanding = 0;
92
93 RequestTable::iterator read = m_readRequestTable.begin();
94 RequestTable::iterator read_end = m_readRequestTable.end();
95 for (; read != read_end; ++read) {
96 SequencerRequest* request = read->second;
97 if (current_time - request->issue_time < m_deadlock_threshold)
98 continue;
99
100 panic("Possible Deadlock detected. Aborting!\n"
101 "version: %d request.paddr: 0x%x m_readRequestTable: %d "
102 "current time: %u issue_time: %d difference: %d\n", m_version,
103 request->pkt->getAddr(), m_readRequestTable.size(),
104 current_time * clockPeriod(), request->issue_time * clockPeriod(),
105 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
106 }
107
108 RequestTable::iterator write = m_writeRequestTable.begin();
109 RequestTable::iterator write_end = m_writeRequestTable.end();
110 for (; write != write_end; ++write) {
111 SequencerRequest* request = write->second;
112 if (current_time - request->issue_time < m_deadlock_threshold)
113 continue;
114
115 panic("Possible Deadlock detected. Aborting!\n"
116 "version: %d request.paddr: 0x%x m_writeRequestTable: %d "
117 "current time: %u issue_time: %d difference: %d\n", m_version,
118 request->pkt->getAddr(), m_writeRequestTable.size(),
119 current_time * clockPeriod(), request->issue_time * clockPeriod(),
120 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
121 }
122
123 total_outstanding += m_writeRequestTable.size();
124 total_outstanding += m_readRequestTable.size();
125
126 assert(m_outstanding_count == total_outstanding);
127
128 if (m_outstanding_count > 0) {
129 // If there are still outstanding requests, keep checking
130 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
131 }
132}
133
134void Sequencer::resetStats()
135{
136 m_latencyHist.reset();
137 m_hitLatencyHist.reset();
138 m_missLatencyHist.reset();
139 for (int i = 0; i < RubyRequestType_NUM; i++) {
140 m_typeLatencyHist[i]->reset();
141 m_hitTypeLatencyHist[i]->reset();
142 m_missTypeLatencyHist[i]->reset();
143 for (int j = 0; j < MachineType_NUM; j++) {
144 m_hitTypeMachLatencyHist[i][j]->reset();
145 m_missTypeMachLatencyHist[i][j]->reset();
146 }
147 }
148
149 for (int i = 0; i < MachineType_NUM; i++) {
150 m_missMachLatencyHist[i]->reset();
151 m_hitMachLatencyHist[i]->reset();
152
153 m_IssueToInitialDelayHist[i]->reset();
154 m_InitialToForwardDelayHist[i]->reset();
155 m_ForwardToFirstResponseDelayHist[i]->reset();
156 m_FirstResponseToCompletionDelayHist[i]->reset();
157
158 m_IncompleteTimes[i] = 0;
159 }
160}
161
162// Insert the request on the correct request table. Return true if
163// the entry was already present.
164RequestStatus
165Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type)
166{
167 assert(m_outstanding_count ==
168 (m_writeRequestTable.size() + m_readRequestTable.size()));
169
170 // See if we should schedule a deadlock check
171 if (!deadlockCheckEvent.scheduled() &&
172 drainState() != DrainState::Draining) {
173 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
174 }
175
176 Addr line_addr = makeLineAddress(pkt->getAddr());
177
178 // Check if the line is blocked for a Locked_RMW
179 if (m_controller->isBlocked(line_addr) &&
180 (request_type != RubyRequestType_Locked_RMW_Write)) {
181 // Return that this request's cache line address aliases with
182 // a prior request that locked the cache line. The request cannot
183 // proceed until the cache line is unlocked by a Locked_RMW_Write
184 return RequestStatus_Aliased;
185 }
186
187 // Create a default entry, mapping the address to NULL, the cast is
188 // there to make gcc 4.4 happy
189 RequestTable::value_type default_entry(line_addr,
190 (SequencerRequest*) NULL);
191
192 if ((request_type == RubyRequestType_ST) ||
193 (request_type == RubyRequestType_RMW_Read) ||
194 (request_type == RubyRequestType_RMW_Write) ||
195 (request_type == RubyRequestType_Load_Linked) ||
196 (request_type == RubyRequestType_Store_Conditional) ||
197 (request_type == RubyRequestType_Locked_RMW_Read) ||
198 (request_type == RubyRequestType_Locked_RMW_Write) ||
199 (request_type == RubyRequestType_FLUSH)) {
200
201 // Check if there is any outstanding read request for the same
202 // cache line.
203 if (m_readRequestTable.count(line_addr) > 0) {
204 m_store_waiting_on_load++;
205 return RequestStatus_Aliased;
206 }
207
208 pair<RequestTable::iterator, bool> r =
209 m_writeRequestTable.insert(default_entry);
210 if (r.second) {
211 RequestTable::iterator i = r.first;
212 i->second = new SequencerRequest(pkt, request_type, curCycle());
213 m_outstanding_count++;
214 } else {
215 // There is an outstanding write request for the cache line
216 m_store_waiting_on_store++;
217 return RequestStatus_Aliased;
218 }
219 } else {
220 // Check if there is any outstanding write request for the same
221 // cache line.
222 if (m_writeRequestTable.count(line_addr) > 0) {
223 m_load_waiting_on_store++;
224 return RequestStatus_Aliased;
225 }
226
227 pair<RequestTable::iterator, bool> r =
228 m_readRequestTable.insert(default_entry);
229
230 if (r.second) {
231 RequestTable::iterator i = r.first;
232 i->second = new SequencerRequest(pkt, request_type, curCycle());
233 m_outstanding_count++;
234 } else {
235 // There is an outstanding read request for the cache line
236 m_load_waiting_on_load++;
237 return RequestStatus_Aliased;
238 }
239 }
240
241 m_outstandReqHist.sample(m_outstanding_count);
242 assert(m_outstanding_count ==
243 (m_writeRequestTable.size() + m_readRequestTable.size()));
244
245 return RequestStatus_Ready;
246}
247
248void
249Sequencer::markRemoved()
250{
251 m_outstanding_count--;
252 assert(m_outstanding_count ==
253 m_writeRequestTable.size() + m_readRequestTable.size());
254}
255
256void
257Sequencer::invalidateSC(Addr address)
258{
259 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
260 // The controller has lost the coherence permissions, hence the lock
261 // on the cache line maintained by the cache should be cleared.
262 if (e && e->isLocked(m_version)) {
263 e->clearLocked();
264 }
265}
266
267bool
268Sequencer::handleLlsc(Addr address, SequencerRequest* request)
269{
270 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
271 if (!e)
272 return true;
273
274 // The success flag indicates whether the LLSC operation was successful.
275 // LL ops will always succeed, but SC may fail if the cache line is no
276 // longer locked.
277 bool success = true;
278 if (request->m_type == RubyRequestType_Store_Conditional) {
279 if (!e->isLocked(m_version)) {
280 //
281 // For failed SC requests, indicate the failure to the cpu by
282 // setting the extra data to zero.
283 //
284 request->pkt->req->setExtraData(0);
285 success = false;
286 } else {
287 //
288 // For successful SC requests, indicate the success to the cpu by
289 // setting the extra data to one.
290 //
291 request->pkt->req->setExtraData(1);
292 }
293 //
294 // Independent of success, all SC operations must clear the lock
295 //
296 e->clearLocked();
297 } else if (request->m_type == RubyRequestType_Load_Linked) {
298 //
299 // Note: To fully follow Alpha LLSC semantics, should the LL clear any
300 // previously locked cache lines?
301 //
302 e->setLocked(m_version);
303 } else if (e->isLocked(m_version)) {
304 //
305 // Normal writes should clear the locked address
306 //
307 e->clearLocked();
308 }
309 return success;
310}
311
312void
313Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type,
314 const MachineType respondingMach,
315 bool isExternalHit, Cycles issuedTime,
316 Cycles initialRequestTime,
317 Cycles forwardRequestTime,
318 Cycles firstResponseTime, Cycles completionTime)
319{
320 m_latencyHist.sample(cycles);
321 m_typeLatencyHist[type]->sample(cycles);
322
323 if (isExternalHit) {
324 m_missLatencyHist.sample(cycles);
325 m_missTypeLatencyHist[type]->sample(cycles);
326
327 if (respondingMach != MachineType_NUM) {
328 m_missMachLatencyHist[respondingMach]->sample(cycles);
329 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles);
330
331 if ((issuedTime <= initialRequestTime) &&
332 (initialRequestTime <= forwardRequestTime) &&
333 (forwardRequestTime <= firstResponseTime) &&
334 (firstResponseTime <= completionTime)) {
335
336 m_IssueToInitialDelayHist[respondingMach]->sample(
337 initialRequestTime - issuedTime);
338 m_InitialToForwardDelayHist[respondingMach]->sample(
339 forwardRequestTime - initialRequestTime);
340 m_ForwardToFirstResponseDelayHist[respondingMach]->sample(
341 firstResponseTime - forwardRequestTime);
342 m_FirstResponseToCompletionDelayHist[respondingMach]->sample(
343 completionTime - firstResponseTime);
344 } else {
345 m_IncompleteTimes[respondingMach]++;
346 }
347 }
348 } else {
349 m_hitLatencyHist.sample(cycles);
350 m_hitTypeLatencyHist[type]->sample(cycles);
351
352 if (respondingMach != MachineType_NUM) {
353 m_hitMachLatencyHist[respondingMach]->sample(cycles);
354 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles);
355 }
356 }
357}
358
359void
360Sequencer::writeCallback(Addr address, DataBlock& data,
361 const bool externalHit, const MachineType mach,
362 const Cycles initialRequestTime,
363 const Cycles forwardRequestTime,
364 const Cycles firstResponseTime)
365{
366 assert(address == makeLineAddress(address));
367 assert(m_writeRequestTable.count(makeLineAddress(address)));
368
369 RequestTable::iterator i = m_writeRequestTable.find(address);
370 assert(i != m_writeRequestTable.end());
371 SequencerRequest* request = i->second;
372
373 m_writeRequestTable.erase(i);
374 markRemoved();
375
376 assert((request->m_type == RubyRequestType_ST) ||
377 (request->m_type == RubyRequestType_ATOMIC) ||
378 (request->m_type == RubyRequestType_RMW_Read) ||
379 (request->m_type == RubyRequestType_RMW_Write) ||
380 (request->m_type == RubyRequestType_Load_Linked) ||
381 (request->m_type == RubyRequestType_Store_Conditional) ||
382 (request->m_type == RubyRequestType_Locked_RMW_Read) ||
383 (request->m_type == RubyRequestType_Locked_RMW_Write) ||
384 (request->m_type == RubyRequestType_FLUSH));
385
386 //
387 // For Alpha, properly handle LL, SC, and write requests with respect to
388 // locked cache blocks.
389 //
390 // Not valid for Garnet_standalone protocl
391 //
392 bool success = true;
393 if (!m_runningGarnetStandalone)
394 success = handleLlsc(address, request);
395
396 // Handle SLICC block_on behavior for Locked_RMW accesses. NOTE: the
397 // address variable here is assumed to be a line address, so when
398 // blocking buffers, must check line addresses.
399 if (request->m_type == RubyRequestType_Locked_RMW_Read) {
400 // blockOnQueue blocks all first-level cache controller queues
401 // waiting on memory accesses for the specified address that go to
402 // the specified queue. In this case, a Locked_RMW_Write must go to
403 // the mandatory_q before unblocking the first-level controller.
404 // This will block standard loads, stores, ifetches, etc.
405 m_controller->blockOnQueue(address, m_mandatory_q_ptr);
406 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) {
407 m_controller->unblock(address);
408 }
409
410 hitCallback(request, data, success, mach, externalHit,
411 initialRequestTime, forwardRequestTime, firstResponseTime);
412}
413
414void
415Sequencer::readCallback(Addr address, DataBlock& data,
416 bool externalHit, const MachineType mach,
417 Cycles initialRequestTime,
418 Cycles forwardRequestTime,
419 Cycles firstResponseTime)
420{
421 assert(address == makeLineAddress(address));
422 assert(m_readRequestTable.count(makeLineAddress(address)));
423
424 RequestTable::iterator i = m_readRequestTable.find(address);
425 assert(i != m_readRequestTable.end());
426 SequencerRequest* request = i->second;
427
428 m_readRequestTable.erase(i);
429 markRemoved();
430
431 assert((request->m_type == RubyRequestType_LD) ||
432 (request->m_type == RubyRequestType_IFETCH));
433
434 hitCallback(request, data, true, mach, externalHit,
435 initialRequestTime, forwardRequestTime, firstResponseTime);
436}
437
438void
439Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data,
440 bool llscSuccess,
441 const MachineType mach, const bool externalHit,
442 const Cycles initialRequestTime,
443 const Cycles forwardRequestTime,
444 const Cycles firstResponseTime)
445{
446 warn_once("Replacement policy updates recently became the responsibility "
447 "of SLICC state machines. Make sure to setMRU() near callbacks "
448 "in .sm files!");
449
450 PacketPtr pkt = srequest->pkt;
451 Addr request_address(pkt->getAddr());
452 RubyRequestType type = srequest->m_type;
453 Cycles issued_time = srequest->issue_time;
454
455 assert(curCycle() >= issued_time);
456 Cycles total_latency = curCycle() - issued_time;
457
458 // Profile the latency for all demand accesses.
459 recordMissLatency(total_latency, type, mach, externalHit, issued_time,
460 initialRequestTime, forwardRequestTime,
461 firstResponseTime, curCycle());
462
463 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n",
464 curTick(), m_version, "Seq",
465 llscSuccess ? "Done" : "SC_Failed", "", "",
466 printAddress(request_address), total_latency);
467
468 // update the data unless it is a non-data-carrying flush
469 if (RubySystem::getWarmupEnabled()) {
470 data.setData(pkt->getConstPtr<uint8_t>(),
471 getOffset(request_address), pkt->getSize());
472 } else if (!pkt->isFlush()) {
473 if ((type == RubyRequestType_LD) ||
474 (type == RubyRequestType_IFETCH) ||
475 (type == RubyRequestType_RMW_Read) ||
476 (type == RubyRequestType_Locked_RMW_Read) ||
477 (type == RubyRequestType_Load_Linked)) {
478 memcpy(pkt->getPtr<uint8_t>(),
479 data.getData(getOffset(request_address), pkt->getSize()),
480 pkt->getSize());
481 DPRINTF(RubySequencer, "read data %s\n", data);
482 } else if (pkt->req->isSwap()) {
483 std::vector<uint8_t> overwrite_val(pkt->getSize());
484 memcpy(&overwrite_val[0], pkt->getConstPtr<uint8_t>(),
485 pkt->getSize());
486 memcpy(pkt->getPtr<uint8_t>(),
487 data.getData(getOffset(request_address), pkt->getSize()),
488 pkt->getSize());
489 data.setData(&overwrite_val[0],
490 getOffset(request_address), pkt->getSize());
491 DPRINTF(RubySequencer, "swap data %s\n", data);
| 1/*
2 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include "mem/ruby/system/Sequencer.hh"
30
31#include "arch/x86/ldstflags.hh"
32#include "base/misc.hh"
33#include "base/str.hh"
34#include "cpu/testers/rubytest/RubyTester.hh"
35#include "debug/MemoryAccess.hh"
36#include "debug/ProtocolTrace.hh"
37#include "debug/RubySequencer.hh"
38#include "debug/RubyStats.hh"
39#include "mem/packet.hh"
40#include "mem/protocol/PrefetchBit.hh"
41#include "mem/protocol/RubyAccessMode.hh"
42#include "mem/ruby/profiler/Profiler.hh"
43#include "mem/ruby/slicc_interface/RubyRequest.hh"
44#include "mem/ruby/system/RubySystem.hh"
45#include "sim/system.hh"
46
47using namespace std;
48
49Sequencer *
50RubySequencerParams::create()
51{
52 return new Sequencer(this);
53}
54
55Sequencer::Sequencer(const Params *p)
56 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this)
57{
58 m_outstanding_count = 0;
59
60 m_instCache_ptr = p->icache;
61 m_dataCache_ptr = p->dcache;
62 m_data_cache_hit_latency = p->dcache_hit_latency;
63 m_inst_cache_hit_latency = p->icache_hit_latency;
64 m_max_outstanding_requests = p->max_outstanding_requests;
65 m_deadlock_threshold = p->deadlock_threshold;
66
67 m_coreId = p->coreid; // for tracking the two CorePair sequencers
68 assert(m_max_outstanding_requests > 0);
69 assert(m_deadlock_threshold > 0);
70 assert(m_instCache_ptr != NULL);
71 assert(m_dataCache_ptr != NULL);
72 assert(m_data_cache_hit_latency > 0);
73 assert(m_inst_cache_hit_latency > 0);
74
75 m_runningGarnetStandalone = p->garnet_standalone;
76}
77
78Sequencer::~Sequencer()
79{
80}
81
82void
83Sequencer::wakeup()
84{
85 assert(drainState() != DrainState::Draining);
86
87 // Check for deadlock of any of the requests
88 Cycles current_time = curCycle();
89
90 // Check across all outstanding requests
91 int total_outstanding = 0;
92
93 RequestTable::iterator read = m_readRequestTable.begin();
94 RequestTable::iterator read_end = m_readRequestTable.end();
95 for (; read != read_end; ++read) {
96 SequencerRequest* request = read->second;
97 if (current_time - request->issue_time < m_deadlock_threshold)
98 continue;
99
100 panic("Possible Deadlock detected. Aborting!\n"
101 "version: %d request.paddr: 0x%x m_readRequestTable: %d "
102 "current time: %u issue_time: %d difference: %d\n", m_version,
103 request->pkt->getAddr(), m_readRequestTable.size(),
104 current_time * clockPeriod(), request->issue_time * clockPeriod(),
105 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
106 }
107
108 RequestTable::iterator write = m_writeRequestTable.begin();
109 RequestTable::iterator write_end = m_writeRequestTable.end();
110 for (; write != write_end; ++write) {
111 SequencerRequest* request = write->second;
112 if (current_time - request->issue_time < m_deadlock_threshold)
113 continue;
114
115 panic("Possible Deadlock detected. Aborting!\n"
116 "version: %d request.paddr: 0x%x m_writeRequestTable: %d "
117 "current time: %u issue_time: %d difference: %d\n", m_version,
118 request->pkt->getAddr(), m_writeRequestTable.size(),
119 current_time * clockPeriod(), request->issue_time * clockPeriod(),
120 (current_time * clockPeriod()) - (request->issue_time * clockPeriod()));
121 }
122
123 total_outstanding += m_writeRequestTable.size();
124 total_outstanding += m_readRequestTable.size();
125
126 assert(m_outstanding_count == total_outstanding);
127
128 if (m_outstanding_count > 0) {
129 // If there are still outstanding requests, keep checking
130 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
131 }
132}
133
134void Sequencer::resetStats()
135{
136 m_latencyHist.reset();
137 m_hitLatencyHist.reset();
138 m_missLatencyHist.reset();
139 for (int i = 0; i < RubyRequestType_NUM; i++) {
140 m_typeLatencyHist[i]->reset();
141 m_hitTypeLatencyHist[i]->reset();
142 m_missTypeLatencyHist[i]->reset();
143 for (int j = 0; j < MachineType_NUM; j++) {
144 m_hitTypeMachLatencyHist[i][j]->reset();
145 m_missTypeMachLatencyHist[i][j]->reset();
146 }
147 }
148
149 for (int i = 0; i < MachineType_NUM; i++) {
150 m_missMachLatencyHist[i]->reset();
151 m_hitMachLatencyHist[i]->reset();
152
153 m_IssueToInitialDelayHist[i]->reset();
154 m_InitialToForwardDelayHist[i]->reset();
155 m_ForwardToFirstResponseDelayHist[i]->reset();
156 m_FirstResponseToCompletionDelayHist[i]->reset();
157
158 m_IncompleteTimes[i] = 0;
159 }
160}
161
162// Insert the request on the correct request table. Return true if
163// the entry was already present.
164RequestStatus
165Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type)
166{
167 assert(m_outstanding_count ==
168 (m_writeRequestTable.size() + m_readRequestTable.size()));
169
170 // See if we should schedule a deadlock check
171 if (!deadlockCheckEvent.scheduled() &&
172 drainState() != DrainState::Draining) {
173 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold));
174 }
175
176 Addr line_addr = makeLineAddress(pkt->getAddr());
177
178 // Check if the line is blocked for a Locked_RMW
179 if (m_controller->isBlocked(line_addr) &&
180 (request_type != RubyRequestType_Locked_RMW_Write)) {
181 // Return that this request's cache line address aliases with
182 // a prior request that locked the cache line. The request cannot
183 // proceed until the cache line is unlocked by a Locked_RMW_Write
184 return RequestStatus_Aliased;
185 }
186
187 // Create a default entry, mapping the address to NULL, the cast is
188 // there to make gcc 4.4 happy
189 RequestTable::value_type default_entry(line_addr,
190 (SequencerRequest*) NULL);
191
192 if ((request_type == RubyRequestType_ST) ||
193 (request_type == RubyRequestType_RMW_Read) ||
194 (request_type == RubyRequestType_RMW_Write) ||
195 (request_type == RubyRequestType_Load_Linked) ||
196 (request_type == RubyRequestType_Store_Conditional) ||
197 (request_type == RubyRequestType_Locked_RMW_Read) ||
198 (request_type == RubyRequestType_Locked_RMW_Write) ||
199 (request_type == RubyRequestType_FLUSH)) {
200
201 // Check if there is any outstanding read request for the same
202 // cache line.
203 if (m_readRequestTable.count(line_addr) > 0) {
204 m_store_waiting_on_load++;
205 return RequestStatus_Aliased;
206 }
207
208 pair<RequestTable::iterator, bool> r =
209 m_writeRequestTable.insert(default_entry);
210 if (r.second) {
211 RequestTable::iterator i = r.first;
212 i->second = new SequencerRequest(pkt, request_type, curCycle());
213 m_outstanding_count++;
214 } else {
215 // There is an outstanding write request for the cache line
216 m_store_waiting_on_store++;
217 return RequestStatus_Aliased;
218 }
219 } else {
220 // Check if there is any outstanding write request for the same
221 // cache line.
222 if (m_writeRequestTable.count(line_addr) > 0) {
223 m_load_waiting_on_store++;
224 return RequestStatus_Aliased;
225 }
226
227 pair<RequestTable::iterator, bool> r =
228 m_readRequestTable.insert(default_entry);
229
230 if (r.second) {
231 RequestTable::iterator i = r.first;
232 i->second = new SequencerRequest(pkt, request_type, curCycle());
233 m_outstanding_count++;
234 } else {
235 // There is an outstanding read request for the cache line
236 m_load_waiting_on_load++;
237 return RequestStatus_Aliased;
238 }
239 }
240
241 m_outstandReqHist.sample(m_outstanding_count);
242 assert(m_outstanding_count ==
243 (m_writeRequestTable.size() + m_readRequestTable.size()));
244
245 return RequestStatus_Ready;
246}
247
248void
249Sequencer::markRemoved()
250{
251 m_outstanding_count--;
252 assert(m_outstanding_count ==
253 m_writeRequestTable.size() + m_readRequestTable.size());
254}
255
256void
257Sequencer::invalidateSC(Addr address)
258{
259 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
260 // The controller has lost the coherence permissions, hence the lock
261 // on the cache line maintained by the cache should be cleared.
262 if (e && e->isLocked(m_version)) {
263 e->clearLocked();
264 }
265}
266
267bool
268Sequencer::handleLlsc(Addr address, SequencerRequest* request)
269{
270 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address);
271 if (!e)
272 return true;
273
274 // The success flag indicates whether the LLSC operation was successful.
275 // LL ops will always succeed, but SC may fail if the cache line is no
276 // longer locked.
277 bool success = true;
278 if (request->m_type == RubyRequestType_Store_Conditional) {
279 if (!e->isLocked(m_version)) {
280 //
281 // For failed SC requests, indicate the failure to the cpu by
282 // setting the extra data to zero.
283 //
284 request->pkt->req->setExtraData(0);
285 success = false;
286 } else {
287 //
288 // For successful SC requests, indicate the success to the cpu by
289 // setting the extra data to one.
290 //
291 request->pkt->req->setExtraData(1);
292 }
293 //
294 // Independent of success, all SC operations must clear the lock
295 //
296 e->clearLocked();
297 } else if (request->m_type == RubyRequestType_Load_Linked) {
298 //
299 // Note: To fully follow Alpha LLSC semantics, should the LL clear any
300 // previously locked cache lines?
301 //
302 e->setLocked(m_version);
303 } else if (e->isLocked(m_version)) {
304 //
305 // Normal writes should clear the locked address
306 //
307 e->clearLocked();
308 }
309 return success;
310}
311
312void
313Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type,
314 const MachineType respondingMach,
315 bool isExternalHit, Cycles issuedTime,
316 Cycles initialRequestTime,
317 Cycles forwardRequestTime,
318 Cycles firstResponseTime, Cycles completionTime)
319{
320 m_latencyHist.sample(cycles);
321 m_typeLatencyHist[type]->sample(cycles);
322
323 if (isExternalHit) {
324 m_missLatencyHist.sample(cycles);
325 m_missTypeLatencyHist[type]->sample(cycles);
326
327 if (respondingMach != MachineType_NUM) {
328 m_missMachLatencyHist[respondingMach]->sample(cycles);
329 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles);
330
331 if ((issuedTime <= initialRequestTime) &&
332 (initialRequestTime <= forwardRequestTime) &&
333 (forwardRequestTime <= firstResponseTime) &&
334 (firstResponseTime <= completionTime)) {
335
336 m_IssueToInitialDelayHist[respondingMach]->sample(
337 initialRequestTime - issuedTime);
338 m_InitialToForwardDelayHist[respondingMach]->sample(
339 forwardRequestTime - initialRequestTime);
340 m_ForwardToFirstResponseDelayHist[respondingMach]->sample(
341 firstResponseTime - forwardRequestTime);
342 m_FirstResponseToCompletionDelayHist[respondingMach]->sample(
343 completionTime - firstResponseTime);
344 } else {
345 m_IncompleteTimes[respondingMach]++;
346 }
347 }
348 } else {
349 m_hitLatencyHist.sample(cycles);
350 m_hitTypeLatencyHist[type]->sample(cycles);
351
352 if (respondingMach != MachineType_NUM) {
353 m_hitMachLatencyHist[respondingMach]->sample(cycles);
354 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles);
355 }
356 }
357}
358
359void
360Sequencer::writeCallback(Addr address, DataBlock& data,
361 const bool externalHit, const MachineType mach,
362 const Cycles initialRequestTime,
363 const Cycles forwardRequestTime,
364 const Cycles firstResponseTime)
365{
366 assert(address == makeLineAddress(address));
367 assert(m_writeRequestTable.count(makeLineAddress(address)));
368
369 RequestTable::iterator i = m_writeRequestTable.find(address);
370 assert(i != m_writeRequestTable.end());
371 SequencerRequest* request = i->second;
372
373 m_writeRequestTable.erase(i);
374 markRemoved();
375
376 assert((request->m_type == RubyRequestType_ST) ||
377 (request->m_type == RubyRequestType_ATOMIC) ||
378 (request->m_type == RubyRequestType_RMW_Read) ||
379 (request->m_type == RubyRequestType_RMW_Write) ||
380 (request->m_type == RubyRequestType_Load_Linked) ||
381 (request->m_type == RubyRequestType_Store_Conditional) ||
382 (request->m_type == RubyRequestType_Locked_RMW_Read) ||
383 (request->m_type == RubyRequestType_Locked_RMW_Write) ||
384 (request->m_type == RubyRequestType_FLUSH));
385
386 //
387 // For Alpha, properly handle LL, SC, and write requests with respect to
388 // locked cache blocks.
389 //
390 // Not valid for Garnet_standalone protocl
391 //
392 bool success = true;
393 if (!m_runningGarnetStandalone)
394 success = handleLlsc(address, request);
395
396 // Handle SLICC block_on behavior for Locked_RMW accesses. NOTE: the
397 // address variable here is assumed to be a line address, so when
398 // blocking buffers, must check line addresses.
399 if (request->m_type == RubyRequestType_Locked_RMW_Read) {
400 // blockOnQueue blocks all first-level cache controller queues
401 // waiting on memory accesses for the specified address that go to
402 // the specified queue. In this case, a Locked_RMW_Write must go to
403 // the mandatory_q before unblocking the first-level controller.
404 // This will block standard loads, stores, ifetches, etc.
405 m_controller->blockOnQueue(address, m_mandatory_q_ptr);
406 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) {
407 m_controller->unblock(address);
408 }
409
410 hitCallback(request, data, success, mach, externalHit,
411 initialRequestTime, forwardRequestTime, firstResponseTime);
412}
413
414void
415Sequencer::readCallback(Addr address, DataBlock& data,
416 bool externalHit, const MachineType mach,
417 Cycles initialRequestTime,
418 Cycles forwardRequestTime,
419 Cycles firstResponseTime)
420{
421 assert(address == makeLineAddress(address));
422 assert(m_readRequestTable.count(makeLineAddress(address)));
423
424 RequestTable::iterator i = m_readRequestTable.find(address);
425 assert(i != m_readRequestTable.end());
426 SequencerRequest* request = i->second;
427
428 m_readRequestTable.erase(i);
429 markRemoved();
430
431 assert((request->m_type == RubyRequestType_LD) ||
432 (request->m_type == RubyRequestType_IFETCH));
433
434 hitCallback(request, data, true, mach, externalHit,
435 initialRequestTime, forwardRequestTime, firstResponseTime);
436}
437
438void
439Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data,
440 bool llscSuccess,
441 const MachineType mach, const bool externalHit,
442 const Cycles initialRequestTime,
443 const Cycles forwardRequestTime,
444 const Cycles firstResponseTime)
445{
446 warn_once("Replacement policy updates recently became the responsibility "
447 "of SLICC state machines. Make sure to setMRU() near callbacks "
448 "in .sm files!");
449
450 PacketPtr pkt = srequest->pkt;
451 Addr request_address(pkt->getAddr());
452 RubyRequestType type = srequest->m_type;
453 Cycles issued_time = srequest->issue_time;
454
455 assert(curCycle() >= issued_time);
456 Cycles total_latency = curCycle() - issued_time;
457
458 // Profile the latency for all demand accesses.
459 recordMissLatency(total_latency, type, mach, externalHit, issued_time,
460 initialRequestTime, forwardRequestTime,
461 firstResponseTime, curCycle());
462
463 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n",
464 curTick(), m_version, "Seq",
465 llscSuccess ? "Done" : "SC_Failed", "", "",
466 printAddress(request_address), total_latency);
467
468 // update the data unless it is a non-data-carrying flush
469 if (RubySystem::getWarmupEnabled()) {
470 data.setData(pkt->getConstPtr<uint8_t>(),
471 getOffset(request_address), pkt->getSize());
472 } else if (!pkt->isFlush()) {
473 if ((type == RubyRequestType_LD) ||
474 (type == RubyRequestType_IFETCH) ||
475 (type == RubyRequestType_RMW_Read) ||
476 (type == RubyRequestType_Locked_RMW_Read) ||
477 (type == RubyRequestType_Load_Linked)) {
478 memcpy(pkt->getPtr<uint8_t>(),
479 data.getData(getOffset(request_address), pkt->getSize()),
480 pkt->getSize());
481 DPRINTF(RubySequencer, "read data %s\n", data);
482 } else if (pkt->req->isSwap()) {
483 std::vector<uint8_t> overwrite_val(pkt->getSize());
484 memcpy(&overwrite_val[0], pkt->getConstPtr<uint8_t>(),
485 pkt->getSize());
486 memcpy(pkt->getPtr<uint8_t>(),
487 data.getData(getOffset(request_address), pkt->getSize()),
488 pkt->getSize());
489 data.setData(&overwrite_val[0],
490 getOffset(request_address), pkt->getSize());
491 DPRINTF(RubySequencer, "swap data %s\n", data);
|
493 data.setData(pkt->getConstPtr<uint8_t>(),
494 getOffset(request_address), pkt->getSize());
495 DPRINTF(RubySequencer, "set data %s\n", data);
496 }
497 }
498
499 // If using the RubyTester, update the RubyTester sender state's
500 // subBlock with the recieved data. The tester will later access
501 // this state.
502 if (m_usingRubyTester) {
503 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n",
504 pkt->cmdString(), pkt->getAddr());
505 RubyTester::SenderState* testerSenderState =
506 pkt->findNextSenderState<RubyTester::SenderState>();
507 assert(testerSenderState);
508 testerSenderState->subBlock.mergeFrom(data);
509 }
510
511 delete srequest;
512
513 RubySystem *rs = m_ruby_system;
514 if (RubySystem::getWarmupEnabled()) {
515 assert(pkt->req);
516 delete pkt->req;
517 delete pkt;
518 rs->m_cache_recorder->enqueueNextFetchRequest();
519 } else if (RubySystem::getCooldownEnabled()) {
520 delete pkt;
521 rs->m_cache_recorder->enqueueNextFlushRequest();
522 } else {
523 ruby_hit_callback(pkt);
524 testDrainComplete();
525 }
526}
527
528bool
529Sequencer::empty() const
530{
531 return m_writeRequestTable.empty() && m_readRequestTable.empty();
532}
533
534RequestStatus
535Sequencer::makeRequest(PacketPtr pkt)
536{
537 if (m_outstanding_count >= m_max_outstanding_requests) {
538 return RequestStatus_BufferFull;
539 }
540
541 RubyRequestType primary_type = RubyRequestType_NULL;
542 RubyRequestType secondary_type = RubyRequestType_NULL;
543
544 if (pkt->isLLSC()) {
545 //
546 // Alpha LL/SC instructions need to be handled carefully by the cache
547 // coherence protocol to ensure they follow the proper semantics. In
548 // particular, by identifying the operations as atomic, the protocol
549 // should understand that migratory sharing optimizations should not
550 // be performed (i.e. a load between the LL and SC should not steal
551 // away exclusive permission).
552 //
553 if (pkt->isWrite()) {
554 DPRINTF(RubySequencer, "Issuing SC\n");
555 primary_type = RubyRequestType_Store_Conditional;
556 } else {
557 DPRINTF(RubySequencer, "Issuing LL\n");
558 assert(pkt->isRead());
559 primary_type = RubyRequestType_Load_Linked;
560 }
561 secondary_type = RubyRequestType_ATOMIC;
562 } else if (pkt->req->isLockedRMW()) {
563 //
564 // x86 locked instructions are translated to store cache coherence
565 // requests because these requests should always be treated as read
566 // exclusive operations and should leverage any migratory sharing
567 // optimization built into the protocol.
568 //
569 if (pkt->isWrite()) {
570 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n");
571 primary_type = RubyRequestType_Locked_RMW_Write;
572 } else {
573 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n");
574 assert(pkt->isRead());
575 primary_type = RubyRequestType_Locked_RMW_Read;
576 }
577 secondary_type = RubyRequestType_ST;
578 } else {
579 //
580 // To support SwapReq, we need to check isWrite() first: a SwapReq
581 // should always be treated like a write, but since a SwapReq implies
582 // both isWrite() and isRead() are true, check isWrite() first here.
583 //
584 if (pkt->isWrite()) {
585 //
586 // Note: M5 packets do not differentiate ST from RMW_Write
587 //
588 primary_type = secondary_type = RubyRequestType_ST;
589 } else if (pkt->isRead()) {
590 if (pkt->req->isInstFetch()) {
591 primary_type = secondary_type = RubyRequestType_IFETCH;
592 } else {
593 bool storeCheck = false;
594 // only X86 need the store check
595 if (system->getArch() == Arch::X86ISA) {
596 uint32_t flags = pkt->req->getFlags();
597 storeCheck = flags &
598 (X86ISA::StoreCheck << X86ISA::FlagShift);
599 }
600 if (storeCheck) {
601 primary_type = RubyRequestType_RMW_Read;
602 secondary_type = RubyRequestType_ST;
603 } else {
604 primary_type = secondary_type = RubyRequestType_LD;
605 }
606 }
607 } else if (pkt->isFlush()) {
608 primary_type = secondary_type = RubyRequestType_FLUSH;
609 } else {
610 panic("Unsupported ruby packet type\n");
611 }
612 }
613
614 RequestStatus status = insertRequest(pkt, primary_type);
615 if (status != RequestStatus_Ready)
616 return status;
617
618 issueRequest(pkt, secondary_type);
619
620 // TODO: issue hardware prefetches here
621 return RequestStatus_Issued;
622}
623
624void
625Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type)
626{
627 assert(pkt != NULL);
628 ContextID proc_id = pkt->req->hasContextId() ?
629 pkt->req->contextId() : InvalidContextID;
630
631 ContextID core_id = coreId();
632
633 // If valid, copy the pc to the ruby request
634 Addr pc = 0;
635 if (pkt->req->hasPC()) {
636 pc = pkt->req->getPC();
637 }
638
639 // check if the packet has data as for example prefetch and flush
640 // requests do not
641 std::shared_ptr<RubyRequest> msg =
642 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(),
643 pkt->isFlush() ?
644 nullptr : pkt->getPtr<uint8_t>(),
645 pkt->getSize(), pc, secondary_type,
646 RubyAccessMode_Supervisor, pkt,
647 PrefetchBit_No, proc_id, core_id);
648
649 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n",
650 curTick(), m_version, "Seq", "Begin", "", "",
651 printAddress(msg->getPhysicalAddress()),
652 RubyRequestType_to_string(secondary_type));
653
654 // The Sequencer currently assesses instruction and data cache hit latency
655 // for the top-level caches at the beginning of a memory access.
656 // TODO: Eventually, this latency should be moved to represent the actual
657 // cache access latency portion of the memory access. This will require
658 // changing cache controller protocol files to assess the latency on the
659 // access response path.
660 Cycles latency(0); // Initialize to zero to catch misconfigured latency
661 if (secondary_type == RubyRequestType_IFETCH)
662 latency = m_inst_cache_hit_latency;
663 else
664 latency = m_data_cache_hit_latency;
665
666 // Send the message to the cache controller
667 assert(latency > 0);
668
669 assert(m_mandatory_q_ptr != NULL);
670 m_mandatory_q_ptr->enqueue(msg, clockEdge(), cyclesToTicks(latency));
671}
672
673template <class KEY, class VALUE>
674std::ostream &
675operator<<(ostream &out, const std::unordered_map<KEY, VALUE> &map)
676{
677 auto i = map.begin();
678 auto end = map.end();
679
680 out << "[";
681 for (; i != end; ++i)
682 out << " " << i->first << "=" << i->second;
683 out << " ]";
684
685 return out;
686}
687
688void
689Sequencer::print(ostream& out) const
690{
691 out << "[Sequencer: " << m_version
692 << ", outstanding requests: " << m_outstanding_count
693 << ", read request table: " << m_readRequestTable
694 << ", write request table: " << m_writeRequestTable
695 << "]";
696}
697
698// this can be called from setState whenever coherence permissions are
699// upgraded when invoked, coherence violations will be checked for the
700// given block
701void
702Sequencer::checkCoherence(Addr addr)
703{
704#ifdef CHECK_COHERENCE
705 m_ruby_system->checkGlobalCoherenceInvariant(addr);
706#endif
707}
708
709void
710Sequencer::recordRequestType(SequencerRequestType requestType) {
711 DPRINTF(RubyStats, "Recorded statistic: %s\n",
712 SequencerRequestType_to_string(requestType));
713}
714
715
716void
717Sequencer::evictionCallback(Addr address)
718{
719 ruby_eviction_callback(address);
720}
721
722void
723Sequencer::regStats()
724{
725 RubyPort::regStats();
726
727 m_store_waiting_on_load
728 .name(name() + ".store_waiting_on_load")
729 .desc("Number of times a store aliased with a pending load")
730 .flags(Stats::nozero);
731 m_store_waiting_on_store
732 .name(name() + ".store_waiting_on_store")
733 .desc("Number of times a store aliased with a pending store")
734 .flags(Stats::nozero);
735 m_load_waiting_on_load
736 .name(name() + ".load_waiting_on_load")
737 .desc("Number of times a load aliased with a pending load")
738 .flags(Stats::nozero);
739 m_load_waiting_on_store
740 .name(name() + ".load_waiting_on_store")
741 .desc("Number of times a load aliased with a pending store")
742 .flags(Stats::nozero);
743
744 // These statistical variables are not for display.
745 // The profiler will collate these across different
746 // sequencers and display those collated statistics.
747 m_outstandReqHist.init(10);
748 m_latencyHist.init(10);
749 m_hitLatencyHist.init(10);
750 m_missLatencyHist.init(10);
751
752 for (int i = 0; i < RubyRequestType_NUM; i++) {
753 m_typeLatencyHist.push_back(new Stats::Histogram());
754 m_typeLatencyHist[i]->init(10);
755
756 m_hitTypeLatencyHist.push_back(new Stats::Histogram());
757 m_hitTypeLatencyHist[i]->init(10);
758
759 m_missTypeLatencyHist.push_back(new Stats::Histogram());
760 m_missTypeLatencyHist[i]->init(10);
761 }
762
763 for (int i = 0; i < MachineType_NUM; i++) {
764 m_hitMachLatencyHist.push_back(new Stats::Histogram());
765 m_hitMachLatencyHist[i]->init(10);
766
767 m_missMachLatencyHist.push_back(new Stats::Histogram());
768 m_missMachLatencyHist[i]->init(10);
769
770 m_IssueToInitialDelayHist.push_back(new Stats::Histogram());
771 m_IssueToInitialDelayHist[i]->init(10);
772
773 m_InitialToForwardDelayHist.push_back(new Stats::Histogram());
774 m_InitialToForwardDelayHist[i]->init(10);
775
776 m_ForwardToFirstResponseDelayHist.push_back(new Stats::Histogram());
777 m_ForwardToFirstResponseDelayHist[i]->init(10);
778
779 m_FirstResponseToCompletionDelayHist.push_back(new Stats::Histogram());
780 m_FirstResponseToCompletionDelayHist[i]->init(10);
781 }
782
783 for (int i = 0; i < RubyRequestType_NUM; i++) {
784 m_hitTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>());
785 m_missTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>());
786
787 for (int j = 0; j < MachineType_NUM; j++) {
788 m_hitTypeMachLatencyHist[i].push_back(new Stats::Histogram());
789 m_hitTypeMachLatencyHist[i][j]->init(10);
790
791 m_missTypeMachLatencyHist[i].push_back(new Stats::Histogram());
792 m_missTypeMachLatencyHist[i][j]->init(10);
793 }
794 }
795}
| 495 data.setData(pkt->getConstPtr<uint8_t>(),
496 getOffset(request_address), pkt->getSize());
497 DPRINTF(RubySequencer, "set data %s\n", data);
498 }
499 }
500
501 // If using the RubyTester, update the RubyTester sender state's
502 // subBlock with the recieved data. The tester will later access
503 // this state.
504 if (m_usingRubyTester) {
505 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n",
506 pkt->cmdString(), pkt->getAddr());
507 RubyTester::SenderState* testerSenderState =
508 pkt->findNextSenderState<RubyTester::SenderState>();
509 assert(testerSenderState);
510 testerSenderState->subBlock.mergeFrom(data);
511 }
512
513 delete srequest;
514
515 RubySystem *rs = m_ruby_system;
516 if (RubySystem::getWarmupEnabled()) {
517 assert(pkt->req);
518 delete pkt->req;
519 delete pkt;
520 rs->m_cache_recorder->enqueueNextFetchRequest();
521 } else if (RubySystem::getCooldownEnabled()) {
522 delete pkt;
523 rs->m_cache_recorder->enqueueNextFlushRequest();
524 } else {
525 ruby_hit_callback(pkt);
526 testDrainComplete();
527 }
528}
529
530bool
531Sequencer::empty() const
532{
533 return m_writeRequestTable.empty() && m_readRequestTable.empty();
534}
535
536RequestStatus
537Sequencer::makeRequest(PacketPtr pkt)
538{
539 if (m_outstanding_count >= m_max_outstanding_requests) {
540 return RequestStatus_BufferFull;
541 }
542
543 RubyRequestType primary_type = RubyRequestType_NULL;
544 RubyRequestType secondary_type = RubyRequestType_NULL;
545
546 if (pkt->isLLSC()) {
547 //
548 // Alpha LL/SC instructions need to be handled carefully by the cache
549 // coherence protocol to ensure they follow the proper semantics. In
550 // particular, by identifying the operations as atomic, the protocol
551 // should understand that migratory sharing optimizations should not
552 // be performed (i.e. a load between the LL and SC should not steal
553 // away exclusive permission).
554 //
555 if (pkt->isWrite()) {
556 DPRINTF(RubySequencer, "Issuing SC\n");
557 primary_type = RubyRequestType_Store_Conditional;
558 } else {
559 DPRINTF(RubySequencer, "Issuing LL\n");
560 assert(pkt->isRead());
561 primary_type = RubyRequestType_Load_Linked;
562 }
563 secondary_type = RubyRequestType_ATOMIC;
564 } else if (pkt->req->isLockedRMW()) {
565 //
566 // x86 locked instructions are translated to store cache coherence
567 // requests because these requests should always be treated as read
568 // exclusive operations and should leverage any migratory sharing
569 // optimization built into the protocol.
570 //
571 if (pkt->isWrite()) {
572 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n");
573 primary_type = RubyRequestType_Locked_RMW_Write;
574 } else {
575 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n");
576 assert(pkt->isRead());
577 primary_type = RubyRequestType_Locked_RMW_Read;
578 }
579 secondary_type = RubyRequestType_ST;
580 } else {
581 //
582 // To support SwapReq, we need to check isWrite() first: a SwapReq
583 // should always be treated like a write, but since a SwapReq implies
584 // both isWrite() and isRead() are true, check isWrite() first here.
585 //
586 if (pkt->isWrite()) {
587 //
588 // Note: M5 packets do not differentiate ST from RMW_Write
589 //
590 primary_type = secondary_type = RubyRequestType_ST;
591 } else if (pkt->isRead()) {
592 if (pkt->req->isInstFetch()) {
593 primary_type = secondary_type = RubyRequestType_IFETCH;
594 } else {
595 bool storeCheck = false;
596 // only X86 need the store check
597 if (system->getArch() == Arch::X86ISA) {
598 uint32_t flags = pkt->req->getFlags();
599 storeCheck = flags &
600 (X86ISA::StoreCheck << X86ISA::FlagShift);
601 }
602 if (storeCheck) {
603 primary_type = RubyRequestType_RMW_Read;
604 secondary_type = RubyRequestType_ST;
605 } else {
606 primary_type = secondary_type = RubyRequestType_LD;
607 }
608 }
609 } else if (pkt->isFlush()) {
610 primary_type = secondary_type = RubyRequestType_FLUSH;
611 } else {
612 panic("Unsupported ruby packet type\n");
613 }
614 }
615
616 RequestStatus status = insertRequest(pkt, primary_type);
617 if (status != RequestStatus_Ready)
618 return status;
619
620 issueRequest(pkt, secondary_type);
621
622 // TODO: issue hardware prefetches here
623 return RequestStatus_Issued;
624}
625
626void
627Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type)
628{
629 assert(pkt != NULL);
630 ContextID proc_id = pkt->req->hasContextId() ?
631 pkt->req->contextId() : InvalidContextID;
632
633 ContextID core_id = coreId();
634
635 // If valid, copy the pc to the ruby request
636 Addr pc = 0;
637 if (pkt->req->hasPC()) {
638 pc = pkt->req->getPC();
639 }
640
641 // check if the packet has data as for example prefetch and flush
642 // requests do not
643 std::shared_ptr<RubyRequest> msg =
644 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(),
645 pkt->isFlush() ?
646 nullptr : pkt->getPtr<uint8_t>(),
647 pkt->getSize(), pc, secondary_type,
648 RubyAccessMode_Supervisor, pkt,
649 PrefetchBit_No, proc_id, core_id);
650
651 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n",
652 curTick(), m_version, "Seq", "Begin", "", "",
653 printAddress(msg->getPhysicalAddress()),
654 RubyRequestType_to_string(secondary_type));
655
656 // The Sequencer currently assesses instruction and data cache hit latency
657 // for the top-level caches at the beginning of a memory access.
658 // TODO: Eventually, this latency should be moved to represent the actual
659 // cache access latency portion of the memory access. This will require
660 // changing cache controller protocol files to assess the latency on the
661 // access response path.
662 Cycles latency(0); // Initialize to zero to catch misconfigured latency
663 if (secondary_type == RubyRequestType_IFETCH)
664 latency = m_inst_cache_hit_latency;
665 else
666 latency = m_data_cache_hit_latency;
667
668 // Send the message to the cache controller
669 assert(latency > 0);
670
671 assert(m_mandatory_q_ptr != NULL);
672 m_mandatory_q_ptr->enqueue(msg, clockEdge(), cyclesToTicks(latency));
673}
674
675template <class KEY, class VALUE>
676std::ostream &
677operator<<(ostream &out, const std::unordered_map<KEY, VALUE> &map)
678{
679 auto i = map.begin();
680 auto end = map.end();
681
682 out << "[";
683 for (; i != end; ++i)
684 out << " " << i->first << "=" << i->second;
685 out << " ]";
686
687 return out;
688}
689
690void
691Sequencer::print(ostream& out) const
692{
693 out << "[Sequencer: " << m_version
694 << ", outstanding requests: " << m_outstanding_count
695 << ", read request table: " << m_readRequestTable
696 << ", write request table: " << m_writeRequestTable
697 << "]";
698}
699
700// this can be called from setState whenever coherence permissions are
701// upgraded when invoked, coherence violations will be checked for the
702// given block
703void
704Sequencer::checkCoherence(Addr addr)
705{
706#ifdef CHECK_COHERENCE
707 m_ruby_system->checkGlobalCoherenceInvariant(addr);
708#endif
709}
710
711void
712Sequencer::recordRequestType(SequencerRequestType requestType) {
713 DPRINTF(RubyStats, "Recorded statistic: %s\n",
714 SequencerRequestType_to_string(requestType));
715}
716
717
718void
719Sequencer::evictionCallback(Addr address)
720{
721 ruby_eviction_callback(address);
722}
723
724void
725Sequencer::regStats()
726{
727 RubyPort::regStats();
728
729 m_store_waiting_on_load
730 .name(name() + ".store_waiting_on_load")
731 .desc("Number of times a store aliased with a pending load")
732 .flags(Stats::nozero);
733 m_store_waiting_on_store
734 .name(name() + ".store_waiting_on_store")
735 .desc("Number of times a store aliased with a pending store")
736 .flags(Stats::nozero);
737 m_load_waiting_on_load
738 .name(name() + ".load_waiting_on_load")
739 .desc("Number of times a load aliased with a pending load")
740 .flags(Stats::nozero);
741 m_load_waiting_on_store
742 .name(name() + ".load_waiting_on_store")
743 .desc("Number of times a load aliased with a pending store")
744 .flags(Stats::nozero);
745
746 // These statistical variables are not for display.
747 // The profiler will collate these across different
748 // sequencers and display those collated statistics.
749 m_outstandReqHist.init(10);
750 m_latencyHist.init(10);
751 m_hitLatencyHist.init(10);
752 m_missLatencyHist.init(10);
753
754 for (int i = 0; i < RubyRequestType_NUM; i++) {
755 m_typeLatencyHist.push_back(new Stats::Histogram());
756 m_typeLatencyHist[i]->init(10);
757
758 m_hitTypeLatencyHist.push_back(new Stats::Histogram());
759 m_hitTypeLatencyHist[i]->init(10);
760
761 m_missTypeLatencyHist.push_back(new Stats::Histogram());
762 m_missTypeLatencyHist[i]->init(10);
763 }
764
765 for (int i = 0; i < MachineType_NUM; i++) {
766 m_hitMachLatencyHist.push_back(new Stats::Histogram());
767 m_hitMachLatencyHist[i]->init(10);
768
769 m_missMachLatencyHist.push_back(new Stats::Histogram());
770 m_missMachLatencyHist[i]->init(10);
771
772 m_IssueToInitialDelayHist.push_back(new Stats::Histogram());
773 m_IssueToInitialDelayHist[i]->init(10);
774
775 m_InitialToForwardDelayHist.push_back(new Stats::Histogram());
776 m_InitialToForwardDelayHist[i]->init(10);
777
778 m_ForwardToFirstResponseDelayHist.push_back(new Stats::Histogram());
779 m_ForwardToFirstResponseDelayHist[i]->init(10);
780
781 m_FirstResponseToCompletionDelayHist.push_back(new Stats::Histogram());
782 m_FirstResponseToCompletionDelayHist[i]->init(10);
783 }
784
785 for (int i = 0; i < RubyRequestType_NUM; i++) {
786 m_hitTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>());
787 m_missTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>());
788
789 for (int j = 0; j < MachineType_NUM; j++) {
790 m_hitTypeMachLatencyHist[i].push_back(new Stats::Histogram());
791 m_hitTypeMachLatencyHist[i][j]->init(10);
792
793 m_missTypeMachLatencyHist[i].push_back(new Stats::Histogram());
794 m_missTypeMachLatencyHist[i][j]->init(10);
795 }
796 }
797}
|