1/*
2 * Copyright (c) 2013-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include "cpu/minor/lsq.hh"
41
42#include <iomanip>
43#include <sstream>
44
45#include "arch/locked_mem.hh"
46#include "arch/mmapped_ipr.hh"
47#include "cpu/minor/cpu.hh"
48#include "cpu/minor/exec_context.hh"
49#include "cpu/minor/execute.hh"
50#include "cpu/minor/pipeline.hh"
51#include "debug/Activity.hh"
52#include "debug/MinorMem.hh"
53
54namespace Minor
55{
56
57/** Returns the offset of addr into an aligned a block of size block_size */
58static Addr
59addrBlockOffset(Addr addr, unsigned int block_size)
60{
61 return addr & (block_size - 1);
62}
63
64/** Returns true if the given [addr .. addr+size-1] transfer needs to be
65 * fragmented across a block size of block_size */
66static bool
67transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
68{
69 return (addrBlockOffset(addr, block_size) + size) > block_size;
70}
71
72LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
73 PacketDataPtr data_, uint64_t *res_) :
74 SenderState(),
75 port(port_),
76 inst(inst_),
77 isLoad(isLoad_),
78 data(data_),
79 packet(NULL),
80 request(),
81 fault(NoFault),
82 res(res_),
83 skipped(false),
84 issuedToMemory(false),
85 state(NotIssued)
86{ }
87
88LSQ::AddrRangeCoverage
89LSQ::LSQRequest::containsAddrRangeOf(
90 Addr req1_addr, unsigned int req1_size,
91 Addr req2_addr, unsigned int req2_size)
92{
93 /* 'end' here means the address of the byte just past the request
94 * blocks */
95 Addr req2_end_addr = req2_addr + req2_size;
96 Addr req1_end_addr = req1_addr + req1_size;
97
98 AddrRangeCoverage ret;
99
100 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
101 ret = NoAddrRangeCoverage;
102 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
103 ret = FullAddrRangeCoverage;
104 else
105 ret = PartialAddrRangeCoverage;
106
107 return ret;
108}
109
110LSQ::AddrRangeCoverage
111LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
112{
113 return containsAddrRangeOf(request.getPaddr(), request.getSize(),
114 other_request->request.getPaddr(), other_request->request.getSize());
115}
116
117bool
118LSQ::LSQRequest::isBarrier()
119{
120 return inst->isInst() && inst->staticInst->isMemBarrier();
121}
122
123bool
124LSQ::LSQRequest::needsToBeSentToStoreBuffer()
125{
126 return state == StoreToStoreBuffer;
127}
128
129void
130LSQ::LSQRequest::setState(LSQRequestState new_state)
131{
132 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
133 " %s\n", state, new_state, *inst);
134 state = new_state;
135}
136
137bool
138LSQ::LSQRequest::isComplete() const
139{
140 /* @todo, There is currently only one 'completed' state. This
141 * may not be a good choice */
142 return state == Complete;
143}
144
145void
146LSQ::LSQRequest::reportData(std::ostream &os) const
147{
148 os << (isLoad ? 'R' : 'W') << ';';
149 inst->reportData(os);
150 os << ';' << state;
151}
152
153std::ostream &
154operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
155{
156 switch (coverage) {
157 case LSQ::PartialAddrRangeCoverage:
158 os << "PartialAddrRangeCoverage";
159 break;
160 case LSQ::FullAddrRangeCoverage:
161 os << "FullAddrRangeCoverage";
162 break;
163 case LSQ::NoAddrRangeCoverage:
164 os << "NoAddrRangeCoverage";
165 break;
166 default:
167 os << "AddrRangeCoverage-" << static_cast<int>(coverage);
168 break;
169 }
170 return os;
171}
172
173std::ostream &
174operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
175{
176 switch (state) {
177 case LSQ::LSQRequest::NotIssued:
178 os << "NotIssued";
179 break;
180 case LSQ::LSQRequest::InTranslation:
181 os << "InTranslation";
182 break;
183 case LSQ::LSQRequest::Translated:
184 os << "Translated";
185 break;
186 case LSQ::LSQRequest::Failed:
187 os << "Failed";
188 break;
189 case LSQ::LSQRequest::RequestIssuing:
190 os << "RequestIssuing";
191 break;
192 case LSQ::LSQRequest::StoreToStoreBuffer:
193 os << "StoreToStoreBuffer";
194 break;
195 case LSQ::LSQRequest::StoreInStoreBuffer:
196 os << "StoreInStoreBuffer";
197 break;
198 case LSQ::LSQRequest::StoreBufferIssuing:
199 os << "StoreBufferIssuing";
200 break;
201 case LSQ::LSQRequest::RequestNeedsRetry:
202 os << "RequestNeedsRetry";
203 break;
204 case LSQ::LSQRequest::StoreBufferNeedsRetry:
205 os << "StoreBufferNeedsRetry";
206 break;
207 case LSQ::LSQRequest::Complete:
208 os << "Complete";
209 break;
210 default:
211 os << "LSQRequestState-" << static_cast<int>(state);
212 break;
213 }
214 return os;
215}
216
217void
218LSQ::clearMemBarrier(MinorDynInstPtr inst)
219{
220 bool is_last_barrier =
221 inst->id.execSeqNum >= lastMemBarrier[inst->id.threadId];
222
223 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
224 (is_last_barrier ? "last" : "a"), *inst);
225
226 if (is_last_barrier)
227 lastMemBarrier[inst->id.threadId] = 0;
228}
229
230void
231LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_,
232 ThreadContext *tc, BaseTLB::Mode mode)
233{
234 fault = fault_;
235
236 port.numAccessesInDTLB--;
237
238 DPRINTFS(MinorMem, (&port), "Received translation response for"
239 " request: %s\n", *inst);
240
241 makePacket();
242
243 setState(Translated);
244 port.tryToSendToTransfers(this);
245
246 /* Let's try and wake up the processor for the next cycle */
247 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
248}
249
250void
251LSQ::SingleDataRequest::startAddrTranslation()
252{
253 ThreadContext *thread = port.cpu.getContext(
254 inst->id.threadId);
255
256 port.numAccessesInDTLB++;
257
258 setState(LSQ::LSQRequest::InTranslation);
259
260 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
261 /* Submit the translation request. The response will come through
262 * finish/markDelayed on the LSQRequest as it bears the Translation
263 * interface */
264 thread->getDTBPtr()->translateTiming(
265 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
266}
267
268void
269LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
270{
271 DPRINTFS(MinorMem, (&port), "Retiring packet\n");
272 packet = packet_;
273 packetInFlight = false;
274 setState(Complete);
275}
276
277void
278LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_,
279 ThreadContext *tc, BaseTLB::Mode mode)
280{
281 fault = fault_;
282
283 port.numAccessesInDTLB--;
284
285 unsigned int M5_VAR_USED expected_fragment_index =
286 numTranslatedFragments;
287
288 numInTranslationFragments--;
289 numTranslatedFragments++;
290
291 DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
292 " %d of request: %s\n", expected_fragment_index, *inst);
293
294 assert(request_ == fragmentRequests[expected_fragment_index]);
295
296 /* Wake up next cycle to get things going again in case the
297 * tryToSendToTransfers does take */
298 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
299
300 if (fault != NoFault) {
301 /* tryToSendToTransfers will handle the fault */
302
303 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
304 " %d of request: %s\n",
305 expected_fragment_index, *inst);
306
307 setState(Translated);
308 port.tryToSendToTransfers(this);
309 } else if (numTranslatedFragments == numFragments) {
310 makeFragmentPackets();
311
312 setState(Translated);
313 port.tryToSendToTransfers(this);
314 } else {
315 /* Avoid calling translateTiming from within ::finish */
316 assert(!translationEvent.scheduled());
317 port.cpu.schedule(translationEvent, curTick());
318 }
319}
320
321LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
322 bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
323 LSQRequest(port_, inst_, isLoad_, data_, res_),
| 1/*
2 * Copyright (c) 2013-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include "cpu/minor/lsq.hh"
41
42#include <iomanip>
43#include <sstream>
44
45#include "arch/locked_mem.hh"
46#include "arch/mmapped_ipr.hh"
47#include "cpu/minor/cpu.hh"
48#include "cpu/minor/exec_context.hh"
49#include "cpu/minor/execute.hh"
50#include "cpu/minor/pipeline.hh"
51#include "debug/Activity.hh"
52#include "debug/MinorMem.hh"
53
54namespace Minor
55{
56
57/** Returns the offset of addr into an aligned a block of size block_size */
58static Addr
59addrBlockOffset(Addr addr, unsigned int block_size)
60{
61 return addr & (block_size - 1);
62}
63
64/** Returns true if the given [addr .. addr+size-1] transfer needs to be
65 * fragmented across a block size of block_size */
66static bool
67transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
68{
69 return (addrBlockOffset(addr, block_size) + size) > block_size;
70}
71
72LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
73 PacketDataPtr data_, uint64_t *res_) :
74 SenderState(),
75 port(port_),
76 inst(inst_),
77 isLoad(isLoad_),
78 data(data_),
79 packet(NULL),
80 request(),
81 fault(NoFault),
82 res(res_),
83 skipped(false),
84 issuedToMemory(false),
85 state(NotIssued)
86{ }
87
88LSQ::AddrRangeCoverage
89LSQ::LSQRequest::containsAddrRangeOf(
90 Addr req1_addr, unsigned int req1_size,
91 Addr req2_addr, unsigned int req2_size)
92{
93 /* 'end' here means the address of the byte just past the request
94 * blocks */
95 Addr req2_end_addr = req2_addr + req2_size;
96 Addr req1_end_addr = req1_addr + req1_size;
97
98 AddrRangeCoverage ret;
99
100 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
101 ret = NoAddrRangeCoverage;
102 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
103 ret = FullAddrRangeCoverage;
104 else
105 ret = PartialAddrRangeCoverage;
106
107 return ret;
108}
109
110LSQ::AddrRangeCoverage
111LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
112{
113 return containsAddrRangeOf(request.getPaddr(), request.getSize(),
114 other_request->request.getPaddr(), other_request->request.getSize());
115}
116
117bool
118LSQ::LSQRequest::isBarrier()
119{
120 return inst->isInst() && inst->staticInst->isMemBarrier();
121}
122
123bool
124LSQ::LSQRequest::needsToBeSentToStoreBuffer()
125{
126 return state == StoreToStoreBuffer;
127}
128
129void
130LSQ::LSQRequest::setState(LSQRequestState new_state)
131{
132 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
133 " %s\n", state, new_state, *inst);
134 state = new_state;
135}
136
137bool
138LSQ::LSQRequest::isComplete() const
139{
140 /* @todo, There is currently only one 'completed' state. This
141 * may not be a good choice */
142 return state == Complete;
143}
144
145void
146LSQ::LSQRequest::reportData(std::ostream &os) const
147{
148 os << (isLoad ? 'R' : 'W') << ';';
149 inst->reportData(os);
150 os << ';' << state;
151}
152
153std::ostream &
154operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
155{
156 switch (coverage) {
157 case LSQ::PartialAddrRangeCoverage:
158 os << "PartialAddrRangeCoverage";
159 break;
160 case LSQ::FullAddrRangeCoverage:
161 os << "FullAddrRangeCoverage";
162 break;
163 case LSQ::NoAddrRangeCoverage:
164 os << "NoAddrRangeCoverage";
165 break;
166 default:
167 os << "AddrRangeCoverage-" << static_cast<int>(coverage);
168 break;
169 }
170 return os;
171}
172
173std::ostream &
174operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
175{
176 switch (state) {
177 case LSQ::LSQRequest::NotIssued:
178 os << "NotIssued";
179 break;
180 case LSQ::LSQRequest::InTranslation:
181 os << "InTranslation";
182 break;
183 case LSQ::LSQRequest::Translated:
184 os << "Translated";
185 break;
186 case LSQ::LSQRequest::Failed:
187 os << "Failed";
188 break;
189 case LSQ::LSQRequest::RequestIssuing:
190 os << "RequestIssuing";
191 break;
192 case LSQ::LSQRequest::StoreToStoreBuffer:
193 os << "StoreToStoreBuffer";
194 break;
195 case LSQ::LSQRequest::StoreInStoreBuffer:
196 os << "StoreInStoreBuffer";
197 break;
198 case LSQ::LSQRequest::StoreBufferIssuing:
199 os << "StoreBufferIssuing";
200 break;
201 case LSQ::LSQRequest::RequestNeedsRetry:
202 os << "RequestNeedsRetry";
203 break;
204 case LSQ::LSQRequest::StoreBufferNeedsRetry:
205 os << "StoreBufferNeedsRetry";
206 break;
207 case LSQ::LSQRequest::Complete:
208 os << "Complete";
209 break;
210 default:
211 os << "LSQRequestState-" << static_cast<int>(state);
212 break;
213 }
214 return os;
215}
216
217void
218LSQ::clearMemBarrier(MinorDynInstPtr inst)
219{
220 bool is_last_barrier =
221 inst->id.execSeqNum >= lastMemBarrier[inst->id.threadId];
222
223 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
224 (is_last_barrier ? "last" : "a"), *inst);
225
226 if (is_last_barrier)
227 lastMemBarrier[inst->id.threadId] = 0;
228}
229
230void
231LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_,
232 ThreadContext *tc, BaseTLB::Mode mode)
233{
234 fault = fault_;
235
236 port.numAccessesInDTLB--;
237
238 DPRINTFS(MinorMem, (&port), "Received translation response for"
239 " request: %s\n", *inst);
240
241 makePacket();
242
243 setState(Translated);
244 port.tryToSendToTransfers(this);
245
246 /* Let's try and wake up the processor for the next cycle */
247 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
248}
249
250void
251LSQ::SingleDataRequest::startAddrTranslation()
252{
253 ThreadContext *thread = port.cpu.getContext(
254 inst->id.threadId);
255
256 port.numAccessesInDTLB++;
257
258 setState(LSQ::LSQRequest::InTranslation);
259
260 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
261 /* Submit the translation request. The response will come through
262 * finish/markDelayed on the LSQRequest as it bears the Translation
263 * interface */
264 thread->getDTBPtr()->translateTiming(
265 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
266}
267
268void
269LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
270{
271 DPRINTFS(MinorMem, (&port), "Retiring packet\n");
272 packet = packet_;
273 packetInFlight = false;
274 setState(Complete);
275}
276
277void
278LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_,
279 ThreadContext *tc, BaseTLB::Mode mode)
280{
281 fault = fault_;
282
283 port.numAccessesInDTLB--;
284
285 unsigned int M5_VAR_USED expected_fragment_index =
286 numTranslatedFragments;
287
288 numInTranslationFragments--;
289 numTranslatedFragments++;
290
291 DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
292 " %d of request: %s\n", expected_fragment_index, *inst);
293
294 assert(request_ == fragmentRequests[expected_fragment_index]);
295
296 /* Wake up next cycle to get things going again in case the
297 * tryToSendToTransfers does take */
298 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
299
300 if (fault != NoFault) {
301 /* tryToSendToTransfers will handle the fault */
302
303 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
304 " %d of request: %s\n",
305 expected_fragment_index, *inst);
306
307 setState(Translated);
308 port.tryToSendToTransfers(this);
309 } else if (numTranslatedFragments == numFragments) {
310 makeFragmentPackets();
311
312 setState(Translated);
313 port.tryToSendToTransfers(this);
314 } else {
315 /* Avoid calling translateTiming from within ::finish */
316 assert(!translationEvent.scheduled());
317 port.cpu.schedule(translationEvent, curTick());
318 }
319}
320
321LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
322 bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
323 LSQRequest(port_, inst_, isLoad_, data_, res_),
|
325 numFragments(0),
326 numInTranslationFragments(0),
327 numTranslatedFragments(0),
328 numIssuedFragments(0),
329 numRetiredFragments(0),
330 fragmentRequests(),
331 fragmentPackets()
332{
333 /* Don't know how many elements are needed until the request is
334 * populated by the caller. */
335}
336
337LSQ::SplitDataRequest::~SplitDataRequest()
338{
339 for (auto i = fragmentRequests.begin();
340 i != fragmentRequests.end(); i++)
341 {
342 delete *i;
343 }
344
345 for (auto i = fragmentPackets.begin();
346 i != fragmentPackets.end(); i++)
347 {
348 delete *i;
349 }
350}
351
352void
353LSQ::SplitDataRequest::makeFragmentRequests()
354{
355 Addr base_addr = request.getVaddr();
356 unsigned int whole_size = request.getSize();
357 unsigned int line_width = port.lineWidth;
358
359 unsigned int fragment_size;
360 Addr fragment_addr;
361
362 /* Assume that this transfer is across potentially many block snap
363 * boundaries:
364 *
365 * | _|________|________|________|___ |
366 * | |0| 1 | 2 | 3 | 4 | |
367 * | |_|________|________|________|___| |
368 * | | | | | |
369 *
370 * The first transfer (0) can be up to lineWidth in size.
371 * All the middle transfers (1-3) are lineWidth in size
372 * The last transfer (4) can be from zero to lineWidth - 1 in size
373 */
374 unsigned int first_fragment_offset =
375 addrBlockOffset(base_addr, line_width);
376 unsigned int last_fragment_size =
377 addrBlockOffset(base_addr + whole_size, line_width);
378 unsigned int first_fragment_size =
379 line_width - first_fragment_offset;
380
381 unsigned int middle_fragments_total_size =
382 whole_size - (first_fragment_size + last_fragment_size);
383
384 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
385
386 unsigned int middle_fragment_count =
387 middle_fragments_total_size / line_width;
388
389 numFragments = 1 /* first */ + middle_fragment_count +
390 (last_fragment_size == 0 ? 0 : 1);
391
392 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
393 " First fragment size: %d Last fragment size: %d\n",
394 numFragments, first_fragment_size,
395 (last_fragment_size == 0 ? line_width : last_fragment_size));
396
397 assert(((middle_fragment_count * line_width) +
398 first_fragment_size + last_fragment_size) == whole_size);
399
400 fragment_addr = base_addr;
401 fragment_size = first_fragment_size;
402
403 /* Just past the last address in the request */
404 Addr end_addr = base_addr + whole_size;
405
406 for (unsigned int fragment_index = 0; fragment_index < numFragments;
407 fragment_index++)
408 {
409 bool M5_VAR_USED is_last_fragment = false;
410
411 if (fragment_addr == base_addr) {
412 /* First fragment */
413 fragment_size = first_fragment_size;
414 } else {
415 if ((fragment_addr + line_width) > end_addr) {
416 /* Adjust size of last fragment */
417 fragment_size = end_addr - fragment_addr;
418 is_last_fragment = true;
419 } else {
420 /* Middle fragments */
421 fragment_size = line_width;
422 }
423 }
424
425 Request *fragment = new Request();
426
427 fragment->setContext(request.contextId());
428 fragment->setVirt(0 /* asid */,
429 fragment_addr, fragment_size, request.getFlags(),
430 request.masterId(),
431 request.getPC());
432
433 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
434 " (whole request addr: 0x%x size: %d) %s\n",
435 fragment_addr, fragment_size, base_addr, whole_size,
436 (is_last_fragment ? "last fragment" : ""));
437
438 fragment_addr += fragment_size;
439
440 fragmentRequests.push_back(fragment);
441 }
442}
443
444void
445LSQ::SplitDataRequest::makeFragmentPackets()
446{
447 Addr base_addr = request.getVaddr();
448
449 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
450
451 for (unsigned int fragment_index = 0; fragment_index < numFragments;
452 fragment_index++)
453 {
454 Request *fragment = fragmentRequests[fragment_index];
455
456 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
457 " (%d, 0x%x)\n",
458 fragment_index, *inst,
459 (fragment->hasPaddr() ? "has paddr" : "no paddr"),
460 (fragment->hasPaddr() ? fragment->getPaddr() : 0));
461
462 Addr fragment_addr = fragment->getVaddr();
463 unsigned int fragment_size = fragment->getSize();
464
465 uint8_t *request_data = NULL;
466
467 if (!isLoad) {
468 /* Split data for Packets. Will become the property of the
469 * outgoing Packets */
470 request_data = new uint8_t[fragment_size];
471 std::memcpy(request_data, data + (fragment_addr - base_addr),
472 fragment_size);
473 }
474
475 assert(fragment->hasPaddr());
476
477 PacketPtr fragment_packet =
478 makePacketForRequest(*fragment, isLoad, this, request_data);
479
480 fragmentPackets.push_back(fragment_packet);
481 /* Accumulate flags in parent request */
482 request.setFlags(fragment->getFlags());
483 }
484
485 /* Might as well make the overall/response packet here */
486 /* Get the physical address for the whole request/packet from the first
487 * fragment */
488 request.setPaddr(fragmentRequests[0]->getPaddr());
489 makePacket();
490}
491
492void
493LSQ::SplitDataRequest::startAddrTranslation()
494{
495 setState(LSQ::LSQRequest::InTranslation);
496
497 makeFragmentRequests();
498
499 numInTranslationFragments = 0;
500 numTranslatedFragments = 0;
501
502 /* @todo, just do these in sequence for now with
503 * a loop of:
504 * do {
505 * sendNextFragmentToTranslation ; translateTiming ; finish
506 * } while (numTranslatedFragments != numFragments);
507 */
508
509 /* Do first translation */
510 sendNextFragmentToTranslation();
511}
512
513PacketPtr
514LSQ::SplitDataRequest::getHeadPacket()
515{
516 assert(numIssuedFragments < numFragments);
517
518 return fragmentPackets[numIssuedFragments];
519}
520
521void
522LSQ::SplitDataRequest::stepToNextPacket()
523{
524 assert(numIssuedFragments < numFragments);
525
526 numIssuedFragments++;
527}
528
529void
530LSQ::SplitDataRequest::retireResponse(PacketPtr response)
531{
532 assert(numRetiredFragments < numFragments);
533
534 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
535 " offset: 0x%x (retired fragment num: %d) %s\n",
536 response->req->getVaddr(), response->req->getSize(),
537 request.getVaddr() - response->req->getVaddr(),
538 numRetiredFragments,
539 (fault == NoFault ? "" : fault->name()));
540
541 numRetiredFragments++;
542
543 if (skipped) {
544 /* Skip because we already knew the request had faulted or been
545 * skipped */
546 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
547 } else if (response->isError()) {
548 /* Mark up the error and leave to execute to handle it */
549 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
550 setSkipped();
551 packet->copyError(response);
552 } else {
553 if (isLoad) {
554 if (!data) {
555 /* For a split transfer, a Packet must be constructed
556 * to contain all returning data. This is that packet's
557 * data */
558 data = new uint8_t[request.getSize()];
559 }
560
561 /* Populate the portion of the overall response data represented
562 * by the response fragment */
563 std::memcpy(
564 data + (response->req->getVaddr() - request.getVaddr()),
565 response->getConstPtr<uint8_t>(),
566 response->req->getSize());
567 }
568 }
569
570 /* Complete early if we're skipping are no more in-flight accesses */
571 if (skipped && !hasPacketsInMemSystem()) {
572 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
573 setState(Complete);
574 if (packet->needsResponse())
575 packet->makeResponse();
576 }
577
578 if (numRetiredFragments == numFragments)
579 setState(Complete);
580
581 if (!skipped && isComplete()) {
582 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
583
584 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
585 " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
586 " %s\n", packet->isRead(), packet->isWrite(),
587 packet->needsResponse(), packet->getSize(), request.getSize(),
588 response->getSize());
589
590 /* A request can become complete by several paths, this is a sanity
591 * check to make sure the packet's data is created */
592 if (!data) {
593 data = new uint8_t[request.getSize()];
594 }
595
596 if (isLoad) {
597 DPRINTFS(MinorMem, (&port), "Copying read data\n");
598 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
599 }
600 packet->makeResponse();
601 }
602
603 /* Packets are all deallocated together in ~SplitLSQRequest */
604}
605
606void
607LSQ::SplitDataRequest::sendNextFragmentToTranslation()
608{
609 unsigned int fragment_index = numTranslatedFragments;
610
611 ThreadContext *thread = port.cpu.getContext(
612 inst->id.threadId);
613
614 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
615 fragment_index);
616
617 port.numAccessesInDTLB++;
618 numInTranslationFragments++;
619
620 thread->getDTBPtr()->translateTiming(
621 fragmentRequests[fragment_index], thread, this, (isLoad ?
622 BaseTLB::Read : BaseTLB::Write));
623}
624
625bool
626LSQ::StoreBuffer::canInsert() const
627{
628 /* @todo, support store amalgamation */
629 return slots.size() < numSlots;
630}
631
632void
633LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
634{
635 auto found = std::find(slots.begin(), slots.end(), request);
636
637 if (found != slots.end()) {
638 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
639 request, *found, *(request->inst));
640 slots.erase(found);
641
642 delete request;
643 }
644}
645
646void
647LSQ::StoreBuffer::insert(LSQRequestPtr request)
648{
649 if (!canInsert()) {
650 warn("%s: store buffer insertion without space to insert from"
651 " inst: %s\n", name(), *(request->inst));
652 }
653
654 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
655
656 numUnissuedAccesses++;
657
658 if (request->state != LSQRequest::Complete)
659 request->setState(LSQRequest::StoreInStoreBuffer);
660
661 slots.push_back(request);
662
663 /* Let's try and wake up the processor for the next cycle to step
664 * the store buffer */
665 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
666}
667
668LSQ::AddrRangeCoverage
669LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
670 unsigned int &found_slot)
671{
672 unsigned int slot_index = slots.size() - 1;
673 auto i = slots.rbegin();
674 AddrRangeCoverage ret = NoAddrRangeCoverage;
675
676 /* Traverse the store buffer in reverse order (most to least recent)
677 * and try to find a slot whose address range overlaps this request */
678 while (ret == NoAddrRangeCoverage && i != slots.rend()) {
679 LSQRequestPtr slot = *i;
680
681 if (slot->packet &&
682 slot->inst->id.threadId == request->inst->id.threadId) {
683 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
684
685 if (coverage != NoAddrRangeCoverage) {
686 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
687 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
688 slot_index, coverage,
689 request->request.getPaddr(), request->request.getSize(),
690 slot->request.getPaddr(), slot->request.getSize());
691
692 found_slot = slot_index;
693 ret = coverage;
694 }
695 }
696
697 i++;
698 slot_index--;
699 }
700
701 return ret;
702}
703
704/** Fill the given packet with appropriate date from slot slot_number */
705void
706LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
707 unsigned int slot_number)
708{
709 assert(slot_number < slots.size());
710 assert(load->packet);
711 assert(load->isLoad);
712
713 LSQRequestPtr store = slots[slot_number];
714
715 assert(store->packet);
716 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
717
718 Addr load_addr = load->request.getPaddr();
719 Addr store_addr = store->request.getPaddr();
720 Addr addr_offset = load_addr - store_addr;
721
722 unsigned int load_size = load->request.getSize();
723
724 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
725 " slot: %d addr: 0x%x addressOffset: 0x%x\n",
726 load_size, load_addr, slot_number,
727 store_addr, addr_offset);
728
729 void *load_packet_data = load->packet->getPtr<void>();
730 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
731
732 std::memcpy(load_packet_data, store_packet_data, load_size);
733}
734
735void
736LSQ::StoreBuffer::countIssuedStore(LSQRequestPtr request)
737{
738 /* Barriers are accounted for as they are cleared from
739 * the queue, not after their transfers are complete */
740 if (!request->isBarrier())
741 numUnissuedAccesses--;
742}
743
744void
745LSQ::StoreBuffer::step()
746{
747 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
748 numUnissuedAccesses);
749
750 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
751 /* Clear all the leading barriers */
752 while (!slots.empty() &&
753 slots.front()->isComplete() && slots.front()->isBarrier())
754 {
755 LSQRequestPtr barrier = slots.front();
756
757 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
758 *(barrier->inst));
759
760 numUnissuedAccesses--;
761 lsq.clearMemBarrier(barrier->inst);
762 slots.pop_front();
763
764 delete barrier;
765 }
766
767 auto i = slots.begin();
768 bool issued = true;
769 unsigned int issue_count = 0;
770
771 /* Skip trying if the memory system is busy */
772 if (lsq.state == LSQ::MemoryNeedsRetry)
773 issued = false;
774
775 /* Try to issue all stores in order starting from the head
776 * of the queue. Responses are allowed to be retired
777 * out of order */
778 while (issued &&
779 issue_count < storeLimitPerCycle &&
780 lsq.canSendToMemorySystem() &&
781 i != slots.end())
782 {
783 LSQRequestPtr request = *i;
784
785 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
786 " state: %s\n",
787 *(request->inst), request->sentAllPackets(),
788 request->state);
789
790 if (request->isBarrier() && request->isComplete()) {
791 /* Give up at barriers */
792 issued = false;
793 } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
794 request->sentAllPackets()))
795 {
796 DPRINTF(MinorMem, "Trying to send request: %s to memory"
797 " system\n", *(request->inst));
798
799 if (lsq.tryToSend(request)) {
800 countIssuedStore(request);
801 issue_count++;
802 } else {
803 /* Don't step on to the next store buffer entry if this
804 * one hasn't issued all its packets as the store
805 * buffer must still enforce ordering */
806 issued = false;
807 }
808 }
809 i++;
810 }
811 }
812}
813
814void
815LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
816 bool committed)
817{
818 if (committed) {
819 /* Not already sent to the store buffer as a store request? */
820 if (!inst->inStoreBuffer) {
821 /* Insert an entry into the store buffer to tick off barriers
822 * until there are none in flight */
823 storeBuffer.insert(new BarrierDataRequest(*this, inst));
824 }
825 } else {
826 /* Clear the barrier anyway if it wasn't actually committed */
827 clearMemBarrier(inst);
828 }
829}
830
831void
832LSQ::StoreBuffer::minorTrace() const
833{
834 unsigned int size = slots.size();
835 unsigned int i = 0;
836 std::ostringstream os;
837
838 while (i < size) {
839 LSQRequestPtr request = slots[i];
840
841 request->reportData(os);
842
843 i++;
844 if (i < numSlots)
845 os << ',';
846 }
847
848 while (i < numSlots) {
849 os << '-';
850
851 i++;
852 if (i < numSlots)
853 os << ',';
854 }
855
856 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
857 numUnissuedAccesses);
858}
859
860void
861LSQ::tryToSendToTransfers(LSQRequestPtr request)
862{
863 if (state == MemoryNeedsRetry) {
864 DPRINTF(MinorMem, "Request needs retry, not issuing to"
865 " memory until retry arrives\n");
866 return;
867 }
868
869 if (request->state == LSQRequest::InTranslation) {
870 DPRINTF(MinorMem, "Request still in translation, not issuing to"
871 " memory\n");
872 return;
873 }
874
875 assert(request->state == LSQRequest::Translated ||
876 request->state == LSQRequest::RequestIssuing ||
877 request->state == LSQRequest::Failed ||
878 request->state == LSQRequest::Complete);
879
880 if (requests.empty() || requests.front() != request) {
881 DPRINTF(MinorMem, "Request not at front of requests queue, can't"
882 " issue to memory\n");
883 return;
884 }
885
886 if (transfers.unreservedRemainingSpace() == 0) {
887 DPRINTF(MinorMem, "No space to insert request into transfers"
888 " queue\n");
889 return;
890 }
891
892 if (request->isComplete() || request->state == LSQRequest::Failed) {
893 DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
894 " queue\n", (request->isComplete() ? "completed" : "failed"));
895 request->setState(LSQRequest::Complete);
896 request->setSkipped();
897 moveFromRequestsToTransfers(request);
898 return;
899 }
900
901 if (!execute.instIsRightStream(request->inst)) {
902 /* Wrong stream, try to abort the transfer but only do so if
903 * there are no packets in flight */
904 if (request->hasPacketsInMemSystem()) {
905 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
906 " waiting for responses before aborting request\n");
907 } else {
908 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
909 " aborting request\n");
910 request->setState(LSQRequest::Complete);
911 request->setSkipped();
912 moveFromRequestsToTransfers(request);
913 }
914 return;
915 }
916
917 if (request->fault != NoFault) {
918 if (request->inst->staticInst->isPrefetch()) {
919 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
920 }
921 DPRINTF(MinorMem, "Moving faulting request into the transfers"
922 " queue\n");
923 request->setState(LSQRequest::Complete);
924 request->setSkipped();
925 moveFromRequestsToTransfers(request);
926 return;
927 }
928
929 bool is_load = request->isLoad;
930 bool is_llsc = request->request.isLLSC();
931 bool is_swap = request->request.isSwap();
932 bool bufferable = !(request->request.isStrictlyOrdered() ||
933 is_llsc || is_swap);
934
935 if (is_load) {
936 if (numStoresInTransfers != 0) {
937 DPRINTF(MinorMem, "Load request with stores still in transfers"
938 " queue, stalling\n");
939 return;
940 }
941 } else {
942 /* Store. Can it be sent to the store buffer? */
943 if (bufferable && !request->request.isMmappedIpr()) {
944 request->setState(LSQRequest::StoreToStoreBuffer);
945 moveFromRequestsToTransfers(request);
946 DPRINTF(MinorMem, "Moving store into transfers queue\n");
947 return;
948 }
949 }
950
951 /* Check if this is the head instruction (and so must be executable as
952 * its stream sequence number was checked above) for loads which must
953 * not be speculatively issued and stores which must be issued here */
954 if (!bufferable) {
955 if (!execute.instIsHeadInst(request->inst)) {
956 DPRINTF(MinorMem, "Memory access not the head inst., can't be"
957 " sure it can be performed, not issuing\n");
958 return;
959 }
960
961 unsigned int forwarding_slot = 0;
962
963 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
964 NoAddrRangeCoverage)
965 {
966 DPRINTF(MinorMem, "Memory access can receive forwarded data"
967 " from the store buffer, need to wait for store buffer to"
968 " drain\n");
969 return;
970 }
971 }
972
973 /* True: submit this packet to the transfers queue to be sent to the
974 * memory system.
975 * False: skip the memory and push a packet for this request onto
976 * requests */
977 bool do_access = true;
978
979 if (!is_llsc) {
980 /* Check for match in the store buffer */
981 if (is_load) {
982 unsigned int forwarding_slot = 0;
983 AddrRangeCoverage forwarding_result =
984 storeBuffer.canForwardDataToLoad(request,
985 forwarding_slot);
986
987 switch (forwarding_result) {
988 case FullAddrRangeCoverage:
989 /* Forward data from the store buffer into this request and
990 * repurpose this request's packet into a response packet */
991 storeBuffer.forwardStoreData(request, forwarding_slot);
992 request->packet->makeResponse();
993
994 /* Just move between queues, no access */
995 do_access = false;
996 break;
997 case PartialAddrRangeCoverage:
998 DPRINTF(MinorMem, "Load partly satisfied by store buffer"
999 " data. Must wait for the store to complete\n");
1000 return;
1001 break;
1002 case NoAddrRangeCoverage:
1003 DPRINTF(MinorMem, "No forwardable data from store buffer\n");
1004 /* Fall through to try access */
1005 break;
1006 }
1007 }
1008 } else {
1009 if (!canSendToMemorySystem()) {
1010 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1011 return;
1012 }
1013
1014 SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1015
1016 TheISA::PCState old_pc = thread.pcState();
1017 ExecContext context(cpu, thread, execute, request->inst);
1018
1019 /* Handle LLSC requests and tests */
1020 if (is_load) {
1021 TheISA::handleLockedRead(&context, &request->request);
1022 } else {
1023 do_access = TheISA::handleLockedWrite(&context,
1024 &request->request, cacheBlockMask);
1025
1026 if (!do_access) {
1027 DPRINTF(MinorMem, "Not perfoming a memory "
1028 "access for store conditional\n");
1029 }
1030 }
1031 thread.pcState(old_pc);
1032 }
1033
1034 /* See the do_access comment above */
1035 if (do_access) {
1036 if (!canSendToMemorySystem()) {
1037 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1038 return;
1039 }
1040
1041 /* Remember if this is an access which can't be idly
1042 * discarded by an interrupt */
1043 if (!bufferable && !request->issuedToMemory) {
1044 numAccessesIssuedToMemory++;
1045 request->issuedToMemory = true;
1046 }
1047
1048 if (tryToSend(request)) {
1049 moveFromRequestsToTransfers(request);
1050 }
1051 } else {
1052 request->setState(LSQRequest::Complete);
1053 moveFromRequestsToTransfers(request);
1054 }
1055}
1056
1057bool
1058LSQ::tryToSend(LSQRequestPtr request)
1059{
1060 bool ret = false;
1061
1062 if (!canSendToMemorySystem()) {
1063 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1064 *(request->inst));
1065 } else {
1066 PacketPtr packet = request->getHeadPacket();
1067
1068 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1069 *(request->inst), packet->req->getVaddr());
1070
1071 /* The sender state of the packet *must* be an LSQRequest
1072 * so the response can be correctly handled */
1073 assert(packet->findNextSenderState<LSQRequest>());
1074
1075 if (request->request.isMmappedIpr()) {
1076 ThreadContext *thread =
1077 cpu.getContext(cpu.contextToThread(
1078 request->request.contextId()));
1079
1080 if (request->isLoad) {
1081 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1082 TheISA::handleIprRead(thread, packet);
1083 } else {
1084 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1085 TheISA::handleIprWrite(thread, packet);
1086 }
1087
1088 request->stepToNextPacket();
1089 ret = request->sentAllPackets();
1090
1091 if (!ret) {
1092 DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1093 *(request->inst));
1094 }
1095
1096 if (ret)
1097 request->setState(LSQRequest::Complete);
1098 else
1099 request->setState(LSQRequest::RequestIssuing);
1100 } else if (dcachePort.sendTimingReq(packet)) {
1101 DPRINTF(MinorMem, "Sent data memory request\n");
1102
1103 numAccessesInMemorySystem++;
1104
1105 request->stepToNextPacket();
1106
1107 ret = request->sentAllPackets();
1108
1109 switch (request->state) {
1110 case LSQRequest::Translated:
1111 case LSQRequest::RequestIssuing:
1112 /* Fully or partially issued a request in the transfers
1113 * queue */
1114 request->setState(LSQRequest::RequestIssuing);
1115 break;
1116 case LSQRequest::StoreInStoreBuffer:
1117 case LSQRequest::StoreBufferIssuing:
1118 /* Fully or partially issued a request in the store
1119 * buffer */
1120 request->setState(LSQRequest::StoreBufferIssuing);
1121 break;
1122 default:
1123 assert(false);
1124 break;
1125 }
1126
1127 state = MemoryRunning;
1128 } else {
1129 DPRINTF(MinorMem,
1130 "Sending data memory request - needs retry\n");
1131
1132 /* Needs to be resent, wait for that */
1133 state = MemoryNeedsRetry;
1134 retryRequest = request;
1135
1136 switch (request->state) {
1137 case LSQRequest::Translated:
1138 case LSQRequest::RequestIssuing:
1139 request->setState(LSQRequest::RequestNeedsRetry);
1140 break;
1141 case LSQRequest::StoreInStoreBuffer:
1142 case LSQRequest::StoreBufferIssuing:
1143 request->setState(LSQRequest::StoreBufferNeedsRetry);
1144 break;
1145 default:
1146 assert(false);
1147 break;
1148 }
1149 }
1150 }
1151
1152 if (ret)
1153 threadSnoop(request);
1154
1155 return ret;
1156}
1157
1158void
1159LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1160{
1161 assert(!requests.empty() && requests.front() == request);
1162 assert(transfers.unreservedRemainingSpace() != 0);
1163
1164 /* Need to count the number of stores in the transfers
1165 * queue so that loads know when their store buffer forwarding
1166 * results will be correct (only when all those stores
1167 * have reached the store buffer) */
1168 if (!request->isLoad)
1169 numStoresInTransfers++;
1170
1171 requests.pop();
1172 transfers.push(request);
1173}
1174
1175bool
1176LSQ::canSendToMemorySystem()
1177{
1178 return state == MemoryRunning &&
1179 numAccessesInMemorySystem < inMemorySystemLimit;
1180}
1181
1182bool
1183LSQ::recvTimingResp(PacketPtr response)
1184{
1185 LSQRequestPtr request =
1186 safe_cast<LSQRequestPtr>(response->popSenderState());
1187
1188 DPRINTF(MinorMem, "Received response packet inst: %s"
1189 " addr: 0x%x cmd: %s\n",
1190 *(request->inst), response->getAddr(),
1191 response->cmd.toString());
1192
1193 numAccessesInMemorySystem--;
1194
1195 if (response->isError()) {
1196 DPRINTF(MinorMem, "Received error response packet: %s\n",
1197 *request->inst);
1198 }
1199
1200 switch (request->state) {
1201 case LSQRequest::RequestIssuing:
1202 case LSQRequest::RequestNeedsRetry:
1203 /* Response to a request from the transfers queue */
1204 request->retireResponse(response);
1205
1206 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1207 request->hasPacketsInMemSystem(), request->isComplete());
1208
1209 break;
1210 case LSQRequest::StoreBufferIssuing:
1211 case LSQRequest::StoreBufferNeedsRetry:
1212 /* Response to a request from the store buffer */
1213 request->retireResponse(response);
1214
1215 /* Remove completed requests unless they are barriers (which will
1216 * need to be removed in order */
1217 if (request->isComplete()) {
1218 if (!request->isBarrier()) {
1219 storeBuffer.deleteRequest(request);
1220 } else {
1221 DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1222 " leaving the request as it is also a barrier\n",
1223 *(request->inst));
1224 }
1225 }
1226 break;
1227 default:
1228 /* Shouldn't be allowed to receive a response from another
1229 * state */
1230 assert(false);
1231 break;
1232 }
1233
1234 /* We go to idle even if there are more things in the requests queue
1235 * as it's the job of step to actually step us on to the next
1236 * transaction */
1237
1238 /* Let's try and wake up the processor for the next cycle */
1239 cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1240
1241 /* Never busy */
1242 return true;
1243}
1244
1245void
1246LSQ::recvReqRetry()
1247{
1248 DPRINTF(MinorMem, "Received retry request\n");
1249
1250 assert(state == MemoryNeedsRetry);
1251
1252 switch (retryRequest->state) {
1253 case LSQRequest::RequestNeedsRetry:
1254 /* Retry in the requests queue */
1255 retryRequest->setState(LSQRequest::Translated);
1256 break;
1257 case LSQRequest::StoreBufferNeedsRetry:
1258 /* Retry in the store buffer */
1259 retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1260 break;
1261 default:
1262 assert(false);
1263 }
1264
1265 /* Set state back to MemoryRunning so that the following
1266 * tryToSend can actually send. Note that this won't
1267 * allow another transfer in as tryToSend should
1268 * issue a memory request and either succeed for this
1269 * request or return the LSQ back to MemoryNeedsRetry */
1270 state = MemoryRunning;
1271
1272 /* Try to resend the request */
1273 if (tryToSend(retryRequest)) {
1274 /* Successfully sent, need to move the request */
1275 switch (retryRequest->state) {
1276 case LSQRequest::RequestIssuing:
1277 /* In the requests queue */
1278 moveFromRequestsToTransfers(retryRequest);
1279 break;
1280 case LSQRequest::StoreBufferIssuing:
1281 /* In the store buffer */
1282 storeBuffer.countIssuedStore(retryRequest);
1283 break;
1284 default:
1285 assert(false);
1286 break;
1287 }
1288
1289 retryRequest = NULL;
1290 }
1291}
1292
1293LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1294 MinorCPU &cpu_, Execute &execute_,
1295 unsigned int in_memory_system_limit, unsigned int line_width,
1296 unsigned int requests_queue_size, unsigned int transfers_queue_size,
1297 unsigned int store_buffer_size,
1298 unsigned int store_buffer_cycle_store_limit) :
1299 Named(name_),
1300 cpu(cpu_),
1301 execute(execute_),
1302 dcachePort(dcache_port_name_, *this, cpu_),
1303 lastMemBarrier(cpu.numThreads, 0),
1304 state(MemoryRunning),
1305 inMemorySystemLimit(in_memory_system_limit),
1306 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1307 requests(name_ + ".requests", "addr", requests_queue_size),
1308 transfers(name_ + ".transfers", "addr", transfers_queue_size),
1309 storeBuffer(name_ + ".storeBuffer",
1310 *this, store_buffer_size, store_buffer_cycle_store_limit),
1311 numAccessesInMemorySystem(0),
1312 numAccessesInDTLB(0),
1313 numStoresInTransfers(0),
1314 numAccessesIssuedToMemory(0),
1315 retryRequest(NULL),
1316 cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1317{
1318 if (in_memory_system_limit < 1) {
1319 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1320 in_memory_system_limit);
1321 }
1322
1323 if (store_buffer_cycle_store_limit < 1) {
1324 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1325 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1326 }
1327
1328 if (requests_queue_size < 1) {
1329 fatal("%s: executeLSQRequestsQueueSize must be"
1330 " >= 1 (%d)\n", name_, requests_queue_size);
1331 }
1332
1333 if (transfers_queue_size < 1) {
1334 fatal("%s: executeLSQTransfersQueueSize must be"
1335 " >= 1 (%d)\n", name_, transfers_queue_size);
1336 }
1337
1338 if (store_buffer_size < 1) {
1339 fatal("%s: executeLSQStoreBufferSize must be"
1340 " >= 1 (%d)\n", name_, store_buffer_size);
1341 }
1342
1343 if ((lineWidth & (lineWidth - 1)) != 0) {
1344 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1345 }
1346}
1347
1348LSQ::~LSQ()
1349{ }
1350
1351LSQ::LSQRequest::~LSQRequest()
1352{
1353 if (packet)
1354 delete packet;
1355 if (data)
1356 delete [] data;
1357}
1358
1359/**
1360 * Step the memory access mechanism on to its next state. In reality, most
1361 * of the stepping is done by the callbacks on the LSQ but this
1362 * function is responsible for issuing memory requests lodged in the
1363 * requests queue.
1364 */
1365void
1366LSQ::step()
1367{
1368 /* Try to move address-translated requests between queues and issue
1369 * them */
1370 if (!requests.empty())
1371 tryToSendToTransfers(requests.front());
1372
1373 storeBuffer.step();
1374}
1375
1376LSQ::LSQRequestPtr
1377LSQ::findResponse(MinorDynInstPtr inst)
1378{
1379 LSQ::LSQRequestPtr ret = NULL;
1380
1381 if (!transfers.empty()) {
1382 LSQRequestPtr request = transfers.front();
1383
1384 /* Same instruction and complete access or a store that's
1385 * capable of being moved to the store buffer */
1386 if (request->inst->id == inst->id) {
1387 bool complete = request->isComplete();
1388 bool can_store = storeBuffer.canInsert();
1389 bool to_store_buffer = request->state ==
1390 LSQRequest::StoreToStoreBuffer;
1391
1392 if ((complete && !(request->isBarrier() && !can_store)) ||
1393 (to_store_buffer && can_store))
1394 {
1395 ret = request;
1396 }
1397 }
1398 }
1399
1400 if (ret) {
1401 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1402 *inst);
1403 } else {
1404 DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1405 *inst);
1406 }
1407
1408 return ret;
1409}
1410
1411void
1412LSQ::popResponse(LSQ::LSQRequestPtr response)
1413{
1414 assert(!transfers.empty() && transfers.front() == response);
1415
1416 transfers.pop();
1417
1418 if (!response->isLoad)
1419 numStoresInTransfers--;
1420
1421 if (response->issuedToMemory)
1422 numAccessesIssuedToMemory--;
1423
1424 if (response->state != LSQRequest::StoreInStoreBuffer) {
1425 DPRINTF(MinorMem, "Deleting %s request: %s\n",
1426 (response->isLoad ? "load" : "store"),
1427 *(response->inst));
1428
1429 delete response;
1430 }
1431}
1432
1433void
1434LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1435{
1436 assert(request->state == LSQRequest::StoreToStoreBuffer);
1437
1438 DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1439 *(request->inst));
1440
1441 request->inst->inStoreBuffer = true;
1442
1443 storeBuffer.insert(request);
1444}
1445
1446bool
1447LSQ::isDrained()
1448{
1449 return requests.empty() && transfers.empty() &&
1450 storeBuffer.isDrained();
1451}
1452
1453bool
1454LSQ::needsToTick()
1455{
1456 bool ret = false;
1457
1458 if (canSendToMemorySystem()) {
1459 bool have_translated_requests = !requests.empty() &&
1460 requests.front()->state != LSQRequest::InTranslation &&
1461 transfers.unreservedRemainingSpace() != 0;
1462
1463 ret = have_translated_requests ||
1464 storeBuffer.numUnissuedStores() != 0;
1465 }
1466
1467 if (ret)
1468 DPRINTF(Activity, "Need to tick\n");
1469
1470 return ret;
1471}
1472
1473void
1474LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1475 unsigned int size, Addr addr, Request::Flags flags,
1476 uint64_t *res)
1477{
1478 bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1479 LSQRequestPtr request;
1480
1481 /* Copy given data into the request. The request will pass this to the
1482 * packet and then it will own the data */
1483 uint8_t *request_data = NULL;
1484
1485 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1486 " 0x%x%s lineWidth : 0x%x\n",
1487 (isLoad ? "load" : "store"), addr, size, flags,
1488 (needs_burst ? " (needs burst)" : ""), lineWidth);
1489
1490 if (!isLoad) {
1491 /* request_data becomes the property of a ...DataRequest (see below)
1492 * and destroyed by its destructor */
1493 request_data = new uint8_t[size];
1494 if (flags & Request::CACHE_BLOCK_ZERO) {
1495 /* For cache zeroing, just use zeroed data */
1496 std::memset(request_data, 0, size);
1497 } else {
1498 std::memcpy(request_data, data, size);
1499 }
1500 }
1501
1502 if (needs_burst) {
1503 request = new SplitDataRequest(
1504 *this, inst, isLoad, request_data, res);
1505 } else {
1506 request = new SingleDataRequest(
1507 *this, inst, isLoad, request_data, res);
1508 }
1509
1510 if (inst->traceData)
1511 inst->traceData->setMem(addr, size, flags);
1512
1513 int cid = cpu.threads[inst->id.threadId]->getTC()->contextId();
1514 request->request.setContext(cid);
1515 request->request.setVirt(0 /* asid */,
1516 addr, size, flags, cpu.dataMasterId(),
1517 /* I've no idea why we need the PC, but give it */
1518 inst->pc.instAddr());
1519
1520 requests.push(request);
1521 request->startAddrTranslation();
1522}
1523
1524void
1525LSQ::pushFailedRequest(MinorDynInstPtr inst)
1526{
1527 LSQRequestPtr request = new FailedDataRequest(*this, inst);
1528 requests.push(request);
1529}
1530
1531void
1532LSQ::minorTrace() const
1533{
1534 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1535 " lastMemBarrier=%d\n",
1536 state, numAccessesInDTLB, numAccessesInMemorySystem,
1537 numStoresInTransfers, lastMemBarrier[0]);
1538 requests.minorTrace();
1539 transfers.minorTrace();
1540 storeBuffer.minorTrace();
1541}
1542
1543LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1544 unsigned int store_buffer_size,
1545 unsigned int store_limit_per_cycle) :
1546 Named(name_), lsq(lsq_),
1547 numSlots(store_buffer_size),
1548 storeLimitPerCycle(store_limit_per_cycle),
1549 slots(),
1550 numUnissuedAccesses(0)
1551{
1552}
1553
1554PacketPtr
1555makePacketForRequest(Request &request, bool isLoad,
1556 Packet::SenderState *sender_state, PacketDataPtr data)
1557{
1558 PacketPtr ret = isLoad ? Packet::createRead(&request)
1559 : Packet::createWrite(&request);
1560
1561 if (sender_state)
1562 ret->pushSenderState(sender_state);
1563
1564 if (isLoad)
1565 ret->allocate();
1566 else
1567 ret->dataDynamic(data);
1568
1569 return ret;
1570}
1571
1572void
1573LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1574{
1575 assert(inst->isInst() && inst->staticInst->isMemBarrier());
1576 assert(inst->id.execSeqNum > lastMemBarrier[inst->id.threadId]);
1577
1578 /* Remember the barrier. We only have a notion of one
1579 * barrier so this may result in some mem refs being
1580 * delayed if they are between barriers */
1581 lastMemBarrier[inst->id.threadId] = inst->id.execSeqNum;
1582}
1583
1584void
1585LSQ::LSQRequest::makePacket()
1586{
1587 /* Make the function idempotent */
1588 if (packet)
1589 return;
1590
1591 // if the translation faulted, do not create a packet
1592 if (fault != NoFault) {
1593 assert(packet == NULL);
1594 return;
1595 }
1596
1597 packet = makePacketForRequest(request, isLoad, this, data);
1598 /* Null the ret data so we know not to deallocate it when the
1599 * ret is destroyed. The data now belongs to the ret and
1600 * the ret is responsible for its destruction */
1601 data = NULL;
1602}
1603
1604std::ostream &
1605operator <<(std::ostream &os, LSQ::MemoryState state)
1606{
1607 switch (state) {
1608 case LSQ::MemoryRunning:
1609 os << "MemoryRunning";
1610 break;
1611 case LSQ::MemoryNeedsRetry:
1612 os << "MemoryNeedsRetry";
1613 break;
1614 default:
1615 os << "MemoryState-" << static_cast<int>(state);
1616 break;
1617 }
1618 return os;
1619}
1620
1621void
1622LSQ::recvTimingSnoopReq(PacketPtr pkt)
1623{
1624 /* LLSC operations in Minor can't be speculative and are executed from
1625 * the head of the requests queue. We shouldn't need to do more than
1626 * this action on snoops. */
1627 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1628 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1629 cpu.wakeup(tid);
1630 }
1631 }
1632
1633 if (pkt->isInvalidate() || pkt->isWrite()) {
1634 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1635 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1636 cacheBlockMask);
1637 }
1638 }
1639}
1640
1641void
1642LSQ::threadSnoop(LSQRequestPtr request)
1643{
1644 /* LLSC operations in Minor can't be speculative and are executed from
1645 * the head of the requests queue. We shouldn't need to do more than
1646 * this action on snoops. */
1647 ThreadID req_tid = request->inst->id.threadId;
1648 PacketPtr pkt = request->packet;
1649
1650 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1651 if (tid != req_tid) {
1652 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1653 cpu.wakeup(tid);
1654 }
1655
1656 if (pkt->isInvalidate() || pkt->isWrite()) {
1657 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1658 cacheBlockMask);
1659 }
1660 }
1661 }
1662}
1663
1664}
| 326 numFragments(0),
327 numInTranslationFragments(0),
328 numTranslatedFragments(0),
329 numIssuedFragments(0),
330 numRetiredFragments(0),
331 fragmentRequests(),
332 fragmentPackets()
333{
334 /* Don't know how many elements are needed until the request is
335 * populated by the caller. */
336}
337
338LSQ::SplitDataRequest::~SplitDataRequest()
339{
340 for (auto i = fragmentRequests.begin();
341 i != fragmentRequests.end(); i++)
342 {
343 delete *i;
344 }
345
346 for (auto i = fragmentPackets.begin();
347 i != fragmentPackets.end(); i++)
348 {
349 delete *i;
350 }
351}
352
353void
354LSQ::SplitDataRequest::makeFragmentRequests()
355{
356 Addr base_addr = request.getVaddr();
357 unsigned int whole_size = request.getSize();
358 unsigned int line_width = port.lineWidth;
359
360 unsigned int fragment_size;
361 Addr fragment_addr;
362
363 /* Assume that this transfer is across potentially many block snap
364 * boundaries:
365 *
366 * | _|________|________|________|___ |
367 * | |0| 1 | 2 | 3 | 4 | |
368 * | |_|________|________|________|___| |
369 * | | | | | |
370 *
371 * The first transfer (0) can be up to lineWidth in size.
372 * All the middle transfers (1-3) are lineWidth in size
373 * The last transfer (4) can be from zero to lineWidth - 1 in size
374 */
375 unsigned int first_fragment_offset =
376 addrBlockOffset(base_addr, line_width);
377 unsigned int last_fragment_size =
378 addrBlockOffset(base_addr + whole_size, line_width);
379 unsigned int first_fragment_size =
380 line_width - first_fragment_offset;
381
382 unsigned int middle_fragments_total_size =
383 whole_size - (first_fragment_size + last_fragment_size);
384
385 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
386
387 unsigned int middle_fragment_count =
388 middle_fragments_total_size / line_width;
389
390 numFragments = 1 /* first */ + middle_fragment_count +
391 (last_fragment_size == 0 ? 0 : 1);
392
393 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
394 " First fragment size: %d Last fragment size: %d\n",
395 numFragments, first_fragment_size,
396 (last_fragment_size == 0 ? line_width : last_fragment_size));
397
398 assert(((middle_fragment_count * line_width) +
399 first_fragment_size + last_fragment_size) == whole_size);
400
401 fragment_addr = base_addr;
402 fragment_size = first_fragment_size;
403
404 /* Just past the last address in the request */
405 Addr end_addr = base_addr + whole_size;
406
407 for (unsigned int fragment_index = 0; fragment_index < numFragments;
408 fragment_index++)
409 {
410 bool M5_VAR_USED is_last_fragment = false;
411
412 if (fragment_addr == base_addr) {
413 /* First fragment */
414 fragment_size = first_fragment_size;
415 } else {
416 if ((fragment_addr + line_width) > end_addr) {
417 /* Adjust size of last fragment */
418 fragment_size = end_addr - fragment_addr;
419 is_last_fragment = true;
420 } else {
421 /* Middle fragments */
422 fragment_size = line_width;
423 }
424 }
425
426 Request *fragment = new Request();
427
428 fragment->setContext(request.contextId());
429 fragment->setVirt(0 /* asid */,
430 fragment_addr, fragment_size, request.getFlags(),
431 request.masterId(),
432 request.getPC());
433
434 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
435 " (whole request addr: 0x%x size: %d) %s\n",
436 fragment_addr, fragment_size, base_addr, whole_size,
437 (is_last_fragment ? "last fragment" : ""));
438
439 fragment_addr += fragment_size;
440
441 fragmentRequests.push_back(fragment);
442 }
443}
444
445void
446LSQ::SplitDataRequest::makeFragmentPackets()
447{
448 Addr base_addr = request.getVaddr();
449
450 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
451
452 for (unsigned int fragment_index = 0; fragment_index < numFragments;
453 fragment_index++)
454 {
455 Request *fragment = fragmentRequests[fragment_index];
456
457 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
458 " (%d, 0x%x)\n",
459 fragment_index, *inst,
460 (fragment->hasPaddr() ? "has paddr" : "no paddr"),
461 (fragment->hasPaddr() ? fragment->getPaddr() : 0));
462
463 Addr fragment_addr = fragment->getVaddr();
464 unsigned int fragment_size = fragment->getSize();
465
466 uint8_t *request_data = NULL;
467
468 if (!isLoad) {
469 /* Split data for Packets. Will become the property of the
470 * outgoing Packets */
471 request_data = new uint8_t[fragment_size];
472 std::memcpy(request_data, data + (fragment_addr - base_addr),
473 fragment_size);
474 }
475
476 assert(fragment->hasPaddr());
477
478 PacketPtr fragment_packet =
479 makePacketForRequest(*fragment, isLoad, this, request_data);
480
481 fragmentPackets.push_back(fragment_packet);
482 /* Accumulate flags in parent request */
483 request.setFlags(fragment->getFlags());
484 }
485
486 /* Might as well make the overall/response packet here */
487 /* Get the physical address for the whole request/packet from the first
488 * fragment */
489 request.setPaddr(fragmentRequests[0]->getPaddr());
490 makePacket();
491}
492
493void
494LSQ::SplitDataRequest::startAddrTranslation()
495{
496 setState(LSQ::LSQRequest::InTranslation);
497
498 makeFragmentRequests();
499
500 numInTranslationFragments = 0;
501 numTranslatedFragments = 0;
502
503 /* @todo, just do these in sequence for now with
504 * a loop of:
505 * do {
506 * sendNextFragmentToTranslation ; translateTiming ; finish
507 * } while (numTranslatedFragments != numFragments);
508 */
509
510 /* Do first translation */
511 sendNextFragmentToTranslation();
512}
513
514PacketPtr
515LSQ::SplitDataRequest::getHeadPacket()
516{
517 assert(numIssuedFragments < numFragments);
518
519 return fragmentPackets[numIssuedFragments];
520}
521
522void
523LSQ::SplitDataRequest::stepToNextPacket()
524{
525 assert(numIssuedFragments < numFragments);
526
527 numIssuedFragments++;
528}
529
530void
531LSQ::SplitDataRequest::retireResponse(PacketPtr response)
532{
533 assert(numRetiredFragments < numFragments);
534
535 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
536 " offset: 0x%x (retired fragment num: %d) %s\n",
537 response->req->getVaddr(), response->req->getSize(),
538 request.getVaddr() - response->req->getVaddr(),
539 numRetiredFragments,
540 (fault == NoFault ? "" : fault->name()));
541
542 numRetiredFragments++;
543
544 if (skipped) {
545 /* Skip because we already knew the request had faulted or been
546 * skipped */
547 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
548 } else if (response->isError()) {
549 /* Mark up the error and leave to execute to handle it */
550 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
551 setSkipped();
552 packet->copyError(response);
553 } else {
554 if (isLoad) {
555 if (!data) {
556 /* For a split transfer, a Packet must be constructed
557 * to contain all returning data. This is that packet's
558 * data */
559 data = new uint8_t[request.getSize()];
560 }
561
562 /* Populate the portion of the overall response data represented
563 * by the response fragment */
564 std::memcpy(
565 data + (response->req->getVaddr() - request.getVaddr()),
566 response->getConstPtr<uint8_t>(),
567 response->req->getSize());
568 }
569 }
570
571 /* Complete early if we're skipping are no more in-flight accesses */
572 if (skipped && !hasPacketsInMemSystem()) {
573 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
574 setState(Complete);
575 if (packet->needsResponse())
576 packet->makeResponse();
577 }
578
579 if (numRetiredFragments == numFragments)
580 setState(Complete);
581
582 if (!skipped && isComplete()) {
583 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
584
585 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
586 " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
587 " %s\n", packet->isRead(), packet->isWrite(),
588 packet->needsResponse(), packet->getSize(), request.getSize(),
589 response->getSize());
590
591 /* A request can become complete by several paths, this is a sanity
592 * check to make sure the packet's data is created */
593 if (!data) {
594 data = new uint8_t[request.getSize()];
595 }
596
597 if (isLoad) {
598 DPRINTFS(MinorMem, (&port), "Copying read data\n");
599 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
600 }
601 packet->makeResponse();
602 }
603
604 /* Packets are all deallocated together in ~SplitLSQRequest */
605}
606
607void
608LSQ::SplitDataRequest::sendNextFragmentToTranslation()
609{
610 unsigned int fragment_index = numTranslatedFragments;
611
612 ThreadContext *thread = port.cpu.getContext(
613 inst->id.threadId);
614
615 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
616 fragment_index);
617
618 port.numAccessesInDTLB++;
619 numInTranslationFragments++;
620
621 thread->getDTBPtr()->translateTiming(
622 fragmentRequests[fragment_index], thread, this, (isLoad ?
623 BaseTLB::Read : BaseTLB::Write));
624}
625
626bool
627LSQ::StoreBuffer::canInsert() const
628{
629 /* @todo, support store amalgamation */
630 return slots.size() < numSlots;
631}
632
633void
634LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
635{
636 auto found = std::find(slots.begin(), slots.end(), request);
637
638 if (found != slots.end()) {
639 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
640 request, *found, *(request->inst));
641 slots.erase(found);
642
643 delete request;
644 }
645}
646
647void
648LSQ::StoreBuffer::insert(LSQRequestPtr request)
649{
650 if (!canInsert()) {
651 warn("%s: store buffer insertion without space to insert from"
652 " inst: %s\n", name(), *(request->inst));
653 }
654
655 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
656
657 numUnissuedAccesses++;
658
659 if (request->state != LSQRequest::Complete)
660 request->setState(LSQRequest::StoreInStoreBuffer);
661
662 slots.push_back(request);
663
664 /* Let's try and wake up the processor for the next cycle to step
665 * the store buffer */
666 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
667}
668
669LSQ::AddrRangeCoverage
670LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
671 unsigned int &found_slot)
672{
673 unsigned int slot_index = slots.size() - 1;
674 auto i = slots.rbegin();
675 AddrRangeCoverage ret = NoAddrRangeCoverage;
676
677 /* Traverse the store buffer in reverse order (most to least recent)
678 * and try to find a slot whose address range overlaps this request */
679 while (ret == NoAddrRangeCoverage && i != slots.rend()) {
680 LSQRequestPtr slot = *i;
681
682 if (slot->packet &&
683 slot->inst->id.threadId == request->inst->id.threadId) {
684 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
685
686 if (coverage != NoAddrRangeCoverage) {
687 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
688 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
689 slot_index, coverage,
690 request->request.getPaddr(), request->request.getSize(),
691 slot->request.getPaddr(), slot->request.getSize());
692
693 found_slot = slot_index;
694 ret = coverage;
695 }
696 }
697
698 i++;
699 slot_index--;
700 }
701
702 return ret;
703}
704
705/** Fill the given packet with appropriate date from slot slot_number */
706void
707LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
708 unsigned int slot_number)
709{
710 assert(slot_number < slots.size());
711 assert(load->packet);
712 assert(load->isLoad);
713
714 LSQRequestPtr store = slots[slot_number];
715
716 assert(store->packet);
717 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
718
719 Addr load_addr = load->request.getPaddr();
720 Addr store_addr = store->request.getPaddr();
721 Addr addr_offset = load_addr - store_addr;
722
723 unsigned int load_size = load->request.getSize();
724
725 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
726 " slot: %d addr: 0x%x addressOffset: 0x%x\n",
727 load_size, load_addr, slot_number,
728 store_addr, addr_offset);
729
730 void *load_packet_data = load->packet->getPtr<void>();
731 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
732
733 std::memcpy(load_packet_data, store_packet_data, load_size);
734}
735
736void
737LSQ::StoreBuffer::countIssuedStore(LSQRequestPtr request)
738{
739 /* Barriers are accounted for as they are cleared from
740 * the queue, not after their transfers are complete */
741 if (!request->isBarrier())
742 numUnissuedAccesses--;
743}
744
745void
746LSQ::StoreBuffer::step()
747{
748 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
749 numUnissuedAccesses);
750
751 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
752 /* Clear all the leading barriers */
753 while (!slots.empty() &&
754 slots.front()->isComplete() && slots.front()->isBarrier())
755 {
756 LSQRequestPtr barrier = slots.front();
757
758 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
759 *(barrier->inst));
760
761 numUnissuedAccesses--;
762 lsq.clearMemBarrier(barrier->inst);
763 slots.pop_front();
764
765 delete barrier;
766 }
767
768 auto i = slots.begin();
769 bool issued = true;
770 unsigned int issue_count = 0;
771
772 /* Skip trying if the memory system is busy */
773 if (lsq.state == LSQ::MemoryNeedsRetry)
774 issued = false;
775
776 /* Try to issue all stores in order starting from the head
777 * of the queue. Responses are allowed to be retired
778 * out of order */
779 while (issued &&
780 issue_count < storeLimitPerCycle &&
781 lsq.canSendToMemorySystem() &&
782 i != slots.end())
783 {
784 LSQRequestPtr request = *i;
785
786 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
787 " state: %s\n",
788 *(request->inst), request->sentAllPackets(),
789 request->state);
790
791 if (request->isBarrier() && request->isComplete()) {
792 /* Give up at barriers */
793 issued = false;
794 } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
795 request->sentAllPackets()))
796 {
797 DPRINTF(MinorMem, "Trying to send request: %s to memory"
798 " system\n", *(request->inst));
799
800 if (lsq.tryToSend(request)) {
801 countIssuedStore(request);
802 issue_count++;
803 } else {
804 /* Don't step on to the next store buffer entry if this
805 * one hasn't issued all its packets as the store
806 * buffer must still enforce ordering */
807 issued = false;
808 }
809 }
810 i++;
811 }
812 }
813}
814
815void
816LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
817 bool committed)
818{
819 if (committed) {
820 /* Not already sent to the store buffer as a store request? */
821 if (!inst->inStoreBuffer) {
822 /* Insert an entry into the store buffer to tick off barriers
823 * until there are none in flight */
824 storeBuffer.insert(new BarrierDataRequest(*this, inst));
825 }
826 } else {
827 /* Clear the barrier anyway if it wasn't actually committed */
828 clearMemBarrier(inst);
829 }
830}
831
832void
833LSQ::StoreBuffer::minorTrace() const
834{
835 unsigned int size = slots.size();
836 unsigned int i = 0;
837 std::ostringstream os;
838
839 while (i < size) {
840 LSQRequestPtr request = slots[i];
841
842 request->reportData(os);
843
844 i++;
845 if (i < numSlots)
846 os << ',';
847 }
848
849 while (i < numSlots) {
850 os << '-';
851
852 i++;
853 if (i < numSlots)
854 os << ',';
855 }
856
857 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
858 numUnissuedAccesses);
859}
860
861void
862LSQ::tryToSendToTransfers(LSQRequestPtr request)
863{
864 if (state == MemoryNeedsRetry) {
865 DPRINTF(MinorMem, "Request needs retry, not issuing to"
866 " memory until retry arrives\n");
867 return;
868 }
869
870 if (request->state == LSQRequest::InTranslation) {
871 DPRINTF(MinorMem, "Request still in translation, not issuing to"
872 " memory\n");
873 return;
874 }
875
876 assert(request->state == LSQRequest::Translated ||
877 request->state == LSQRequest::RequestIssuing ||
878 request->state == LSQRequest::Failed ||
879 request->state == LSQRequest::Complete);
880
881 if (requests.empty() || requests.front() != request) {
882 DPRINTF(MinorMem, "Request not at front of requests queue, can't"
883 " issue to memory\n");
884 return;
885 }
886
887 if (transfers.unreservedRemainingSpace() == 0) {
888 DPRINTF(MinorMem, "No space to insert request into transfers"
889 " queue\n");
890 return;
891 }
892
893 if (request->isComplete() || request->state == LSQRequest::Failed) {
894 DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
895 " queue\n", (request->isComplete() ? "completed" : "failed"));
896 request->setState(LSQRequest::Complete);
897 request->setSkipped();
898 moveFromRequestsToTransfers(request);
899 return;
900 }
901
902 if (!execute.instIsRightStream(request->inst)) {
903 /* Wrong stream, try to abort the transfer but only do so if
904 * there are no packets in flight */
905 if (request->hasPacketsInMemSystem()) {
906 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
907 " waiting for responses before aborting request\n");
908 } else {
909 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
910 " aborting request\n");
911 request->setState(LSQRequest::Complete);
912 request->setSkipped();
913 moveFromRequestsToTransfers(request);
914 }
915 return;
916 }
917
918 if (request->fault != NoFault) {
919 if (request->inst->staticInst->isPrefetch()) {
920 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
921 }
922 DPRINTF(MinorMem, "Moving faulting request into the transfers"
923 " queue\n");
924 request->setState(LSQRequest::Complete);
925 request->setSkipped();
926 moveFromRequestsToTransfers(request);
927 return;
928 }
929
930 bool is_load = request->isLoad;
931 bool is_llsc = request->request.isLLSC();
932 bool is_swap = request->request.isSwap();
933 bool bufferable = !(request->request.isStrictlyOrdered() ||
934 is_llsc || is_swap);
935
936 if (is_load) {
937 if (numStoresInTransfers != 0) {
938 DPRINTF(MinorMem, "Load request with stores still in transfers"
939 " queue, stalling\n");
940 return;
941 }
942 } else {
943 /* Store. Can it be sent to the store buffer? */
944 if (bufferable && !request->request.isMmappedIpr()) {
945 request->setState(LSQRequest::StoreToStoreBuffer);
946 moveFromRequestsToTransfers(request);
947 DPRINTF(MinorMem, "Moving store into transfers queue\n");
948 return;
949 }
950 }
951
952 /* Check if this is the head instruction (and so must be executable as
953 * its stream sequence number was checked above) for loads which must
954 * not be speculatively issued and stores which must be issued here */
955 if (!bufferable) {
956 if (!execute.instIsHeadInst(request->inst)) {
957 DPRINTF(MinorMem, "Memory access not the head inst., can't be"
958 " sure it can be performed, not issuing\n");
959 return;
960 }
961
962 unsigned int forwarding_slot = 0;
963
964 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
965 NoAddrRangeCoverage)
966 {
967 DPRINTF(MinorMem, "Memory access can receive forwarded data"
968 " from the store buffer, need to wait for store buffer to"
969 " drain\n");
970 return;
971 }
972 }
973
974 /* True: submit this packet to the transfers queue to be sent to the
975 * memory system.
976 * False: skip the memory and push a packet for this request onto
977 * requests */
978 bool do_access = true;
979
980 if (!is_llsc) {
981 /* Check for match in the store buffer */
982 if (is_load) {
983 unsigned int forwarding_slot = 0;
984 AddrRangeCoverage forwarding_result =
985 storeBuffer.canForwardDataToLoad(request,
986 forwarding_slot);
987
988 switch (forwarding_result) {
989 case FullAddrRangeCoverage:
990 /* Forward data from the store buffer into this request and
991 * repurpose this request's packet into a response packet */
992 storeBuffer.forwardStoreData(request, forwarding_slot);
993 request->packet->makeResponse();
994
995 /* Just move between queues, no access */
996 do_access = false;
997 break;
998 case PartialAddrRangeCoverage:
999 DPRINTF(MinorMem, "Load partly satisfied by store buffer"
1000 " data. Must wait for the store to complete\n");
1001 return;
1002 break;
1003 case NoAddrRangeCoverage:
1004 DPRINTF(MinorMem, "No forwardable data from store buffer\n");
1005 /* Fall through to try access */
1006 break;
1007 }
1008 }
1009 } else {
1010 if (!canSendToMemorySystem()) {
1011 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1012 return;
1013 }
1014
1015 SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1016
1017 TheISA::PCState old_pc = thread.pcState();
1018 ExecContext context(cpu, thread, execute, request->inst);
1019
1020 /* Handle LLSC requests and tests */
1021 if (is_load) {
1022 TheISA::handleLockedRead(&context, &request->request);
1023 } else {
1024 do_access = TheISA::handleLockedWrite(&context,
1025 &request->request, cacheBlockMask);
1026
1027 if (!do_access) {
1028 DPRINTF(MinorMem, "Not perfoming a memory "
1029 "access for store conditional\n");
1030 }
1031 }
1032 thread.pcState(old_pc);
1033 }
1034
1035 /* See the do_access comment above */
1036 if (do_access) {
1037 if (!canSendToMemorySystem()) {
1038 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1039 return;
1040 }
1041
1042 /* Remember if this is an access which can't be idly
1043 * discarded by an interrupt */
1044 if (!bufferable && !request->issuedToMemory) {
1045 numAccessesIssuedToMemory++;
1046 request->issuedToMemory = true;
1047 }
1048
1049 if (tryToSend(request)) {
1050 moveFromRequestsToTransfers(request);
1051 }
1052 } else {
1053 request->setState(LSQRequest::Complete);
1054 moveFromRequestsToTransfers(request);
1055 }
1056}
1057
1058bool
1059LSQ::tryToSend(LSQRequestPtr request)
1060{
1061 bool ret = false;
1062
1063 if (!canSendToMemorySystem()) {
1064 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1065 *(request->inst));
1066 } else {
1067 PacketPtr packet = request->getHeadPacket();
1068
1069 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1070 *(request->inst), packet->req->getVaddr());
1071
1072 /* The sender state of the packet *must* be an LSQRequest
1073 * so the response can be correctly handled */
1074 assert(packet->findNextSenderState<LSQRequest>());
1075
1076 if (request->request.isMmappedIpr()) {
1077 ThreadContext *thread =
1078 cpu.getContext(cpu.contextToThread(
1079 request->request.contextId()));
1080
1081 if (request->isLoad) {
1082 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1083 TheISA::handleIprRead(thread, packet);
1084 } else {
1085 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1086 TheISA::handleIprWrite(thread, packet);
1087 }
1088
1089 request->stepToNextPacket();
1090 ret = request->sentAllPackets();
1091
1092 if (!ret) {
1093 DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1094 *(request->inst));
1095 }
1096
1097 if (ret)
1098 request->setState(LSQRequest::Complete);
1099 else
1100 request->setState(LSQRequest::RequestIssuing);
1101 } else if (dcachePort.sendTimingReq(packet)) {
1102 DPRINTF(MinorMem, "Sent data memory request\n");
1103
1104 numAccessesInMemorySystem++;
1105
1106 request->stepToNextPacket();
1107
1108 ret = request->sentAllPackets();
1109
1110 switch (request->state) {
1111 case LSQRequest::Translated:
1112 case LSQRequest::RequestIssuing:
1113 /* Fully or partially issued a request in the transfers
1114 * queue */
1115 request->setState(LSQRequest::RequestIssuing);
1116 break;
1117 case LSQRequest::StoreInStoreBuffer:
1118 case LSQRequest::StoreBufferIssuing:
1119 /* Fully or partially issued a request in the store
1120 * buffer */
1121 request->setState(LSQRequest::StoreBufferIssuing);
1122 break;
1123 default:
1124 assert(false);
1125 break;
1126 }
1127
1128 state = MemoryRunning;
1129 } else {
1130 DPRINTF(MinorMem,
1131 "Sending data memory request - needs retry\n");
1132
1133 /* Needs to be resent, wait for that */
1134 state = MemoryNeedsRetry;
1135 retryRequest = request;
1136
1137 switch (request->state) {
1138 case LSQRequest::Translated:
1139 case LSQRequest::RequestIssuing:
1140 request->setState(LSQRequest::RequestNeedsRetry);
1141 break;
1142 case LSQRequest::StoreInStoreBuffer:
1143 case LSQRequest::StoreBufferIssuing:
1144 request->setState(LSQRequest::StoreBufferNeedsRetry);
1145 break;
1146 default:
1147 assert(false);
1148 break;
1149 }
1150 }
1151 }
1152
1153 if (ret)
1154 threadSnoop(request);
1155
1156 return ret;
1157}
1158
1159void
1160LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1161{
1162 assert(!requests.empty() && requests.front() == request);
1163 assert(transfers.unreservedRemainingSpace() != 0);
1164
1165 /* Need to count the number of stores in the transfers
1166 * queue so that loads know when their store buffer forwarding
1167 * results will be correct (only when all those stores
1168 * have reached the store buffer) */
1169 if (!request->isLoad)
1170 numStoresInTransfers++;
1171
1172 requests.pop();
1173 transfers.push(request);
1174}
1175
1176bool
1177LSQ::canSendToMemorySystem()
1178{
1179 return state == MemoryRunning &&
1180 numAccessesInMemorySystem < inMemorySystemLimit;
1181}
1182
1183bool
1184LSQ::recvTimingResp(PacketPtr response)
1185{
1186 LSQRequestPtr request =
1187 safe_cast<LSQRequestPtr>(response->popSenderState());
1188
1189 DPRINTF(MinorMem, "Received response packet inst: %s"
1190 " addr: 0x%x cmd: %s\n",
1191 *(request->inst), response->getAddr(),
1192 response->cmd.toString());
1193
1194 numAccessesInMemorySystem--;
1195
1196 if (response->isError()) {
1197 DPRINTF(MinorMem, "Received error response packet: %s\n",
1198 *request->inst);
1199 }
1200
1201 switch (request->state) {
1202 case LSQRequest::RequestIssuing:
1203 case LSQRequest::RequestNeedsRetry:
1204 /* Response to a request from the transfers queue */
1205 request->retireResponse(response);
1206
1207 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1208 request->hasPacketsInMemSystem(), request->isComplete());
1209
1210 break;
1211 case LSQRequest::StoreBufferIssuing:
1212 case LSQRequest::StoreBufferNeedsRetry:
1213 /* Response to a request from the store buffer */
1214 request->retireResponse(response);
1215
1216 /* Remove completed requests unless they are barriers (which will
1217 * need to be removed in order */
1218 if (request->isComplete()) {
1219 if (!request->isBarrier()) {
1220 storeBuffer.deleteRequest(request);
1221 } else {
1222 DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1223 " leaving the request as it is also a barrier\n",
1224 *(request->inst));
1225 }
1226 }
1227 break;
1228 default:
1229 /* Shouldn't be allowed to receive a response from another
1230 * state */
1231 assert(false);
1232 break;
1233 }
1234
1235 /* We go to idle even if there are more things in the requests queue
1236 * as it's the job of step to actually step us on to the next
1237 * transaction */
1238
1239 /* Let's try and wake up the processor for the next cycle */
1240 cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1241
1242 /* Never busy */
1243 return true;
1244}
1245
1246void
1247LSQ::recvReqRetry()
1248{
1249 DPRINTF(MinorMem, "Received retry request\n");
1250
1251 assert(state == MemoryNeedsRetry);
1252
1253 switch (retryRequest->state) {
1254 case LSQRequest::RequestNeedsRetry:
1255 /* Retry in the requests queue */
1256 retryRequest->setState(LSQRequest::Translated);
1257 break;
1258 case LSQRequest::StoreBufferNeedsRetry:
1259 /* Retry in the store buffer */
1260 retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1261 break;
1262 default:
1263 assert(false);
1264 }
1265
1266 /* Set state back to MemoryRunning so that the following
1267 * tryToSend can actually send. Note that this won't
1268 * allow another transfer in as tryToSend should
1269 * issue a memory request and either succeed for this
1270 * request or return the LSQ back to MemoryNeedsRetry */
1271 state = MemoryRunning;
1272
1273 /* Try to resend the request */
1274 if (tryToSend(retryRequest)) {
1275 /* Successfully sent, need to move the request */
1276 switch (retryRequest->state) {
1277 case LSQRequest::RequestIssuing:
1278 /* In the requests queue */
1279 moveFromRequestsToTransfers(retryRequest);
1280 break;
1281 case LSQRequest::StoreBufferIssuing:
1282 /* In the store buffer */
1283 storeBuffer.countIssuedStore(retryRequest);
1284 break;
1285 default:
1286 assert(false);
1287 break;
1288 }
1289
1290 retryRequest = NULL;
1291 }
1292}
1293
1294LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1295 MinorCPU &cpu_, Execute &execute_,
1296 unsigned int in_memory_system_limit, unsigned int line_width,
1297 unsigned int requests_queue_size, unsigned int transfers_queue_size,
1298 unsigned int store_buffer_size,
1299 unsigned int store_buffer_cycle_store_limit) :
1300 Named(name_),
1301 cpu(cpu_),
1302 execute(execute_),
1303 dcachePort(dcache_port_name_, *this, cpu_),
1304 lastMemBarrier(cpu.numThreads, 0),
1305 state(MemoryRunning),
1306 inMemorySystemLimit(in_memory_system_limit),
1307 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1308 requests(name_ + ".requests", "addr", requests_queue_size),
1309 transfers(name_ + ".transfers", "addr", transfers_queue_size),
1310 storeBuffer(name_ + ".storeBuffer",
1311 *this, store_buffer_size, store_buffer_cycle_store_limit),
1312 numAccessesInMemorySystem(0),
1313 numAccessesInDTLB(0),
1314 numStoresInTransfers(0),
1315 numAccessesIssuedToMemory(0),
1316 retryRequest(NULL),
1317 cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1318{
1319 if (in_memory_system_limit < 1) {
1320 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1321 in_memory_system_limit);
1322 }
1323
1324 if (store_buffer_cycle_store_limit < 1) {
1325 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1326 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1327 }
1328
1329 if (requests_queue_size < 1) {
1330 fatal("%s: executeLSQRequestsQueueSize must be"
1331 " >= 1 (%d)\n", name_, requests_queue_size);
1332 }
1333
1334 if (transfers_queue_size < 1) {
1335 fatal("%s: executeLSQTransfersQueueSize must be"
1336 " >= 1 (%d)\n", name_, transfers_queue_size);
1337 }
1338
1339 if (store_buffer_size < 1) {
1340 fatal("%s: executeLSQStoreBufferSize must be"
1341 " >= 1 (%d)\n", name_, store_buffer_size);
1342 }
1343
1344 if ((lineWidth & (lineWidth - 1)) != 0) {
1345 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1346 }
1347}
1348
1349LSQ::~LSQ()
1350{ }
1351
1352LSQ::LSQRequest::~LSQRequest()
1353{
1354 if (packet)
1355 delete packet;
1356 if (data)
1357 delete [] data;
1358}
1359
1360/**
1361 * Step the memory access mechanism on to its next state. In reality, most
1362 * of the stepping is done by the callbacks on the LSQ but this
1363 * function is responsible for issuing memory requests lodged in the
1364 * requests queue.
1365 */
1366void
1367LSQ::step()
1368{
1369 /* Try to move address-translated requests between queues and issue
1370 * them */
1371 if (!requests.empty())
1372 tryToSendToTransfers(requests.front());
1373
1374 storeBuffer.step();
1375}
1376
1377LSQ::LSQRequestPtr
1378LSQ::findResponse(MinorDynInstPtr inst)
1379{
1380 LSQ::LSQRequestPtr ret = NULL;
1381
1382 if (!transfers.empty()) {
1383 LSQRequestPtr request = transfers.front();
1384
1385 /* Same instruction and complete access or a store that's
1386 * capable of being moved to the store buffer */
1387 if (request->inst->id == inst->id) {
1388 bool complete = request->isComplete();
1389 bool can_store = storeBuffer.canInsert();
1390 bool to_store_buffer = request->state ==
1391 LSQRequest::StoreToStoreBuffer;
1392
1393 if ((complete && !(request->isBarrier() && !can_store)) ||
1394 (to_store_buffer && can_store))
1395 {
1396 ret = request;
1397 }
1398 }
1399 }
1400
1401 if (ret) {
1402 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1403 *inst);
1404 } else {
1405 DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1406 *inst);
1407 }
1408
1409 return ret;
1410}
1411
1412void
1413LSQ::popResponse(LSQ::LSQRequestPtr response)
1414{
1415 assert(!transfers.empty() && transfers.front() == response);
1416
1417 transfers.pop();
1418
1419 if (!response->isLoad)
1420 numStoresInTransfers--;
1421
1422 if (response->issuedToMemory)
1423 numAccessesIssuedToMemory--;
1424
1425 if (response->state != LSQRequest::StoreInStoreBuffer) {
1426 DPRINTF(MinorMem, "Deleting %s request: %s\n",
1427 (response->isLoad ? "load" : "store"),
1428 *(response->inst));
1429
1430 delete response;
1431 }
1432}
1433
1434void
1435LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1436{
1437 assert(request->state == LSQRequest::StoreToStoreBuffer);
1438
1439 DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1440 *(request->inst));
1441
1442 request->inst->inStoreBuffer = true;
1443
1444 storeBuffer.insert(request);
1445}
1446
1447bool
1448LSQ::isDrained()
1449{
1450 return requests.empty() && transfers.empty() &&
1451 storeBuffer.isDrained();
1452}
1453
1454bool
1455LSQ::needsToTick()
1456{
1457 bool ret = false;
1458
1459 if (canSendToMemorySystem()) {
1460 bool have_translated_requests = !requests.empty() &&
1461 requests.front()->state != LSQRequest::InTranslation &&
1462 transfers.unreservedRemainingSpace() != 0;
1463
1464 ret = have_translated_requests ||
1465 storeBuffer.numUnissuedStores() != 0;
1466 }
1467
1468 if (ret)
1469 DPRINTF(Activity, "Need to tick\n");
1470
1471 return ret;
1472}
1473
1474void
1475LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1476 unsigned int size, Addr addr, Request::Flags flags,
1477 uint64_t *res)
1478{
1479 bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1480 LSQRequestPtr request;
1481
1482 /* Copy given data into the request. The request will pass this to the
1483 * packet and then it will own the data */
1484 uint8_t *request_data = NULL;
1485
1486 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1487 " 0x%x%s lineWidth : 0x%x\n",
1488 (isLoad ? "load" : "store"), addr, size, flags,
1489 (needs_burst ? " (needs burst)" : ""), lineWidth);
1490
1491 if (!isLoad) {
1492 /* request_data becomes the property of a ...DataRequest (see below)
1493 * and destroyed by its destructor */
1494 request_data = new uint8_t[size];
1495 if (flags & Request::CACHE_BLOCK_ZERO) {
1496 /* For cache zeroing, just use zeroed data */
1497 std::memset(request_data, 0, size);
1498 } else {
1499 std::memcpy(request_data, data, size);
1500 }
1501 }
1502
1503 if (needs_burst) {
1504 request = new SplitDataRequest(
1505 *this, inst, isLoad, request_data, res);
1506 } else {
1507 request = new SingleDataRequest(
1508 *this, inst, isLoad, request_data, res);
1509 }
1510
1511 if (inst->traceData)
1512 inst->traceData->setMem(addr, size, flags);
1513
1514 int cid = cpu.threads[inst->id.threadId]->getTC()->contextId();
1515 request->request.setContext(cid);
1516 request->request.setVirt(0 /* asid */,
1517 addr, size, flags, cpu.dataMasterId(),
1518 /* I've no idea why we need the PC, but give it */
1519 inst->pc.instAddr());
1520
1521 requests.push(request);
1522 request->startAddrTranslation();
1523}
1524
1525void
1526LSQ::pushFailedRequest(MinorDynInstPtr inst)
1527{
1528 LSQRequestPtr request = new FailedDataRequest(*this, inst);
1529 requests.push(request);
1530}
1531
1532void
1533LSQ::minorTrace() const
1534{
1535 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1536 " lastMemBarrier=%d\n",
1537 state, numAccessesInDTLB, numAccessesInMemorySystem,
1538 numStoresInTransfers, lastMemBarrier[0]);
1539 requests.minorTrace();
1540 transfers.minorTrace();
1541 storeBuffer.minorTrace();
1542}
1543
1544LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1545 unsigned int store_buffer_size,
1546 unsigned int store_limit_per_cycle) :
1547 Named(name_), lsq(lsq_),
1548 numSlots(store_buffer_size),
1549 storeLimitPerCycle(store_limit_per_cycle),
1550 slots(),
1551 numUnissuedAccesses(0)
1552{
1553}
1554
1555PacketPtr
1556makePacketForRequest(Request &request, bool isLoad,
1557 Packet::SenderState *sender_state, PacketDataPtr data)
1558{
1559 PacketPtr ret = isLoad ? Packet::createRead(&request)
1560 : Packet::createWrite(&request);
1561
1562 if (sender_state)
1563 ret->pushSenderState(sender_state);
1564
1565 if (isLoad)
1566 ret->allocate();
1567 else
1568 ret->dataDynamic(data);
1569
1570 return ret;
1571}
1572
1573void
1574LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1575{
1576 assert(inst->isInst() && inst->staticInst->isMemBarrier());
1577 assert(inst->id.execSeqNum > lastMemBarrier[inst->id.threadId]);
1578
1579 /* Remember the barrier. We only have a notion of one
1580 * barrier so this may result in some mem refs being
1581 * delayed if they are between barriers */
1582 lastMemBarrier[inst->id.threadId] = inst->id.execSeqNum;
1583}
1584
1585void
1586LSQ::LSQRequest::makePacket()
1587{
1588 /* Make the function idempotent */
1589 if (packet)
1590 return;
1591
1592 // if the translation faulted, do not create a packet
1593 if (fault != NoFault) {
1594 assert(packet == NULL);
1595 return;
1596 }
1597
1598 packet = makePacketForRequest(request, isLoad, this, data);
1599 /* Null the ret data so we know not to deallocate it when the
1600 * ret is destroyed. The data now belongs to the ret and
1601 * the ret is responsible for its destruction */
1602 data = NULL;
1603}
1604
1605std::ostream &
1606operator <<(std::ostream &os, LSQ::MemoryState state)
1607{
1608 switch (state) {
1609 case LSQ::MemoryRunning:
1610 os << "MemoryRunning";
1611 break;
1612 case LSQ::MemoryNeedsRetry:
1613 os << "MemoryNeedsRetry";
1614 break;
1615 default:
1616 os << "MemoryState-" << static_cast<int>(state);
1617 break;
1618 }
1619 return os;
1620}
1621
1622void
1623LSQ::recvTimingSnoopReq(PacketPtr pkt)
1624{
1625 /* LLSC operations in Minor can't be speculative and are executed from
1626 * the head of the requests queue. We shouldn't need to do more than
1627 * this action on snoops. */
1628 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1629 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1630 cpu.wakeup(tid);
1631 }
1632 }
1633
1634 if (pkt->isInvalidate() || pkt->isWrite()) {
1635 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1636 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1637 cacheBlockMask);
1638 }
1639 }
1640}
1641
1642void
1643LSQ::threadSnoop(LSQRequestPtr request)
1644{
1645 /* LLSC operations in Minor can't be speculative and are executed from
1646 * the head of the requests queue. We shouldn't need to do more than
1647 * this action on snoops. */
1648 ThreadID req_tid = request->inst->id.threadId;
1649 PacketPtr pkt = request->packet;
1650
1651 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) {
1652 if (tid != req_tid) {
1653 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) {
1654 cpu.wakeup(tid);
1655 }
1656
1657 if (pkt->isInvalidate() || pkt->isWrite()) {
1658 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt,
1659 cacheBlockMask);
1660 }
1661 }
1662 }
1663}
1664
1665}
|