3c3
< * Copyright (c) 2010-2014, 2017 ARM Limited
---
> * Copyright (c) 2010-2014, 2017-2018 ARM Limited
68a69,70
> assert(_inst->savedReq);
> _inst->savedReq->writebackScheduled();
79,81c81,82
< if (pkt->senderState)
< delete pkt->senderState;
<
---
> assert(inst->savedReq);
> inst->savedReq->writebackDone();
91a93,110
> template <class Impl>
> bool
> LSQUnit<Impl>::recvTimingResp(PacketPtr pkt)
> {
> auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
> LSQRequest* req = senderState->request();
> assert(req != nullptr);
> bool ret = true;
> /* Check that the request is still alive before any further action. */
> if (senderState->alive()) {
> ret = req->recvTimingResp(pkt);
> } else {
> senderState->outstanding--;
> }
> return ret;
>
> }
>
98,99d116
< DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
< DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
101,108c118
< if (state->cacheBlocked) {
< // This is the first half of a previous split load,
< // where the 2nd half blocked, ignore this response
< DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier "
< "blocked split load recieved. Ignoring.\n", inst->seqNum);
< delete state;
< return;
< }
---
> cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
110,113c120,122
< // If this is a split access, wait until all packets are received.
< if (TheISA::HasUnalignedMemAcc && !state->complete()) {
< return;
< }
---
> /* Notify the sender state that the access is complete (for ownership
> * tracking). */
> state->complete();
117c126
< if (!state->noWB) {
---
> if (state->needWB) {
121,125c130,134
< if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
< !state->isLoad) {
< writeback(inst, pkt);
< } else {
< writeback(inst, state->mainPkt);
---
> writeback(inst, state->request()->mainPacket());
> if (inst->isStore()) {
> auto ss = dynamic_cast<SQSenderState*>(state);
> ss->writebackDone();
> completeStore(ss->idx);
126a136,137
> } else if (inst->isStore()) {
> completeStore(dynamic_cast<SQSenderState*>(state)->idx);
128,131d138
<
< if (inst->isStore()) {
< completeStore(state->idx);
< }
133,141d139
<
< if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
< delete state->mainPkt;
< }
<
< pkt->req->setAccessLatency();
< cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
<
< delete state;
147d144
< LQEntries(lqEntries+1), SQEntries(sqEntries+1),
149,150c146,147
< isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false),
< pendingPkt(nullptr)
---
> isStoreBlocked(false), storeInFlight(false), hasPendingRequest(false),
> pendingRequest(nullptr)
170d166
< cacheStorePorts = params->cacheStorePorts;
183d178
< loadHead = loadTail = 0;
185c180
< storeHead = storeWBIdx = storeTail = 0;
---
> storeWBIt = storeQueue.begin();
187,188d181
< usedStorePorts = 0;
<
262,275d254
< LSQUnit<Impl>::clearLQ()
< {
< loadQueue.clear();
< }
<
< template<class Impl>
< void
< LSQUnit<Impl>::clearSQ()
< {
< storeQueue.clear();
< }
<
< template<class Impl>
< void
278,279c257,258
< for (int i = 0; i < loadQueue.size(); ++i)
< assert(!loadQueue[i]);
---
> for (int i = 0; i < loadQueue.capacity(); ++i)
> assert(!loadQueue[i].valid());
292,329d270
< template<class Impl>
< void
< LSQUnit<Impl>::resizeLQ(unsigned size)
< {
< unsigned size_plus_sentinel = size + 1;
< assert(size_plus_sentinel >= LQEntries);
<
< if (size_plus_sentinel > LQEntries) {
< while (size_plus_sentinel > loadQueue.size()) {
< DynInstPtr dummy;
< loadQueue.push_back(dummy);
< LQEntries++;
< }
< } else {
< LQEntries = size_plus_sentinel;
< }
<
< assert(LQEntries <= 256);
< }
<
< template<class Impl>
< void
< LSQUnit<Impl>::resizeSQ(unsigned size)
< {
< unsigned size_plus_sentinel = size + 1;
< if (size_plus_sentinel > SQEntries) {
< while (size_plus_sentinel > storeQueue.size()) {
< SQEntry dummy;
< storeQueue.push_back(dummy);
< SQEntries++;
< }
< } else {
< SQEntries = size_plus_sentinel;
< }
<
< assert(SQEntries <= 256);
< }
<
351,352c292,293
< assert((loadTail + 1) % LQEntries != loadHead);
< assert(loads < LQEntries);
---
> assert(!loadQueue.full());
> assert(loads < loadQueue.capacity());
355c296
< load_inst->pcState(), loadTail, load_inst->seqNum);
---
> load_inst->pcState(), loadQueue.tail(), load_inst->seqNum);
357c298,299
< load_inst->lqIdx = loadTail;
---
> /* Grow the queue. */
> loadQueue.advance_tail();
359,363c301
< if (stores == 0) {
< load_inst->sqIdx = -1;
< } else {
< load_inst->sqIdx = storeTail;
< }
---
> load_inst->sqIt = storeQueue.end();
365c303,306
< loadQueue[loadTail] = load_inst;
---
> assert(!loadQueue.back().valid());
> loadQueue.back().set(load_inst);
> load_inst->lqIdx = loadQueue.tail();
> load_inst->lqIt = loadQueue.getIterator(load_inst->lqIdx);
367,368d307
< incrLdIdx(loadTail);
<
374c313
< LSQUnit<Impl>::insertStore(const DynInstPtr &store_inst)
---
> LSQUnit<Impl>::insertStore(const DynInstPtr& store_inst)
377,378c316,317
< assert((storeTail + 1) % SQEntries != storeHead);
< assert(stores < SQEntries);
---
> assert(!storeQueue.full());
> assert(stores < storeQueue.capacity());
381c320,321
< store_inst->pcState(), storeTail, store_inst->seqNum);
---
> store_inst->pcState(), storeQueue.tail(), store_inst->seqNum);
> storeQueue.advance_tail();
383,384c323,325
< store_inst->sqIdx = storeTail;
< store_inst->lqIdx = loadTail;
---
> store_inst->sqIdx = storeQueue.tail();
> store_inst->lqIdx = loadQueue.moduloAdd(loadQueue.tail(), 1);
> store_inst->lqIt = loadQueue.end();
386c327
< storeQueue[storeTail] = SQEntry(store_inst);
---
> storeQueue.back().set(store_inst);
388,389d328
< incrStIdx(storeTail);
<
410,411c349,351
< DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads);
< return LQEntries - loads - 1;
---
> DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n",
> 1 + loadQueue.capacity(), loads);
> return loadQueue.capacity() - loads;
420,421c360,362
< DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores);
< return SQEntries - stores - 1;
---
> DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n",
> 1 + storeQueue.capacity(), stores);
> return storeQueue.capacity() - stores;
432d372
< int load_idx = loadHead;
435,436d374
< // Only Invalidate packet calls checkSnoop
< assert(pkt->isInvalidate());
445c383,384
< Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
---
> if (loadQueue.empty())
> return;
447,450c386
< DynInstPtr ld_inst = loadQueue[load_idx];
< if (ld_inst) {
< Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
< Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
---
> auto iter = loadQueue.begin();
452,457c388
< // Check that this snoop didn't just invalidate our lock flag
< if (ld_inst->effAddrValid() && (load_addr_low == invalidate_addr
< || load_addr_high == invalidate_addr)
< && ld_inst->memReqFlags & Request::LLSC)
< TheISA::handleLockedSnoopHit(ld_inst.get());
< }
---
> Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
459,461c390,392
< // If this is the only load in the LSQ we don't care
< if (load_idx == loadTail)
< return;
---
> DynInstPtr ld_inst = iter->instruction();
> assert(ld_inst);
> LSQRequest *req = iter->request();
463c394,398
< incrLdIdx(load_idx);
---
> // Check that this snoop didn't just invalidate our lock flag
> if (ld_inst->effAddrValid() &&
> req->isCacheBlockHit(invalidate_addr, cacheBlockMask)
> && ld_inst->memReqFlags & Request::LLSC)
> TheISA::handleLockedSnoopHit(ld_inst.get());
467,471c402,406
< while (load_idx != loadTail) {
< DynInstPtr ld_inst = loadQueue[load_idx];
<
< if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
< incrLdIdx(load_idx);
---
> while (++iter != loadQueue.end()) {
> ld_inst = iter->instruction();
> assert(ld_inst);
> req = iter->request();
> if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered())
473d407
< }
475,476c409,410
< Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
< Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
---
> DPRINTF(LSQUnit, "-- inst [sn:%lli] to pktAddr:%#x\n",
> ld_inst->seqNum, invalidate_addr);
478,482c412,413
< DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
< ld_inst->seqNum, load_addr_low, invalidate_addr);
<
< if ((load_addr_low == invalidate_addr
< || load_addr_high == invalidate_addr) || force_squash) {
---
> if (force_squash ||
> req->isCacheBlockHit(invalidate_addr, cacheBlockMask)) {
511d441
< incrLdIdx(load_idx);
518c448,449
< LSQUnit<Impl>::checkViolations(int load_idx, const DynInstPtr &inst)
---
> LSQUnit<Impl>::checkViolations(typename LoadQueue::iterator& loadIt,
> const DynInstPtr& inst)
528,529c459,460
< while (load_idx != loadTail) {
< DynInstPtr ld_inst = loadQueue[load_idx];
---
> while (loadIt != loadQueue.end()) {
> DynInstPtr ld_inst = loadIt->instruction();
531c462
< incrLdIdx(load_idx);
---
> ++loadIt;
588c519
< incrLdIdx(load_idx);
---
> ++loadIt;
611,612c542
< if (inst->isTranslationDelayed() &&
< load_fault == NoFault)
---
> if (inst->isTranslationDelayed() && load_fault == NoFault)
634,636c564,566
< assert(inst->effAddrValid());
< int load_idx = inst->lqIdx;
< incrLdIdx(load_idx);
---
> if (inst->effAddrValid()) {
> auto it = inst->lqIt;
> ++it;
638,639c568,570
< if (checkLoads)
< return checkViolations(load_idx, inst);
---
> if (checkLoads)
> return checkViolations(it, inst);
> }
662c593
< int load_idx = store_inst->lqIdx;
---
> typename LoadQueue::iterator loadIt = store_inst->lqIt;
677c608
< if (storeQueue[store_idx].size == 0) {
---
> if (storeQueue[store_idx].size() == 0) {
689c620
< storeQueue[store_idx].canWB = true;
---
> storeQueue[store_idx].canWB() = true;
694c625
< return checkViolations(load_idx, store_inst);
---
> return checkViolations(loadIt, store_inst);
702c633
< assert(loadQueue[loadHead]);
---
> assert(loadQueue.front().valid());
705c636
< loadQueue[loadHead]->pcState());
---
> loadQueue.front().instruction()->pcState());
707c638,639
< loadQueue[loadHead] = NULL;
---
> loadQueue.front().clear();
> loadQueue.pop_front();
709,710d640
< incrLdIdx(loadHead);
<
718c648
< assert(loads == 0 || loadQueue[loadHead]);
---
> assert(loads == 0 || loadQueue.front().valid());
720c650,651
< while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
---
> while (loads != 0 && loadQueue.front().instruction()->seqNum
> <= youngest_inst) {
729c660
< assert(stores == 0 || storeQueue[storeHead].inst);
---
> assert(stores == 0 || storeQueue.front().valid());
731,734c662,664
< int store_idx = storeHead;
<
< while (store_idx != storeTail) {
< assert(storeQueue[store_idx].inst);
---
> /* Forward iterate the store queue (age order). */
> for (auto& x : storeQueue) {
> assert(x.valid());
737,738c667,668
< if (!storeQueue[store_idx].canWB) {
< if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
---
> if (!x.canWB()) {
> if (x.instruction()->seqNum > youngest_inst) {
743,744c673,674
< storeQueue[store_idx].inst->pcState(),
< storeQueue[store_idx].inst->seqNum);
---
> x.instruction()->pcState(),
> x.instruction()->seqNum);
746c676
< storeQueue[store_idx].canWB = true;
---
> x.canWB() = true;
750,751d679
<
< incrStIdx(store_idx);
757c685
< LSQUnit<Impl>::writebackPendingStore()
---
> LSQUnit<Impl>::writebackBlockedStore()
759,767c687,690
< if (hasPendingPkt) {
< assert(pendingPkt != NULL);
<
< // If the cache is blocked, this will store the packet for retry.
< if (sendStore(pendingPkt)) {
< storePostSend(pendingPkt);
< }
< pendingPkt = NULL;
< hasPendingPkt = false;
---
> assert(isStoreBlocked);
> storeWBIt->request()->sendPacketToCache();
> if (storeWBIt->request()->isSent()){
> storePostSend();
775,778c698,700
< // First writeback the second packet from any split store that didn't
< // complete last cycle because there weren't enough cache ports available.
< if (TheISA::HasUnalignedMemAcc) {
< writebackPendingStore();
---
> if (isStoreBlocked) {
> DPRINTF(LSQUnit, "Writing back blocked store\n");
> writebackBlockedStore();
782,784c704,706
< storeWBIdx != storeTail &&
< storeQueue[storeWBIdx].inst &&
< storeQueue[storeWBIdx].canWB &&
---
> storeWBIt.dereferenceable() &&
> storeWBIt->valid() &&
> storeWBIt->canWB() &&
786c708
< usedStorePorts < cacheStorePorts) {
---
> lsq->storePortAvailable()) {
796,800c718,722
< if (storeQueue[storeWBIdx].size == 0) {
< completeStore(storeWBIdx);
<
< incrStIdx(storeWBIdx);
<
---
> if (storeWBIt->size() == 0) {
> /* It is important that the preincrement happens at (or before)
> * the call, as the the code of completeStore checks
> * storeWBIt. */
> completeStore(storeWBIt++);
804,808c726,727
< ++usedStorePorts;
<
< if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
< incrStIdx(storeWBIdx);
<
---
> if (storeWBIt->instruction()->isDataPrefetch()) {
> storeWBIt++;
812,813c731,732
< assert(storeQueue[storeWBIdx].req);
< assert(!storeQueue[storeWBIdx].committed);
---
> assert(storeWBIt->hasRequest());
> assert(!storeWBIt->committed());
815,818c734,736
< if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
< assert(storeQueue[storeWBIdx].sreqLow);
< assert(storeQueue[storeWBIdx].sreqHigh);
< }
---
> DynInstPtr inst = storeWBIt->instruction();
> LSQRequest* req = storeWBIt->request();
> storeWBIt->committed() = true;
820,827d737
< DynInstPtr inst = storeQueue[storeWBIdx].inst;
<
< RequestPtr &req = storeQueue[storeWBIdx].req;
< const RequestPtr &sreqLow = storeQueue[storeWBIdx].sreqLow;
< const RequestPtr &sreqHigh = storeQueue[storeWBIdx].sreqHigh;
<
< storeQueue[storeWBIdx].committed = true;
<
829c739
< inst->memData = new uint8_t[req->getSize()];
---
> inst->memData = new uint8_t[req->_size];
831,832c741,742
< if (storeQueue[storeWBIdx].isAllZeros)
< memset(inst->memData, 0, req->getSize());
---
> if (storeWBIt->isAllZeros())
> memset(inst->memData, 0, req->_size);
834c744
< memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
---
> memcpy(inst->memData, storeWBIt->data(), req->_size);
836,837d745
< PacketPtr data_pkt;
< PacketPtr snd_data_pkt = NULL;
839,842c747,751
< LSQSenderState *state = new LSQSenderState;
< state->isLoad = false;
< state->idx = storeWBIdx;
< state->inst = inst;
---
> if (req->senderState() == nullptr) {
> SQSenderState *state = new SQSenderState(storeWBIt);
> state->isLoad = false;
> state->needWB = false;
> state->inst = inst;
844,865c753,757
< if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
<
< // Build a single data packet if the store isn't split.
< data_pkt = Packet::createWrite(req);
< data_pkt->dataStatic(inst->memData);
< data_pkt->senderState = state;
< } else {
< // Create two packets if the store is split in two.
< data_pkt = Packet::createWrite(sreqLow);
< snd_data_pkt = Packet::createWrite(sreqHigh);
<
< data_pkt->dataStatic(inst->memData);
< snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
<
< data_pkt->senderState = state;
< snd_data_pkt->senderState = state;
<
< state->isSplit = true;
< state->outstanding = 2;
<
< // Can delete the main request now.
< req = sreqLow;
---
> req->senderState(state);
> if (inst->isStoreConditional()) {
> /* Only store conditionals need a writeback. */
> state->needWB = true;
> }
866a759
> req->buildPackets();
870,871c763,764
< storeWBIdx, inst->pcState(),
< req->getPaddr(), (int)*(inst->memData),
---
> storeWBIt.idx(), inst->pcState(),
> req->request()->getPaddr(), (int)*(inst->memData),
876d768
< assert(!storeQueue[storeWBIdx].isSplit);
881c773,774
< bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask);
---
> bool success = TheISA::handleLockedWrite(inst.get(),
> req->request(), cacheBlockMask);
882a776
> req->packetSent();
884a779
> req->complete();
889c784,786
< WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
---
> PacketPtr new_pkt = new Packet(*req->packet());
> WritebackEvent *wb = new WritebackEvent(inst,
> new_pkt, this);
891,892c788,792
< completeStore(storeWBIdx);
< incrStIdx(storeWBIdx);
---
> completeStore(storeWBIt);
> if (!storeQueue.empty())
> storeWBIt++;
> else
> storeWBIt = storeQueue.end();
895,897d794
< } else {
< // Non-store conditionals do not need a writeback.
< state->noWB = true;
900,905c797
< bool split =
< TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
<
< ThreadContext *thread = cpu->tcBase(lsqID);
<
< if (req->isMmappedIpr()) {
---
> if (req->request()->isMmappedIpr()) {
907,920c799,810
< TheISA::handleIprWrite(thread, data_pkt);
< delete data_pkt;
< if (split) {
< assert(snd_data_pkt->req->isMmappedIpr());
< TheISA::handleIprWrite(thread, snd_data_pkt);
< delete snd_data_pkt;
< }
< delete state;
< completeStore(storeWBIdx);
< incrStIdx(storeWBIdx);
< } else if (!sendStore(data_pkt)) {
< DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
< "retry later\n",
< inst->seqNum);
---
> ThreadContext *thread = cpu->tcBase(lsqID);
> PacketPtr main_pkt = new Packet(req->mainRequest(),
> MemCmd::WriteReq);
> main_pkt->dataStatic(inst->memData);
> req->handleIprWrite(thread, main_pkt);
> delete main_pkt;
> completeStore(storeWBIt);
> storeWBIt++;
> continue;
> }
> /* Send to cache */
> req->sendPacketToCache();
922,926c812,814
< // Need to store the second packet, if split.
< if (split) {
< state->pktToSend = true;
< state->pendingPacket = snd_data_pkt;
< }
---
> /* If successful, do the post send */
> if (req->isSent()) {
> storePostSend();
928,954c816,818
<
< // If split, try to send the second packet too
< if (split) {
< assert(snd_data_pkt);
<
< // Ensure there are enough ports to use.
< if (usedStorePorts < cacheStorePorts) {
< ++usedStorePorts;
< if (sendStore(snd_data_pkt)) {
< storePostSend(snd_data_pkt);
< } else {
< DPRINTF(IEW, "D-Cache became blocked when writing"
< " [sn:%lli] second packet, will retry later\n",
< inst->seqNum);
< }
< } else {
<
< // Store the packet for when there's free ports.
< assert(pendingPkt == NULL);
< pendingPkt = snd_data_pkt;
< hasPendingPkt = true;
< }
< } else {
<
< // Not a split store.
< storePostSend(data_pkt);
< }
---
> DPRINTF(LSQUnit, "D-Cache became blocked when writing [sn:%lli], "
> "will retry later\n",
> inst->seqNum);
957,960d820
<
< // Not sure this should set it to 0.
< usedStorePorts = 0;
<
964,977d823
< /*template <class Impl>
< void
< LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
< {
< list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
< mshrSeqNums.end(),
< seqNum);
<
< if (mshr_it != mshrSeqNums.end()) {
< mshrSeqNums.erase(mshr_it);
< DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
< }
< }*/
<
985,988c831,832
< int load_idx = loadTail;
< decrLdIdx(load_idx);
<
< while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
---
> while (loads != 0 &&
> loadQueue.back().instruction()->seqNum > squashed_num) {
991,992c835,836
< loadQueue[load_idx]->pcState(),
< loadQueue[load_idx]->seqNum);
---
> loadQueue.back().instruction()->pcState(),
> loadQueue.back().instruction()->seqNum);
994c838
< if (isStalled() && load_idx == stallingLoadIdx) {
---
> if (isStalled() && loadQueue.tail() == stallingLoadIdx) {
1001,1002c845,847
< loadQueue[load_idx]->setSquashed();
< loadQueue[load_idx] = NULL;
---
> loadQueue.back().instruction()->setSquashed();
> loadQueue.back().clear();
>
1005,1008c850
< // Inefficient!
< loadTail = load_idx;
<
< decrLdIdx(load_idx);
---
> loadQueue.pop_back();
1016,1018d857
< int store_idx = storeTail;
< decrStIdx(store_idx);
<
1020c859
< storeQueue[store_idx].inst->seqNum > squashed_num) {
---
> storeQueue.back().instruction()->seqNum > squashed_num) {
1022c861
< if (storeQueue[store_idx].canWB) {
---
> if (storeQueue.back().canWB()) {
1028,1029c867,868
< storeQueue[store_idx].inst->pcState(),
< store_idx, storeQueue[store_idx].inst->seqNum);
---
> storeQueue.back().instruction()->pcState(),
> storeQueue.tail(), storeQueue.back().instruction()->seqNum);
1034c873
< storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
---
> storeQueue.back().instruction()->seqNum == stallingStoreIsn) {
1041,1043c880
< storeQueue[store_idx].inst->setSquashed();
< storeQueue[store_idx].inst = NULL;
< storeQueue[store_idx].canWB = 0;
---
> storeQueue.back().instruction()->setSquashed();
1048,1053c885
< storeQueue[store_idx].req.reset();
< if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
< storeQueue[store_idx].sreqLow.reset();
< storeQueue[store_idx].sreqHigh.reset();
< }
<
---
> storeQueue.back().clear();
1056,1059c888
< // Inefficient!
< storeTail = store_idx;
<
< decrStIdx(store_idx);
---
> storeQueue.pop_back();
1066c895
< LSQUnit<Impl>::storePostSend(PacketPtr pkt)
---
> LSQUnit<Impl>::storePostSend()
1069c898
< storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
---
> storeWBIt->instruction()->seqNum == stallingStoreIsn) {
1075c904
< iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
---
> iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
1078c907
< if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
---
> if (!storeWBIt->instruction()->isStoreConditional()) {
1082c911
< storeQueue[storeWBIdx].inst->setCompleted();
---
> storeWBIt->instruction()->setCompleted();
1085c914
< cpu->checker->verify(storeQueue[storeWBIdx].inst);
---
> cpu->checker->verify(storeWBIt->instruction());
1093c922
< incrStIdx(storeWBIdx);
---
> storeWBIt++;
1139c968
< LSQUnit<Impl>::completeStore(int store_idx)
---
> LSQUnit<Impl>::completeStore(typename StoreQueue::iterator store_idx)
1141,1142c970,971
< assert(storeQueue[store_idx].inst);
< storeQueue[store_idx].completed = true;
---
> assert(store_idx->valid());
> store_idx->completed() = true;
1150c979,982
< if (store_idx == storeHead) {
---
> /* We 'need' a copy here because we may clear the entry from the
> * store queue. */
> DynInstPtr store_inst = store_idx->instruction();
> if (store_idx == storeQueue.begin()) {
1152,1153c984,985
< incrStIdx(storeHead);
<
---
> storeQueue.front().clear();
> storeQueue.pop_front();
1155,1156c987,988
< } while (storeQueue[storeHead].completed &&
< storeHead != storeTail);
---
> } while (storeQueue.front().completed() &&
> !storeQueue.empty());
1163c995
< storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
---
> store_inst->seqNum, store_idx.idx() - 1, storeQueue.head() - 1);
1167,1168c999,1000
< storeQueue[store_idx].inst->storeTick =
< curTick() - storeQueue[store_idx].inst->fetchTick;
---
> store_idx->instruction()->storeTick =
> curTick() - store_idx->instruction()->fetchTick;
1173c1005
< storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
---
> store_inst->seqNum == stallingStoreIsn) {
1179c1011
< iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
---
> iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
1182c1014
< storeQueue[store_idx].inst->setCompleted();
---
> store_inst->setCompleted();
1191d1022
<
1195,1196c1026,1027
< if (cpu->checker && !storeQueue[store_idx].inst->isStoreConditional()) {
< cpu->checker->verify(storeQueue[store_idx].inst);
---
> if (cpu->checker && !store_inst->isStoreConditional()) {
> cpu->checker->verify(store_inst);
1202c1033
< LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
---
> LSQUnit<Impl>::trySendPacket(bool isLoad, PacketPtr data_pkt)
1204,1213c1035,1036
< if (!dcachePort->sendTimingReq(data_pkt)) {
< // Need to handle becoming blocked on a store.
< isStoreBlocked = true;
< ++lsqCacheBlocked;
< assert(retryPkt == NULL);
< retryPkt = data_pkt;
< return false;
< }
< return true;
< }
---
> bool ret = true;
> bool cache_got_blocked = false;
1215,1221c1038
< template <class Impl>
< void
< LSQUnit<Impl>::recvRetry()
< {
< if (isStoreBlocked) {
< DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
< assert(retryPkt != NULL);
---
> auto state = dynamic_cast<LSQSenderState*>(data_pkt->senderState);
1223,1224c1040,1047
< LSQSenderState *state =
< dynamic_cast<LSQSenderState *>(retryPkt->senderState);
---
> if (!lsq->cacheBlocked() && (isLoad || lsq->storePortAvailable())) {
> if (!dcachePort->sendTimingReq(data_pkt)) {
> ret = false;
> cache_got_blocked = true;
> }
> } else {
> ret = false;
> }
1226,1233c1049,1051
< if (dcachePort->sendTimingReq(retryPkt)) {
< // Don't finish the store unless this is the last packet.
< if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
< state->pendingPacket == retryPkt) {
< state->pktToSend = false;
< storePostSend(retryPkt);
< }
< retryPkt = NULL;
---
> if (ret) {
> if (!isLoad) {
> lsq->storePortBusy();
1235,1244c1053,1058
<
< // Send any outstanding packet.
< if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
< assert(state->pendingPacket);
< if (sendStore(state->pendingPacket)) {
< storePostSend(state->pendingPacket);
< }
< }
< } else {
< // Still blocked!
---
> }
> state->outstanding++;
> state->request()->packetSent();
> } else {
> if (cache_got_blocked) {
> lsq->cacheBlocked(true);
1246a1061,1065
> if (!isLoad) {
> assert(state->request() == storeWBIt->request());
> isStoreBlocked = true;
> }
> state->request()->packetNotSent();
1248d1066
< }
1250,1255c1068
< template <class Impl>
< inline void
< LSQUnit<Impl>::incrStIdx(int &store_idx) const
< {
< if (++store_idx >= SQEntries)
< store_idx = 0;
---
> return ret;
1259,1260c1072,1073
< inline void
< LSQUnit<Impl>::decrStIdx(int &store_idx) const
---
> void
> LSQUnit<Impl>::recvRetry()
1262,1263c1075,1078
< if (--store_idx < 0)
< store_idx += SQEntries;
---
> if (isStoreBlocked) {
> DPRINTF(LSQUnit, "Receiving retry: blocked store\n");
> writebackBlockedStore();
> }
1267,1282d1081
< inline void
< LSQUnit<Impl>::incrLdIdx(int &load_idx) const
< {
< if (++load_idx >= LQEntries)
< load_idx = 0;
< }
<
< template <class Impl>
< inline void
< LSQUnit<Impl>::decrLdIdx(int &load_idx) const
< {
< if (--load_idx < 0)
< load_idx += LQEntries;
< }
<
< template <class Impl>
1290,1293c1089,1090
< int load_idx = loadHead;
<
< while (load_idx != loadTail && loadQueue[load_idx]) {
< const DynInstPtr &inst(loadQueue[load_idx]);
---
> for (const auto& e: loadQueue) {
> const DynInstPtr &inst(e.instruction());
1295,1296d1091
<
< incrLdIdx(load_idx);
1303,1306c1098,1099
< int store_idx = storeHead;
<
< while (store_idx != storeTail && storeQueue[store_idx].inst) {
< const DynInstPtr &inst(storeQueue[store_idx].inst);
---
> for (const auto& e: storeQueue) {
> const DynInstPtr &inst(e.instruction());
1308,1309d1100
<
< incrStIdx(store_idx);
1314a1106,1112
> template <class Impl>
> unsigned int
> LSQUnit<Impl>::cacheLineSize()
> {
> return cpu->cacheLineSize();
> }
>
< * Copyright (c) 2010-2014, 2017 ARM Limited
---
> * Copyright (c) 2010-2014, 2017-2018 ARM Limited
68a69,70
> assert(_inst->savedReq);
> _inst->savedReq->writebackScheduled();
79,81c81,82
< if (pkt->senderState)
< delete pkt->senderState;
<
---
> assert(inst->savedReq);
> inst->savedReq->writebackDone();
91a93,110
> template <class Impl>
> bool
> LSQUnit<Impl>::recvTimingResp(PacketPtr pkt)
> {
> auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
> LSQRequest* req = senderState->request();
> assert(req != nullptr);
> bool ret = true;
> /* Check that the request is still alive before any further action. */
> if (senderState->alive()) {
> ret = req->recvTimingResp(pkt);
> } else {
> senderState->outstanding--;
> }
> return ret;
>
> }
>
98,99d116
< DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
< DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
101,108c118
< if (state->cacheBlocked) {
< // This is the first half of a previous split load,
< // where the 2nd half blocked, ignore this response
< DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier "
< "blocked split load recieved. Ignoring.\n", inst->seqNum);
< delete state;
< return;
< }
---
> cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
110,113c120,122
< // If this is a split access, wait until all packets are received.
< if (TheISA::HasUnalignedMemAcc && !state->complete()) {
< return;
< }
---
> /* Notify the sender state that the access is complete (for ownership
> * tracking). */
> state->complete();
117c126
< if (!state->noWB) {
---
> if (state->needWB) {
121,125c130,134
< if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
< !state->isLoad) {
< writeback(inst, pkt);
< } else {
< writeback(inst, state->mainPkt);
---
> writeback(inst, state->request()->mainPacket());
> if (inst->isStore()) {
> auto ss = dynamic_cast<SQSenderState*>(state);
> ss->writebackDone();
> completeStore(ss->idx);
126a136,137
> } else if (inst->isStore()) {
> completeStore(dynamic_cast<SQSenderState*>(state)->idx);
128,131d138
<
< if (inst->isStore()) {
< completeStore(state->idx);
< }
133,141d139
<
< if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
< delete state->mainPkt;
< }
<
< pkt->req->setAccessLatency();
< cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
<
< delete state;
147d144
< LQEntries(lqEntries+1), SQEntries(sqEntries+1),
149,150c146,147
< isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false),
< pendingPkt(nullptr)
---
> isStoreBlocked(false), storeInFlight(false), hasPendingRequest(false),
> pendingRequest(nullptr)
170d166
< cacheStorePorts = params->cacheStorePorts;
183d178
< loadHead = loadTail = 0;
185c180
< storeHead = storeWBIdx = storeTail = 0;
---
> storeWBIt = storeQueue.begin();
187,188d181
< usedStorePorts = 0;
<
262,275d254
< LSQUnit<Impl>::clearLQ()
< {
< loadQueue.clear();
< }
<
< template<class Impl>
< void
< LSQUnit<Impl>::clearSQ()
< {
< storeQueue.clear();
< }
<
< template<class Impl>
< void
278,279c257,258
< for (int i = 0; i < loadQueue.size(); ++i)
< assert(!loadQueue[i]);
---
> for (int i = 0; i < loadQueue.capacity(); ++i)
> assert(!loadQueue[i].valid());
292,329d270
< template<class Impl>
< void
< LSQUnit<Impl>::resizeLQ(unsigned size)
< {
< unsigned size_plus_sentinel = size + 1;
< assert(size_plus_sentinel >= LQEntries);
<
< if (size_plus_sentinel > LQEntries) {
< while (size_plus_sentinel > loadQueue.size()) {
< DynInstPtr dummy;
< loadQueue.push_back(dummy);
< LQEntries++;
< }
< } else {
< LQEntries = size_plus_sentinel;
< }
<
< assert(LQEntries <= 256);
< }
<
< template<class Impl>
< void
< LSQUnit<Impl>::resizeSQ(unsigned size)
< {
< unsigned size_plus_sentinel = size + 1;
< if (size_plus_sentinel > SQEntries) {
< while (size_plus_sentinel > storeQueue.size()) {
< SQEntry dummy;
< storeQueue.push_back(dummy);
< SQEntries++;
< }
< } else {
< SQEntries = size_plus_sentinel;
< }
<
< assert(SQEntries <= 256);
< }
<
351,352c292,293
< assert((loadTail + 1) % LQEntries != loadHead);
< assert(loads < LQEntries);
---
> assert(!loadQueue.full());
> assert(loads < loadQueue.capacity());
355c296
< load_inst->pcState(), loadTail, load_inst->seqNum);
---
> load_inst->pcState(), loadQueue.tail(), load_inst->seqNum);
357c298,299
< load_inst->lqIdx = loadTail;
---
> /* Grow the queue. */
> loadQueue.advance_tail();
359,363c301
< if (stores == 0) {
< load_inst->sqIdx = -1;
< } else {
< load_inst->sqIdx = storeTail;
< }
---
> load_inst->sqIt = storeQueue.end();
365c303,306
< loadQueue[loadTail] = load_inst;
---
> assert(!loadQueue.back().valid());
> loadQueue.back().set(load_inst);
> load_inst->lqIdx = loadQueue.tail();
> load_inst->lqIt = loadQueue.getIterator(load_inst->lqIdx);
367,368d307
< incrLdIdx(loadTail);
<
374c313
< LSQUnit<Impl>::insertStore(const DynInstPtr &store_inst)
---
> LSQUnit<Impl>::insertStore(const DynInstPtr& store_inst)
377,378c316,317
< assert((storeTail + 1) % SQEntries != storeHead);
< assert(stores < SQEntries);
---
> assert(!storeQueue.full());
> assert(stores < storeQueue.capacity());
381c320,321
< store_inst->pcState(), storeTail, store_inst->seqNum);
---
> store_inst->pcState(), storeQueue.tail(), store_inst->seqNum);
> storeQueue.advance_tail();
383,384c323,325
< store_inst->sqIdx = storeTail;
< store_inst->lqIdx = loadTail;
---
> store_inst->sqIdx = storeQueue.tail();
> store_inst->lqIdx = loadQueue.moduloAdd(loadQueue.tail(), 1);
> store_inst->lqIt = loadQueue.end();
386c327
< storeQueue[storeTail] = SQEntry(store_inst);
---
> storeQueue.back().set(store_inst);
388,389d328
< incrStIdx(storeTail);
<
410,411c349,351
< DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads);
< return LQEntries - loads - 1;
---
> DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n",
> 1 + loadQueue.capacity(), loads);
> return loadQueue.capacity() - loads;
420,421c360,362
< DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores);
< return SQEntries - stores - 1;
---
> DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n",
> 1 + storeQueue.capacity(), stores);
> return storeQueue.capacity() - stores;
432d372
< int load_idx = loadHead;
435,436d374
< // Only Invalidate packet calls checkSnoop
< assert(pkt->isInvalidate());
445c383,384
< Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
---
> if (loadQueue.empty())
> return;
447,450c386
< DynInstPtr ld_inst = loadQueue[load_idx];
< if (ld_inst) {
< Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
< Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
---
> auto iter = loadQueue.begin();
452,457c388
< // Check that this snoop didn't just invalidate our lock flag
< if (ld_inst->effAddrValid() && (load_addr_low == invalidate_addr
< || load_addr_high == invalidate_addr)
< && ld_inst->memReqFlags & Request::LLSC)
< TheISA::handleLockedSnoopHit(ld_inst.get());
< }
---
> Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
459,461c390,392
< // If this is the only load in the LSQ we don't care
< if (load_idx == loadTail)
< return;
---
> DynInstPtr ld_inst = iter->instruction();
> assert(ld_inst);
> LSQRequest *req = iter->request();
463c394,398
< incrLdIdx(load_idx);
---
> // Check that this snoop didn't just invalidate our lock flag
> if (ld_inst->effAddrValid() &&
> req->isCacheBlockHit(invalidate_addr, cacheBlockMask)
> && ld_inst->memReqFlags & Request::LLSC)
> TheISA::handleLockedSnoopHit(ld_inst.get());
467,471c402,406
< while (load_idx != loadTail) {
< DynInstPtr ld_inst = loadQueue[load_idx];
<
< if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
< incrLdIdx(load_idx);
---
> while (++iter != loadQueue.end()) {
> ld_inst = iter->instruction();
> assert(ld_inst);
> req = iter->request();
> if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered())
473d407
< }
475,476c409,410
< Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
< Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
---
> DPRINTF(LSQUnit, "-- inst [sn:%lli] to pktAddr:%#x\n",
> ld_inst->seqNum, invalidate_addr);
478,482c412,413
< DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
< ld_inst->seqNum, load_addr_low, invalidate_addr);
<
< if ((load_addr_low == invalidate_addr
< || load_addr_high == invalidate_addr) || force_squash) {
---
> if (force_squash ||
> req->isCacheBlockHit(invalidate_addr, cacheBlockMask)) {
511d441
< incrLdIdx(load_idx);
518c448,449
< LSQUnit<Impl>::checkViolations(int load_idx, const DynInstPtr &inst)
---
> LSQUnit<Impl>::checkViolations(typename LoadQueue::iterator& loadIt,
> const DynInstPtr& inst)
528,529c459,460
< while (load_idx != loadTail) {
< DynInstPtr ld_inst = loadQueue[load_idx];
---
> while (loadIt != loadQueue.end()) {
> DynInstPtr ld_inst = loadIt->instruction();
531c462
< incrLdIdx(load_idx);
---
> ++loadIt;
588c519
< incrLdIdx(load_idx);
---
> ++loadIt;
611,612c542
< if (inst->isTranslationDelayed() &&
< load_fault == NoFault)
---
> if (inst->isTranslationDelayed() && load_fault == NoFault)
634,636c564,566
< assert(inst->effAddrValid());
< int load_idx = inst->lqIdx;
< incrLdIdx(load_idx);
---
> if (inst->effAddrValid()) {
> auto it = inst->lqIt;
> ++it;
638,639c568,570
< if (checkLoads)
< return checkViolations(load_idx, inst);
---
> if (checkLoads)
> return checkViolations(it, inst);
> }
662c593
< int load_idx = store_inst->lqIdx;
---
> typename LoadQueue::iterator loadIt = store_inst->lqIt;
677c608
< if (storeQueue[store_idx].size == 0) {
---
> if (storeQueue[store_idx].size() == 0) {
689c620
< storeQueue[store_idx].canWB = true;
---
> storeQueue[store_idx].canWB() = true;
694c625
< return checkViolations(load_idx, store_inst);
---
> return checkViolations(loadIt, store_inst);
702c633
< assert(loadQueue[loadHead]);
---
> assert(loadQueue.front().valid());
705c636
< loadQueue[loadHead]->pcState());
---
> loadQueue.front().instruction()->pcState());
707c638,639
< loadQueue[loadHead] = NULL;
---
> loadQueue.front().clear();
> loadQueue.pop_front();
709,710d640
< incrLdIdx(loadHead);
<
718c648
< assert(loads == 0 || loadQueue[loadHead]);
---
> assert(loads == 0 || loadQueue.front().valid());
720c650,651
< while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
---
> while (loads != 0 && loadQueue.front().instruction()->seqNum
> <= youngest_inst) {
729c660
< assert(stores == 0 || storeQueue[storeHead].inst);
---
> assert(stores == 0 || storeQueue.front().valid());
731,734c662,664
< int store_idx = storeHead;
<
< while (store_idx != storeTail) {
< assert(storeQueue[store_idx].inst);
---
> /* Forward iterate the store queue (age order). */
> for (auto& x : storeQueue) {
> assert(x.valid());
737,738c667,668
< if (!storeQueue[store_idx].canWB) {
< if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
---
> if (!x.canWB()) {
> if (x.instruction()->seqNum > youngest_inst) {
743,744c673,674
< storeQueue[store_idx].inst->pcState(),
< storeQueue[store_idx].inst->seqNum);
---
> x.instruction()->pcState(),
> x.instruction()->seqNum);
746c676
< storeQueue[store_idx].canWB = true;
---
> x.canWB() = true;
750,751d679
<
< incrStIdx(store_idx);
757c685
< LSQUnit<Impl>::writebackPendingStore()
---
> LSQUnit<Impl>::writebackBlockedStore()
759,767c687,690
< if (hasPendingPkt) {
< assert(pendingPkt != NULL);
<
< // If the cache is blocked, this will store the packet for retry.
< if (sendStore(pendingPkt)) {
< storePostSend(pendingPkt);
< }
< pendingPkt = NULL;
< hasPendingPkt = false;
---
> assert(isStoreBlocked);
> storeWBIt->request()->sendPacketToCache();
> if (storeWBIt->request()->isSent()){
> storePostSend();
775,778c698,700
< // First writeback the second packet from any split store that didn't
< // complete last cycle because there weren't enough cache ports available.
< if (TheISA::HasUnalignedMemAcc) {
< writebackPendingStore();
---
> if (isStoreBlocked) {
> DPRINTF(LSQUnit, "Writing back blocked store\n");
> writebackBlockedStore();
782,784c704,706
< storeWBIdx != storeTail &&
< storeQueue[storeWBIdx].inst &&
< storeQueue[storeWBIdx].canWB &&
---
> storeWBIt.dereferenceable() &&
> storeWBIt->valid() &&
> storeWBIt->canWB() &&
786c708
< usedStorePorts < cacheStorePorts) {
---
> lsq->storePortAvailable()) {
796,800c718,722
< if (storeQueue[storeWBIdx].size == 0) {
< completeStore(storeWBIdx);
<
< incrStIdx(storeWBIdx);
<
---
> if (storeWBIt->size() == 0) {
> /* It is important that the preincrement happens at (or before)
> * the call, as the the code of completeStore checks
> * storeWBIt. */
> completeStore(storeWBIt++);
804,808c726,727
< ++usedStorePorts;
<
< if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
< incrStIdx(storeWBIdx);
<
---
> if (storeWBIt->instruction()->isDataPrefetch()) {
> storeWBIt++;
812,813c731,732
< assert(storeQueue[storeWBIdx].req);
< assert(!storeQueue[storeWBIdx].committed);
---
> assert(storeWBIt->hasRequest());
> assert(!storeWBIt->committed());
815,818c734,736
< if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
< assert(storeQueue[storeWBIdx].sreqLow);
< assert(storeQueue[storeWBIdx].sreqHigh);
< }
---
> DynInstPtr inst = storeWBIt->instruction();
> LSQRequest* req = storeWBIt->request();
> storeWBIt->committed() = true;
820,827d737
< DynInstPtr inst = storeQueue[storeWBIdx].inst;
<
< RequestPtr &req = storeQueue[storeWBIdx].req;
< const RequestPtr &sreqLow = storeQueue[storeWBIdx].sreqLow;
< const RequestPtr &sreqHigh = storeQueue[storeWBIdx].sreqHigh;
<
< storeQueue[storeWBIdx].committed = true;
<
829c739
< inst->memData = new uint8_t[req->getSize()];
---
> inst->memData = new uint8_t[req->_size];
831,832c741,742
< if (storeQueue[storeWBIdx].isAllZeros)
< memset(inst->memData, 0, req->getSize());
---
> if (storeWBIt->isAllZeros())
> memset(inst->memData, 0, req->_size);
834c744
< memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
---
> memcpy(inst->memData, storeWBIt->data(), req->_size);
836,837d745
< PacketPtr data_pkt;
< PacketPtr snd_data_pkt = NULL;
839,842c747,751
< LSQSenderState *state = new LSQSenderState;
< state->isLoad = false;
< state->idx = storeWBIdx;
< state->inst = inst;
---
> if (req->senderState() == nullptr) {
> SQSenderState *state = new SQSenderState(storeWBIt);
> state->isLoad = false;
> state->needWB = false;
> state->inst = inst;
844,865c753,757
< if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
<
< // Build a single data packet if the store isn't split.
< data_pkt = Packet::createWrite(req);
< data_pkt->dataStatic(inst->memData);
< data_pkt->senderState = state;
< } else {
< // Create two packets if the store is split in two.
< data_pkt = Packet::createWrite(sreqLow);
< snd_data_pkt = Packet::createWrite(sreqHigh);
<
< data_pkt->dataStatic(inst->memData);
< snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
<
< data_pkt->senderState = state;
< snd_data_pkt->senderState = state;
<
< state->isSplit = true;
< state->outstanding = 2;
<
< // Can delete the main request now.
< req = sreqLow;
---
> req->senderState(state);
> if (inst->isStoreConditional()) {
> /* Only store conditionals need a writeback. */
> state->needWB = true;
> }
866a759
> req->buildPackets();
870,871c763,764
< storeWBIdx, inst->pcState(),
< req->getPaddr(), (int)*(inst->memData),
---
> storeWBIt.idx(), inst->pcState(),
> req->request()->getPaddr(), (int)*(inst->memData),
876d768
< assert(!storeQueue[storeWBIdx].isSplit);
881c773,774
< bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask);
---
> bool success = TheISA::handleLockedWrite(inst.get(),
> req->request(), cacheBlockMask);
882a776
> req->packetSent();
884a779
> req->complete();
889c784,786
< WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
---
> PacketPtr new_pkt = new Packet(*req->packet());
> WritebackEvent *wb = new WritebackEvent(inst,
> new_pkt, this);
891,892c788,792
< completeStore(storeWBIdx);
< incrStIdx(storeWBIdx);
---
> completeStore(storeWBIt);
> if (!storeQueue.empty())
> storeWBIt++;
> else
> storeWBIt = storeQueue.end();
895,897d794
< } else {
< // Non-store conditionals do not need a writeback.
< state->noWB = true;
900,905c797
< bool split =
< TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
<
< ThreadContext *thread = cpu->tcBase(lsqID);
<
< if (req->isMmappedIpr()) {
---
> if (req->request()->isMmappedIpr()) {
907,920c799,810
< TheISA::handleIprWrite(thread, data_pkt);
< delete data_pkt;
< if (split) {
< assert(snd_data_pkt->req->isMmappedIpr());
< TheISA::handleIprWrite(thread, snd_data_pkt);
< delete snd_data_pkt;
< }
< delete state;
< completeStore(storeWBIdx);
< incrStIdx(storeWBIdx);
< } else if (!sendStore(data_pkt)) {
< DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
< "retry later\n",
< inst->seqNum);
---
> ThreadContext *thread = cpu->tcBase(lsqID);
> PacketPtr main_pkt = new Packet(req->mainRequest(),
> MemCmd::WriteReq);
> main_pkt->dataStatic(inst->memData);
> req->handleIprWrite(thread, main_pkt);
> delete main_pkt;
> completeStore(storeWBIt);
> storeWBIt++;
> continue;
> }
> /* Send to cache */
> req->sendPacketToCache();
922,926c812,814
< // Need to store the second packet, if split.
< if (split) {
< state->pktToSend = true;
< state->pendingPacket = snd_data_pkt;
< }
---
> /* If successful, do the post send */
> if (req->isSent()) {
> storePostSend();
928,954c816,818
<
< // If split, try to send the second packet too
< if (split) {
< assert(snd_data_pkt);
<
< // Ensure there are enough ports to use.
< if (usedStorePorts < cacheStorePorts) {
< ++usedStorePorts;
< if (sendStore(snd_data_pkt)) {
< storePostSend(snd_data_pkt);
< } else {
< DPRINTF(IEW, "D-Cache became blocked when writing"
< " [sn:%lli] second packet, will retry later\n",
< inst->seqNum);
< }
< } else {
<
< // Store the packet for when there's free ports.
< assert(pendingPkt == NULL);
< pendingPkt = snd_data_pkt;
< hasPendingPkt = true;
< }
< } else {
<
< // Not a split store.
< storePostSend(data_pkt);
< }
---
> DPRINTF(LSQUnit, "D-Cache became blocked when writing [sn:%lli], "
> "will retry later\n",
> inst->seqNum);
957,960d820
<
< // Not sure this should set it to 0.
< usedStorePorts = 0;
<
964,977d823
< /*template <class Impl>
< void
< LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
< {
< list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
< mshrSeqNums.end(),
< seqNum);
<
< if (mshr_it != mshrSeqNums.end()) {
< mshrSeqNums.erase(mshr_it);
< DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
< }
< }*/
<
985,988c831,832
< int load_idx = loadTail;
< decrLdIdx(load_idx);
<
< while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
---
> while (loads != 0 &&
> loadQueue.back().instruction()->seqNum > squashed_num) {
991,992c835,836
< loadQueue[load_idx]->pcState(),
< loadQueue[load_idx]->seqNum);
---
> loadQueue.back().instruction()->pcState(),
> loadQueue.back().instruction()->seqNum);
994c838
< if (isStalled() && load_idx == stallingLoadIdx) {
---
> if (isStalled() && loadQueue.tail() == stallingLoadIdx) {
1001,1002c845,847
< loadQueue[load_idx]->setSquashed();
< loadQueue[load_idx] = NULL;
---
> loadQueue.back().instruction()->setSquashed();
> loadQueue.back().clear();
>
1005,1008c850
< // Inefficient!
< loadTail = load_idx;
<
< decrLdIdx(load_idx);
---
> loadQueue.pop_back();
1016,1018d857
< int store_idx = storeTail;
< decrStIdx(store_idx);
<
1020c859
< storeQueue[store_idx].inst->seqNum > squashed_num) {
---
> storeQueue.back().instruction()->seqNum > squashed_num) {
1022c861
< if (storeQueue[store_idx].canWB) {
---
> if (storeQueue.back().canWB()) {
1028,1029c867,868
< storeQueue[store_idx].inst->pcState(),
< store_idx, storeQueue[store_idx].inst->seqNum);
---
> storeQueue.back().instruction()->pcState(),
> storeQueue.tail(), storeQueue.back().instruction()->seqNum);
1034c873
< storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
---
> storeQueue.back().instruction()->seqNum == stallingStoreIsn) {
1041,1043c880
< storeQueue[store_idx].inst->setSquashed();
< storeQueue[store_idx].inst = NULL;
< storeQueue[store_idx].canWB = 0;
---
> storeQueue.back().instruction()->setSquashed();
1048,1053c885
< storeQueue[store_idx].req.reset();
< if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
< storeQueue[store_idx].sreqLow.reset();
< storeQueue[store_idx].sreqHigh.reset();
< }
<
---
> storeQueue.back().clear();
1056,1059c888
< // Inefficient!
< storeTail = store_idx;
<
< decrStIdx(store_idx);
---
> storeQueue.pop_back();
1066c895
< LSQUnit<Impl>::storePostSend(PacketPtr pkt)
---
> LSQUnit<Impl>::storePostSend()
1069c898
< storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
---
> storeWBIt->instruction()->seqNum == stallingStoreIsn) {
1075c904
< iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
---
> iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
1078c907
< if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
---
> if (!storeWBIt->instruction()->isStoreConditional()) {
1082c911
< storeQueue[storeWBIdx].inst->setCompleted();
---
> storeWBIt->instruction()->setCompleted();
1085c914
< cpu->checker->verify(storeQueue[storeWBIdx].inst);
---
> cpu->checker->verify(storeWBIt->instruction());
1093c922
< incrStIdx(storeWBIdx);
---
> storeWBIt++;
1139c968
< LSQUnit<Impl>::completeStore(int store_idx)
---
> LSQUnit<Impl>::completeStore(typename StoreQueue::iterator store_idx)
1141,1142c970,971
< assert(storeQueue[store_idx].inst);
< storeQueue[store_idx].completed = true;
---
> assert(store_idx->valid());
> store_idx->completed() = true;
1150c979,982
< if (store_idx == storeHead) {
---
> /* We 'need' a copy here because we may clear the entry from the
> * store queue. */
> DynInstPtr store_inst = store_idx->instruction();
> if (store_idx == storeQueue.begin()) {
1152,1153c984,985
< incrStIdx(storeHead);
<
---
> storeQueue.front().clear();
> storeQueue.pop_front();
1155,1156c987,988
< } while (storeQueue[storeHead].completed &&
< storeHead != storeTail);
---
> } while (storeQueue.front().completed() &&
> !storeQueue.empty());
1163c995
< storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
---
> store_inst->seqNum, store_idx.idx() - 1, storeQueue.head() - 1);
1167,1168c999,1000
< storeQueue[store_idx].inst->storeTick =
< curTick() - storeQueue[store_idx].inst->fetchTick;
---
> store_idx->instruction()->storeTick =
> curTick() - store_idx->instruction()->fetchTick;
1173c1005
< storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
---
> store_inst->seqNum == stallingStoreIsn) {
1179c1011
< iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
---
> iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
1182c1014
< storeQueue[store_idx].inst->setCompleted();
---
> store_inst->setCompleted();
1191d1022
<
1195,1196c1026,1027
< if (cpu->checker && !storeQueue[store_idx].inst->isStoreConditional()) {
< cpu->checker->verify(storeQueue[store_idx].inst);
---
> if (cpu->checker && !store_inst->isStoreConditional()) {
> cpu->checker->verify(store_inst);
1202c1033
< LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
---
> LSQUnit<Impl>::trySendPacket(bool isLoad, PacketPtr data_pkt)
1204,1213c1035,1036
< if (!dcachePort->sendTimingReq(data_pkt)) {
< // Need to handle becoming blocked on a store.
< isStoreBlocked = true;
< ++lsqCacheBlocked;
< assert(retryPkt == NULL);
< retryPkt = data_pkt;
< return false;
< }
< return true;
< }
---
> bool ret = true;
> bool cache_got_blocked = false;
1215,1221c1038
< template <class Impl>
< void
< LSQUnit<Impl>::recvRetry()
< {
< if (isStoreBlocked) {
< DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
< assert(retryPkt != NULL);
---
> auto state = dynamic_cast<LSQSenderState*>(data_pkt->senderState);
1223,1224c1040,1047
< LSQSenderState *state =
< dynamic_cast<LSQSenderState *>(retryPkt->senderState);
---
> if (!lsq->cacheBlocked() && (isLoad || lsq->storePortAvailable())) {
> if (!dcachePort->sendTimingReq(data_pkt)) {
> ret = false;
> cache_got_blocked = true;
> }
> } else {
> ret = false;
> }
1226,1233c1049,1051
< if (dcachePort->sendTimingReq(retryPkt)) {
< // Don't finish the store unless this is the last packet.
< if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
< state->pendingPacket == retryPkt) {
< state->pktToSend = false;
< storePostSend(retryPkt);
< }
< retryPkt = NULL;
---
> if (ret) {
> if (!isLoad) {
> lsq->storePortBusy();
1235,1244c1053,1058
<
< // Send any outstanding packet.
< if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
< assert(state->pendingPacket);
< if (sendStore(state->pendingPacket)) {
< storePostSend(state->pendingPacket);
< }
< }
< } else {
< // Still blocked!
---
> }
> state->outstanding++;
> state->request()->packetSent();
> } else {
> if (cache_got_blocked) {
> lsq->cacheBlocked(true);
1246a1061,1065
> if (!isLoad) {
> assert(state->request() == storeWBIt->request());
> isStoreBlocked = true;
> }
> state->request()->packetNotSent();
1248d1066
< }
1250,1255c1068
< template <class Impl>
< inline void
< LSQUnit<Impl>::incrStIdx(int &store_idx) const
< {
< if (++store_idx >= SQEntries)
< store_idx = 0;
---
> return ret;
1259,1260c1072,1073
< inline void
< LSQUnit<Impl>::decrStIdx(int &store_idx) const
---
> void
> LSQUnit<Impl>::recvRetry()
1262,1263c1075,1078
< if (--store_idx < 0)
< store_idx += SQEntries;
---
> if (isStoreBlocked) {
> DPRINTF(LSQUnit, "Receiving retry: blocked store\n");
> writebackBlockedStore();
> }
1267,1282d1081
< inline void
< LSQUnit<Impl>::incrLdIdx(int &load_idx) const
< {
< if (++load_idx >= LQEntries)
< load_idx = 0;
< }
<
< template <class Impl>
< inline void
< LSQUnit<Impl>::decrLdIdx(int &load_idx) const
< {
< if (--load_idx < 0)
< load_idx += LQEntries;
< }
<
< template <class Impl>
1290,1293c1089,1090
< int load_idx = loadHead;
<
< while (load_idx != loadTail && loadQueue[load_idx]) {
< const DynInstPtr &inst(loadQueue[load_idx]);
---
> for (const auto& e: loadQueue) {
> const DynInstPtr &inst(e.instruction());
1295,1296d1091
<
< incrLdIdx(load_idx);
1303,1306c1098,1099
< int store_idx = storeHead;
<
< while (store_idx != storeTail && storeQueue[store_idx].inst) {
< const DynInstPtr &inst(storeQueue[store_idx].inst);
---
> for (const auto& e: storeQueue) {
> const DynInstPtr &inst(e.instruction());
1308,1309d1100
<
< incrStIdx(store_idx);
1314a1106,1112
> template <class Impl>
> unsigned int
> LSQUnit<Impl>::cacheLineSize()
> {
> return cpu->cacheLineSize();
> }
>