src/mem/coherent_xbar.cc

143 CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
149     bool is_express_snoop = pkt->isExpressSnoop();
150     bool cache_responding = pkt->cacheResponding();
156     PortID master_port_id = findPort(pkt->getAddrRange());
162                 src_port->name(), pkt->print());
167             src_port->name(), pkt->print());
171     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
172     unsigned int pkt_cmd = pkt->cmdToIndex();
175     Tick old_header_delay = pkt->headerDelay;
181     calcPacketTiming(pkt, xbar_delay);
184     Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay;
190     const bool is_destination = isDestination(pkt);
193         pkt->cmd != MemCmd::WriteClean;
195         assert(pkt->snoopDelay == 0);
197         if (pkt->isClean() && !is_destination) {
201             if (!masterPorts[master_port_id]->tryTiming(pkt)) {
203                         src_port->name(), pkt->print());
217             auto sf_res = snoopFilter->lookupRequest(pkt, *src_port);
221             pkt->headerDelay += sf_res.second * clockPeriod();
223                     __func__, src_port->name(), pkt->print(),
226             if (pkt->isEviction()) {
233                     pkt->setBlockCached();
235                 forwardTiming(pkt, slave_port_id, sf_res.first);
238             forwardTiming(pkt, slave_port_id);
242         pkt->headerDelay += pkt->snoopDelay;
243         pkt->snoopDelay = 0;
252     const bool expect_snoop_resp = !cache_responding && pkt->cacheResponding();
253     bool expect_response = pkt->needsResponse() && !pkt->cacheResponding();
255     const bool sink_packet = sinkPacket(pkt);
261     const Addr addr(pkt->getAddr());
264                 pkt->print());
268         if (forwardPacket(pkt)) {
274             if (pkt->cacheResponding()) {
275                 pkt->setExpressSnoop();
281             if (pkt->isWrite() && is_destination) {
282                 pkt->clearWriteThrough();
286             success = masterPorts[master_port_id]->sendTimingReq(pkt);
290             assert(pkt->needsResponse());
300         snoopFilter->finishRequest(!success, addr, pkt->isSecure());
309         pkt->headerDelay = old_header_delay;
312                 src_port->name(), pkt->print());
324                 assert(outstandingSnoop.find(pkt->req) ==
326                 outstandingSnoop.insert(pkt->req);
336                 assert(routeTo.find(pkt->req) == routeTo.end());
337                 routeTo[pkt->req] = slave_port_id;
361         pendingDelete.reset(pkt);
364     PacketPtr rsp_pkt = pkt;
378         ((pkt->isClean() && pkt->satisfied()) ||
379          pkt->cmd == MemCmd::WriteClean) &&
381         PacketPtr deferred_rsp = pkt->isWrite() ? nullptr : pkt;
382         auto cmo_lookup = outstandingCMO.find(pkt->id);
386             if (pkt->isWrite()) {
402             outstandingCMO.emplace(pkt->id, deferred_rsp);
403             if (!pkt->isWrite()) {
404                 assert(routeTo.find(pkt->req) == routeTo.end());
405                 routeTo[pkt->req] = slave_port_id;
430         Tick response_time = clockEdge() + pkt->headerDelay;
440 CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
446     const auto route_lookup = routeTo.find(pkt->req);
456                 src_port->name(), pkt->print());
461             src_port->name(), pkt->print());
465     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
466     unsigned int pkt_cmd = pkt->cmdToIndex();
472     calcPacketTiming(pkt, xbar_delay);
475     Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay;
479         snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
484     Tick latency = pkt->headerDelay;
485     pkt->headerDelay = 0;
486     slavePorts[slave_port_id]->schedTimingResp(pkt, curTick() + latency);
502 CoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id)
505             masterPorts[master_port_id]->name(), pkt->print());
508     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
509     transDist[pkt->cmdToIndex()]++;
514     assert(pkt->isExpressSnoop());
518     calcPacketTiming(pkt, forwardLatency * clockPeriod());
522     const bool cache_responding = pkt->cacheResponding();
524     assert(pkt->snoopDelay == 0);
528         auto sf_res = snoopFilter->lookupSnoop(pkt);
532         pkt->headerDelay += sf_res.second * clockPeriod();
534                 __func__, masterPorts[master_port_id]->name(), pkt->print(),
538         forwardTiming(pkt, InvalidPortID, sf_res.first);
540         forwardTiming(pkt, InvalidPortID);
544     pkt->headerDelay += pkt->snoopDelay;
545     pkt->snoopDelay = 0;
548     if (!cache_responding && pkt->cacheResponding()) {
549         assert(routeTo.find(pkt->req) == routeTo.end());
550         routeTo[pkt->req] = master_port_id;
558     assert(findPort(pkt->getAddrRange()) == master_port_id);
562 CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
568     const auto route_lookup = routeTo.find(pkt->req);
577     const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
588                     src_port->name(), pkt->print());
597                     snoop_port->name(), pkt->print());
603             src_port->name(), pkt->print());
607     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
608     unsigned int pkt_cmd = pkt->cmdToIndex();
611     assert(!pkt->isExpressSnoop());
620     calcPacketTiming(pkt, xbar_delay);
623     Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay;
633             snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id],
638             masterPorts[dest_port_id]->sendTimingSnoopResp(pkt);
649         outstandingSnoop.erase(pkt->req);
658             snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id],
663                 src_port->name(), pkt->print());
668         Tick latency = pkt->headerDelay;
669         pkt->headerDelay = 0;
670         slavePorts[dest_port_id]->schedTimingResp(pkt, curTick() + latency);
688 CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id,
691     DPRINTF(CoherentXBar, "%s for %s\n", __func__, pkt->print());
706             p->sendTimingSnoopReq(pkt);
725 CoherentXBar::recvAtomicBackdoor(PacketPtr pkt, PortID slave_port_id,
729             slavePorts[slave_port_id]->name(), pkt->print());
731     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
732     unsigned int pkt_cmd = pkt->cmdToIndex();
741     const bool is_destination = isDestination(pkt);
744         pkt->cmd != MemCmd::WriteClean;
751                 snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]);
754                     __func__, slavePorts[slave_port_id]->name(), pkt->print(),
761             snoopFilter->finishRequest(false, pkt->getAddr(), pkt->isSecure());
763             if (pkt->isEviction()) {
770                     pkt->setBlockCached();
772                 snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID,
776             snoop_result = forwardAtomic(pkt, slave_port_id);
785     const bool sink_packet = sinkPacket(pkt);
789     PortID master_port_id = findPort(pkt->getAddrRange());
793                 pkt->print());
795         if (forwardPacket(pkt)) {
799             if (pkt->isWrite() && is_destination) {
800                 pkt->clearWriteThrough();
806                 master->sendAtomicBackdoor(pkt, *backdoor) :
807                 master->sendAtomic(pkt);
810             assert(pkt->needsResponse());
812             pkt->makeResponse();
823     if (!system->bypassCaches() && snoopFilter && pkt->isResponse()) {
824         snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
830         assert(!pkt->isResponse());
831         pkt->cmd = snoop_response_cmd;
844     if (pkt->isClean() && isDestination(pkt) && pkt->satisfied()) {
845         auto it = outstandingCMO.find(pkt->id);
849     } else if (pkt->cmd == MemCmd::WriteClean && isDestination(pkt)) {
853         auto M5_VAR_USED ret = outstandingCMO.emplace(pkt->id, nullptr);
860     if (pkt->isResponse()) {
861         pkt_size = pkt->hasData() ? pkt->getSize() : 0;
862         pkt_cmd = pkt->cmdToIndex();
871     pkt->payloadDelay = response_latency;
876 CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id)
879             masterPorts[master_port_id]->name(), pkt->print());
882     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
890         auto sf_res = snoopFilter->lookupSnoop(pkt);
893                 __func__, masterPorts[master_port_id]->name(), pkt->print(),
895         snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id,
898         snoop_result = forwardAtomic(pkt, InvalidPortID);
904         pkt->cmd = snoop_response_cmd;
907     if (pkt->isResponse()) {
912     pkt->payloadDelay = snoop_response_latency;
917 CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id,
924     MemCmd orig_cmd = pkt->cmd;
942         Tick latency = p->sendAtomicSnoop(pkt);
948         if (!pkt->isResponse())
952         assert(pkt->cmd != orig_cmd);
953         assert(pkt->cacheResponding());
957         snoop_response_cmd = pkt->cmd;
966                 snoopFilter->updateSnoopForward(pkt, *p,
971                 snoopFilter->updateSnoopResponse(pkt, *p,
976         pkt->cmd = orig_cmd;
988 CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id)
990     if (!pkt->isPrint()) {
993                 slavePorts[slave_port_id]->name(), pkt->print());
998         forwardFunctional(pkt, slave_port_id);
1003     if (!pkt->isResponse()) {
1009             if (p->trySatisfyFunctional(pkt)) {
1010                 if (pkt->needsResponse())
1011                     pkt->makeResponse();
1016         PortID dest_id = findPort(pkt->getAddrRange());
1018         masterPorts[dest_id]->sendFunctional(pkt);
1023 CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id)
1025     if (!pkt->isPrint()) {
1028                 masterPorts[master_port_id]->name(), pkt->print());
1032         if (p->trySatisfyFunctional(pkt)) {
1033             if (pkt->needsResponse())
1034                 pkt->makeResponse();
1040     forwardFunctional(pkt, InvalidPortID);
1044 CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id)
1056             p->sendFunctionalSnoop(pkt);
1059         if (pkt->isResponse()) {
1066 CoherentXBar::sinkPacket(const PacketPtr pkt) const
1081     return (pointOfCoherency && pkt->cacheResponding()) ||
1082         (pointOfCoherency && !(pkt->isRead() || pkt->isWrite()) &&
1083          !pkt->needsResponse()) ||
1084         (pkt->isCleanEviction() && pkt->isBlockCached()) ||
1085         (pkt->cacheResponding() &&
1086          (!pkt->needsWritable() || pkt->responderHadWritable()));
1090 CoherentXBar::forwardPacket(const PacketPtr pkt)
1097     if (pkt->isClean()) {
1098         return !isDestination(pkt);
1100     return pkt->isRead() || pkt->isWrite() || !pointOfCoherency;