coherent_xbar.cc revision 10405
11689SN/A/*
212106SRekai.GonzalezAlberquilla@arm.com * Copyright (c) 2011-2014 ARM Limited
310715SRekai.GonzalezAlberquilla@arm.com * All rights reserved
410715SRekai.GonzalezAlberquilla@arm.com *
510715SRekai.GonzalezAlberquilla@arm.com * The license below extends only to copyright in the software and shall
610715SRekai.GonzalezAlberquilla@arm.com * not be construed as granting a license to any other intellectual
710715SRekai.GonzalezAlberquilla@arm.com * property including but not limited to intellectual property relating
810715SRekai.GonzalezAlberquilla@arm.com * to a hardware implementation of the functionality of the software
910715SRekai.GonzalezAlberquilla@arm.com * licensed hereunder.  You may use the software subject to the license
1010715SRekai.GonzalezAlberquilla@arm.com * terms below provided that you ensure that this notice is replicated
1110715SRekai.GonzalezAlberquilla@arm.com * unmodified and in its entirety in all distributions of the software,
1210715SRekai.GonzalezAlberquilla@arm.com * modified or unmodified, in source code or in binary form.
1310715SRekai.GonzalezAlberquilla@arm.com *
141689SN/A * Copyright (c) 2006 The Regents of The University of Michigan
159919Ssteve.reinhardt@amd.com * All rights reserved.
161689SN/A *
171689SN/A * Redistribution and use in source and binary forms, with or without
181689SN/A * modification, are permitted provided that the following conditions are
191689SN/A * met: redistributions of source code must retain the above copyright
201689SN/A * notice, this list of conditions and the following disclaimer;
211689SN/A * redistributions in binary form must reproduce the above copyright
221689SN/A * notice, this list of conditions and the following disclaimer in the
231689SN/A * documentation and/or other materials provided with the distribution;
241689SN/A * neither the name of the copyright holders nor the names of its
251689SN/A * contributors may be used to endorse or promote products derived from
261689SN/A * this software without specific prior written permission.
271689SN/A *
281689SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
291689SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
301689SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
311689SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
321689SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
331689SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
341689SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
351689SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
361689SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
371689SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
381689SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
391689SN/A *
402665Ssaidi@eecs.umich.edu * Authors: Ali Saidi
412665Ssaidi@eecs.umich.edu *          Andreas Hansson
429919Ssteve.reinhardt@amd.com *          William Wang
431689SN/A */
441689SN/A
452292SN/A/**
462292SN/A * @file
471060SN/A * Definition of a crossbar object.
481060SN/A */
491461SN/A
501060SN/A#include "base/misc.hh"
511060SN/A#include "base/trace.hh"
526658Snate@binkert.org#include "debug/AddrRanges.hh"
536658Snate@binkert.org#include "debug/CoherentXBar.hh"
541717SN/A#include "mem/coherent_xbar.hh"
559919Ssteve.reinhardt@amd.com#include "sim/system.hh"
569919Ssteve.reinhardt@amd.com
5712109SRekai.GonzalezAlberquilla@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p)
581060SN/A    : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter)
599919Ssteve.reinhardt@amd.com{
609919Ssteve.reinhardt@amd.com    // create the ports based on the size of the master and slave
619919Ssteve.reinhardt@amd.com    // vector ports, and the presence of the default port, the ports
629919Ssteve.reinhardt@amd.com    // are enumerated starting from zero
639919Ssteve.reinhardt@amd.com    for (int i = 0; i < p->port_master_connection_count; ++i) {
649919Ssteve.reinhardt@amd.com        std::string portName = csprintf("%s.master[%d]", name(), i);
659919Ssteve.reinhardt@amd.com        MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i);
661060SN/A        masterPorts.push_back(bp);
671060SN/A        reqLayers.push_back(new ReqLayer(*bp, *this,
689919Ssteve.reinhardt@amd.com                                         csprintf(".reqLayer%d", i)));
6912106SRekai.GonzalezAlberquilla@arm.com        snoopLayers.push_back(new SnoopLayer(*bp, *this,
709919Ssteve.reinhardt@amd.com                                             csprintf(".snoopLayer%d", i)));
7112106SRekai.GonzalezAlberquilla@arm.com    }
7212109SRekai.GonzalezAlberquilla@arm.com
7312109SRekai.GonzalezAlberquilla@arm.com    // see if we have a default slave device connected and if so add
7412109SRekai.GonzalezAlberquilla@arm.com    // our corresponding master port
7512109SRekai.GonzalezAlberquilla@arm.com    if (p->port_default_connection_count) {
769919Ssteve.reinhardt@amd.com        defaultPortID = masterPorts.size();
779919Ssteve.reinhardt@amd.com        std::string portName = name() + ".default";
789919Ssteve.reinhardt@amd.com        MasterPort* bp = new CoherentXBarMasterPort(portName, *this,
799919Ssteve.reinhardt@amd.com                                                   defaultPortID);
809919Ssteve.reinhardt@amd.com        masterPorts.push_back(bp);
819919Ssteve.reinhardt@amd.com        reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d",
829919Ssteve.reinhardt@amd.com                                             defaultPortID)));
839919Ssteve.reinhardt@amd.com        snoopLayers.push_back(new SnoopLayer(*bp, *this,
849919Ssteve.reinhardt@amd.com                                             csprintf(".snoopLayer%d",
859919Ssteve.reinhardt@amd.com                                                      defaultPortID)));
869919Ssteve.reinhardt@amd.com    }
879919Ssteve.reinhardt@amd.com
889919Ssteve.reinhardt@amd.com    // create the slave ports, once again starting at zero
899919Ssteve.reinhardt@amd.com    for (int i = 0; i < p->port_slave_connection_count; ++i) {
9012106SRekai.GonzalezAlberquilla@arm.com        std::string portName = csprintf("%s.slave[%d]", name(), i);
919919Ssteve.reinhardt@amd.com        SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i);
921060SN/A        slavePorts.push_back(bp);
939919Ssteve.reinhardt@amd.com        respLayers.push_back(new RespLayer(*bp, *this,
949919Ssteve.reinhardt@amd.com                                           csprintf(".respLayer%d", i)));
959919Ssteve.reinhardt@amd.com        snoopRespPorts.push_back(new SnoopRespPort(*bp, *this));
969919Ssteve.reinhardt@amd.com    }
979919Ssteve.reinhardt@amd.com
981060SN/A    if (snoopFilter)
999919Ssteve.reinhardt@amd.com        snoopFilter->setSlavePorts(slavePorts);
1009919Ssteve.reinhardt@amd.com
1019919Ssteve.reinhardt@amd.com    clearPortCache();
1021060SN/A}
1039919Ssteve.reinhardt@amd.com
1041060SN/ACoherentXBar::~CoherentXBar()
1051060SN/A{
1061060SN/A    for (auto l: reqLayers)
1071060SN/A        delete l;
1081060SN/A    for (auto l: respLayers)
1091060SN/A        delete l;
1101060SN/A    for (auto l: snoopLayers)
11112105Snathanael.premillieu@arm.com        delete l;
1121060SN/A    for (auto p: snoopRespPorts)
1139919Ssteve.reinhardt@amd.com        delete p;
1149919Ssteve.reinhardt@amd.com}
1159919Ssteve.reinhardt@amd.com
1169919Ssteve.reinhardt@amd.comvoid
1179919Ssteve.reinhardt@amd.comCoherentXBar::init()
1189919Ssteve.reinhardt@amd.com{
1199919Ssteve.reinhardt@amd.com    // the base class is responsible for determining the block size
12012106SRekai.GonzalezAlberquilla@arm.com    BaseXBar::init();
1219919Ssteve.reinhardt@amd.com
1229919Ssteve.reinhardt@amd.com    // iterate over our slave ports and determine which of our
1239919Ssteve.reinhardt@amd.com    // neighbouring master ports are snooping and add them as snoopers
1249919Ssteve.reinhardt@amd.com    for (const auto& p: slavePorts) {
1259919Ssteve.reinhardt@amd.com        // check if the connected master port is snooping
1269919Ssteve.reinhardt@amd.com        if (p->isSnooping()) {
12712106SRekai.GonzalezAlberquilla@arm.com            DPRINTF(AddrRanges, "Adding snooping master %s\n",
1289919Ssteve.reinhardt@amd.com                    p->getMasterPort().name());
12912106SRekai.GonzalezAlberquilla@arm.com            snoopPorts.push_back(p);
13012106SRekai.GonzalezAlberquilla@arm.com        }
1319919Ssteve.reinhardt@amd.com    }
1329919Ssteve.reinhardt@amd.com
1339919Ssteve.reinhardt@amd.com    if (snoopPorts.empty())
1349919Ssteve.reinhardt@amd.com        warn("CoherentXBar %s has no snooping ports attached!\n", name());
1359919Ssteve.reinhardt@amd.com}
1369919Ssteve.reinhardt@amd.com
1379919Ssteve.reinhardt@amd.combool
1389919Ssteve.reinhardt@amd.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
13912106SRekai.GonzalezAlberquilla@arm.com{
1409919Ssteve.reinhardt@amd.com    // determine the source port based on the id
14112106SRekai.GonzalezAlberquilla@arm.com    SlavePort *src_port = slavePorts[slave_port_id];
14212106SRekai.GonzalezAlberquilla@arm.com
1439919Ssteve.reinhardt@amd.com    // remember if the packet is an express snoop
1449919Ssteve.reinhardt@amd.com    bool is_express_snoop = pkt->isExpressSnoop();
1459919Ssteve.reinhardt@amd.com
1469919Ssteve.reinhardt@amd.com    // determine the destination based on the address
14712109SRekai.GonzalezAlberquilla@arm.com    PortID master_port_id = findPort(pkt->getAddr());
14812109SRekai.GonzalezAlberquilla@arm.com
14912109SRekai.GonzalezAlberquilla@arm.com    // test if the crossbar should be considered occupied for the current
15012109SRekai.GonzalezAlberquilla@arm.com    // port, and exclude express snoops from the check
15112109SRekai.GonzalezAlberquilla@arm.com    if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) {
15212109SRekai.GonzalezAlberquilla@arm.com        DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n",
15312109SRekai.GonzalezAlberquilla@arm.com                src_port->name(), pkt->cmdString(), pkt->getAddr());
15412109SRekai.GonzalezAlberquilla@arm.com        return false;
15512109SRekai.GonzalezAlberquilla@arm.com    }
15612109SRekai.GonzalezAlberquilla@arm.com
15712109SRekai.GonzalezAlberquilla@arm.com    DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n",
15812109SRekai.GonzalezAlberquilla@arm.com            src_port->name(), pkt->cmdString(), is_express_snoop,
15912109SRekai.GonzalezAlberquilla@arm.com            pkt->getAddr());
16012109SRekai.GonzalezAlberquilla@arm.com
1619919Ssteve.reinhardt@amd.com    // store size and command as they might be modified when
1629919Ssteve.reinhardt@amd.com    // forwarding the packet
1639919Ssteve.reinhardt@amd.com    unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
1649919Ssteve.reinhardt@amd.com    unsigned int pkt_cmd = pkt->cmdToIndex();
1659919Ssteve.reinhardt@amd.com
1669919Ssteve.reinhardt@amd.com    // set the source port for routing of the response
16712104Snathanael.premillieu@arm.com    pkt->setSrc(slave_port_id);
1689919Ssteve.reinhardt@amd.com
16912104Snathanael.premillieu@arm.com    calcPacketTiming(pkt);
1709919Ssteve.reinhardt@amd.com    Tick packetFinishTime = pkt->lastWordDelay + curTick();
1719919Ssteve.reinhardt@amd.com
1729919Ssteve.reinhardt@amd.com    // uncacheable requests need never be snooped
1739919Ssteve.reinhardt@amd.com    if (!pkt->req->isUncacheable() && !system->bypassCaches()) {
17412109SRekai.GonzalezAlberquilla@arm.com        // the packet is a memory-mapped request and should be
17512109SRekai.GonzalezAlberquilla@arm.com        // broadcasted to our snoopers but the source
1769919Ssteve.reinhardt@amd.com        if (snoopFilter) {
1779919Ssteve.reinhardt@amd.com            // check with the snoop filter where to forward this packet
1789919Ssteve.reinhardt@amd.com            auto sf_res = snoopFilter->lookupRequest(pkt, *src_port);
1799919Ssteve.reinhardt@amd.com            packetFinishTime += sf_res.second * clockPeriod();
1809919Ssteve.reinhardt@amd.com            DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\
1819919Ssteve.reinhardt@amd.com                    " SF size: %i lat: %i\n", src_port->name(),
1829919Ssteve.reinhardt@amd.com                    pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
18312105Snathanael.premillieu@arm.com                    sf_res.second);
18412105Snathanael.premillieu@arm.com            forwardTiming(pkt, slave_port_id, sf_res.first);
18512105Snathanael.premillieu@arm.com        } else {
18612109SRekai.GonzalezAlberquilla@arm.com            forwardTiming(pkt, slave_port_id);
18712109SRekai.GonzalezAlberquilla@arm.com        }
18812109SRekai.GonzalezAlberquilla@arm.com    }
18912109SRekai.GonzalezAlberquilla@arm.com
19012109SRekai.GonzalezAlberquilla@arm.com    // remember if we add an outstanding req so we can undo it if
19112109SRekai.GonzalezAlberquilla@arm.com    // necessary, if the packet needs a response, we should add it
19212109SRekai.GonzalezAlberquilla@arm.com    // as outstanding and express snoops never fail so there is
19312109SRekai.GonzalezAlberquilla@arm.com    // not need to worry about them
19412109SRekai.GonzalezAlberquilla@arm.com    bool add_outstanding = !is_express_snoop && pkt->needsResponse();
1959919Ssteve.reinhardt@amd.com
19612105Snathanael.premillieu@arm.com    // keep track that we have an outstanding request packet
19712105Snathanael.premillieu@arm.com    // matching this request, this is used by the coherency
1989919Ssteve.reinhardt@amd.com    // mechanism in determining what to do with snoop responses
1999919Ssteve.reinhardt@amd.com    // (in recvTimingSnoop)
2009919Ssteve.reinhardt@amd.com    if (add_outstanding) {
2011060SN/A        // we should never have an exsiting request outstanding
2029919Ssteve.reinhardt@amd.com        assert(outstandingReq.find(pkt->req) == outstandingReq.end());
2039919Ssteve.reinhardt@amd.com        outstandingReq.insert(pkt->req);
2049919Ssteve.reinhardt@amd.com    }
2052348SN/A
20610537Sandreas.hansson@arm.com    // Note: Cannot create a copy of the full packet, here.
2071060SN/A    MemCmd orig_cmd(pkt->cmd);
2081061SN/A
2099919Ssteve.reinhardt@amd.com    // since it is a normal request, attempt to send the packet
2101061SN/A    bool success = masterPorts[master_port_id]->sendTimingReq(pkt);
2112348SN/A
2129919Ssteve.reinhardt@amd.com    if (snoopFilter && !pkt->req->isUncacheable()
2132292SN/A        && !system->bypassCaches()) {
2142292SN/A        // The packet may already be overwritten by the sendTimingReq function.
21512109SRekai.GonzalezAlberquilla@arm.com        // The snoop filter needs to see the original request *and* the return
21612109SRekai.GonzalezAlberquilla@arm.com        // status of the send operation, so we need to recreate the original
2172292SN/A        // request.  Atomic mode does not have the issue, as there the send
2189919Ssteve.reinhardt@amd.com        // operation and the response happen instantaneously and don't need two
2199919Ssteve.reinhardt@amd.com        // phase tracking.
22012104Snathanael.premillieu@arm.com        MemCmd tmp_cmd(pkt->cmd);
22112106SRekai.GonzalezAlberquilla@arm.com        pkt->cmd = orig_cmd;
22212106SRekai.GonzalezAlberquilla@arm.com        // Let the snoop filter know about the success of the send operation
2239919Ssteve.reinhardt@amd.com        snoopFilter->updateRequest(pkt, *src_port, !success);
2249919Ssteve.reinhardt@amd.com        pkt->cmd = tmp_cmd;
2259919Ssteve.reinhardt@amd.com    }
22612106SRekai.GonzalezAlberquilla@arm.com
22712106SRekai.GonzalezAlberquilla@arm.com    // if this is an express snoop, we are done at this point
22812106SRekai.GonzalezAlberquilla@arm.com    if (is_express_snoop) {
22912106SRekai.GonzalezAlberquilla@arm.com        assert(success);
23012106SRekai.GonzalezAlberquilla@arm.com        snoops++;
23112106SRekai.GonzalezAlberquilla@arm.com    } else {
23212106SRekai.GonzalezAlberquilla@arm.com        // for normal requests, check if successful
23312109SRekai.GonzalezAlberquilla@arm.com        if (!success)  {
23412109SRekai.GonzalezAlberquilla@arm.com            // inhibited packets should never be forced to retry
23512109SRekai.GonzalezAlberquilla@arm.com            assert(!pkt->memInhibitAsserted());
23612109SRekai.GonzalezAlberquilla@arm.com
23712109SRekai.GonzalezAlberquilla@arm.com            // if it was added as outstanding and the send failed, then
23812109SRekai.GonzalezAlberquilla@arm.com            // erase it again
23912106SRekai.GonzalezAlberquilla@arm.com            if (add_outstanding)
24012106SRekai.GonzalezAlberquilla@arm.com                outstandingReq.erase(pkt->req);
24112106SRekai.GonzalezAlberquilla@arm.com
24212106SRekai.GonzalezAlberquilla@arm.com            // undo the calculation so we can check for 0 again
24312106SRekai.GonzalezAlberquilla@arm.com            pkt->firstWordDelay = pkt->lastWordDelay = 0;
24412106SRekai.GonzalezAlberquilla@arm.com
24512106SRekai.GonzalezAlberquilla@arm.com            DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n",
24612106SRekai.GonzalezAlberquilla@arm.com                    src_port->name(), pkt->cmdString(), pkt->getAddr());
24712106SRekai.GonzalezAlberquilla@arm.com
24812106SRekai.GonzalezAlberquilla@arm.com            // update the layer state and schedule an idle event
2491060SN/A            reqLayers[master_port_id]->failedTiming(src_port,
25012106SRekai.GonzalezAlberquilla@arm.com                                                    clockEdge(headerCycles));
25112106SRekai.GonzalezAlberquilla@arm.com        } else {
25212106SRekai.GonzalezAlberquilla@arm.com            // update the layer state and schedule an idle event
25312106SRekai.GonzalezAlberquilla@arm.com            reqLayers[master_port_id]->succeededTiming(packetFinishTime);
2549919Ssteve.reinhardt@amd.com        }
2551060SN/A    }
2561060SN/A
2579919Ssteve.reinhardt@amd.com    // stats updates only consider packets that were successfully sent
25812104Snathanael.premillieu@arm.com    if (success) {
2599919Ssteve.reinhardt@amd.com        pktCount[slave_port_id][master_port_id]++;
26012104Snathanael.premillieu@arm.com        pktSize[slave_port_id][master_port_id] += pkt_size;
2619919Ssteve.reinhardt@amd.com        transDist[pkt_cmd]++;
2629919Ssteve.reinhardt@amd.com    }
26312106SRekai.GonzalezAlberquilla@arm.com
26412106SRekai.GonzalezAlberquilla@arm.com    return success;
26512106SRekai.GonzalezAlberquilla@arm.com}
26612106SRekai.GonzalezAlberquilla@arm.com
26712106SRekai.GonzalezAlberquilla@arm.combool
2681060SN/ACoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
26912106SRekai.GonzalezAlberquilla@arm.com{
27012106SRekai.GonzalezAlberquilla@arm.com    // determine the source port based on the id
2711060SN/A    MasterPort *src_port = masterPorts[master_port_id];
27212109SRekai.GonzalezAlberquilla@arm.com
27312109SRekai.GonzalezAlberquilla@arm.com    // determine the destination based on what is stored in the packet
27412109SRekai.GonzalezAlberquilla@arm.com    PortID slave_port_id = pkt->getDest();
27512109SRekai.GonzalezAlberquilla@arm.com
27612109SRekai.GonzalezAlberquilla@arm.com    // test if the crossbar should be considered occupied for the
27712109SRekai.GonzalezAlberquilla@arm.com    // current port
27812109SRekai.GonzalezAlberquilla@arm.com    if (!respLayers[slave_port_id]->tryTiming(src_port)) {
27912109SRekai.GonzalezAlberquilla@arm.com        DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n",
28012106SRekai.GonzalezAlberquilla@arm.com                src_port->name(), pkt->cmdString(), pkt->getAddr());
28112106SRekai.GonzalezAlberquilla@arm.com        return false;
2821060SN/A    }
28312106SRekai.GonzalezAlberquilla@arm.com
28412106SRekai.GonzalezAlberquilla@arm.com    DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n",
28512106SRekai.GonzalezAlberquilla@arm.com            src_port->name(), pkt->cmdString(), pkt->getAddr());
28612106SRekai.GonzalezAlberquilla@arm.com
2879920Syasuko.eckert@amd.com    // store size and command as they might be modified when
28812106SRekai.GonzalezAlberquilla@arm.com    // forwarding the packet
28912106SRekai.GonzalezAlberquilla@arm.com    unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
29012106SRekai.GonzalezAlberquilla@arm.com    unsigned int pkt_cmd = pkt->cmdToIndex();
29112106SRekai.GonzalezAlberquilla@arm.com
2929919Ssteve.reinhardt@amd.com    calcPacketTiming(pkt);
2931060SN/A    Tick packetFinishTime = pkt->lastWordDelay + curTick();
2949919Ssteve.reinhardt@amd.com
2959919Ssteve.reinhardt@amd.com    // the packet is a normal response to a request that we should
2969919Ssteve.reinhardt@amd.com    // have seen passing through the crossbar
29712104Snathanael.premillieu@arm.com    assert(outstandingReq.find(pkt->req) != outstandingReq.end());
2989919Ssteve.reinhardt@amd.com
29912104Snathanael.premillieu@arm.com    if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) {
3009919Ssteve.reinhardt@amd.com        // let the snoop filter inspect the response and update its state
3019919Ssteve.reinhardt@amd.com        snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
30212106SRekai.GonzalezAlberquilla@arm.com    }
30312106SRekai.GonzalezAlberquilla@arm.com
30412106SRekai.GonzalezAlberquilla@arm.com    // remove it as outstanding
30512106SRekai.GonzalezAlberquilla@arm.com    outstandingReq.erase(pkt->req);
30612106SRekai.GonzalezAlberquilla@arm.com
30712106SRekai.GonzalezAlberquilla@arm.com    // send the packet through the destination slave port
3081060SN/A    bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt);
30912106SRekai.GonzalezAlberquilla@arm.com
31012106SRekai.GonzalezAlberquilla@arm.com    // currently it is illegal to block responses... can lead to
31112106SRekai.GonzalezAlberquilla@arm.com    // deadlock
3121060SN/A    assert(success);
31312109SRekai.GonzalezAlberquilla@arm.com
31412109SRekai.GonzalezAlberquilla@arm.com    respLayers[slave_port_id]->succeededTiming(packetFinishTime);
31512109SRekai.GonzalezAlberquilla@arm.com
31612109SRekai.GonzalezAlberquilla@arm.com    // stats updates
31712109SRekai.GonzalezAlberquilla@arm.com    pktCount[slave_port_id][master_port_id]++;
31812109SRekai.GonzalezAlberquilla@arm.com    pktSize[slave_port_id][master_port_id] += pkt_size;
31912109SRekai.GonzalezAlberquilla@arm.com    transDist[pkt_cmd]++;
32012109SRekai.GonzalezAlberquilla@arm.com
32112109SRekai.GonzalezAlberquilla@arm.com    return true;
32212109SRekai.GonzalezAlberquilla@arm.com}
32312106SRekai.GonzalezAlberquilla@arm.com
32412106SRekai.GonzalezAlberquilla@arm.comvoid
32512106SRekai.GonzalezAlberquilla@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id)
3261060SN/A{
32712106SRekai.GonzalezAlberquilla@arm.com    DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n",
32812106SRekai.GonzalezAlberquilla@arm.com            masterPorts[master_port_id]->name(), pkt->cmdString(),
32912106SRekai.GonzalezAlberquilla@arm.com            pkt->getAddr());
33012106SRekai.GonzalezAlberquilla@arm.com
33112106SRekai.GonzalezAlberquilla@arm.com    // update stats here as we know the forwarding will succeed
33212106SRekai.GonzalezAlberquilla@arm.com    transDist[pkt->cmdToIndex()]++;
33312106SRekai.GonzalezAlberquilla@arm.com    snoops++;
33412106SRekai.GonzalezAlberquilla@arm.com
33512106SRekai.GonzalezAlberquilla@arm.com    // we should only see express snoops from caches
33612106SRekai.GonzalezAlberquilla@arm.com    assert(pkt->isExpressSnoop());
33712106SRekai.GonzalezAlberquilla@arm.com
33812106SRekai.GonzalezAlberquilla@arm.com    // set the source port for routing of the response
3399920Syasuko.eckert@amd.com    pkt->setSrc(master_port_id);
3409920Syasuko.eckert@amd.com
3419920Syasuko.eckert@amd.com    if (snoopFilter) {
3429919Ssteve.reinhardt@amd.com        // let the Snoop Filter work its magic and guide probing
3439919Ssteve.reinhardt@amd.com        auto sf_res = snoopFilter->lookupSnoop(pkt);
3449919Ssteve.reinhardt@amd.com        // No timing here: packetFinishTime += sf_res.second * clockPeriod();
3459919Ssteve.reinhardt@amd.com        DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\
3461060SN/A                " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(),
3479919Ssteve.reinhardt@amd.com                pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
3481060SN/A                sf_res.second);
34912109SRekai.GonzalezAlberquilla@arm.com
35012109SRekai.GonzalezAlberquilla@arm.com        // forward to all snoopers
35112109SRekai.GonzalezAlberquilla@arm.com        forwardTiming(pkt, InvalidPortID, sf_res.first);
35212109SRekai.GonzalezAlberquilla@arm.com    } else {
3539919Ssteve.reinhardt@amd.com        forwardTiming(pkt, InvalidPortID);
35410715SRekai.GonzalezAlberquilla@arm.com    }
35512109SRekai.GonzalezAlberquilla@arm.com
35612109SRekai.GonzalezAlberquilla@arm.com    // a snoop request came from a connected slave device (one of
35712109SRekai.GonzalezAlberquilla@arm.com    // our master ports), and if it is not coming from the slave
35812109SRekai.GonzalezAlberquilla@arm.com    // device responsible for the address range something is
35912109SRekai.GonzalezAlberquilla@arm.com    // wrong, hence there is nothing further to do as the packet
36012109SRekai.GonzalezAlberquilla@arm.com    // would be going back to where it came from
36112109SRekai.GonzalezAlberquilla@arm.com    assert(master_port_id == findPort(pkt->getAddr()));
36212109SRekai.GonzalezAlberquilla@arm.com}
36312109SRekai.GonzalezAlberquilla@arm.com
36412109SRekai.GonzalezAlberquilla@arm.combool
36510715SRekai.GonzalezAlberquilla@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
36610715SRekai.GonzalezAlberquilla@arm.com{
36710715SRekai.GonzalezAlberquilla@arm.com    // determine the source port based on the id
36812109SRekai.GonzalezAlberquilla@arm.com    SlavePort* src_port = slavePorts[slave_port_id];
36912109SRekai.GonzalezAlberquilla@arm.com
37010715SRekai.GonzalezAlberquilla@arm.com    // get the destination from the packet
37110715SRekai.GonzalezAlberquilla@arm.com    PortID dest_port_id = pkt->getDest();
37210715SRekai.GonzalezAlberquilla@arm.com
37312109SRekai.GonzalezAlberquilla@arm.com    // determine if the response is from a snoop request we
37412109SRekai.GonzalezAlberquilla@arm.com    // created as the result of a normal request (in which case it
37510935Snilay@cs.wisc.edu    // should be in the outstandingReq), or if we merely forwarded
37610715SRekai.GonzalezAlberquilla@arm.com    // someone else's snoop request
37712109SRekai.GonzalezAlberquilla@arm.com    bool forwardAsSnoop = outstandingReq.find(pkt->req) ==
37812109SRekai.GonzalezAlberquilla@arm.com        outstandingReq.end();
37912109SRekai.GonzalezAlberquilla@arm.com
38012109SRekai.GonzalezAlberquilla@arm.com    // test if the crossbar should be considered occupied for the
38112109SRekai.GonzalezAlberquilla@arm.com    // current port, note that the check is bypassed if the response
38210715SRekai.GonzalezAlberquilla@arm.com    // is being passed on as a normal response since this is occupying
3831060SN/A    // the response layer rather than the snoop response layer
3841060SN/A    if (forwardAsSnoop) {
3852292SN/A        if (!snoopLayers[dest_port_id]->tryTiming(src_port)) {
386            DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
387                    src_port->name(), pkt->cmdString(), pkt->getAddr());
388            return false;
389        }
390    } else {
391        // get the master port that mirrors this slave port internally
392        MasterPort* snoop_port = snoopRespPorts[slave_port_id];
393        if (!respLayers[dest_port_id]->tryTiming(snoop_port)) {
394            DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
395                    snoop_port->name(), pkt->cmdString(), pkt->getAddr());
396            return false;
397        }
398    }
399
400    DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n",
401            src_port->name(), pkt->cmdString(), pkt->getAddr());
402
403    // store size and command as they might be modified when
404    // forwarding the packet
405    unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
406    unsigned int pkt_cmd = pkt->cmdToIndex();
407
408    // responses are never express snoops
409    assert(!pkt->isExpressSnoop());
410
411    calcPacketTiming(pkt);
412    Tick packetFinishTime = pkt->lastWordDelay + curTick();
413
414    // forward it either as a snoop response or a normal response
415    if (forwardAsSnoop) {
416        // this is a snoop response to a snoop request we forwarded,
417        // e.g. coming from the L1 and going to the L2, and it should
418        // be forwarded as a snoop response
419
420        if (snoopFilter) {
421            // update the probe filter so that it can properly track the line
422            snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id],
423                                            *masterPorts[dest_port_id]);
424        }
425
426        bool success M5_VAR_USED =
427            masterPorts[dest_port_id]->sendTimingSnoopResp(pkt);
428        pktCount[slave_port_id][dest_port_id]++;
429        pktSize[slave_port_id][dest_port_id] += pkt_size;
430        assert(success);
431
432        snoopLayers[dest_port_id]->succeededTiming(packetFinishTime);
433    } else {
434        // we got a snoop response on one of our slave ports,
435        // i.e. from a coherent master connected to the crossbar, and
436        // since we created the snoop request as part of recvTiming,
437        // this should now be a normal response again
438        outstandingReq.erase(pkt->req);
439
440        // this is a snoop response from a coherent master, with a
441        // destination field set on its way through the crossbar as
442        // request, hence it should never go back to where the snoop
443        // response came from, but instead to where the original
444        // request came from
445        assert(slave_port_id != dest_port_id);
446
447        if (snoopFilter) {
448            // update the probe filter so that it can properly track the line
449            snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id],
450                                    *slavePorts[dest_port_id]);
451        }
452
453        DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\
454                " FWD RESP\n", src_port->name(), pkt->cmdString(),
455                pkt->getAddr());
456
457        // as a normal response, it should go back to a master through
458        // one of our slave ports, at this point we are ignoring the
459        // fact that the response layer could be busy and do not touch
460        // its state
461        bool success M5_VAR_USED =
462            slavePorts[dest_port_id]->sendTimingResp(pkt);
463
464        // @todo Put the response in an internal FIFO and pass it on
465        // to the response layer from there
466
467        // currently it is illegal to block responses... can lead
468        // to deadlock
469        assert(success);
470
471        respLayers[dest_port_id]->succeededTiming(packetFinishTime);
472    }
473
474    // stats updates
475    transDist[pkt_cmd]++;
476    snoops++;
477
478    return true;
479}
480
481
482void
483CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id,
484                           const std::vector<SlavePort*>& dests)
485{
486    DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__,
487            pkt->cmdString(), pkt->getAddr(), pkt->getSize());
488
489    // snoops should only happen if the system isn't bypassing caches
490    assert(!system->bypassCaches());
491
492    unsigned fanout = 0;
493
494    for (const auto& p: dests) {
495        // we could have gotten this request from a snooping master
496        // (corresponding to our own slave port that is also in
497        // snoopPorts) and should not send it back to where it came
498        // from
499        if (exclude_slave_port_id == InvalidPortID ||
500            p->getId() != exclude_slave_port_id) {
501            // cache is not allowed to refuse snoop
502            p->sendTimingSnoopReq(pkt);
503            fanout++;
504        }
505    }
506
507    // Stats for fanout of this forward operation
508    snoopFanout.sample(fanout);
509}
510
511void
512CoherentXBar::recvRetry(PortID master_port_id)
513{
514    // responses and snoop responses never block on forwarding them,
515    // so the retry will always be coming from a port to which we
516    // tried to forward a request
517    reqLayers[master_port_id]->recvRetry();
518}
519
520Tick
521CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id)
522{
523    DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n",
524            slavePorts[slave_port_id]->name(), pkt->getAddr(),
525            pkt->cmdString());
526
527    unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
528    unsigned int pkt_cmd = pkt->cmdToIndex();
529
530    MemCmd snoop_response_cmd = MemCmd::InvalidCmd;
531    Tick snoop_response_latency = 0;
532
533    // uncacheable requests need never be snooped
534    if (!pkt->req->isUncacheable() && !system->bypassCaches()) {
535        // forward to all snoopers but the source
536        std::pair<MemCmd, Tick> snoop_result;
537        if (snoopFilter) {
538            // check with the snoop filter where to forward this packet
539            auto sf_res =
540                snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]);
541            snoop_response_latency += sf_res.second * clockPeriod();
542            DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\
543                    " SF size: %i lat: %i\n", __func__,
544                    slavePorts[slave_port_id]->name(), pkt->cmdString(),
545                    pkt->getAddr(), sf_res.first.size(), sf_res.second);
546            snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID,
547                                         sf_res.first);
548        } else {
549            snoop_result = forwardAtomic(pkt, slave_port_id);
550        }
551        snoop_response_cmd = snoop_result.first;
552        snoop_response_latency += snoop_result.second;
553    }
554
555    // even if we had a snoop response, we must continue and also
556    // perform the actual request at the destination
557    PortID master_port_id = findPort(pkt->getAddr());
558
559    // stats updates for the request
560    pktCount[slave_port_id][master_port_id]++;
561    pktSize[slave_port_id][master_port_id] += pkt_size;
562    transDist[pkt_cmd]++;
563
564    // forward the request to the appropriate destination
565    Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt);
566
567    // Lower levels have replied, tell the snoop filter
568    if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() &&
569        pkt->isResponse()) {
570        snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
571    }
572
573    // if we got a response from a snooper, restore it here
574    if (snoop_response_cmd != MemCmd::InvalidCmd) {
575        // no one else should have responded
576        assert(!pkt->isResponse());
577        pkt->cmd = snoop_response_cmd;
578        response_latency = snoop_response_latency;
579    }
580
581    // add the response data
582    if (pkt->isResponse()) {
583        pkt_size = pkt->hasData() ? pkt->getSize() : 0;
584        pkt_cmd = pkt->cmdToIndex();
585
586        // stats updates
587        pktCount[slave_port_id][master_port_id]++;
588        pktSize[slave_port_id][master_port_id] += pkt_size;
589        transDist[pkt_cmd]++;
590    }
591
592    // @todo: Not setting first-word time
593    pkt->lastWordDelay = response_latency;
594    return response_latency;
595}
596
597Tick
598CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id)
599{
600    DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n",
601            masterPorts[master_port_id]->name(), pkt->getAddr(),
602            pkt->cmdString());
603
604    // add the request snoop data
605    snoops++;
606
607    // forward to all snoopers
608    std::pair<MemCmd, Tick> snoop_result;
609    Tick snoop_response_latency = 0;
610    if (snoopFilter) {
611        auto sf_res = snoopFilter->lookupSnoop(pkt);
612        snoop_response_latency += sf_res.second * clockPeriod();
613        DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n",
614                __func__, masterPorts[master_port_id]->name(), pkt->cmdString(),
615                pkt->getAddr(), sf_res.first.size(), sf_res.second);
616        snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id,
617                                     sf_res.first);
618    } else {
619        snoop_result = forwardAtomic(pkt, InvalidPortID);
620    }
621    MemCmd snoop_response_cmd = snoop_result.first;
622    snoop_response_latency += snoop_result.second;
623
624    if (snoop_response_cmd != MemCmd::InvalidCmd)
625        pkt->cmd = snoop_response_cmd;
626
627    // add the response snoop data
628    if (pkt->isResponse()) {
629        snoops++;
630    }
631
632    // @todo: Not setting first-word time
633    pkt->lastWordDelay = snoop_response_latency;
634    return snoop_response_latency;
635}
636
637std::pair<MemCmd, Tick>
638CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id,
639                           PortID source_master_port_id,
640                           const std::vector<SlavePort*>& dests)
641{
642    // the packet may be changed on snoops, record the original
643    // command to enable us to restore it between snoops so that
644    // additional snoops can take place properly
645    MemCmd orig_cmd = pkt->cmd;
646    MemCmd snoop_response_cmd = MemCmd::InvalidCmd;
647    Tick snoop_response_latency = 0;
648
649    // snoops should only happen if the system isn't bypassing caches
650    assert(!system->bypassCaches());
651
652    unsigned fanout = 0;
653
654    for (const auto& p: dests) {
655        // we could have gotten this request from a snooping master
656        // (corresponding to our own slave port that is also in
657        // snoopPorts) and should not send it back to where it came
658        // from
659        if (exclude_slave_port_id != InvalidPortID &&
660            p->getId() == exclude_slave_port_id)
661            continue;
662
663        Tick latency = p->sendAtomicSnoop(pkt);
664        fanout++;
665
666        // in contrast to a functional access, we have to keep on
667        // going as all snoopers must be updated even if we get a
668        // response
669        if (!pkt->isResponse())
670            continue;
671
672        // response from snoop agent
673        assert(pkt->cmd != orig_cmd);
674        assert(pkt->memInhibitAsserted());
675        // should only happen once
676        assert(snoop_response_cmd == MemCmd::InvalidCmd);
677        // save response state
678        snoop_response_cmd = pkt->cmd;
679        snoop_response_latency = latency;
680
681        if (snoopFilter) {
682            // Handle responses by the snoopers and differentiate between
683            // responses to requests from above and snoops from below
684            if (source_master_port_id != InvalidPortID) {
685                // Getting a response for a snoop from below
686                assert(exclude_slave_port_id == InvalidPortID);
687                snoopFilter->updateSnoopForward(pkt, *p,
688                             *masterPorts[source_master_port_id]);
689            } else {
690                // Getting a response for a request from above
691                assert(source_master_port_id == InvalidPortID);
692                snoopFilter->updateSnoopResponse(pkt, *p,
693                             *slavePorts[exclude_slave_port_id]);
694            }
695        }
696        // restore original packet state for remaining snoopers
697        pkt->cmd = orig_cmd;
698    }
699
700    // Stats for fanout
701    snoopFanout.sample(fanout);
702
703    // the packet is restored as part of the loop and any potential
704    // snoop response is part of the returned pair
705    return std::make_pair(snoop_response_cmd, snoop_response_latency);
706}
707
708void
709CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id)
710{
711    if (!pkt->isPrint()) {
712        // don't do DPRINTFs on PrintReq as it clutters up the output
713        DPRINTF(CoherentXBar,
714                "recvFunctional: packet src %s addr 0x%x cmd %s\n",
715                slavePorts[slave_port_id]->name(), pkt->getAddr(),
716                pkt->cmdString());
717    }
718
719    // uncacheable requests need never be snooped
720    if (!pkt->req->isUncacheable() && !system->bypassCaches()) {
721        // forward to all snoopers but the source
722        forwardFunctional(pkt, slave_port_id);
723    }
724
725    // there is no need to continue if the snooping has found what we
726    // were looking for and the packet is already a response
727    if (!pkt->isResponse()) {
728        PortID dest_id = findPort(pkt->getAddr());
729
730        masterPorts[dest_id]->sendFunctional(pkt);
731    }
732}
733
734void
735CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id)
736{
737    if (!pkt->isPrint()) {
738        // don't do DPRINTFs on PrintReq as it clutters up the output
739        DPRINTF(CoherentXBar,
740                "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n",
741                masterPorts[master_port_id]->name(), pkt->getAddr(),
742                pkt->cmdString());
743    }
744
745    // forward to all snoopers
746    forwardFunctional(pkt, InvalidPortID);
747}
748
749void
750CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id)
751{
752    // snoops should only happen if the system isn't bypassing caches
753    assert(!system->bypassCaches());
754
755    for (const auto& p: snoopPorts) {
756        // we could have gotten this request from a snooping master
757        // (corresponding to our own slave port that is also in
758        // snoopPorts) and should not send it back to where it came
759        // from
760        if (exclude_slave_port_id == InvalidPortID ||
761            p->getId() != exclude_slave_port_id)
762            p->sendFunctionalSnoop(pkt);
763
764        // if we get a response we are done
765        if (pkt->isResponse()) {
766            break;
767        }
768    }
769}
770
771unsigned int
772CoherentXBar::drain(DrainManager *dm)
773{
774    // sum up the individual layers
775    unsigned int total = 0;
776    for (auto l: reqLayers)
777        total += l->drain(dm);
778    for (auto l: respLayers)
779        total += l->drain(dm);
780    for (auto l: snoopLayers)
781        total += l->drain(dm);
782    return total;
783}
784
785void
786CoherentXBar::regStats()
787{
788    // register the stats of the base class and our layers
789    BaseXBar::regStats();
790    for (auto l: reqLayers)
791        l->regStats();
792    for (auto l: respLayers)
793        l->regStats();
794    for (auto l: snoopLayers)
795        l->regStats();
796
797    snoops
798        .name(name() + ".snoops")
799        .desc("Total snoops (count)")
800    ;
801
802    snoopFanout
803        .init(0, snoopPorts.size(), 1)
804        .name(name() + ".snoop_fanout")
805        .desc("Request fanout histogram")
806    ;
807}
808
809CoherentXBar *
810CoherentXBarParams::create()
811{
812    return new CoherentXBar(this);
813}
814