GarnetNetwork.cc revision 11666:10d59d546ea2 
1/*
2 * Copyright (c) 2008 Princeton University
3 * Copyright (c) 2016 Georgia Institute of Technology
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met: redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer;
10 * redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution;
13 * neither the name of the copyright holders nor the names of its
14 * contributors may be used to endorse or promote products derived from
15 * this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * Authors: Niket Agarwal
30 *          Tushar Krishna
31 */
32
33
34#include "mem/ruby/network/garnet2.0/GarnetNetwork.hh"
35
36#include <cassert>
37
38#include "base/cast.hh"
39#include "base/stl_helpers.hh"
40#include "mem/ruby/common/NetDest.hh"
41#include "mem/ruby/network/MessageBuffer.hh"
42#include "mem/ruby/network/garnet2.0/CommonTypes.hh"
43#include "mem/ruby/network/garnet2.0/CreditLink.hh"
44#include "mem/ruby/network/garnet2.0/GarnetLink.hh"
45#include "mem/ruby/network/garnet2.0/NetworkInterface.hh"
46#include "mem/ruby/network/garnet2.0/NetworkLink.hh"
47#include "mem/ruby/network/garnet2.0/Router.hh"
48#include "mem/ruby/system/RubySystem.hh"
49
50using namespace std;
51using m5::stl_helpers::deletePointers;
52
53/*
54 * GarnetNetwork sets up the routers and links and collects stats.
55 * Default parameters (GarnetNetwork.py) can be overwritten from command line
56 * (see configs/network/Network.py)
57 */
58
59GarnetNetwork::GarnetNetwork(const Params *p)
60    : Network(p)
61{
62    m_num_rows = p->num_rows;
63    m_ni_flit_size = p->ni_flit_size;
64    m_vcs_per_vnet = p->vcs_per_vnet;
65    m_buffers_per_data_vc = p->buffers_per_data_vc;
66    m_buffers_per_ctrl_vc = p->buffers_per_ctrl_vc;
67    m_routing_algorithm = p->routing_algorithm;
68
69    m_enable_fault_model = p->enable_fault_model;
70    if (m_enable_fault_model)
71        fault_model = p->fault_model;
72
73    m_vnet_type.resize(m_virtual_networks);
74
75    for (int i = 0 ; i < m_virtual_networks ; i++) {
76        if (m_vnet_type_names[i] == "response")
77            m_vnet_type[i] = DATA_VNET_; // carries data (and ctrl) packets
78        else
79            m_vnet_type[i] = CTRL_VNET_; // carries only ctrl packets
80    }
81
82    // record the routers
83    for (vector<BasicRouter*>::const_iterator i =  p->routers.begin();
84         i != p->routers.end(); ++i) {
85        Router* router = safe_cast<Router*>(*i);
86        m_routers.push_back(router);
87
88        // initialize the router's network pointers
89        router->init_net_ptr(this);
90    }
91
92    // record the network interfaces
93    for (vector<ClockedObject*>::const_iterator i = p->netifs.begin();
94         i != p->netifs.end(); ++i) {
95        NetworkInterface *ni = safe_cast<NetworkInterface *>(*i);
96        m_nis.push_back(ni);
97        ni->init_net_ptr(this);
98    }
99}
100
101void
102GarnetNetwork::init()
103{
104    Network::init();
105
106    for (int i=0; i < m_nodes; i++) {
107        m_nis[i]->addNode(m_toNetQueues[i], m_fromNetQueues[i]);
108    }
109
110    // The topology pointer should have already been initialized in the
111    // parent network constructor
112    assert(m_topology_ptr != NULL);
113    m_topology_ptr->createLinks(this);
114
115    // Initialize topology specific parameters
116    if (getNumRows() > 0) {
117        // Only for Mesh topology
118        // m_num_rows and m_num_cols are only used for
119        // implementing XY or custom routing in RoutingUnit.cc
120        m_num_rows = getNumRows();
121        m_num_cols = m_routers.size() / m_num_rows;
122        assert(m_num_rows * m_num_cols == m_routers.size());
123    } else {
124        m_num_rows = -1;
125        m_num_cols = -1;
126    }
127
128    // FaultModel: declare each router to the fault model
129    if (isFaultModelEnabled()) {
130        for (vector<Router*>::const_iterator i= m_routers.begin();
131             i != m_routers.end(); ++i) {
132            Router* router = safe_cast<Router*>(*i);
133            int router_id M5_VAR_USED =
134                fault_model->declare_router(router->get_num_inports(),
135                                            router->get_num_outports(),
136                                            router->get_vc_per_vnet(),
137                                            getBuffersPerDataVC(),
138                                            getBuffersPerCtrlVC());
139            assert(router_id == router->get_id());
140            router->printAggregateFaultProbability(cout);
141            router->printFaultVector(cout);
142        }
143    }
144}
145
146GarnetNetwork::~GarnetNetwork()
147{
148    deletePointers(m_routers);
149    deletePointers(m_nis);
150    deletePointers(m_networklinks);
151    deletePointers(m_creditlinks);
152}
153
154/*
155 * This function creates a link from the Network Interface (NI)
156 * into the Network.
157 * It creates a Network Link from the NI to a Router and a Credit Link from
158 * the Router to the NI
159*/
160
161void
162GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
163                            const NetDest& routing_table_entry)
164{
165    assert(src < m_nodes);
166
167    GarnetExtLink* garnet_link = safe_cast<GarnetExtLink*>(link);
168
169    // GarnetExtLink is bi-directional
170    NetworkLink* net_link = garnet_link->m_network_links[LinkDirection_In];
171    net_link->setType(EXT_IN_);
172    CreditLink* credit_link = garnet_link->m_credit_links[LinkDirection_In];
173
174    m_networklinks.push_back(net_link);
175    m_creditlinks.push_back(credit_link);
176
177    PortDirection dst_inport_dirn = "Local";
178    m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link);
179    m_nis[src]->addOutPort(net_link, credit_link, dest);
180}
181
182/*
183 * This function creates a link from the Network to a NI.
184 * It creates a Network Link from a Router to the NI and
185 * a Credit Link from NI to the Router
186*/
187
188void
189GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
190                             const NetDest& routing_table_entry)
191{
192    assert(dest < m_nodes);
193    assert(src < m_routers.size());
194    assert(m_routers[src] != NULL);
195
196    GarnetExtLink* garnet_link = safe_cast<GarnetExtLink*>(link);
197
198    // GarnetExtLink is bi-directional
199    NetworkLink* net_link = garnet_link->m_network_links[LinkDirection_Out];
200    net_link->setType(EXT_OUT_);
201    CreditLink* credit_link = garnet_link->m_credit_links[LinkDirection_Out];
202
203    m_networklinks.push_back(net_link);
204    m_creditlinks.push_back(credit_link);
205
206    PortDirection src_outport_dirn = "Local";
207    m_routers[src]->addOutPort(src_outport_dirn, net_link,
208                               routing_table_entry,
209                               link->m_weight, credit_link);
210    m_nis[dest]->addInPort(net_link, credit_link);
211}
212
213/*
214 * This function creates an internal network link between two routers.
215 * It adds both the network link and an opposite credit link.
216*/
217
218void
219GarnetNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
220                                const NetDest& routing_table_entry,
221                                PortDirection src_outport_dirn,
222                                PortDirection dst_inport_dirn)
223{
224    GarnetIntLink* garnet_link = safe_cast<GarnetIntLink*>(link);
225
226    // GarnetIntLink is unidirectional
227    NetworkLink* net_link = garnet_link->m_network_link;
228    net_link->setType(INT_);
229    CreditLink* credit_link = garnet_link->m_credit_link;
230
231    m_networklinks.push_back(net_link);
232    m_creditlinks.push_back(credit_link);
233
234    m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link);
235    m_routers[src]->addOutPort(src_outport_dirn, net_link,
236                               routing_table_entry,
237                               link->m_weight, credit_link);
238}
239
240// Total routers in the network
241int
242GarnetNetwork::getNumRouters()
243{
244    return m_routers.size();
245}
246
247// Get ID of router connected to a NI.
248int
249GarnetNetwork::get_router_id(int ni)
250{
251    return m_nis[ni]->get_router_id();
252}
253
254void
255GarnetNetwork::regStats()
256{
257    Network::regStats();
258
259    // Packets
260    m_packets_received
261        .init(m_virtual_networks)
262        .name(name() + ".packets_received")
263        .flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
264        ;
265
266    m_packets_injected
267        .init(m_virtual_networks)
268        .name(name() + ".packets_injected")
269        .flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
270        ;
271
272    m_packet_network_latency
273        .init(m_virtual_networks)
274        .name(name() + ".packet_network_latency")
275        .flags(Stats::oneline)
276        ;
277
278    m_packet_queueing_latency
279        .init(m_virtual_networks)
280        .name(name() + ".packet_queueing_latency")
281        .flags(Stats::oneline)
282        ;
283
284    for (int i = 0; i < m_virtual_networks; i++) {
285        m_packets_received.subname(i, csprintf("vnet-%i", i));
286        m_packets_injected.subname(i, csprintf("vnet-%i", i));
287        m_packet_network_latency.subname(i, csprintf("vnet-%i", i));
288        m_packet_queueing_latency.subname(i, csprintf("vnet-%i", i));
289    }
290
291    m_avg_packet_vnet_latency
292        .name(name() + ".average_packet_vnet_latency")
293        .flags(Stats::oneline);
294    m_avg_packet_vnet_latency =
295        m_packet_network_latency / m_packets_received;
296
297    m_avg_packet_vqueue_latency
298        .name(name() + ".average_packet_vqueue_latency")
299        .flags(Stats::oneline);
300    m_avg_packet_vqueue_latency =
301        m_packet_queueing_latency / m_packets_received;
302
303    m_avg_packet_network_latency
304        .name(name() + ".average_packet_network_latency");
305    m_avg_packet_network_latency =
306        sum(m_packet_network_latency) / sum(m_packets_received);
307
308    m_avg_packet_queueing_latency
309        .name(name() + ".average_packet_queueing_latency");
310    m_avg_packet_queueing_latency
311        = sum(m_packet_queueing_latency) / sum(m_packets_received);
312
313    m_avg_packet_latency
314        .name(name() + ".average_packet_latency");
315    m_avg_packet_latency
316        = m_avg_packet_network_latency + m_avg_packet_queueing_latency;
317
318    // Flits
319    m_flits_received
320        .init(m_virtual_networks)
321        .name(name() + ".flits_received")
322        .flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
323        ;
324
325    m_flits_injected
326        .init(m_virtual_networks)
327        .name(name() + ".flits_injected")
328        .flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
329        ;
330
331    m_flit_network_latency
332        .init(m_virtual_networks)
333        .name(name() + ".flit_network_latency")
334        .flags(Stats::oneline)
335        ;
336
337    m_flit_queueing_latency
338        .init(m_virtual_networks)
339        .name(name() + ".flit_queueing_latency")
340        .flags(Stats::oneline)
341        ;
342
343    for (int i = 0; i < m_virtual_networks; i++) {
344        m_flits_received.subname(i, csprintf("vnet-%i", i));
345        m_flits_injected.subname(i, csprintf("vnet-%i", i));
346        m_flit_network_latency.subname(i, csprintf("vnet-%i", i));
347        m_flit_queueing_latency.subname(i, csprintf("vnet-%i", i));
348    }
349
350    m_avg_flit_vnet_latency
351        .name(name() + ".average_flit_vnet_latency")
352        .flags(Stats::oneline);
353    m_avg_flit_vnet_latency = m_flit_network_latency / m_flits_received;
354
355    m_avg_flit_vqueue_latency
356        .name(name() + ".average_flit_vqueue_latency")
357        .flags(Stats::oneline);
358    m_avg_flit_vqueue_latency =
359        m_flit_queueing_latency / m_flits_received;
360
361    m_avg_flit_network_latency
362        .name(name() + ".average_flit_network_latency");
363    m_avg_flit_network_latency =
364        sum(m_flit_network_latency) / sum(m_flits_received);
365
366    m_avg_flit_queueing_latency
367        .name(name() + ".average_flit_queueing_latency");
368    m_avg_flit_queueing_latency =
369        sum(m_flit_queueing_latency) / sum(m_flits_received);
370
371    m_avg_flit_latency
372        .name(name() + ".average_flit_latency");
373    m_avg_flit_latency =
374        m_avg_flit_network_latency + m_avg_flit_queueing_latency;
375
376
377    // Hops
378    m_avg_hops.name(name() + ".average_hops");
379    m_avg_hops = m_total_hops / sum(m_flits_received);
380
381    // Links
382    m_total_ext_in_link_utilization
383        .name(name() + ".ext_in_link_utilization");
384    m_total_ext_out_link_utilization
385        .name(name() + ".ext_out_link_utilization");
386    m_total_int_link_utilization
387        .name(name() + ".int_link_utilization");
388    m_average_link_utilization
389        .name(name() + ".avg_link_utilization");
390
391    m_average_vc_load
392        .init(m_virtual_networks * m_vcs_per_vnet)
393        .name(name() + ".avg_vc_load")
394        .flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
395        ;
396}
397
398void
399GarnetNetwork::collateStats()
400{
401    RubySystem *rs = params()->ruby_system;
402    double time_delta = double(curCycle() - rs->getStartCycle());
403
404    for (int i = 0; i < m_networklinks.size(); i++) {
405        link_type type = m_networklinks[i]->getType();
406        int activity = m_networklinks[i]->getLinkUtilization();
407
408        if (type == EXT_IN_)
409            m_total_ext_in_link_utilization += activity;
410        else if (type == EXT_OUT_)
411            m_total_ext_out_link_utilization += activity;
412        else if (type == INT_)
413            m_total_int_link_utilization += activity;
414
415        m_average_link_utilization +=
416            (double(activity) / time_delta);
417
418        vector<unsigned int> vc_load = m_networklinks[i]->getVcLoad();
419        for (int j = 0; j < vc_load.size(); j++) {
420            m_average_vc_load[j] += ((double)vc_load[j] / time_delta);
421        }
422    }
423
424    // Ask the routers to collate their statistics
425    for (int i = 0; i < m_routers.size(); i++) {
426        m_routers[i]->collateStats();
427    }
428}
429
430void
431GarnetNetwork::print(ostream& out) const
432{
433    out << "[GarnetNetwork]";
434}
435
436GarnetNetwork *
437GarnetNetworkParams::create()
438{
439    return new GarnetNetwork(this);
440}
441
442uint32_t
443GarnetNetwork::functionalWrite(Packet *pkt)
444{
445    uint32_t num_functional_writes = 0;
446
447    for (unsigned int i = 0; i < m_routers.size(); i++) {
448        num_functional_writes += m_routers[i]->functionalWrite(pkt);
449    }
450
451    for (unsigned int i = 0; i < m_nis.size(); ++i) {
452        num_functional_writes += m_nis[i]->functionalWrite(pkt);
453    }
454
455    for (unsigned int i = 0; i < m_networklinks.size(); ++i) {
456        num_functional_writes += m_networklinks[i]->functionalWrite(pkt);
457    }
458
459    return num_functional_writes;
460}
461