NetworkInterface.cc revision 11762:29d401db3746
1/* 2 * Copyright (c) 2008 Princeton University 3 * Copyright (c) 2016 Georgia Institute of Technology 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are 8 * met: redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer; 10 * redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution; 13 * neither the name of the copyright holders nor the names of its 14 * contributors may be used to endorse or promote products derived from 15 * this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * Authors: Niket Agarwal 30 * Tushar Krishna 31 */ 32 33 34#include "mem/ruby/network/garnet2.0/NetworkInterface.hh" 35 36#include <cassert> 37#include <cmath> 38 39#include "base/cast.hh" 40#include "base/stl_helpers.hh" 41#include "debug/RubyNetwork.hh" 42#include "mem/ruby/network/MessageBuffer.hh" 43#include "mem/ruby/network/garnet2.0/Credit.hh" 44#include "mem/ruby/network/garnet2.0/flitBuffer.hh" 45#include "mem/ruby/slicc_interface/Message.hh" 46 47using namespace std; 48using m5::stl_helpers::deletePointers; 49 50NetworkInterface::NetworkInterface(const Params *p) 51 : ClockedObject(p), Consumer(this), m_id(p->id), 52 m_virtual_networks(p->virt_nets), m_vc_per_vnet(p->vcs_per_vnet), 53 m_num_vcs(m_vc_per_vnet * m_virtual_networks), 54 m_deadlock_threshold(p->garnet_deadlock_threshold), 55 vc_busy_counter(m_virtual_networks, 0) 56{ 57 m_router_id = -1; 58 m_vc_round_robin = 0; 59 m_ni_out_vcs.resize(m_num_vcs); 60 m_ni_out_vcs_enqueue_time.resize(m_num_vcs); 61 outCreditQueue = new flitBuffer(); 62 63 // instantiating the NI flit buffers 64 for (int i = 0; i < m_num_vcs; i++) { 65 m_ni_out_vcs[i] = new flitBuffer(); 66 m_ni_out_vcs_enqueue_time[i] = Cycles(INFINITE_); 67 } 68 69 m_vc_allocator.resize(m_virtual_networks); // 1 allocator per vnet 70 for (int i = 0; i < m_virtual_networks; i++) { 71 m_vc_allocator[i] = 0; 72 } 73} 74 75void 76NetworkInterface::init() 77{ 78 for (int i = 0; i < m_num_vcs; i++) { 79 m_out_vc_state.push_back(new OutVcState(i, m_net_ptr)); 80 } 81} 82 83NetworkInterface::~NetworkInterface() 84{ 85 deletePointers(m_out_vc_state); 86 deletePointers(m_ni_out_vcs); 87 delete outCreditQueue; 88 delete outFlitQueue; 89} 90 91void 92NetworkInterface::addInPort(NetworkLink *in_link, 93 CreditLink *credit_link) 94{ 95 inNetLink = in_link; 96 in_link->setLinkConsumer(this); 97 outCreditLink = credit_link; 98 credit_link->setSourceQueue(outCreditQueue); 99} 100 101void 102NetworkInterface::addOutPort(NetworkLink *out_link, 103 CreditLink *credit_link, 104 SwitchID router_id) 105{ 106 inCreditLink = credit_link; 107 credit_link->setLinkConsumer(this); 108 109 outNetLink = out_link; 110 outFlitQueue = new flitBuffer(); 111 out_link->setSourceQueue(outFlitQueue); 112 113 m_router_id = router_id; 114} 115 116void 117NetworkInterface::addNode(vector<MessageBuffer *>& in, 118 vector<MessageBuffer *>& out) 119{ 120 inNode_ptr = in; 121 outNode_ptr = out; 122 123 for (auto& it : in) { 124 if (it != nullptr) { 125 it->setConsumer(this); 126 } 127 } 128} 129 130 131/* 132 * The NI wakeup checks whether there are any ready messages in the protocol 133 * buffer. If yes, it picks that up, flitisizes it into a number of flits and 134 * puts it into an output buffer and schedules the output link. On a wakeup 135 * it also checks whether there are flits in the input link. If yes, it picks 136 * them up and if the flit is a tail, the NI inserts the corresponding message 137 * into the protocol buffer. It also checks for credits being sent by the 138 * downstream router. 139 */ 140 141void 142NetworkInterface::wakeup() 143{ 144 DPRINTF(RubyNetwork, "Network Interface %d connected to router %d " 145 "woke up at time: %lld\n", m_id, m_router_id, curCycle()); 146 147 MsgPtr msg_ptr; 148 Tick curTime = clockEdge(); 149 150 // Checking for messages coming from the protocol 151 // can pick up a message/cycle for each virtual net 152 for (int vnet = 0; vnet < inNode_ptr.size(); ++vnet) { 153 MessageBuffer *b = inNode_ptr[vnet]; 154 if (b == nullptr) { 155 continue; 156 } 157 158 if (b->isReady(curTime)) { // Is there a message waiting 159 msg_ptr = b->peekMsgPtr(); 160 if (flitisizeMessage(msg_ptr, vnet)) { 161 b->dequeue(curTime); 162 } else { 163 break; 164 } 165 } 166 } 167 168 scheduleOutputLink(); 169 checkReschedule(); 170 171 /*********** Check the incoming flit link **********/ 172 173 if (inNetLink->isReady(curCycle())) { 174 flit *t_flit = inNetLink->consumeLink(); 175 bool free_signal = false; 176 if (t_flit->get_type() == TAIL_ || t_flit->get_type() == HEAD_TAIL_) { 177 free_signal = true; 178 179 // enqueue into the protocol buffers 180 outNode_ptr[t_flit->get_vnet()]->enqueue( 181 t_flit->get_msg_ptr(), curTime, cyclesToTicks(Cycles(1))); 182 } 183 // Simply send a credit back since we are not buffering 184 // this flit in the NI 185 Credit *t_credit = new Credit(t_flit->get_vc(), free_signal, 186 curCycle()); 187 outCreditQueue->insert(t_credit); 188 outCreditLink-> 189 scheduleEventAbsolute(clockEdge(Cycles(1))); 190 191 int vnet = t_flit->get_vnet(); 192 193 // Update Stats 194 195 // Latency 196 m_net_ptr->increment_received_flits(vnet); 197 Cycles network_delay = curCycle() - t_flit->get_enqueue_time(); 198 Cycles queueing_delay = t_flit->get_src_delay(); 199 200 m_net_ptr->increment_flit_network_latency(network_delay, vnet); 201 m_net_ptr->increment_flit_queueing_latency(queueing_delay, vnet); 202 203 if (t_flit->get_type() == TAIL_ || t_flit->get_type() == HEAD_TAIL_) { 204 m_net_ptr->increment_received_packets(vnet); 205 m_net_ptr->increment_packet_network_latency(network_delay, vnet); 206 m_net_ptr->increment_packet_queueing_latency(queueing_delay, vnet); 207 } 208 209 // Hops 210 m_net_ptr->increment_total_hops(t_flit->get_route().hops_traversed); 211 212 delete t_flit; 213 } 214 215 /****************** Check the incoming credit link *******/ 216 217 if (inCreditLink->isReady(curCycle())) { 218 Credit *t_credit = (Credit*) inCreditLink->consumeLink(); 219 m_out_vc_state[t_credit->get_vc()]->increment_credit(); 220 if (t_credit->is_free_signal()) { 221 m_out_vc_state[t_credit->get_vc()]->setState(IDLE_, curCycle()); 222 } 223 delete t_credit; 224 } 225} 226 227 228// Embed the protocol message into flits 229bool 230NetworkInterface::flitisizeMessage(MsgPtr msg_ptr, int vnet) 231{ 232 Message *net_msg_ptr = msg_ptr.get(); 233 NetDest net_msg_dest = net_msg_ptr->getDestination(); 234 235 // gets all the destinations associated with this message. 236 vector<NodeID> dest_nodes = net_msg_dest.getAllDest(); 237 238 // Number of flits is dependent on the link bandwidth available. 239 // This is expressed in terms of bytes/cycle or the flit size 240 int num_flits = (int) ceil((double) m_net_ptr->MessageSizeType_to_int( 241 net_msg_ptr->getMessageSize())/m_net_ptr->getNiFlitSize()); 242 243 // loop to convert all multicast messages into unicast messages 244 for (int ctr = 0; ctr < dest_nodes.size(); ctr++) { 245 246 // this will return a free output virtual channel 247 int vc = calculateVC(vnet); 248 249 if (vc == -1) { 250 return false ; 251 } 252 MsgPtr new_msg_ptr = msg_ptr->clone(); 253 NodeID destID = dest_nodes[ctr]; 254 255 Message *new_net_msg_ptr = new_msg_ptr.get(); 256 if (dest_nodes.size() > 1) { 257 NetDest personal_dest; 258 for (int m = 0; m < (int) MachineType_NUM; m++) { 259 if ((destID >= MachineType_base_number((MachineType) m)) && 260 destID < MachineType_base_number((MachineType) (m+1))) { 261 // calculating the NetDest associated with this destID 262 personal_dest.clear(); 263 personal_dest.add((MachineID) {(MachineType) m, (destID - 264 MachineType_base_number((MachineType) m))}); 265 new_net_msg_ptr->getDestination() = personal_dest; 266 break; 267 } 268 } 269 net_msg_dest.removeNetDest(personal_dest); 270 // removing the destination from the original message to reflect 271 // that a message with this particular destination has been 272 // flitisized and an output vc is acquired 273 net_msg_ptr->getDestination().removeNetDest(personal_dest); 274 } 275 276 // Embed Route into the flits 277 // NetDest format is used by the routing table 278 // Custom routing algorithms just need destID 279 RouteInfo route; 280 route.vnet = vnet; 281 route.net_dest = new_net_msg_ptr->getDestination(); 282 route.src_ni = m_id; 283 route.src_router = m_router_id; 284 route.dest_ni = destID; 285 route.dest_router = m_net_ptr->get_router_id(destID); 286 287 // initialize hops_traversed to -1 288 // so that the first router increments it to 0 289 route.hops_traversed = -1; 290 291 m_net_ptr->increment_injected_packets(vnet); 292 for (int i = 0; i < num_flits; i++) { 293 m_net_ptr->increment_injected_flits(vnet); 294 flit *fl = new flit(i, vc, vnet, route, num_flits, new_msg_ptr, 295 curCycle()); 296 297 fl->set_src_delay(curCycle() - ticksToCycles(msg_ptr->getTime())); 298 m_ni_out_vcs[vc]->insert(fl); 299 } 300 301 m_ni_out_vcs_enqueue_time[vc] = curCycle(); 302 m_out_vc_state[vc]->setState(ACTIVE_, curCycle()); 303 } 304 return true ; 305} 306 307// Looking for a free output vc 308int 309NetworkInterface::calculateVC(int vnet) 310{ 311 for (int i = 0; i < m_vc_per_vnet; i++) { 312 int delta = m_vc_allocator[vnet]; 313 m_vc_allocator[vnet]++; 314 if (m_vc_allocator[vnet] == m_vc_per_vnet) 315 m_vc_allocator[vnet] = 0; 316 317 if (m_out_vc_state[(vnet*m_vc_per_vnet) + delta]->isInState( 318 IDLE_, curCycle())) { 319 vc_busy_counter[vnet] = 0; 320 return ((vnet*m_vc_per_vnet) + delta); 321 } 322 } 323 324 vc_busy_counter[vnet] += 1; 325 panic_if(vc_busy_counter[vnet] > m_deadlock_threshold, 326 "%s: Possible network deadlock in vnet: %d at time: %llu \n", 327 name(), vnet, curTick()); 328 329 return -1; 330} 331 332 333/** This function looks at the NI buffers 334 * if some buffer has flits which are ready to traverse the link in the next 335 * cycle, and the downstream output vc associated with this flit has buffers 336 * left, the link is scheduled for the next cycle 337 */ 338 339void 340NetworkInterface::scheduleOutputLink() 341{ 342 int vc = m_vc_round_robin; 343 m_vc_round_robin++; 344 if (m_vc_round_robin == m_num_vcs) 345 m_vc_round_robin = 0; 346 347 for (int i = 0; i < m_num_vcs; i++) { 348 vc++; 349 if (vc == m_num_vcs) 350 vc = 0; 351 352 // model buffer backpressure 353 if (m_ni_out_vcs[vc]->isReady(curCycle()) && 354 m_out_vc_state[vc]->has_credit()) { 355 356 bool is_candidate_vc = true; 357 int t_vnet = get_vnet(vc); 358 int vc_base = t_vnet * m_vc_per_vnet; 359 360 if (m_net_ptr->isVNetOrdered(t_vnet)) { 361 for (int vc_offset = 0; vc_offset < m_vc_per_vnet; 362 vc_offset++) { 363 int t_vc = vc_base + vc_offset; 364 if (m_ni_out_vcs[t_vc]->isReady(curCycle())) { 365 if (m_ni_out_vcs_enqueue_time[t_vc] < 366 m_ni_out_vcs_enqueue_time[vc]) { 367 is_candidate_vc = false; 368 break; 369 } 370 } 371 } 372 } 373 if (!is_candidate_vc) 374 continue; 375 376 m_out_vc_state[vc]->decrement_credit(); 377 // Just removing the flit 378 flit *t_flit = m_ni_out_vcs[vc]->getTopFlit(); 379 t_flit->set_time(curCycle() + Cycles(1)); 380 outFlitQueue->insert(t_flit); 381 // schedule the out link 382 outNetLink->scheduleEventAbsolute(clockEdge(Cycles(1))); 383 384 if (t_flit->get_type() == TAIL_ || 385 t_flit->get_type() == HEAD_TAIL_) { 386 m_ni_out_vcs_enqueue_time[vc] = Cycles(INFINITE_); 387 } 388 return; 389 } 390 } 391} 392 393int 394NetworkInterface::get_vnet(int vc) 395{ 396 for (int i = 0; i < m_virtual_networks; i++) { 397 if (vc >= (i*m_vc_per_vnet) && vc < ((i+1)*m_vc_per_vnet)) { 398 return i; 399 } 400 } 401 fatal("Could not determine vc"); 402} 403 404 405// Wakeup the NI in the next cycle if there are waiting 406// messages in the protocol buffer, or waiting flits in the 407// output VC buffer 408void 409NetworkInterface::checkReschedule() 410{ 411 for (const auto& it : inNode_ptr) { 412 if (it == nullptr) { 413 continue; 414 } 415 416 while (it->isReady(clockEdge())) { // Is there a message waiting 417 scheduleEvent(Cycles(1)); 418 return; 419 } 420 } 421 422 for (int vc = 0; vc < m_num_vcs; vc++) { 423 if (m_ni_out_vcs[vc]->isReady(curCycle() + Cycles(1))) { 424 scheduleEvent(Cycles(1)); 425 return; 426 } 427 } 428} 429 430void 431NetworkInterface::print(std::ostream& out) const 432{ 433 out << "[Network Interface]"; 434} 435 436uint32_t 437NetworkInterface::functionalWrite(Packet *pkt) 438{ 439 uint32_t num_functional_writes = 0; 440 for (unsigned int i = 0; i < m_num_vcs; ++i) { 441 num_functional_writes += m_ni_out_vcs[i]->functionalWrite(pkt); 442 } 443 444 num_functional_writes += outFlitQueue->functionalWrite(pkt); 445 return num_functional_writes; 446} 447 448NetworkInterface * 449GarnetNetworkInterfaceParams::create() 450{ 451 return new NetworkInterface(this); 452} 453