dist_iface.cc revision 11290
110923SN/A/* 210923SN/A * Copyright (c) 2015 ARM Limited 310923SN/A * All rights reserved 410923SN/A * 510923SN/A * The license below extends only to copyright in the software and shall 610923SN/A * not be construed as granting a license to any other intellectual 710923SN/A * property including but not limited to intellectual property relating 810923SN/A * to a hardware implementation of the functionality of the software 910923SN/A * licensed hereunder. You may use the software subject to the license 1010923SN/A * terms below provided that you ensure that this notice is replicated 1110923SN/A * unmodified and in its entirety in all distributions of the software, 1210923SN/A * modified or unmodified, in source code or in binary form. 1310923SN/A * 1410923SN/A * Redistribution and use in source and binary forms, with or without 1510923SN/A * modification, are permitted provided that the following conditions are 1610923SN/A * met: redistributions of source code must retain the above copyright 1710923SN/A * notice, this list of conditions and the following disclaimer; 1810923SN/A * redistributions in binary form must reproduce the above copyright 1910923SN/A * notice, this list of conditions and the following disclaimer in the 2010923SN/A * documentation and/or other materials provided with the distribution; 2110923SN/A * neither the name of the copyright holders nor the names of its 2210923SN/A * contributors may be used to endorse or promote products derived from 2310923SN/A * this software without specific prior written permission. 2410923SN/A * 2510923SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2610923SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2710923SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2810923SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2910923SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3010923SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3110923SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3210923SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3310923SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3410923SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3510923SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3610923SN/A * 3710923SN/A * Authors: Gabor Dozsa 3810923SN/A */ 3910923SN/A 4010923SN/A/* @file 4111290Sgabor.dozsa@arm.com * The interface class for dist-gem5 simulations. 4210923SN/A */ 4310923SN/A 4411290Sgabor.dozsa@arm.com#include "dev/net/dist_iface.hh" 4510923SN/A 4610923SN/A#include <queue> 4710923SN/A#include <thread> 4810923SN/A 4910923SN/A#include "base/random.hh" 5010923SN/A#include "base/trace.hh" 5111290Sgabor.dozsa@arm.com#include "debug/DistEthernet.hh" 5211290Sgabor.dozsa@arm.com#include "debug/DistEthernetPkt.hh" 5311263SN/A#include "dev/net/etherpkt.hh" 5410923SN/A#include "sim/sim_exit.hh" 5510923SN/A#include "sim/sim_object.hh" 5610923SN/A 5711290Sgabor.dozsa@arm.comusing namespace std; 5811290Sgabor.dozsa@arm.comDistIface::Sync *DistIface::sync = nullptr; 5911290Sgabor.dozsa@arm.comDistIface::SyncEvent *DistIface::syncEvent = nullptr; 6011290Sgabor.dozsa@arm.comunsigned DistIface::distIfaceNum = 0; 6111290Sgabor.dozsa@arm.comunsigned DistIface::recvThreadsNum = 0; 6211290Sgabor.dozsa@arm.comDistIface *DistIface::master = nullptr; 6310923SN/A 6411290Sgabor.dozsa@arm.comvoid 6511290Sgabor.dozsa@arm.comDistIface::Sync::init(Tick start_tick, Tick repeat_tick) 6611290Sgabor.dozsa@arm.com{ 6711290Sgabor.dozsa@arm.com if (start_tick < firstAt) { 6811290Sgabor.dozsa@arm.com firstAt = start_tick; 6911290Sgabor.dozsa@arm.com inform("Next dist synchronisation tick is changed to %lu.\n", nextAt); 7011290Sgabor.dozsa@arm.com } 7110923SN/A 7211290Sgabor.dozsa@arm.com if (repeat_tick == 0) 7311290Sgabor.dozsa@arm.com panic("Dist synchronisation interval must be greater than zero"); 7411290Sgabor.dozsa@arm.com 7511290Sgabor.dozsa@arm.com if (repeat_tick < nextRepeat) { 7611290Sgabor.dozsa@arm.com nextRepeat = repeat_tick; 7711290Sgabor.dozsa@arm.com inform("Dist synchronisation interval is changed to %lu.\n", 7811290Sgabor.dozsa@arm.com nextRepeat); 7911290Sgabor.dozsa@arm.com } 8011290Sgabor.dozsa@arm.com} 8111290Sgabor.dozsa@arm.com 8211290Sgabor.dozsa@arm.comDistIface::SyncSwitch::SyncSwitch(int num_nodes) 8311290Sgabor.dozsa@arm.com{ 8411290Sgabor.dozsa@arm.com numNodes = num_nodes; 8511290Sgabor.dozsa@arm.com waitNum = num_nodes; 8611290Sgabor.dozsa@arm.com numExitReq = 0; 8711290Sgabor.dozsa@arm.com numCkptReq = 0; 8811290Sgabor.dozsa@arm.com doExit = false; 8911290Sgabor.dozsa@arm.com doCkpt = false; 9011290Sgabor.dozsa@arm.com firstAt = std::numeric_limits<Tick>::max(); 9111290Sgabor.dozsa@arm.com nextAt = 0; 9211290Sgabor.dozsa@arm.com nextRepeat = std::numeric_limits<Tick>::max(); 9311290Sgabor.dozsa@arm.com} 9411290Sgabor.dozsa@arm.com 9511290Sgabor.dozsa@arm.comDistIface::SyncNode::SyncNode() 9611290Sgabor.dozsa@arm.com{ 9711290Sgabor.dozsa@arm.com waitNum = 0; 9811290Sgabor.dozsa@arm.com needExit = ReqType::none; 9911290Sgabor.dozsa@arm.com needCkpt = ReqType::none; 10011290Sgabor.dozsa@arm.com doExit = false; 10111290Sgabor.dozsa@arm.com doCkpt = false; 10211290Sgabor.dozsa@arm.com firstAt = std::numeric_limits<Tick>::max(); 10311290Sgabor.dozsa@arm.com nextAt = 0; 10411290Sgabor.dozsa@arm.com nextRepeat = std::numeric_limits<Tick>::max(); 10511290Sgabor.dozsa@arm.com} 10611290Sgabor.dozsa@arm.com 10711290Sgabor.dozsa@arm.comvoid 10811290Sgabor.dozsa@arm.comDistIface::SyncNode::run(bool same_tick) 10910923SN/A{ 11010923SN/A std::unique_lock<std::mutex> sync_lock(lock); 11111290Sgabor.dozsa@arm.com Header header; 11210923SN/A 11311290Sgabor.dozsa@arm.com assert(waitNum == 0); 11411290Sgabor.dozsa@arm.com waitNum = DistIface::recvThreadsNum; 11511290Sgabor.dozsa@arm.com // initiate the global synchronisation 11611290Sgabor.dozsa@arm.com header.msgType = MsgType::cmdSyncReq; 11711290Sgabor.dozsa@arm.com header.sendTick = curTick(); 11811290Sgabor.dozsa@arm.com header.syncRepeat = nextRepeat; 11911290Sgabor.dozsa@arm.com header.needCkpt = needCkpt; 12011290Sgabor.dozsa@arm.com if (needCkpt != ReqType::none) 12111290Sgabor.dozsa@arm.com needCkpt = ReqType::pending; 12211290Sgabor.dozsa@arm.com header.needExit = needExit; 12311290Sgabor.dozsa@arm.com if (needExit != ReqType::none) 12411290Sgabor.dozsa@arm.com needExit = ReqType::pending; 12511290Sgabor.dozsa@arm.com DistIface::master->sendCmd(header); 12610923SN/A // now wait until all receiver threads complete the synchronisation 12710923SN/A auto lf = [this]{ return waitNum == 0; }; 12810923SN/A cv.wait(sync_lock, lf); 12911290Sgabor.dozsa@arm.com // global synchronisation is done 13011290Sgabor.dozsa@arm.com assert(!same_tick || (nextAt == curTick())); 13111290Sgabor.dozsa@arm.com} 13210923SN/A 13311290Sgabor.dozsa@arm.com 13411290Sgabor.dozsa@arm.comvoid 13511290Sgabor.dozsa@arm.comDistIface::SyncSwitch::run(bool same_tick) 13611290Sgabor.dozsa@arm.com{ 13711290Sgabor.dozsa@arm.com std::unique_lock<std::mutex> sync_lock(lock); 13811290Sgabor.dozsa@arm.com Header header; 13911290Sgabor.dozsa@arm.com // Wait for the sync requests from the nodes 14011290Sgabor.dozsa@arm.com if (waitNum > 0) { 14111290Sgabor.dozsa@arm.com auto lf = [this]{ return waitNum == 0; }; 14211290Sgabor.dozsa@arm.com cv.wait(sync_lock, lf); 14311290Sgabor.dozsa@arm.com } 14411290Sgabor.dozsa@arm.com assert(waitNum == 0); 14511290Sgabor.dozsa@arm.com assert(!same_tick || (nextAt == curTick())); 14611290Sgabor.dozsa@arm.com waitNum = numNodes; 14711290Sgabor.dozsa@arm.com // Complete the global synchronisation 14811290Sgabor.dozsa@arm.com header.msgType = MsgType::cmdSyncAck; 14911290Sgabor.dozsa@arm.com header.sendTick = nextAt; 15011290Sgabor.dozsa@arm.com header.syncRepeat = nextRepeat; 15111290Sgabor.dozsa@arm.com if (doCkpt || numCkptReq == numNodes) { 15211290Sgabor.dozsa@arm.com doCkpt = true; 15311290Sgabor.dozsa@arm.com header.needCkpt = ReqType::immediate; 15411290Sgabor.dozsa@arm.com numCkptReq = 0; 15511290Sgabor.dozsa@arm.com } else { 15611290Sgabor.dozsa@arm.com header.needCkpt = ReqType::none; 15711290Sgabor.dozsa@arm.com } 15811290Sgabor.dozsa@arm.com if (doExit || numExitReq == numNodes) { 15911290Sgabor.dozsa@arm.com doExit = true; 16011290Sgabor.dozsa@arm.com header.needExit = ReqType::immediate; 16111290Sgabor.dozsa@arm.com } else { 16211290Sgabor.dozsa@arm.com header.needExit = ReqType::none; 16311290Sgabor.dozsa@arm.com } 16411290Sgabor.dozsa@arm.com DistIface::master->sendCmd(header); 16510923SN/A} 16610923SN/A 16710923SN/Avoid 16811290Sgabor.dozsa@arm.comDistIface::SyncSwitch::progress(Tick send_tick, 16911290Sgabor.dozsa@arm.com Tick sync_repeat, 17011290Sgabor.dozsa@arm.com ReqType need_ckpt, 17111290Sgabor.dozsa@arm.com ReqType need_exit) 17210923SN/A{ 17310923SN/A std::unique_lock<std::mutex> sync_lock(lock); 17411290Sgabor.dozsa@arm.com assert(waitNum > 0); 17510923SN/A 17611290Sgabor.dozsa@arm.com if (send_tick > nextAt) 17711290Sgabor.dozsa@arm.com nextAt = send_tick; 17811290Sgabor.dozsa@arm.com if (nextRepeat > sync_repeat) 17911290Sgabor.dozsa@arm.com nextRepeat = sync_repeat; 18011290Sgabor.dozsa@arm.com 18111290Sgabor.dozsa@arm.com if (need_ckpt == ReqType::collective) 18211290Sgabor.dozsa@arm.com numCkptReq++; 18311290Sgabor.dozsa@arm.com else if (need_ckpt == ReqType::immediate) 18411290Sgabor.dozsa@arm.com doCkpt = true; 18511290Sgabor.dozsa@arm.com if (need_exit == ReqType::collective) 18611290Sgabor.dozsa@arm.com numExitReq++; 18711290Sgabor.dozsa@arm.com else if (need_exit == ReqType::immediate) 18811290Sgabor.dozsa@arm.com doExit = true; 18911290Sgabor.dozsa@arm.com 19010923SN/A waitNum--; 19111290Sgabor.dozsa@arm.com // Notify the simulation thread if the on-going sync is complete 19211290Sgabor.dozsa@arm.com if (waitNum == 0) { 19310923SN/A sync_lock.unlock(); 19410923SN/A cv.notify_one(); 19510923SN/A } 19610923SN/A} 19710923SN/A 19811290Sgabor.dozsa@arm.comvoid 19911290Sgabor.dozsa@arm.comDistIface::SyncNode::progress(Tick max_send_tick, 20011290Sgabor.dozsa@arm.com Tick next_repeat, 20111290Sgabor.dozsa@arm.com ReqType do_ckpt, 20211290Sgabor.dozsa@arm.com ReqType do_exit) 20310923SN/A{ 20411290Sgabor.dozsa@arm.com std::unique_lock<std::mutex> sync_lock(lock); 20511290Sgabor.dozsa@arm.com assert(waitNum > 0); 20611290Sgabor.dozsa@arm.com 20711290Sgabor.dozsa@arm.com nextAt = max_send_tick; 20811290Sgabor.dozsa@arm.com nextRepeat = next_repeat; 20911290Sgabor.dozsa@arm.com doCkpt = (do_ckpt != ReqType::none); 21011290Sgabor.dozsa@arm.com doExit = (do_exit != ReqType::none); 21111290Sgabor.dozsa@arm.com 21211290Sgabor.dozsa@arm.com waitNum--; 21311290Sgabor.dozsa@arm.com // Notify the simulation thread if the on-going sync is complete 21411290Sgabor.dozsa@arm.com if (waitNum == 0) { 21511290Sgabor.dozsa@arm.com sync_lock.unlock(); 21611290Sgabor.dozsa@arm.com cv.notify_one(); 21711290Sgabor.dozsa@arm.com } 21810923SN/A} 21910923SN/A 22010923SN/Avoid 22111290Sgabor.dozsa@arm.comDistIface::SyncNode::requestCkpt(ReqType req) 22210923SN/A{ 22311290Sgabor.dozsa@arm.com std::lock_guard<std::mutex> sync_lock(lock); 22411290Sgabor.dozsa@arm.com assert(req != ReqType::none); 22511290Sgabor.dozsa@arm.com if (needCkpt != ReqType::none) 22611290Sgabor.dozsa@arm.com warn("Ckpt requested multiple times (req:%d)\n", static_cast<int>(req)); 22711290Sgabor.dozsa@arm.com if (needCkpt == ReqType::none || req == ReqType::immediate) 22811290Sgabor.dozsa@arm.com needCkpt = req; 22910923SN/A} 23010923SN/A 23110923SN/Avoid 23211290Sgabor.dozsa@arm.comDistIface::SyncNode::requestExit(ReqType req) 23310923SN/A{ 23411290Sgabor.dozsa@arm.com std::lock_guard<std::mutex> sync_lock(lock); 23511290Sgabor.dozsa@arm.com assert(req != ReqType::none); 23611290Sgabor.dozsa@arm.com if (needExit != ReqType::none) 23711290Sgabor.dozsa@arm.com warn("Exit requested multiple times (req:%d)\n", static_cast<int>(req)); 23811290Sgabor.dozsa@arm.com if (needExit == ReqType::none || req == ReqType::immediate) 23911290Sgabor.dozsa@arm.com needExit = req; 24011290Sgabor.dozsa@arm.com} 24111290Sgabor.dozsa@arm.com 24211290Sgabor.dozsa@arm.comvoid 24311290Sgabor.dozsa@arm.comDistIface::Sync::drainComplete() 24411290Sgabor.dozsa@arm.com{ 24511290Sgabor.dozsa@arm.com if (doCkpt) { 24611290Sgabor.dozsa@arm.com // The first DistIface object called this right before writing the 24711290Sgabor.dozsa@arm.com // checkpoint. We need to drain the underlying physical network here. 24811290Sgabor.dozsa@arm.com // Note that other gem5 peers may enter this barrier at different 24911290Sgabor.dozsa@arm.com // ticks due to draining. 25011290Sgabor.dozsa@arm.com run(false); 25111290Sgabor.dozsa@arm.com // Only the "first" DistIface object has to perform the sync 25211290Sgabor.dozsa@arm.com doCkpt = false; 25311290Sgabor.dozsa@arm.com } 25411290Sgabor.dozsa@arm.com} 25511290Sgabor.dozsa@arm.com 25611290Sgabor.dozsa@arm.comvoid 25711290Sgabor.dozsa@arm.comDistIface::SyncNode::serialize(CheckpointOut &cp) const 25811290Sgabor.dozsa@arm.com{ 25911290Sgabor.dozsa@arm.com int need_exit = static_cast<int>(needExit); 26011290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(need_exit); 26111290Sgabor.dozsa@arm.com} 26211290Sgabor.dozsa@arm.com 26311290Sgabor.dozsa@arm.comvoid 26411290Sgabor.dozsa@arm.comDistIface::SyncNode::unserialize(CheckpointIn &cp) 26511290Sgabor.dozsa@arm.com{ 26611290Sgabor.dozsa@arm.com int need_exit; 26711290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(need_exit); 26811290Sgabor.dozsa@arm.com needExit = static_cast<ReqType>(need_exit); 26911290Sgabor.dozsa@arm.com} 27011290Sgabor.dozsa@arm.com 27111290Sgabor.dozsa@arm.comvoid 27211290Sgabor.dozsa@arm.comDistIface::SyncSwitch::serialize(CheckpointOut &cp) const 27311290Sgabor.dozsa@arm.com{ 27411290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(numExitReq); 27511290Sgabor.dozsa@arm.com} 27611290Sgabor.dozsa@arm.com 27711290Sgabor.dozsa@arm.comvoid 27811290Sgabor.dozsa@arm.comDistIface::SyncSwitch::unserialize(CheckpointIn &cp) 27911290Sgabor.dozsa@arm.com{ 28011290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(numExitReq); 28111290Sgabor.dozsa@arm.com} 28211290Sgabor.dozsa@arm.com 28311290Sgabor.dozsa@arm.comvoid 28411290Sgabor.dozsa@arm.comDistIface::SyncEvent::start() 28511290Sgabor.dozsa@arm.com{ 28611290Sgabor.dozsa@arm.com // Note that this may be called either from startup() or drainResume() 28711290Sgabor.dozsa@arm.com 28811290Sgabor.dozsa@arm.com // At this point, all DistIface objects has already called Sync::init() so 28911290Sgabor.dozsa@arm.com // we have a local minimum of the start tick and repeat for the periodic 29011290Sgabor.dozsa@arm.com // sync. 29111290Sgabor.dozsa@arm.com Tick firstAt = DistIface::sync->firstAt; 29211290Sgabor.dozsa@arm.com repeat = DistIface::sync->nextRepeat; 29311290Sgabor.dozsa@arm.com // Do a global barrier to agree on a common repeat value (the smallest 29411290Sgabor.dozsa@arm.com // one from all participating nodes 29511290Sgabor.dozsa@arm.com DistIface::sync->run(curTick() == 0); 29611290Sgabor.dozsa@arm.com 29711290Sgabor.dozsa@arm.com assert(!DistIface::sync->doCkpt); 29811290Sgabor.dozsa@arm.com assert(!DistIface::sync->doExit); 29911290Sgabor.dozsa@arm.com assert(DistIface::sync->nextAt >= curTick()); 30011290Sgabor.dozsa@arm.com assert(DistIface::sync->nextRepeat <= repeat); 30111290Sgabor.dozsa@arm.com 30211290Sgabor.dozsa@arm.com // if this is called at tick 0 then we use the config start param otherwise 30311290Sgabor.dozsa@arm.com // the maximum of the current tick of all participating nodes 30411290Sgabor.dozsa@arm.com if (curTick() == 0) { 30511290Sgabor.dozsa@arm.com assert(!scheduled()); 30611290Sgabor.dozsa@arm.com assert(DistIface::sync->nextAt == 0); 30711290Sgabor.dozsa@arm.com schedule(firstAt); 30811290Sgabor.dozsa@arm.com } else { 30911290Sgabor.dozsa@arm.com if (scheduled()) 31011290Sgabor.dozsa@arm.com reschedule(DistIface::sync->nextAt); 31111290Sgabor.dozsa@arm.com else 31211290Sgabor.dozsa@arm.com schedule(DistIface::sync->nextAt); 31311290Sgabor.dozsa@arm.com } 31411290Sgabor.dozsa@arm.com inform("Dist sync scheduled at %lu and repeats %lu\n", when(), 31511290Sgabor.dozsa@arm.com DistIface::sync->nextRepeat); 31611290Sgabor.dozsa@arm.com} 31711290Sgabor.dozsa@arm.com 31811290Sgabor.dozsa@arm.comvoid 31911290Sgabor.dozsa@arm.comDistIface::SyncEvent::process() 32011290Sgabor.dozsa@arm.com{ 32111290Sgabor.dozsa@arm.com // We may not start a global periodic sync while draining before taking a 32211290Sgabor.dozsa@arm.com // checkpoint. This is due to the possibility that peer gem5 processes 32311290Sgabor.dozsa@arm.com // may not hit the same periodic sync before they complete draining and 32411290Sgabor.dozsa@arm.com // that would make this periodic sync clash with sync called from 32511290Sgabor.dozsa@arm.com // DistIface::serialize() by other gem5 processes. 32611290Sgabor.dozsa@arm.com // We would need a 'distributed drain' solution to eliminate this 32711290Sgabor.dozsa@arm.com // restriction. 32811290Sgabor.dozsa@arm.com // Note that if draining was not triggered by checkpointing then we are 32911290Sgabor.dozsa@arm.com // fine since no extra global sync will happen (i.e. all peer gem5 will 33011290Sgabor.dozsa@arm.com // hit this periodic sync eventually). 33111290Sgabor.dozsa@arm.com panic_if(_draining && DistIface::sync->doCkpt, 33211290Sgabor.dozsa@arm.com "Distributed sync is hit while draining"); 33310923SN/A /* 33410923SN/A * Note that this is a global event so this process method will be called 33510923SN/A * by only exactly one thread. 33610923SN/A */ 33710923SN/A /* 33810923SN/A * We hold the eventq lock at this point but the receiver thread may 33910923SN/A * need the lock to schedule new recv events while waiting for the 34011290Sgabor.dozsa@arm.com * dist sync to complete. 34110923SN/A * Note that the other simulation threads also release their eventq 34210923SN/A * locks while waiting for us due to the global event semantics. 34310923SN/A */ 34411290Sgabor.dozsa@arm.com { 34511290Sgabor.dozsa@arm.com EventQueue::ScopedRelease sr(curEventQueue()); 34611290Sgabor.dozsa@arm.com // we do a global sync here that is supposed to happen at the same 34711290Sgabor.dozsa@arm.com // tick in all gem5 peers 34811290Sgabor.dozsa@arm.com DistIface::sync->run(true); 34911290Sgabor.dozsa@arm.com // global sync completed 35011290Sgabor.dozsa@arm.com } 35111290Sgabor.dozsa@arm.com if (DistIface::sync->doCkpt) 35211290Sgabor.dozsa@arm.com exitSimLoop("checkpoint"); 35311290Sgabor.dozsa@arm.com if (DistIface::sync->doExit) 35411290Sgabor.dozsa@arm.com exitSimLoop("exit request from gem5 peers"); 35511290Sgabor.dozsa@arm.com 35611290Sgabor.dozsa@arm.com // schedule the next periodic sync 35711290Sgabor.dozsa@arm.com repeat = DistIface::sync->nextRepeat; 35811290Sgabor.dozsa@arm.com schedule(curTick() + repeat); 35910923SN/A} 36010923SN/A 36111290Sgabor.dozsa@arm.comvoid 36211290Sgabor.dozsa@arm.comDistIface::RecvScheduler::init(Event *recv_done, Tick link_delay) 36310923SN/A{ 36411290Sgabor.dozsa@arm.com // This is called from the receiver thread when it starts running. The new 36511290Sgabor.dozsa@arm.com // receiver thread shares the event queue with the simulation thread 36611290Sgabor.dozsa@arm.com // (associated with the simulated Ethernet link). 36711290Sgabor.dozsa@arm.com curEventQueue(eventManager->eventQueue()); 36811290Sgabor.dozsa@arm.com 36911290Sgabor.dozsa@arm.com recvDone = recv_done; 37011290Sgabor.dozsa@arm.com linkDelay = link_delay; 37110923SN/A} 37210923SN/A 37311290Sgabor.dozsa@arm.comTick 37411290Sgabor.dozsa@arm.comDistIface::RecvScheduler::calcReceiveTick(Tick send_tick, 37511290Sgabor.dozsa@arm.com Tick send_delay, 37611290Sgabor.dozsa@arm.com Tick prev_recv_tick) 37710923SN/A{ 37811290Sgabor.dozsa@arm.com Tick recv_tick = send_tick + send_delay + linkDelay; 37911290Sgabor.dozsa@arm.com // sanity check (we need atleast a send delay long window) 38011290Sgabor.dozsa@arm.com assert(recv_tick >= prev_recv_tick + send_delay); 38111290Sgabor.dozsa@arm.com panic_if(prev_recv_tick + send_delay > recv_tick, 38211290Sgabor.dozsa@arm.com "Receive window is smaller than send delay"); 38311290Sgabor.dozsa@arm.com panic_if(recv_tick <= curTick(), 38411290Sgabor.dozsa@arm.com "Simulators out of sync - missed packet receive by %llu ticks" 38511290Sgabor.dozsa@arm.com "(rev_recv_tick: %lu send_tick: %lu send_delay: %lu " 38611290Sgabor.dozsa@arm.com "linkDelay: %lu )", 38711290Sgabor.dozsa@arm.com curTick() - recv_tick, prev_recv_tick, send_tick, send_delay, 38811290Sgabor.dozsa@arm.com linkDelay); 38911290Sgabor.dozsa@arm.com 39011290Sgabor.dozsa@arm.com return recv_tick; 39110923SN/A} 39210923SN/A 39311290Sgabor.dozsa@arm.comvoid 39411290Sgabor.dozsa@arm.comDistIface::RecvScheduler::resumeRecvTicks() 39510923SN/A{ 39611290Sgabor.dozsa@arm.com // Schedule pending packets asap in case link speed/delay changed when 39711290Sgabor.dozsa@arm.com // restoring from the checkpoint. 39811290Sgabor.dozsa@arm.com // This may be done during unserialize except that curTick() is unknown 39911290Sgabor.dozsa@arm.com // so we call this during drainResume(). 40011290Sgabor.dozsa@arm.com // If we are not restoring from a checkppint then link latency could not 40111290Sgabor.dozsa@arm.com // change so we just return. 40211290Sgabor.dozsa@arm.com if (!ckptRestore) 40311290Sgabor.dozsa@arm.com return; 40411290Sgabor.dozsa@arm.com 40511290Sgabor.dozsa@arm.com std::vector<Desc> v; 40611290Sgabor.dozsa@arm.com while (!descQueue.empty()) { 40711290Sgabor.dozsa@arm.com Desc d = descQueue.front(); 40811290Sgabor.dozsa@arm.com descQueue.pop(); 40911290Sgabor.dozsa@arm.com d.sendTick = curTick(); 41011290Sgabor.dozsa@arm.com d.sendDelay = d.packet->size(); // assume 1 tick/byte max link speed 41111290Sgabor.dozsa@arm.com v.push_back(d); 41211290Sgabor.dozsa@arm.com } 41311290Sgabor.dozsa@arm.com 41411290Sgabor.dozsa@arm.com for (auto &d : v) 41511290Sgabor.dozsa@arm.com descQueue.push(d); 41611290Sgabor.dozsa@arm.com 41711290Sgabor.dozsa@arm.com if (recvDone->scheduled()) { 41811290Sgabor.dozsa@arm.com assert(!descQueue.empty()); 41911290Sgabor.dozsa@arm.com eventManager->reschedule(recvDone, curTick()); 42011290Sgabor.dozsa@arm.com } else { 42111290Sgabor.dozsa@arm.com assert(descQueue.empty() && v.empty()); 42211290Sgabor.dozsa@arm.com } 42311290Sgabor.dozsa@arm.com ckptRestore = false; 42410923SN/A} 42510923SN/A 42611290Sgabor.dozsa@arm.comvoid 42711290Sgabor.dozsa@arm.comDistIface::RecvScheduler::pushPacket(EthPacketPtr new_packet, 42811290Sgabor.dozsa@arm.com Tick send_tick, 42911290Sgabor.dozsa@arm.com Tick send_delay) 43011290Sgabor.dozsa@arm.com{ 43111290Sgabor.dozsa@arm.com // Note : this is called from the receiver thread 43211290Sgabor.dozsa@arm.com curEventQueue()->lock(); 43311290Sgabor.dozsa@arm.com Tick recv_tick = calcReceiveTick(send_tick, send_delay, prevRecvTick); 43411290Sgabor.dozsa@arm.com 43511290Sgabor.dozsa@arm.com DPRINTF(DistEthernetPkt, "DistIface::recvScheduler::pushPacket " 43611290Sgabor.dozsa@arm.com "send_tick:%llu send_delay:%llu link_delay:%llu recv_tick:%llu\n", 43711290Sgabor.dozsa@arm.com send_tick, send_delay, linkDelay, recv_tick); 43811290Sgabor.dozsa@arm.com // Every packet must be sent and arrive in the same quantum 43911290Sgabor.dozsa@arm.com assert(send_tick > master->syncEvent->when() - 44011290Sgabor.dozsa@arm.com master->syncEvent->repeat); 44111290Sgabor.dozsa@arm.com // No packet may be scheduled for receive in the arrival quantum 44211290Sgabor.dozsa@arm.com assert(send_tick + send_delay + linkDelay > master->syncEvent->when()); 44311290Sgabor.dozsa@arm.com 44411290Sgabor.dozsa@arm.com // Now we are about to schedule a recvDone event for the new data packet. 44511290Sgabor.dozsa@arm.com // We use the same recvDone object for all incoming data packets. Packet 44611290Sgabor.dozsa@arm.com // descriptors are saved in the ordered queue. The currently scheduled 44711290Sgabor.dozsa@arm.com // packet is always on the top of the queue. 44811290Sgabor.dozsa@arm.com // NOTE: we use the event queue lock to protect the receive desc queue, 44911290Sgabor.dozsa@arm.com // too, which is accessed both by the receiver thread and the simulation 45011290Sgabor.dozsa@arm.com // thread. 45111290Sgabor.dozsa@arm.com descQueue.emplace(new_packet, send_tick, send_delay); 45211290Sgabor.dozsa@arm.com if (descQueue.size() == 1) { 45311290Sgabor.dozsa@arm.com assert(!recvDone->scheduled()); 45411290Sgabor.dozsa@arm.com eventManager->schedule(recvDone, recv_tick); 45511290Sgabor.dozsa@arm.com } else { 45611290Sgabor.dozsa@arm.com assert(recvDone->scheduled()); 45711290Sgabor.dozsa@arm.com panic_if(descQueue.front().sendTick + descQueue.front().sendDelay > recv_tick, 45811290Sgabor.dozsa@arm.com "Out of order packet received (recv_tick: %lu top(): %lu\n", 45911290Sgabor.dozsa@arm.com recv_tick, descQueue.front().sendTick + descQueue.front().sendDelay); 46011290Sgabor.dozsa@arm.com } 46111290Sgabor.dozsa@arm.com curEventQueue()->unlock(); 46211290Sgabor.dozsa@arm.com} 46311290Sgabor.dozsa@arm.com 46411290Sgabor.dozsa@arm.comEthPacketPtr 46511290Sgabor.dozsa@arm.comDistIface::RecvScheduler::popPacket() 46611290Sgabor.dozsa@arm.com{ 46711290Sgabor.dozsa@arm.com // Note : this is called from the simulation thread when a receive done 46811290Sgabor.dozsa@arm.com // event is being processed for the link. We assume that the thread holds 46911290Sgabor.dozsa@arm.com // the event queue queue lock when this is called! 47011290Sgabor.dozsa@arm.com EthPacketPtr next_packet = descQueue.front().packet; 47111290Sgabor.dozsa@arm.com descQueue.pop(); 47211290Sgabor.dozsa@arm.com 47311290Sgabor.dozsa@arm.com if (descQueue.size() > 0) { 47411290Sgabor.dozsa@arm.com Tick recv_tick = calcReceiveTick(descQueue.front().sendTick, 47511290Sgabor.dozsa@arm.com descQueue.front().sendDelay, 47611290Sgabor.dozsa@arm.com curTick()); 47711290Sgabor.dozsa@arm.com eventManager->schedule(recvDone, recv_tick); 47811290Sgabor.dozsa@arm.com } 47911290Sgabor.dozsa@arm.com prevRecvTick = curTick(); 48011290Sgabor.dozsa@arm.com return next_packet; 48111290Sgabor.dozsa@arm.com} 48211290Sgabor.dozsa@arm.com 48311290Sgabor.dozsa@arm.comvoid 48411290Sgabor.dozsa@arm.comDistIface::RecvScheduler::Desc::serialize(CheckpointOut &cp) const 48511290Sgabor.dozsa@arm.com{ 48611290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(sendTick); 48711290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(sendDelay); 48811290Sgabor.dozsa@arm.com packet->serialize("rxPacket", cp); 48911290Sgabor.dozsa@arm.com} 49011290Sgabor.dozsa@arm.com 49111290Sgabor.dozsa@arm.comvoid 49211290Sgabor.dozsa@arm.comDistIface::RecvScheduler::Desc::unserialize(CheckpointIn &cp) 49311290Sgabor.dozsa@arm.com{ 49411290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(sendTick); 49511290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(sendDelay); 49611290Sgabor.dozsa@arm.com packet = std::make_shared<EthPacketData>(16384); 49711290Sgabor.dozsa@arm.com packet->unserialize("rxPacket", cp); 49811290Sgabor.dozsa@arm.com} 49911290Sgabor.dozsa@arm.com 50011290Sgabor.dozsa@arm.comvoid 50111290Sgabor.dozsa@arm.comDistIface::RecvScheduler::serialize(CheckpointOut &cp) const 50211290Sgabor.dozsa@arm.com{ 50311290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(prevRecvTick); 50411290Sgabor.dozsa@arm.com // serialize the receive desc queue 50511290Sgabor.dozsa@arm.com std::queue<Desc> tmp_queue(descQueue); 50611290Sgabor.dozsa@arm.com unsigned n_desc_queue = descQueue.size(); 50711290Sgabor.dozsa@arm.com assert(tmp_queue.size() == descQueue.size()); 50811290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(n_desc_queue); 50911290Sgabor.dozsa@arm.com for (int i = 0; i < n_desc_queue; i++) { 51011290Sgabor.dozsa@arm.com tmp_queue.front().serializeSection(cp, csprintf("rxDesc_%d", i)); 51111290Sgabor.dozsa@arm.com tmp_queue.pop(); 51211290Sgabor.dozsa@arm.com } 51311290Sgabor.dozsa@arm.com assert(tmp_queue.empty()); 51411290Sgabor.dozsa@arm.com} 51511290Sgabor.dozsa@arm.com 51611290Sgabor.dozsa@arm.comvoid 51711290Sgabor.dozsa@arm.comDistIface::RecvScheduler::unserialize(CheckpointIn &cp) 51811290Sgabor.dozsa@arm.com{ 51911290Sgabor.dozsa@arm.com assert(descQueue.size() == 0); 52011290Sgabor.dozsa@arm.com assert(recvDone->scheduled() == false); 52111290Sgabor.dozsa@arm.com assert(ckptRestore == false); 52211290Sgabor.dozsa@arm.com 52311290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(prevRecvTick); 52411290Sgabor.dozsa@arm.com // unserialize the receive desc queue 52511290Sgabor.dozsa@arm.com unsigned n_desc_queue; 52611290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(n_desc_queue); 52711290Sgabor.dozsa@arm.com for (int i = 0; i < n_desc_queue; i++) { 52811290Sgabor.dozsa@arm.com Desc recv_desc; 52911290Sgabor.dozsa@arm.com recv_desc.unserializeSection(cp, csprintf("rxDesc_%d", i)); 53011290Sgabor.dozsa@arm.com descQueue.push(recv_desc); 53111290Sgabor.dozsa@arm.com } 53211290Sgabor.dozsa@arm.com ckptRestore = true; 53311290Sgabor.dozsa@arm.com} 53411290Sgabor.dozsa@arm.com 53511290Sgabor.dozsa@arm.comDistIface::DistIface(unsigned dist_rank, 53611290Sgabor.dozsa@arm.com unsigned dist_size, 53711290Sgabor.dozsa@arm.com Tick sync_start, 53811290Sgabor.dozsa@arm.com Tick sync_repeat, 53911290Sgabor.dozsa@arm.com EventManager *em, 54011290Sgabor.dozsa@arm.com bool is_switch, int num_nodes) : 54110923SN/A syncStart(sync_start), syncRepeat(sync_repeat), 54211290Sgabor.dozsa@arm.com recvThread(nullptr), recvScheduler(em), 54311290Sgabor.dozsa@arm.com rank(dist_rank), size(dist_size) 54410923SN/A{ 54511290Sgabor.dozsa@arm.com DPRINTF(DistEthernet, "DistIface() ctor rank:%d\n",dist_rank); 54611290Sgabor.dozsa@arm.com isMaster = false; 54710923SN/A if (master == nullptr) { 54810923SN/A assert(sync == nullptr); 54910923SN/A assert(syncEvent == nullptr); 55011290Sgabor.dozsa@arm.com if (is_switch) 55111290Sgabor.dozsa@arm.com sync = new SyncSwitch(num_nodes); 55211290Sgabor.dozsa@arm.com else 55311290Sgabor.dozsa@arm.com sync = new SyncNode(); 55410923SN/A syncEvent = new SyncEvent(); 55510923SN/A master = this; 55611290Sgabor.dozsa@arm.com isMaster = true; 55710923SN/A } 55811290Sgabor.dozsa@arm.com distIfaceId = distIfaceNum; 55911290Sgabor.dozsa@arm.com distIfaceNum++; 56010923SN/A} 56110923SN/A 56211290Sgabor.dozsa@arm.comDistIface::~DistIface() 56310923SN/A{ 56410923SN/A assert(recvThread); 56510923SN/A delete recvThread; 56610923SN/A if (this == master) { 56710923SN/A assert(syncEvent); 56810923SN/A delete syncEvent; 56910923SN/A assert(sync); 57010923SN/A delete sync; 57111290Sgabor.dozsa@arm.com master = nullptr; 57210923SN/A } 57310923SN/A} 57410923SN/A 57510923SN/Avoid 57611290Sgabor.dozsa@arm.comDistIface::packetOut(EthPacketPtr pkt, Tick send_delay) 57710923SN/A{ 57811290Sgabor.dozsa@arm.com Header header; 57910923SN/A 58011290Sgabor.dozsa@arm.com // Prepare a dist header packet for the Ethernet packet we want to 58110923SN/A // send out. 58211290Sgabor.dozsa@arm.com header.msgType = MsgType::dataDescriptor; 58311290Sgabor.dozsa@arm.com header.sendTick = curTick(); 58411290Sgabor.dozsa@arm.com header.sendDelay = send_delay; 58510923SN/A 58611290Sgabor.dozsa@arm.com header.dataPacketLength = pkt->size(); 58710923SN/A 58811290Sgabor.dozsa@arm.com // Send out the packet and the meta info. 58911290Sgabor.dozsa@arm.com sendPacket(header, pkt); 59010923SN/A 59111290Sgabor.dozsa@arm.com DPRINTF(DistEthernetPkt, 59211290Sgabor.dozsa@arm.com "DistIface::sendDataPacket() done size:%d send_delay:%llu\n", 59311290Sgabor.dozsa@arm.com pkt->size(), send_delay); 59410923SN/A} 59510923SN/A 59610923SN/Avoid 59711290Sgabor.dozsa@arm.comDistIface::recvThreadFunc(Event *recv_done, Tick link_delay) 59810923SN/A{ 59910923SN/A EthPacketPtr new_packet; 60011290Sgabor.dozsa@arm.com DistHeaderPkt::Header header; 60110923SN/A 60211290Sgabor.dozsa@arm.com // Initialize receive scheduler parameters 60311290Sgabor.dozsa@arm.com recvScheduler.init(recv_done, link_delay); 60411290Sgabor.dozsa@arm.com 60510923SN/A // Main loop to wait for and process any incoming message. 60610923SN/A for (;;) { 60711290Sgabor.dozsa@arm.com // recvHeader() blocks until the next dist header packet comes in. 60810923SN/A if (!recvHeader(header)) { 60910923SN/A // We lost connection to the peer gem5 processes most likely 61010923SN/A // because one of them called m5 exit. So we stop here. 61111290Sgabor.dozsa@arm.com // Grab the eventq lock to stop the simulation thread 61211290Sgabor.dozsa@arm.com curEventQueue()->lock(); 61311290Sgabor.dozsa@arm.com exit_message("info", 61411290Sgabor.dozsa@arm.com 0, 61511290Sgabor.dozsa@arm.com "Message server closed connection, " 61610923SN/A "simulation is exiting"); 61710923SN/A } 61811290Sgabor.dozsa@arm.com 61911290Sgabor.dozsa@arm.com // We got a valid dist header packet, let's process it 62010923SN/A if (header.msgType == MsgType::dataDescriptor) { 62111290Sgabor.dozsa@arm.com recvPacket(header, new_packet); 62211290Sgabor.dozsa@arm.com recvScheduler.pushPacket(new_packet, 62311290Sgabor.dozsa@arm.com header.sendTick, 62411290Sgabor.dozsa@arm.com header.sendDelay); 62510923SN/A } else { 62610923SN/A // everything else must be synchronisation related command 62711290Sgabor.dozsa@arm.com sync->progress(header.sendTick, 62811290Sgabor.dozsa@arm.com header.syncRepeat, 62911290Sgabor.dozsa@arm.com header.needCkpt, 63011290Sgabor.dozsa@arm.com header.needExit); 63110923SN/A } 63210923SN/A } 63310923SN/A} 63410923SN/A 63510923SN/Avoid 63611290Sgabor.dozsa@arm.comDistIface::spawnRecvThread(const Event *recv_done, Tick link_delay) 63710923SN/A{ 63810923SN/A assert(recvThread == nullptr); 63910923SN/A 64011290Sgabor.dozsa@arm.com recvThread = new std::thread(&DistIface::recvThreadFunc, 64111290Sgabor.dozsa@arm.com this, 64211290Sgabor.dozsa@arm.com const_cast<Event *>(recv_done), 64311290Sgabor.dozsa@arm.com link_delay); 64410923SN/A recvThreadsNum++; 64510923SN/A} 64610923SN/A 64710923SN/ADrainState 64811290Sgabor.dozsa@arm.comDistIface::drain() 64910923SN/A{ 65011290Sgabor.dozsa@arm.com DPRINTF(DistEthernet,"DistIFace::drain() called\n"); 65110923SN/A // This can be called multiple times in the same drain cycle. 65211290Sgabor.dozsa@arm.com if (this == master) 65311290Sgabor.dozsa@arm.com syncEvent->draining(true); 65410923SN/A return DrainState::Drained; 65510923SN/A} 65610923SN/A 65711290Sgabor.dozsa@arm.comvoid 65811290Sgabor.dozsa@arm.comDistIface::drainResume() { 65911290Sgabor.dozsa@arm.com DPRINTF(DistEthernet,"DistIFace::drainResume() called\n"); 66011290Sgabor.dozsa@arm.com if (this == master) 66111290Sgabor.dozsa@arm.com syncEvent->draining(false); 66211290Sgabor.dozsa@arm.com recvScheduler.resumeRecvTicks(); 66311290Sgabor.dozsa@arm.com} 66411290Sgabor.dozsa@arm.com 66511290Sgabor.dozsa@arm.comvoid 66611290Sgabor.dozsa@arm.comDistIface::serialize(CheckpointOut &cp) const 66711290Sgabor.dozsa@arm.com{ 66811290Sgabor.dozsa@arm.com // Drain the dist interface before the checkpoint is taken. We cannot call 66911290Sgabor.dozsa@arm.com // this as part of the normal drain cycle because this dist sync has to be 67011290Sgabor.dozsa@arm.com // called exactly once after the system is fully drained. 67111290Sgabor.dozsa@arm.com sync->drainComplete(); 67211290Sgabor.dozsa@arm.com 67311290Sgabor.dozsa@arm.com unsigned rank_orig = rank, dist_iface_id_orig = distIfaceId; 67411290Sgabor.dozsa@arm.com 67511290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(rank_orig); 67611290Sgabor.dozsa@arm.com SERIALIZE_SCALAR(dist_iface_id_orig); 67711290Sgabor.dozsa@arm.com 67811290Sgabor.dozsa@arm.com recvScheduler.serializeSection(cp, "recvScheduler"); 67911290Sgabor.dozsa@arm.com if (this == master) { 68011290Sgabor.dozsa@arm.com sync->serializeSection(cp, "Sync"); 68110923SN/A } 68210923SN/A} 68310923SN/A 68411290Sgabor.dozsa@arm.comvoid 68511290Sgabor.dozsa@arm.comDistIface::unserialize(CheckpointIn &cp) 68610923SN/A{ 68711290Sgabor.dozsa@arm.com unsigned rank_orig, dist_iface_id_orig; 68811290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(rank_orig); 68911290Sgabor.dozsa@arm.com UNSERIALIZE_SCALAR(dist_iface_id_orig); 69010923SN/A 69111290Sgabor.dozsa@arm.com panic_if(rank != rank_orig, "Rank mismatch at resume (rank=%d, orig=%d)", 69211290Sgabor.dozsa@arm.com rank, rank_orig); 69311290Sgabor.dozsa@arm.com panic_if(distIfaceId != dist_iface_id_orig, "Dist iface ID mismatch " 69411290Sgabor.dozsa@arm.com "at resume (distIfaceId=%d, orig=%d)", distIfaceId, 69511290Sgabor.dozsa@arm.com dist_iface_id_orig); 69610923SN/A 69711290Sgabor.dozsa@arm.com recvScheduler.unserializeSection(cp, "recvScheduler"); 69811290Sgabor.dozsa@arm.com if (this == master) { 69911290Sgabor.dozsa@arm.com sync->unserializeSection(cp, "Sync"); 70010923SN/A } 70110923SN/A} 70210923SN/A 70311290Sgabor.dozsa@arm.comvoid 70411290Sgabor.dozsa@arm.comDistIface::init(const Event *done_event, Tick link_delay) 70510923SN/A{ 70611290Sgabor.dozsa@arm.com // Init hook for the underlaying message transport to setup/finalize 70711290Sgabor.dozsa@arm.com // communication channels 70811290Sgabor.dozsa@arm.com initTransport(); 70910923SN/A 71011290Sgabor.dozsa@arm.com // Spawn a new receiver thread that will process messages 71111290Sgabor.dozsa@arm.com // coming in from peer gem5 processes. 71211290Sgabor.dozsa@arm.com // The receive thread will also schedule a (receive) doneEvent 71311290Sgabor.dozsa@arm.com // for each incoming data packet. 71411290Sgabor.dozsa@arm.com spawnRecvThread(done_event, link_delay); 71510923SN/A 71610923SN/A 71711290Sgabor.dozsa@arm.com // Adjust the periodic sync start and interval. Different DistIface 71811290Sgabor.dozsa@arm.com // might have different requirements. The singleton sync object 71911290Sgabor.dozsa@arm.com // will select the minimum values for both params. 72011290Sgabor.dozsa@arm.com assert(sync != nullptr); 72111290Sgabor.dozsa@arm.com sync->init(syncStart, syncRepeat); 72210923SN/A 72310923SN/A // Initialize the seed for random generator to avoid the same sequence 72410923SN/A // in all gem5 peer processes 72510923SN/A assert(master != nullptr); 72610923SN/A if (this == master) 72710923SN/A random_mt.init(5489 * (rank+1) + 257); 72810923SN/A} 72910923SN/A 73011290Sgabor.dozsa@arm.comvoid 73111290Sgabor.dozsa@arm.comDistIface::startup() 73210923SN/A{ 73311290Sgabor.dozsa@arm.com DPRINTF(DistEthernet, "DistIface::startup() started\n"); 73411290Sgabor.dozsa@arm.com if (this == master) 73511290Sgabor.dozsa@arm.com syncEvent->start(); 73611290Sgabor.dozsa@arm.com DPRINTF(DistEthernet, "DistIface::startup() done\n"); 73711290Sgabor.dozsa@arm.com} 73810923SN/A 73911290Sgabor.dozsa@arm.combool 74011290Sgabor.dozsa@arm.comDistIface::readyToCkpt(Tick delay, Tick period) 74111290Sgabor.dozsa@arm.com{ 74211290Sgabor.dozsa@arm.com bool ret = true; 74311290Sgabor.dozsa@arm.com DPRINTF(DistEthernet, "DistIface::readyToCkpt() called, delay:%lu " 74411290Sgabor.dozsa@arm.com "period:%lu\n", delay, period); 74511290Sgabor.dozsa@arm.com if (master) { 74611290Sgabor.dozsa@arm.com if (delay == 0) { 74711290Sgabor.dozsa@arm.com inform("m5 checkpoint called with zero delay => triggering collaborative " 74811290Sgabor.dozsa@arm.com "checkpoint\n"); 74911290Sgabor.dozsa@arm.com sync->requestCkpt(ReqType::collective); 75010923SN/A } else { 75111290Sgabor.dozsa@arm.com inform("m5 checkpoint called with non-zero delay => triggering immediate " 75211290Sgabor.dozsa@arm.com "checkpoint (at the next sync)\n"); 75311290Sgabor.dozsa@arm.com sync->requestCkpt(ReqType::immediate); 75410923SN/A } 75511290Sgabor.dozsa@arm.com if (period != 0) 75611290Sgabor.dozsa@arm.com inform("Non-zero period for m5_ckpt is ignored in " 75711290Sgabor.dozsa@arm.com "distributed gem5 runs\n"); 75811290Sgabor.dozsa@arm.com ret = false; 75911290Sgabor.dozsa@arm.com } 76011290Sgabor.dozsa@arm.com return ret; 76111290Sgabor.dozsa@arm.com} 76211290Sgabor.dozsa@arm.com 76311290Sgabor.dozsa@arm.combool 76411290Sgabor.dozsa@arm.comDistIface::readyToExit(Tick delay) 76511290Sgabor.dozsa@arm.com{ 76611290Sgabor.dozsa@arm.com bool ret = true; 76711290Sgabor.dozsa@arm.com DPRINTF(DistEthernet, "DistIface::readyToExit() called, delay:%lu\n", 76811290Sgabor.dozsa@arm.com delay); 76911290Sgabor.dozsa@arm.com if (master) { 77011290Sgabor.dozsa@arm.com if (delay == 0) { 77111290Sgabor.dozsa@arm.com inform("m5 exit called with zero delay => triggering collaborative " 77211290Sgabor.dozsa@arm.com "exit\n"); 77311290Sgabor.dozsa@arm.com sync->requestExit(ReqType::collective); 77411290Sgabor.dozsa@arm.com } else { 77511290Sgabor.dozsa@arm.com inform("m5 exit called with non-zero delay => triggering immediate " 77611290Sgabor.dozsa@arm.com "exit (at the next sync)\n"); 77711290Sgabor.dozsa@arm.com sync->requestExit(ReqType::immediate); 77811290Sgabor.dozsa@arm.com } 77911290Sgabor.dozsa@arm.com ret = false; 78011290Sgabor.dozsa@arm.com } 78111290Sgabor.dozsa@arm.com return ret; 78211290Sgabor.dozsa@arm.com} 78311290Sgabor.dozsa@arm.com 78411290Sgabor.dozsa@arm.comuint64_t 78511290Sgabor.dozsa@arm.comDistIface::rankParam() 78611290Sgabor.dozsa@arm.com{ 78711290Sgabor.dozsa@arm.com uint64_t val; 78811290Sgabor.dozsa@arm.com if (master) { 78911290Sgabor.dozsa@arm.com val = master->rank; 79010923SN/A } else { 79111290Sgabor.dozsa@arm.com warn("Dist-rank parameter is queried in single gem5 simulation."); 79211290Sgabor.dozsa@arm.com val = 0; 79310923SN/A } 79411290Sgabor.dozsa@arm.com return val; 79510923SN/A} 79611290Sgabor.dozsa@arm.com 79711290Sgabor.dozsa@arm.comuint64_t 79811290Sgabor.dozsa@arm.comDistIface::sizeParam() 79911290Sgabor.dozsa@arm.com{ 80011290Sgabor.dozsa@arm.com uint64_t val; 80111290Sgabor.dozsa@arm.com if (master) { 80211290Sgabor.dozsa@arm.com val = master->size; 80311290Sgabor.dozsa@arm.com } else { 80411290Sgabor.dozsa@arm.com warn("Dist-size parameter is queried in single gem5 simulation."); 80511290Sgabor.dozsa@arm.com val = 1; 80611290Sgabor.dozsa@arm.com } 80711290Sgabor.dozsa@arm.com return val; 80811290Sgabor.dozsa@arm.com} 809