dist_iface.hh revision 11757
110923SN/A/* 211757Sgabor.dozsa@arm.com * Copyright (c) 2015-2016 ARM Limited 310923SN/A * All rights reserved 410923SN/A * 510923SN/A * The license below extends only to copyright in the software and shall 610923SN/A * not be construed as granting a license to any other intellectual 710923SN/A * property including but not limited to intellectual property relating 810923SN/A * to a hardware implementation of the functionality of the software 910923SN/A * licensed hereunder. You may use the software subject to the license 1010923SN/A * terms below provided that you ensure that this notice is replicated 1110923SN/A * unmodified and in its entirety in all distributions of the software, 1210923SN/A * modified or unmodified, in source code or in binary form. 1310923SN/A * 1410923SN/A * Redistribution and use in source and binary forms, with or without 1510923SN/A * modification, are permitted provided that the following conditions are 1610923SN/A * met: redistributions of source code must retain the above copyright 1710923SN/A * notice, this list of conditions and the following disclaimer; 1810923SN/A * redistributions in binary form must reproduce the above copyright 1910923SN/A * notice, this list of conditions and the following disclaimer in the 2010923SN/A * documentation and/or other materials provided with the distribution; 2110923SN/A * neither the name of the copyright holders nor the names of its 2210923SN/A * contributors may be used to endorse or promote products derived from 2310923SN/A * this software without specific prior written permission. 2410923SN/A * 2510923SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2610923SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2710923SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2810923SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2910923SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3010923SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3110923SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3210923SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3310923SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3410923SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3510923SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3610923SN/A * 3710923SN/A * Authors: Gabor Dozsa 3810923SN/A */ 3910923SN/A 4010923SN/A/* @file 4111290Sgabor.dozsa@arm.com * The interface class for dist gem5 simulations. 4210923SN/A * 4311290Sgabor.dozsa@arm.com * dist-gem5 is an extension to gem5 to enable parallel simulation of a 4410923SN/A * distributed system (e.g. simulation of a pool of machines 4511290Sgabor.dozsa@arm.com * connected by Ethernet links). A dist gem5 run consists of seperate gem5 4610923SN/A * processes running in parallel. Each gem5 process executes 4710923SN/A * the simulation of a component of the simulated distributed system. 4811290Sgabor.dozsa@arm.com * (An example component can be a dist-core board with an Ethernet NIC.) 4911290Sgabor.dozsa@arm.com * The DistIface class below provides services to transfer data and 5010923SN/A * control messages among the gem5 processes. The main such services are 5110923SN/A * as follows. 5210923SN/A * 5310923SN/A * 1. Send a data packet coming from a simulated Ethernet link. The packet 5410923SN/A * will be transferred to (all) the target(s) gem5 processes. The send 5510923SN/A * operation is always performed by the simulation thread, i.e. the gem5 5610923SN/A * thread that is processing the event queue associated with the simulated 5710923SN/A * Ethernet link. 5810923SN/A * 5910923SN/A * 2. Spawn a receiver thread to process messages coming in from the 6010923SN/A * from other gem5 processes. Each simulated Ethernet link has its own 6110923SN/A * associated receiver thread. The receiver thread saves the incoming packet 6210923SN/A * and schedule an appropriate receive event in the event queue. 6310923SN/A * 6410923SN/A * 3. Schedule a global barrier event periodically to keep the gem5 6510923SN/A * processes in sync. 6610923SN/A * Periodic barrier event to keep peer gem5 processes in sync. The basic idea 6710923SN/A * is that no gem5 process can go ahead further than the simulated link 6810923SN/A * transmission delay to ensure that a corresponding receive event can always 6910923SN/A * be scheduled for any message coming in from a peer gem5 process. 7010923SN/A * 7110923SN/A * 7210923SN/A * 7311290Sgabor.dozsa@arm.com * This interface is an abstract class. It can work with various low level 7410923SN/A * send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP 7511290Sgabor.dozsa@arm.com * stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc]. 7610923SN/A */ 7711290Sgabor.dozsa@arm.com#ifndef __DEV_DIST_IFACE_HH__ 7811290Sgabor.dozsa@arm.com#define __DEV_DIST_IFACE_HH__ 7910923SN/A 8010923SN/A#include <array> 8110923SN/A#include <mutex> 8210923SN/A#include <queue> 8310923SN/A#include <thread> 8410923SN/A#include <utility> 8510923SN/A 8611290Sgabor.dozsa@arm.com#include "dev/net/dist_packet.hh" 8711263SN/A#include "dev/net/etherpkt.hh" 8810923SN/A#include "sim/core.hh" 8910923SN/A#include "sim/drain.hh" 9010923SN/A#include "sim/global_event.hh" 9111290Sgabor.dozsa@arm.com#include "sim/serialize.hh" 9210923SN/A 9310923SN/Aclass EventManager; 9411703Smichael.lebeane@amd.comclass System; 9511703Smichael.lebeane@amd.comclass ThreadContext; 9610923SN/A 9710923SN/A/** 9810923SN/A * The interface class to talk to peer gem5 processes. 9910923SN/A */ 10011290Sgabor.dozsa@arm.comclass DistIface : public Drainable, public Serializable 10110923SN/A{ 10210923SN/A public: 10311290Sgabor.dozsa@arm.com typedef DistHeaderPkt::Header Header; 10411290Sgabor.dozsa@arm.com 10511290Sgabor.dozsa@arm.com protected: 10611290Sgabor.dozsa@arm.com typedef DistHeaderPkt::MsgType MsgType; 10711290Sgabor.dozsa@arm.com typedef DistHeaderPkt::ReqType ReqType; 10810923SN/A 10910923SN/A private: 11011290Sgabor.dozsa@arm.com class SyncEvent; 11110923SN/A /** @class Sync 11210923SN/A * This class implements global sync operations among gem5 peer processes. 11310923SN/A * 11411290Sgabor.dozsa@arm.com * @note This class is used as a singleton object (shared by all DistIface 11510923SN/A * objects). 11610923SN/A */ 11711290Sgabor.dozsa@arm.com class Sync : public Serializable 11810923SN/A { 11911290Sgabor.dozsa@arm.com protected: 12010923SN/A /** 12111290Sgabor.dozsa@arm.com * The lock to protect access to the Sync object. 12210923SN/A */ 12310923SN/A std::mutex lock; 12410923SN/A /** 12510923SN/A * Condition variable for the simulation thread to wait on 12610923SN/A * until all receiver threads completes the current global 12710923SN/A * synchronisation. 12810923SN/A */ 12910923SN/A std::condition_variable cv; 13010923SN/A /** 13110923SN/A * Number of receiver threads that not yet completed the current global 13210923SN/A * synchronisation. 13310923SN/A */ 13410923SN/A unsigned waitNum; 13510923SN/A /** 13611290Sgabor.dozsa@arm.com * Flag is set if exit is permitted upon sync completion 13710923SN/A */ 13811290Sgabor.dozsa@arm.com bool doExit; 13910923SN/A /** 14011290Sgabor.dozsa@arm.com * Flag is set if taking a ckpt is permitted upon sync completion 14110923SN/A */ 14211290Sgabor.dozsa@arm.com bool doCkpt; 14311290Sgabor.dozsa@arm.com /** 14411703Smichael.lebeane@amd.com * Flag is set if sync is to stop upon sync completion 14511703Smichael.lebeane@amd.com */ 14611703Smichael.lebeane@amd.com bool doStopSync; 14711703Smichael.lebeane@amd.com /** 14811290Sgabor.dozsa@arm.com * The repeat value for the next periodic sync 14911290Sgabor.dozsa@arm.com */ 15011290Sgabor.dozsa@arm.com Tick nextRepeat; 15111290Sgabor.dozsa@arm.com /** 15211290Sgabor.dozsa@arm.com * Tick for the next periodic sync (if the event is not scheduled yet) 15311290Sgabor.dozsa@arm.com */ 15411290Sgabor.dozsa@arm.com Tick nextAt; 15511757Sgabor.dozsa@arm.com /** 15611757Sgabor.dozsa@arm.com * Flag is set if the sync is aborted (e.g. due to connection lost) 15711757Sgabor.dozsa@arm.com */ 15811757Sgabor.dozsa@arm.com bool isAbort; 15910923SN/A 16011290Sgabor.dozsa@arm.com friend class SyncEvent; 16110923SN/A 16210923SN/A public: 16310923SN/A /** 16411290Sgabor.dozsa@arm.com * Initialize periodic sync params. 16510923SN/A * 16611290Sgabor.dozsa@arm.com * @param start Start tick for dist synchronisation 16711290Sgabor.dozsa@arm.com * @param repeat Frequency of dist synchronisation 16810923SN/A * 16910923SN/A */ 17011290Sgabor.dozsa@arm.com void init(Tick start, Tick repeat); 17110923SN/A /** 17211290Sgabor.dozsa@arm.com * Core method to perform a full dist sync. 17311757Sgabor.dozsa@arm.com * 17411757Sgabor.dozsa@arm.com * @return true if the sync completes, false if it gets aborted 17510923SN/A */ 17611757Sgabor.dozsa@arm.com virtual bool run(bool same_tick) = 0; 17711290Sgabor.dozsa@arm.com /** 17811290Sgabor.dozsa@arm.com * Callback when the receiver thread gets a sync ack message. 17911757Sgabor.dozsa@arm.com * 18011757Sgabor.dozsa@arm.com * @return false if the receiver thread needs to stop (e.g. 18111757Sgabor.dozsa@arm.com * simulation is to exit) 18211290Sgabor.dozsa@arm.com */ 18311757Sgabor.dozsa@arm.com virtual bool progress(Tick send_tick, 18411290Sgabor.dozsa@arm.com Tick next_repeat, 18511290Sgabor.dozsa@arm.com ReqType do_ckpt, 18611703Smichael.lebeane@amd.com ReqType do_exit, 18711703Smichael.lebeane@amd.com ReqType do_stop_sync) = 0; 18811757Sgabor.dozsa@arm.com /** 18911757Sgabor.dozsa@arm.com * Abort processing an on-going sync event (in case of an error, e.g. 19011757Sgabor.dozsa@arm.com * lost connection to a peer gem5) 19111757Sgabor.dozsa@arm.com */ 19211757Sgabor.dozsa@arm.com void abort(); 19310923SN/A 19411290Sgabor.dozsa@arm.com virtual void requestCkpt(ReqType req) = 0; 19511290Sgabor.dozsa@arm.com virtual void requestExit(ReqType req) = 0; 19611703Smichael.lebeane@amd.com virtual void requestStopSync(ReqType req) = 0; 19711290Sgabor.dozsa@arm.com 19811290Sgabor.dozsa@arm.com void drainComplete(); 19911290Sgabor.dozsa@arm.com 20011290Sgabor.dozsa@arm.com virtual void serialize(CheckpointOut &cp) const override = 0; 20111290Sgabor.dozsa@arm.com virtual void unserialize(CheckpointIn &cp) override = 0; 20210923SN/A }; 20310923SN/A 20411290Sgabor.dozsa@arm.com class SyncNode: public Sync 20511290Sgabor.dozsa@arm.com { 20611290Sgabor.dozsa@arm.com private: 20711290Sgabor.dozsa@arm.com /** 20811290Sgabor.dozsa@arm.com * Exit requested 20911290Sgabor.dozsa@arm.com */ 21011290Sgabor.dozsa@arm.com ReqType needExit; 21111290Sgabor.dozsa@arm.com /** 21211290Sgabor.dozsa@arm.com * Ckpt requested 21311290Sgabor.dozsa@arm.com */ 21411290Sgabor.dozsa@arm.com ReqType needCkpt; 21511703Smichael.lebeane@amd.com /** 21611703Smichael.lebeane@amd.com * Sync stop requested 21711703Smichael.lebeane@amd.com */ 21811703Smichael.lebeane@amd.com ReqType needStopSync; 21911290Sgabor.dozsa@arm.com 22011290Sgabor.dozsa@arm.com public: 22111290Sgabor.dozsa@arm.com 22211290Sgabor.dozsa@arm.com SyncNode(); 22311290Sgabor.dozsa@arm.com ~SyncNode() {} 22411757Sgabor.dozsa@arm.com bool run(bool same_tick) override; 22511757Sgabor.dozsa@arm.com bool progress(Tick max_req_tick, 22611290Sgabor.dozsa@arm.com Tick next_repeat, 22711290Sgabor.dozsa@arm.com ReqType do_ckpt, 22811703Smichael.lebeane@amd.com ReqType do_exit, 22911703Smichael.lebeane@amd.com ReqType do_stop_sync) override; 23011290Sgabor.dozsa@arm.com 23111290Sgabor.dozsa@arm.com void requestCkpt(ReqType req) override; 23211290Sgabor.dozsa@arm.com void requestExit(ReqType req) override; 23311703Smichael.lebeane@amd.com void requestStopSync(ReqType req) override; 23411290Sgabor.dozsa@arm.com 23511290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 23611290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 23711290Sgabor.dozsa@arm.com }; 23811290Sgabor.dozsa@arm.com 23911290Sgabor.dozsa@arm.com class SyncSwitch: public Sync 24011290Sgabor.dozsa@arm.com { 24111290Sgabor.dozsa@arm.com private: 24211290Sgabor.dozsa@arm.com /** 24311290Sgabor.dozsa@arm.com * Counter for recording exit requests 24411290Sgabor.dozsa@arm.com */ 24511290Sgabor.dozsa@arm.com unsigned numExitReq; 24611290Sgabor.dozsa@arm.com /** 24711290Sgabor.dozsa@arm.com * Counter for recording ckpt requests 24811290Sgabor.dozsa@arm.com */ 24911290Sgabor.dozsa@arm.com unsigned numCkptReq; 25011290Sgabor.dozsa@arm.com /** 25111703Smichael.lebeane@amd.com * Counter for recording stop sync requests 25211703Smichael.lebeane@amd.com */ 25311703Smichael.lebeane@amd.com unsigned numStopSyncReq; 25411703Smichael.lebeane@amd.com /** 25511290Sgabor.dozsa@arm.com * Number of connected simulated nodes 25611290Sgabor.dozsa@arm.com */ 25711290Sgabor.dozsa@arm.com unsigned numNodes; 25811290Sgabor.dozsa@arm.com 25911290Sgabor.dozsa@arm.com public: 26011290Sgabor.dozsa@arm.com SyncSwitch(int num_nodes); 26111290Sgabor.dozsa@arm.com ~SyncSwitch() {} 26211290Sgabor.dozsa@arm.com 26311757Sgabor.dozsa@arm.com bool run(bool same_tick) override; 26411757Sgabor.dozsa@arm.com bool progress(Tick max_req_tick, 26511290Sgabor.dozsa@arm.com Tick next_repeat, 26611290Sgabor.dozsa@arm.com ReqType do_ckpt, 26711703Smichael.lebeane@amd.com ReqType do_exit, 26811703Smichael.lebeane@amd.com ReqType do_stop_sync) override; 26911290Sgabor.dozsa@arm.com 27011290Sgabor.dozsa@arm.com void requestCkpt(ReqType) override { 27111290Sgabor.dozsa@arm.com panic("Switch requested checkpoint"); 27211290Sgabor.dozsa@arm.com } 27311290Sgabor.dozsa@arm.com void requestExit(ReqType) override { 27411290Sgabor.dozsa@arm.com panic("Switch requested exit"); 27511290Sgabor.dozsa@arm.com } 27611703Smichael.lebeane@amd.com void requestStopSync(ReqType) override { 27711703Smichael.lebeane@amd.com panic("Switch requested stop sync"); 27811703Smichael.lebeane@amd.com } 27911290Sgabor.dozsa@arm.com 28011290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 28111290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 28211290Sgabor.dozsa@arm.com }; 28310923SN/A 28410923SN/A /** 28511290Sgabor.dozsa@arm.com * The global event to schedule periodic dist sync. It is used as a 28610923SN/A * singleton object. 28710923SN/A * 28810923SN/A * The periodic synchronisation works as follows. 28911290Sgabor.dozsa@arm.com * 1. A SyncEvent is scheduled as a global event when startup() is 29010923SN/A * called. 29111290Sgabor.dozsa@arm.com * 2. The process() method of the SyncEvent initiates a new barrier 29211290Sgabor.dozsa@arm.com * for each simulated Ethernet link. 29310923SN/A * 3. Simulation thread(s) then waits until all receiver threads 29411290Sgabor.dozsa@arm.com * complete the ongoing barrier. The global sync event is done. 29510923SN/A */ 29610923SN/A class SyncEvent : public GlobalSyncEvent 29710923SN/A { 29811290Sgabor.dozsa@arm.com private: 29911290Sgabor.dozsa@arm.com /** 30011290Sgabor.dozsa@arm.com * Flag to set when the system is draining 30111290Sgabor.dozsa@arm.com */ 30211290Sgabor.dozsa@arm.com bool _draining; 30310923SN/A public: 30410923SN/A /** 30511290Sgabor.dozsa@arm.com * Only the firstly instantiated DistIface object will 30611290Sgabor.dozsa@arm.com * call this constructor. 30710923SN/A */ 30811290Sgabor.dozsa@arm.com SyncEvent() : GlobalSyncEvent(Sim_Exit_Pri, 0), _draining(false) {} 30911290Sgabor.dozsa@arm.com 31011290Sgabor.dozsa@arm.com ~SyncEvent() {} 31110923SN/A /** 31211290Sgabor.dozsa@arm.com * Schedule the first periodic sync event. 31310923SN/A */ 31411290Sgabor.dozsa@arm.com void start(); 31510923SN/A /** 31611290Sgabor.dozsa@arm.com * This is a global event so process() will only be called by 31711290Sgabor.dozsa@arm.com * exactly one simulation thread. (See further comments in the .cc 31811290Sgabor.dozsa@arm.com * file.) 31910923SN/A */ 32011290Sgabor.dozsa@arm.com void process() override; 32111290Sgabor.dozsa@arm.com 32211290Sgabor.dozsa@arm.com bool draining() const { return _draining; } 32311290Sgabor.dozsa@arm.com void draining(bool fl) { _draining = fl; } 32411290Sgabor.dozsa@arm.com }; 32511290Sgabor.dozsa@arm.com /** 32611290Sgabor.dozsa@arm.com * Class to encapsulate information about data packets received. 32711290Sgabor.dozsa@arm.com 32811290Sgabor.dozsa@arm.com * @note The main purpose of the class to take care of scheduling receive 32911290Sgabor.dozsa@arm.com * done events for the simulated network link and store incoming packets 33011290Sgabor.dozsa@arm.com * until they can be received by the simulated network link. 33111290Sgabor.dozsa@arm.com */ 33211290Sgabor.dozsa@arm.com class RecvScheduler : public Serializable 33311290Sgabor.dozsa@arm.com { 33411290Sgabor.dozsa@arm.com private: 33511290Sgabor.dozsa@arm.com /** 33611290Sgabor.dozsa@arm.com * Received packet descriptor. This information is used by the receive 33711290Sgabor.dozsa@arm.com * thread to schedule receive events and by the simulation thread to 33811290Sgabor.dozsa@arm.com * process those events. 33911290Sgabor.dozsa@arm.com */ 34011290Sgabor.dozsa@arm.com struct Desc : public Serializable 34111290Sgabor.dozsa@arm.com { 34211290Sgabor.dozsa@arm.com EthPacketPtr packet; 34311290Sgabor.dozsa@arm.com Tick sendTick; 34411290Sgabor.dozsa@arm.com Tick sendDelay; 34511290Sgabor.dozsa@arm.com 34611290Sgabor.dozsa@arm.com Desc() : sendTick(0), sendDelay(0) {} 34711290Sgabor.dozsa@arm.com Desc(EthPacketPtr p, Tick s, Tick d) : 34811290Sgabor.dozsa@arm.com packet(p), sendTick(s), sendDelay(d) {} 34911290Sgabor.dozsa@arm.com Desc(const Desc &d) : 35011290Sgabor.dozsa@arm.com packet(d.packet), sendTick(d.sendTick), sendDelay(d.sendDelay) {} 35111290Sgabor.dozsa@arm.com 35211290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 35311290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 35411290Sgabor.dozsa@arm.com }; 35511290Sgabor.dozsa@arm.com /** 35611290Sgabor.dozsa@arm.com * The queue to store the receive descriptors. 35711290Sgabor.dozsa@arm.com */ 35811290Sgabor.dozsa@arm.com std::queue<Desc> descQueue; 35911290Sgabor.dozsa@arm.com /** 36011290Sgabor.dozsa@arm.com * The tick when the most recent receive event was processed. 36111290Sgabor.dozsa@arm.com * 36211290Sgabor.dozsa@arm.com * @note This information is necessary to simulate possible receiver 36311290Sgabor.dozsa@arm.com * link contention when calculating the receive tick for the next 36411290Sgabor.dozsa@arm.com * incoming data packet (see the calcReceiveTick() method) 36511290Sgabor.dozsa@arm.com */ 36611290Sgabor.dozsa@arm.com Tick prevRecvTick; 36711290Sgabor.dozsa@arm.com /** 36811290Sgabor.dozsa@arm.com * The receive done event for the simulated Ethernet link. 36911290Sgabor.dozsa@arm.com * 37011290Sgabor.dozsa@arm.com * @note This object is constructed by the simulated network link. We 37111290Sgabor.dozsa@arm.com * schedule this object for each incoming data packet. 37211290Sgabor.dozsa@arm.com */ 37311290Sgabor.dozsa@arm.com Event *recvDone; 37411290Sgabor.dozsa@arm.com /** 37511290Sgabor.dozsa@arm.com * The link delay in ticks for the simulated Ethernet link. 37611290Sgabor.dozsa@arm.com * 37711290Sgabor.dozsa@arm.com * @note This value is used for calculating the receive ticks for 37811290Sgabor.dozsa@arm.com * incoming data packets. 37911290Sgabor.dozsa@arm.com */ 38011290Sgabor.dozsa@arm.com Tick linkDelay; 38111290Sgabor.dozsa@arm.com /** 38211290Sgabor.dozsa@arm.com * The event manager associated with the simulated Ethernet link. 38311290Sgabor.dozsa@arm.com * 38411290Sgabor.dozsa@arm.com * @note It is used to access the event queue for scheduling receive 38511290Sgabor.dozsa@arm.com * done events for the link. 38611290Sgabor.dozsa@arm.com */ 38711290Sgabor.dozsa@arm.com EventManager *eventManager; 38811290Sgabor.dozsa@arm.com /** 38911290Sgabor.dozsa@arm.com * Calculate the tick to schedule the next receive done event. 39011290Sgabor.dozsa@arm.com * 39111290Sgabor.dozsa@arm.com * @param send_tick The tick the packet was sent. 39211290Sgabor.dozsa@arm.com * @param send_delay The simulated delay at the sender side. 39311290Sgabor.dozsa@arm.com * @param prev_recv_tick Tick when the last receive event was 39411290Sgabor.dozsa@arm.com * processed. 39511290Sgabor.dozsa@arm.com * 39611290Sgabor.dozsa@arm.com * @note This method tries to take into account possible receiver link 39711290Sgabor.dozsa@arm.com * contention and adjust receive tick for the incoming packets 39811290Sgabor.dozsa@arm.com * accordingly. 39911290Sgabor.dozsa@arm.com */ 40011290Sgabor.dozsa@arm.com Tick calcReceiveTick(Tick send_tick, 40111290Sgabor.dozsa@arm.com Tick send_delay, 40211290Sgabor.dozsa@arm.com Tick prev_recv_tick); 40311290Sgabor.dozsa@arm.com 40411290Sgabor.dozsa@arm.com /** 40511290Sgabor.dozsa@arm.com * Flag to set if receive ticks for pending packets need to be 40611290Sgabor.dozsa@arm.com * recalculated due to changed link latencies at a resume 40711290Sgabor.dozsa@arm.com */ 40811290Sgabor.dozsa@arm.com bool ckptRestore; 40910923SN/A 41010923SN/A public: 41110923SN/A /** 41211290Sgabor.dozsa@arm.com * Scheduler for the incoming data packets. 41311290Sgabor.dozsa@arm.com * 41411290Sgabor.dozsa@arm.com * @param em The event manager associated with the simulated Ethernet 41511290Sgabor.dozsa@arm.com * link. 41610923SN/A */ 41711290Sgabor.dozsa@arm.com RecvScheduler(EventManager *em) : 41811290Sgabor.dozsa@arm.com prevRecvTick(0), recvDone(nullptr), linkDelay(0), 41911290Sgabor.dozsa@arm.com eventManager(em), ckptRestore(false) {} 42010923SN/A 42110923SN/A /** 42211290Sgabor.dozsa@arm.com * Initialize network link parameters. 42310923SN/A * 42411290Sgabor.dozsa@arm.com * @note This method is called from the receiver thread (see 42511290Sgabor.dozsa@arm.com * recvThreadFunc()). 42611290Sgabor.dozsa@arm.com */ 42711290Sgabor.dozsa@arm.com void init(Event *recv_done, Tick link_delay); 42811290Sgabor.dozsa@arm.com /** 42911290Sgabor.dozsa@arm.com * Fetch the next packet that is to be received by the simulated network 43011290Sgabor.dozsa@arm.com * link. 43110923SN/A * 43211290Sgabor.dozsa@arm.com * @note This method is called from the process() method of the receive 43311290Sgabor.dozsa@arm.com * done event associated with the network link. 43410923SN/A */ 43511290Sgabor.dozsa@arm.com EthPacketPtr popPacket(); 43610923SN/A /** 43711290Sgabor.dozsa@arm.com * Push a newly arrived packet into the desc queue. 43811290Sgabor.dozsa@arm.com */ 43911290Sgabor.dozsa@arm.com void pushPacket(EthPacketPtr new_packet, 44011290Sgabor.dozsa@arm.com Tick send_tick, 44111290Sgabor.dozsa@arm.com Tick send_delay); 44211290Sgabor.dozsa@arm.com 44311290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 44411290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 44511290Sgabor.dozsa@arm.com /** 44611290Sgabor.dozsa@arm.com * Adjust receive ticks for pending packets when restoring from a 44711290Sgabor.dozsa@arm.com * checkpoint 44810923SN/A * 44911290Sgabor.dozsa@arm.com * @note Link speed and delay parameters may change at resume. 45010923SN/A */ 45111290Sgabor.dozsa@arm.com void resumeRecvTicks(); 45210923SN/A }; 45310923SN/A /** 45411290Sgabor.dozsa@arm.com * Tick to schedule the first dist sync event. 45511290Sgabor.dozsa@arm.com * This is just as optimization : we do not need any dist sync 45610923SN/A * event until the simulated NIC is brought up by the OS. 45710923SN/A */ 45810923SN/A Tick syncStart; 45910923SN/A /** 46011290Sgabor.dozsa@arm.com * Frequency of dist sync events in ticks. 46110923SN/A */ 46210923SN/A Tick syncRepeat; 46310923SN/A /** 46410923SN/A * Receiver thread pointer. 46511290Sgabor.dozsa@arm.com * Each DistIface object must have exactly one receiver thread. 46610923SN/A */ 46710923SN/A std::thread *recvThread; 46810923SN/A /** 46911290Sgabor.dozsa@arm.com * Meta information about data packets received. 47010923SN/A */ 47111290Sgabor.dozsa@arm.com RecvScheduler recvScheduler; 47211703Smichael.lebeane@amd.com /** 47311703Smichael.lebeane@amd.com * Use pseudoOp to start synchronization. 47411703Smichael.lebeane@amd.com */ 47511703Smichael.lebeane@amd.com bool syncStartOnPseudoOp; 47610923SN/A 47711290Sgabor.dozsa@arm.com protected: 47810923SN/A /** 47910923SN/A * The rank of this process among the gem5 peers. 48010923SN/A */ 48110923SN/A unsigned rank; 48210923SN/A /** 48311290Sgabor.dozsa@arm.com * The number of gem5 processes comprising this dist simulation. 48411290Sgabor.dozsa@arm.com */ 48511290Sgabor.dozsa@arm.com unsigned size; 48611290Sgabor.dozsa@arm.com /** 48711290Sgabor.dozsa@arm.com * Number of DistIface objects (i.e. dist links in this gem5 process) 48811290Sgabor.dozsa@arm.com */ 48911290Sgabor.dozsa@arm.com static unsigned distIfaceNum; 49011290Sgabor.dozsa@arm.com /** 49111290Sgabor.dozsa@arm.com * Unique id for the dist link 49211290Sgabor.dozsa@arm.com */ 49311290Sgabor.dozsa@arm.com unsigned distIfaceId; 49411290Sgabor.dozsa@arm.com 49511290Sgabor.dozsa@arm.com bool isMaster; 49611290Sgabor.dozsa@arm.com 49711290Sgabor.dozsa@arm.com private: 49811290Sgabor.dozsa@arm.com /** 49911290Sgabor.dozsa@arm.com * Number of receiver threads (in this gem5 process) 50010923SN/A */ 50110923SN/A static unsigned recvThreadsNum; 50210923SN/A /** 50311290Sgabor.dozsa@arm.com * The singleton Sync object to perform dist synchronisation. 50411290Sgabor.dozsa@arm.com */ 50511290Sgabor.dozsa@arm.com static Sync *sync; 50611290Sgabor.dozsa@arm.com /** 50711290Sgabor.dozsa@arm.com * The singleton SyncEvent object to schedule periodic dist sync. 50811290Sgabor.dozsa@arm.com */ 50911290Sgabor.dozsa@arm.com static SyncEvent *syncEvent; 51011290Sgabor.dozsa@arm.com /** 51111290Sgabor.dozsa@arm.com * The very first DistIface object created becomes the master. We need 51210923SN/A * a master to co-ordinate the global synchronisation. 51310923SN/A */ 51411290Sgabor.dozsa@arm.com static DistIface *master; 51511703Smichael.lebeane@amd.com /** 51611703Smichael.lebeane@amd.com * System pointer used to wakeup sleeping threads when stopping sync. 51711703Smichael.lebeane@amd.com */ 51811703Smichael.lebeane@amd.com static System *sys; 51911703Smichael.lebeane@amd.com /** 52011703Smichael.lebeane@amd.com * Is this node a switch? 52111703Smichael.lebeane@amd.com */ 52211703Smichael.lebeane@amd.com static bool isSwitch; 52310923SN/A 52411290Sgabor.dozsa@arm.com private: 52510923SN/A /** 52611290Sgabor.dozsa@arm.com * Send out a data packet to the remote end. 52711290Sgabor.dozsa@arm.com * @param header Meta info about the packet (which needs to be transferred 52811290Sgabor.dozsa@arm.com * to the destination alongside the packet). 52911290Sgabor.dozsa@arm.com * @param packet Pointer to the packet to send. 53010923SN/A */ 53111290Sgabor.dozsa@arm.com virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0; 53210923SN/A /** 53311290Sgabor.dozsa@arm.com * Send out a control command to the remote end. 53411290Sgabor.dozsa@arm.com * @param header Meta info describing the command (e.g. sync request) 53510923SN/A */ 53611290Sgabor.dozsa@arm.com virtual void sendCmd(const Header &header) = 0; 53710923SN/A /** 53811290Sgabor.dozsa@arm.com * Receive a header (i.e. meta info describing a data packet or a control command) 53911290Sgabor.dozsa@arm.com * from the remote end. 54011290Sgabor.dozsa@arm.com * @param header The meta info structure to store the incoming header. 54110923SN/A */ 54211290Sgabor.dozsa@arm.com virtual bool recvHeader(Header &header) = 0; 54311290Sgabor.dozsa@arm.com /** 54411290Sgabor.dozsa@arm.com * Receive a packet from the remote end. 54511290Sgabor.dozsa@arm.com * @param header Meta info about the incoming packet (obtanied by a previous 54611290Sgabor.dozsa@arm.com * call to the recvHedaer() method). 54711290Sgabor.dozsa@arm.com * @param Pointer to packet received. 54811290Sgabor.dozsa@arm.com */ 54911290Sgabor.dozsa@arm.com virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0; 55011290Sgabor.dozsa@arm.com /** 55111290Sgabor.dozsa@arm.com * Init hook for the underlaying transport 55211290Sgabor.dozsa@arm.com */ 55311290Sgabor.dozsa@arm.com virtual void initTransport() = 0; 55411290Sgabor.dozsa@arm.com /** 55511290Sgabor.dozsa@arm.com * spawn the receiver thread. 55611290Sgabor.dozsa@arm.com * @param recv_done The receive done event associated with the simulated 55711290Sgabor.dozsa@arm.com * Ethernet link. 55811290Sgabor.dozsa@arm.com * @param link_delay The link delay for the simulated Ethernet link. 55911290Sgabor.dozsa@arm.com */ 56011290Sgabor.dozsa@arm.com void spawnRecvThread(const Event *recv_done, Tick link_delay); 56110923SN/A /** 56210923SN/A * The function executed by a receiver thread. 56310923SN/A */ 56411290Sgabor.dozsa@arm.com void recvThreadFunc(Event *recv_done, Tick link_delay); 56510923SN/A 56610923SN/A public: 56710923SN/A 56810923SN/A /** 56910923SN/A * ctor 57011290Sgabor.dozsa@arm.com * @param dist_rank Rank of this gem5 process within the dist run 57111290Sgabor.dozsa@arm.com * @param sync_start Start tick for dist synchronisation 57211290Sgabor.dozsa@arm.com * @param sync_repeat Frequency for dist synchronisation 57310923SN/A * @param em The event manager associated with the simulated Ethernet link 57410923SN/A */ 57511290Sgabor.dozsa@arm.com DistIface(unsigned dist_rank, 57611290Sgabor.dozsa@arm.com unsigned dist_size, 57711290Sgabor.dozsa@arm.com Tick sync_start, 57811290Sgabor.dozsa@arm.com Tick sync_repeat, 57911290Sgabor.dozsa@arm.com EventManager *em, 58011703Smichael.lebeane@amd.com bool use_pseudo_op, 58111290Sgabor.dozsa@arm.com bool is_switch, 58211290Sgabor.dozsa@arm.com int num_nodes); 58310923SN/A 58411290Sgabor.dozsa@arm.com virtual ~DistIface(); 58510923SN/A /** 58610923SN/A * Send out an Ethernet packet. 58710923SN/A * @param pkt The Ethernet packet to send. 58810923SN/A * @param send_delay The delay in ticks for the send completion event. 58910923SN/A */ 59010923SN/A void packetOut(EthPacketPtr pkt, Tick send_delay); 59110923SN/A /** 59211290Sgabor.dozsa@arm.com * Fetch the packet scheduled to be received next by the simulated 59311290Sgabor.dozsa@arm.com * network link. 59411290Sgabor.dozsa@arm.com * 59511290Sgabor.dozsa@arm.com * @note This method is called within the process() method of the link 59611290Sgabor.dozsa@arm.com * receive done event. It also schedules the next receive event if the 59711290Sgabor.dozsa@arm.com * receive queue is not empty. 59810923SN/A */ 59911290Sgabor.dozsa@arm.com EthPacketPtr packetIn() { return recvScheduler.popPacket(); } 60010923SN/A 60111168SN/A DrainState drain() override; 60211290Sgabor.dozsa@arm.com void drainResume() override; 60311290Sgabor.dozsa@arm.com void init(const Event *e, Tick link_delay); 60411290Sgabor.dozsa@arm.com void startup(); 60510923SN/A 60611290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 60711290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 60810923SN/A /** 60911290Sgabor.dozsa@arm.com * Initiate the exit from the simulation. 61011290Sgabor.dozsa@arm.com * @param delay Delay param from the m5 exit command. If Delay is zero 61111290Sgabor.dozsa@arm.com * then a collaborative exit is requested (i.e. all nodes have to call 61211290Sgabor.dozsa@arm.com * this method before the distributed simulation can exit). If Delay is 61311290Sgabor.dozsa@arm.com * not zero then exit is requested asap (and it will happen at the next 61411290Sgabor.dozsa@arm.com * sync tick). 61511290Sgabor.dozsa@arm.com * @return False if we are in distributed mode (i.e. exit can happen only 61611290Sgabor.dozsa@arm.com * at sync), True otherwise. 61710923SN/A */ 61811290Sgabor.dozsa@arm.com static bool readyToExit(Tick delay); 61911290Sgabor.dozsa@arm.com /** 62011290Sgabor.dozsa@arm.com * Initiate taking a checkpoint 62111290Sgabor.dozsa@arm.com * @param delay Delay param from the m5 checkpoint command. If Delay is 62211290Sgabor.dozsa@arm.com * zero then a collaborative checkpoint is requested (i.e. all nodes have 62311290Sgabor.dozsa@arm.com * to call this method before the checkpoint can be taken). If Delay is 62411290Sgabor.dozsa@arm.com * not zero then a checkpoint is requested asap (and it will happen at the 62511290Sgabor.dozsa@arm.com * next sync tick). 62611290Sgabor.dozsa@arm.com * @return False if we are in dist mode (i.e. exit can happen only at 62711290Sgabor.dozsa@arm.com * sync), True otherwise. 62811290Sgabor.dozsa@arm.com */ 62911290Sgabor.dozsa@arm.com static bool readyToCkpt(Tick delay, Tick period); 63011290Sgabor.dozsa@arm.com /** 63111290Sgabor.dozsa@arm.com * Getter for the dist rank param. 63211290Sgabor.dozsa@arm.com */ 63311290Sgabor.dozsa@arm.com static uint64_t rankParam(); 63411290Sgabor.dozsa@arm.com /** 63511290Sgabor.dozsa@arm.com * Getter for the dist size param. 63611290Sgabor.dozsa@arm.com */ 63711290Sgabor.dozsa@arm.com static uint64_t sizeParam(); 63811703Smichael.lebeane@amd.com /** 63911703Smichael.lebeane@amd.com * Trigger the master to start/stop synchronization. 64011703Smichael.lebeane@amd.com */ 64111703Smichael.lebeane@amd.com static void toggleSync(ThreadContext *tc); 64211290Sgabor.dozsa@arm.com }; 64310923SN/A 64411290Sgabor.dozsa@arm.com#endif 645