dist_iface.hh revision 11703
110923SN/A/* 210923SN/A * Copyright (c) 2015 ARM Limited 310923SN/A * All rights reserved 410923SN/A * 510923SN/A * The license below extends only to copyright in the software and shall 610923SN/A * not be construed as granting a license to any other intellectual 710923SN/A * property including but not limited to intellectual property relating 810923SN/A * to a hardware implementation of the functionality of the software 910923SN/A * licensed hereunder. You may use the software subject to the license 1010923SN/A * terms below provided that you ensure that this notice is replicated 1110923SN/A * unmodified and in its entirety in all distributions of the software, 1210923SN/A * modified or unmodified, in source code or in binary form. 1310923SN/A * 1410923SN/A * Redistribution and use in source and binary forms, with or without 1510923SN/A * modification, are permitted provided that the following conditions are 1610923SN/A * met: redistributions of source code must retain the above copyright 1710923SN/A * notice, this list of conditions and the following disclaimer; 1810923SN/A * redistributions in binary form must reproduce the above copyright 1910923SN/A * notice, this list of conditions and the following disclaimer in the 2010923SN/A * documentation and/or other materials provided with the distribution; 2110923SN/A * neither the name of the copyright holders nor the names of its 2210923SN/A * contributors may be used to endorse or promote products derived from 2310923SN/A * this software without specific prior written permission. 2410923SN/A * 2510923SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2610923SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2710923SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2810923SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2910923SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3010923SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3110923SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3210923SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3310923SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3410923SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3510923SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3610923SN/A * 3710923SN/A * Authors: Gabor Dozsa 3810923SN/A */ 3910923SN/A 4010923SN/A/* @file 4111290Sgabor.dozsa@arm.com * The interface class for dist gem5 simulations. 4210923SN/A * 4311290Sgabor.dozsa@arm.com * dist-gem5 is an extension to gem5 to enable parallel simulation of a 4410923SN/A * distributed system (e.g. simulation of a pool of machines 4511290Sgabor.dozsa@arm.com * connected by Ethernet links). A dist gem5 run consists of seperate gem5 4610923SN/A * processes running in parallel. Each gem5 process executes 4710923SN/A * the simulation of a component of the simulated distributed system. 4811290Sgabor.dozsa@arm.com * (An example component can be a dist-core board with an Ethernet NIC.) 4911290Sgabor.dozsa@arm.com * The DistIface class below provides services to transfer data and 5010923SN/A * control messages among the gem5 processes. The main such services are 5110923SN/A * as follows. 5210923SN/A * 5310923SN/A * 1. Send a data packet coming from a simulated Ethernet link. The packet 5410923SN/A * will be transferred to (all) the target(s) gem5 processes. The send 5510923SN/A * operation is always performed by the simulation thread, i.e. the gem5 5610923SN/A * thread that is processing the event queue associated with the simulated 5710923SN/A * Ethernet link. 5810923SN/A * 5910923SN/A * 2. Spawn a receiver thread to process messages coming in from the 6010923SN/A * from other gem5 processes. Each simulated Ethernet link has its own 6110923SN/A * associated receiver thread. The receiver thread saves the incoming packet 6210923SN/A * and schedule an appropriate receive event in the event queue. 6310923SN/A * 6410923SN/A * 3. Schedule a global barrier event periodically to keep the gem5 6510923SN/A * processes in sync. 6610923SN/A * Periodic barrier event to keep peer gem5 processes in sync. The basic idea 6710923SN/A * is that no gem5 process can go ahead further than the simulated link 6810923SN/A * transmission delay to ensure that a corresponding receive event can always 6910923SN/A * be scheduled for any message coming in from a peer gem5 process. 7010923SN/A * 7110923SN/A * 7210923SN/A * 7311290Sgabor.dozsa@arm.com * This interface is an abstract class. It can work with various low level 7410923SN/A * send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP 7511290Sgabor.dozsa@arm.com * stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc]. 7610923SN/A */ 7711290Sgabor.dozsa@arm.com#ifndef __DEV_DIST_IFACE_HH__ 7811290Sgabor.dozsa@arm.com#define __DEV_DIST_IFACE_HH__ 7910923SN/A 8010923SN/A#include <array> 8110923SN/A#include <mutex> 8210923SN/A#include <queue> 8310923SN/A#include <thread> 8410923SN/A#include <utility> 8510923SN/A 8611290Sgabor.dozsa@arm.com#include "dev/net/dist_packet.hh" 8711263SN/A#include "dev/net/etherpkt.hh" 8810923SN/A#include "sim/core.hh" 8910923SN/A#include "sim/drain.hh" 9010923SN/A#include "sim/global_event.hh" 9111290Sgabor.dozsa@arm.com#include "sim/serialize.hh" 9210923SN/A 9310923SN/Aclass EventManager; 9411703Smichael.lebeane@amd.comclass System; 9511703Smichael.lebeane@amd.comclass ThreadContext; 9610923SN/A 9710923SN/A/** 9810923SN/A * The interface class to talk to peer gem5 processes. 9910923SN/A */ 10011290Sgabor.dozsa@arm.comclass DistIface : public Drainable, public Serializable 10110923SN/A{ 10210923SN/A public: 10311290Sgabor.dozsa@arm.com typedef DistHeaderPkt::Header Header; 10411290Sgabor.dozsa@arm.com 10511290Sgabor.dozsa@arm.com protected: 10611290Sgabor.dozsa@arm.com typedef DistHeaderPkt::MsgType MsgType; 10711290Sgabor.dozsa@arm.com typedef DistHeaderPkt::ReqType ReqType; 10810923SN/A 10910923SN/A private: 11011290Sgabor.dozsa@arm.com class SyncEvent; 11110923SN/A /** @class Sync 11210923SN/A * This class implements global sync operations among gem5 peer processes. 11310923SN/A * 11411290Sgabor.dozsa@arm.com * @note This class is used as a singleton object (shared by all DistIface 11510923SN/A * objects). 11610923SN/A */ 11711290Sgabor.dozsa@arm.com class Sync : public Serializable 11810923SN/A { 11911290Sgabor.dozsa@arm.com protected: 12010923SN/A /** 12111290Sgabor.dozsa@arm.com * The lock to protect access to the Sync object. 12210923SN/A */ 12310923SN/A std::mutex lock; 12410923SN/A /** 12510923SN/A * Condition variable for the simulation thread to wait on 12610923SN/A * until all receiver threads completes the current global 12710923SN/A * synchronisation. 12810923SN/A */ 12910923SN/A std::condition_variable cv; 13010923SN/A /** 13110923SN/A * Number of receiver threads that not yet completed the current global 13210923SN/A * synchronisation. 13310923SN/A */ 13410923SN/A unsigned waitNum; 13510923SN/A /** 13611290Sgabor.dozsa@arm.com * Flag is set if exit is permitted upon sync completion 13710923SN/A */ 13811290Sgabor.dozsa@arm.com bool doExit; 13910923SN/A /** 14011290Sgabor.dozsa@arm.com * Flag is set if taking a ckpt is permitted upon sync completion 14110923SN/A */ 14211290Sgabor.dozsa@arm.com bool doCkpt; 14311290Sgabor.dozsa@arm.com /** 14411703Smichael.lebeane@amd.com * Flag is set if sync is to stop upon sync completion 14511703Smichael.lebeane@amd.com */ 14611703Smichael.lebeane@amd.com bool doStopSync; 14711703Smichael.lebeane@amd.com /** 14811290Sgabor.dozsa@arm.com * The repeat value for the next periodic sync 14911290Sgabor.dozsa@arm.com */ 15011290Sgabor.dozsa@arm.com Tick nextRepeat; 15111290Sgabor.dozsa@arm.com /** 15211290Sgabor.dozsa@arm.com * Tick for the next periodic sync (if the event is not scheduled yet) 15311290Sgabor.dozsa@arm.com */ 15411290Sgabor.dozsa@arm.com Tick nextAt; 15510923SN/A 15611290Sgabor.dozsa@arm.com friend class SyncEvent; 15710923SN/A 15810923SN/A public: 15910923SN/A /** 16011290Sgabor.dozsa@arm.com * Initialize periodic sync params. 16110923SN/A * 16211290Sgabor.dozsa@arm.com * @param start Start tick for dist synchronisation 16311290Sgabor.dozsa@arm.com * @param repeat Frequency of dist synchronisation 16410923SN/A * 16510923SN/A */ 16611290Sgabor.dozsa@arm.com void init(Tick start, Tick repeat); 16710923SN/A /** 16811290Sgabor.dozsa@arm.com * Core method to perform a full dist sync. 16910923SN/A */ 17011290Sgabor.dozsa@arm.com virtual void run(bool same_tick) = 0; 17111290Sgabor.dozsa@arm.com /** 17211290Sgabor.dozsa@arm.com * Callback when the receiver thread gets a sync ack message. 17311290Sgabor.dozsa@arm.com */ 17411290Sgabor.dozsa@arm.com virtual void progress(Tick send_tick, 17511290Sgabor.dozsa@arm.com Tick next_repeat, 17611290Sgabor.dozsa@arm.com ReqType do_ckpt, 17711703Smichael.lebeane@amd.com ReqType do_exit, 17811703Smichael.lebeane@amd.com ReqType do_stop_sync) = 0; 17910923SN/A 18011290Sgabor.dozsa@arm.com virtual void requestCkpt(ReqType req) = 0; 18111290Sgabor.dozsa@arm.com virtual void requestExit(ReqType req) = 0; 18211703Smichael.lebeane@amd.com virtual void requestStopSync(ReqType req) = 0; 18311290Sgabor.dozsa@arm.com 18411290Sgabor.dozsa@arm.com void drainComplete(); 18511290Sgabor.dozsa@arm.com 18611290Sgabor.dozsa@arm.com virtual void serialize(CheckpointOut &cp) const override = 0; 18711290Sgabor.dozsa@arm.com virtual void unserialize(CheckpointIn &cp) override = 0; 18810923SN/A }; 18910923SN/A 19011290Sgabor.dozsa@arm.com class SyncNode: public Sync 19111290Sgabor.dozsa@arm.com { 19211290Sgabor.dozsa@arm.com private: 19311290Sgabor.dozsa@arm.com /** 19411290Sgabor.dozsa@arm.com * Exit requested 19511290Sgabor.dozsa@arm.com */ 19611290Sgabor.dozsa@arm.com ReqType needExit; 19711290Sgabor.dozsa@arm.com /** 19811290Sgabor.dozsa@arm.com * Ckpt requested 19911290Sgabor.dozsa@arm.com */ 20011290Sgabor.dozsa@arm.com ReqType needCkpt; 20111703Smichael.lebeane@amd.com /** 20211703Smichael.lebeane@amd.com * Sync stop requested 20311703Smichael.lebeane@amd.com */ 20411703Smichael.lebeane@amd.com ReqType needStopSync; 20511290Sgabor.dozsa@arm.com 20611290Sgabor.dozsa@arm.com public: 20711290Sgabor.dozsa@arm.com 20811290Sgabor.dozsa@arm.com SyncNode(); 20911290Sgabor.dozsa@arm.com ~SyncNode() {} 21011290Sgabor.dozsa@arm.com void run(bool same_tick) override; 21111290Sgabor.dozsa@arm.com void progress(Tick max_req_tick, 21211290Sgabor.dozsa@arm.com Tick next_repeat, 21311290Sgabor.dozsa@arm.com ReqType do_ckpt, 21411703Smichael.lebeane@amd.com ReqType do_exit, 21511703Smichael.lebeane@amd.com ReqType do_stop_sync) override; 21611290Sgabor.dozsa@arm.com 21711290Sgabor.dozsa@arm.com void requestCkpt(ReqType req) override; 21811290Sgabor.dozsa@arm.com void requestExit(ReqType req) override; 21911703Smichael.lebeane@amd.com void requestStopSync(ReqType req) override; 22011290Sgabor.dozsa@arm.com 22111290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 22211290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 22311290Sgabor.dozsa@arm.com }; 22411290Sgabor.dozsa@arm.com 22511290Sgabor.dozsa@arm.com class SyncSwitch: public Sync 22611290Sgabor.dozsa@arm.com { 22711290Sgabor.dozsa@arm.com private: 22811290Sgabor.dozsa@arm.com /** 22911290Sgabor.dozsa@arm.com * Counter for recording exit requests 23011290Sgabor.dozsa@arm.com */ 23111290Sgabor.dozsa@arm.com unsigned numExitReq; 23211290Sgabor.dozsa@arm.com /** 23311290Sgabor.dozsa@arm.com * Counter for recording ckpt requests 23411290Sgabor.dozsa@arm.com */ 23511290Sgabor.dozsa@arm.com unsigned numCkptReq; 23611290Sgabor.dozsa@arm.com /** 23711703Smichael.lebeane@amd.com * Counter for recording stop sync requests 23811703Smichael.lebeane@amd.com */ 23911703Smichael.lebeane@amd.com unsigned numStopSyncReq; 24011703Smichael.lebeane@amd.com /** 24111290Sgabor.dozsa@arm.com * Number of connected simulated nodes 24211290Sgabor.dozsa@arm.com */ 24311290Sgabor.dozsa@arm.com unsigned numNodes; 24411290Sgabor.dozsa@arm.com 24511290Sgabor.dozsa@arm.com public: 24611290Sgabor.dozsa@arm.com SyncSwitch(int num_nodes); 24711290Sgabor.dozsa@arm.com ~SyncSwitch() {} 24811290Sgabor.dozsa@arm.com 24911290Sgabor.dozsa@arm.com void run(bool same_tick) override; 25011290Sgabor.dozsa@arm.com void progress(Tick max_req_tick, 25111290Sgabor.dozsa@arm.com Tick next_repeat, 25211290Sgabor.dozsa@arm.com ReqType do_ckpt, 25311703Smichael.lebeane@amd.com ReqType do_exit, 25411703Smichael.lebeane@amd.com ReqType do_stop_sync) override; 25511290Sgabor.dozsa@arm.com 25611290Sgabor.dozsa@arm.com void requestCkpt(ReqType) override { 25711290Sgabor.dozsa@arm.com panic("Switch requested checkpoint"); 25811290Sgabor.dozsa@arm.com } 25911290Sgabor.dozsa@arm.com void requestExit(ReqType) override { 26011290Sgabor.dozsa@arm.com panic("Switch requested exit"); 26111290Sgabor.dozsa@arm.com } 26211703Smichael.lebeane@amd.com void requestStopSync(ReqType) override { 26311703Smichael.lebeane@amd.com panic("Switch requested stop sync"); 26411703Smichael.lebeane@amd.com } 26511290Sgabor.dozsa@arm.com 26611290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 26711290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 26811290Sgabor.dozsa@arm.com }; 26910923SN/A 27010923SN/A /** 27111290Sgabor.dozsa@arm.com * The global event to schedule periodic dist sync. It is used as a 27210923SN/A * singleton object. 27310923SN/A * 27410923SN/A * The periodic synchronisation works as follows. 27511290Sgabor.dozsa@arm.com * 1. A SyncEvent is scheduled as a global event when startup() is 27610923SN/A * called. 27711290Sgabor.dozsa@arm.com * 2. The process() method of the SyncEvent initiates a new barrier 27811290Sgabor.dozsa@arm.com * for each simulated Ethernet link. 27910923SN/A * 3. Simulation thread(s) then waits until all receiver threads 28011290Sgabor.dozsa@arm.com * complete the ongoing barrier. The global sync event is done. 28110923SN/A */ 28210923SN/A class SyncEvent : public GlobalSyncEvent 28310923SN/A { 28411290Sgabor.dozsa@arm.com private: 28511290Sgabor.dozsa@arm.com /** 28611290Sgabor.dozsa@arm.com * Flag to set when the system is draining 28711290Sgabor.dozsa@arm.com */ 28811290Sgabor.dozsa@arm.com bool _draining; 28910923SN/A public: 29010923SN/A /** 29111290Sgabor.dozsa@arm.com * Only the firstly instantiated DistIface object will 29211290Sgabor.dozsa@arm.com * call this constructor. 29310923SN/A */ 29411290Sgabor.dozsa@arm.com SyncEvent() : GlobalSyncEvent(Sim_Exit_Pri, 0), _draining(false) {} 29511290Sgabor.dozsa@arm.com 29611290Sgabor.dozsa@arm.com ~SyncEvent() {} 29710923SN/A /** 29811290Sgabor.dozsa@arm.com * Schedule the first periodic sync event. 29910923SN/A */ 30011290Sgabor.dozsa@arm.com void start(); 30110923SN/A /** 30211290Sgabor.dozsa@arm.com * This is a global event so process() will only be called by 30311290Sgabor.dozsa@arm.com * exactly one simulation thread. (See further comments in the .cc 30411290Sgabor.dozsa@arm.com * file.) 30510923SN/A */ 30611290Sgabor.dozsa@arm.com void process() override; 30711290Sgabor.dozsa@arm.com 30811290Sgabor.dozsa@arm.com bool draining() const { return _draining; } 30911290Sgabor.dozsa@arm.com void draining(bool fl) { _draining = fl; } 31011290Sgabor.dozsa@arm.com }; 31111290Sgabor.dozsa@arm.com /** 31211290Sgabor.dozsa@arm.com * Class to encapsulate information about data packets received. 31311290Sgabor.dozsa@arm.com 31411290Sgabor.dozsa@arm.com * @note The main purpose of the class to take care of scheduling receive 31511290Sgabor.dozsa@arm.com * done events for the simulated network link and store incoming packets 31611290Sgabor.dozsa@arm.com * until they can be received by the simulated network link. 31711290Sgabor.dozsa@arm.com */ 31811290Sgabor.dozsa@arm.com class RecvScheduler : public Serializable 31911290Sgabor.dozsa@arm.com { 32011290Sgabor.dozsa@arm.com private: 32111290Sgabor.dozsa@arm.com /** 32211290Sgabor.dozsa@arm.com * Received packet descriptor. This information is used by the receive 32311290Sgabor.dozsa@arm.com * thread to schedule receive events and by the simulation thread to 32411290Sgabor.dozsa@arm.com * process those events. 32511290Sgabor.dozsa@arm.com */ 32611290Sgabor.dozsa@arm.com struct Desc : public Serializable 32711290Sgabor.dozsa@arm.com { 32811290Sgabor.dozsa@arm.com EthPacketPtr packet; 32911290Sgabor.dozsa@arm.com Tick sendTick; 33011290Sgabor.dozsa@arm.com Tick sendDelay; 33111290Sgabor.dozsa@arm.com 33211290Sgabor.dozsa@arm.com Desc() : sendTick(0), sendDelay(0) {} 33311290Sgabor.dozsa@arm.com Desc(EthPacketPtr p, Tick s, Tick d) : 33411290Sgabor.dozsa@arm.com packet(p), sendTick(s), sendDelay(d) {} 33511290Sgabor.dozsa@arm.com Desc(const Desc &d) : 33611290Sgabor.dozsa@arm.com packet(d.packet), sendTick(d.sendTick), sendDelay(d.sendDelay) {} 33711290Sgabor.dozsa@arm.com 33811290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 33911290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 34011290Sgabor.dozsa@arm.com }; 34111290Sgabor.dozsa@arm.com /** 34211290Sgabor.dozsa@arm.com * The queue to store the receive descriptors. 34311290Sgabor.dozsa@arm.com */ 34411290Sgabor.dozsa@arm.com std::queue<Desc> descQueue; 34511290Sgabor.dozsa@arm.com /** 34611290Sgabor.dozsa@arm.com * The tick when the most recent receive event was processed. 34711290Sgabor.dozsa@arm.com * 34811290Sgabor.dozsa@arm.com * @note This information is necessary to simulate possible receiver 34911290Sgabor.dozsa@arm.com * link contention when calculating the receive tick for the next 35011290Sgabor.dozsa@arm.com * incoming data packet (see the calcReceiveTick() method) 35111290Sgabor.dozsa@arm.com */ 35211290Sgabor.dozsa@arm.com Tick prevRecvTick; 35311290Sgabor.dozsa@arm.com /** 35411290Sgabor.dozsa@arm.com * The receive done event for the simulated Ethernet link. 35511290Sgabor.dozsa@arm.com * 35611290Sgabor.dozsa@arm.com * @note This object is constructed by the simulated network link. We 35711290Sgabor.dozsa@arm.com * schedule this object for each incoming data packet. 35811290Sgabor.dozsa@arm.com */ 35911290Sgabor.dozsa@arm.com Event *recvDone; 36011290Sgabor.dozsa@arm.com /** 36111290Sgabor.dozsa@arm.com * The link delay in ticks for the simulated Ethernet link. 36211290Sgabor.dozsa@arm.com * 36311290Sgabor.dozsa@arm.com * @note This value is used for calculating the receive ticks for 36411290Sgabor.dozsa@arm.com * incoming data packets. 36511290Sgabor.dozsa@arm.com */ 36611290Sgabor.dozsa@arm.com Tick linkDelay; 36711290Sgabor.dozsa@arm.com /** 36811290Sgabor.dozsa@arm.com * The event manager associated with the simulated Ethernet link. 36911290Sgabor.dozsa@arm.com * 37011290Sgabor.dozsa@arm.com * @note It is used to access the event queue for scheduling receive 37111290Sgabor.dozsa@arm.com * done events for the link. 37211290Sgabor.dozsa@arm.com */ 37311290Sgabor.dozsa@arm.com EventManager *eventManager; 37411290Sgabor.dozsa@arm.com /** 37511290Sgabor.dozsa@arm.com * Calculate the tick to schedule the next receive done event. 37611290Sgabor.dozsa@arm.com * 37711290Sgabor.dozsa@arm.com * @param send_tick The tick the packet was sent. 37811290Sgabor.dozsa@arm.com * @param send_delay The simulated delay at the sender side. 37911290Sgabor.dozsa@arm.com * @param prev_recv_tick Tick when the last receive event was 38011290Sgabor.dozsa@arm.com * processed. 38111290Sgabor.dozsa@arm.com * 38211290Sgabor.dozsa@arm.com * @note This method tries to take into account possible receiver link 38311290Sgabor.dozsa@arm.com * contention and adjust receive tick for the incoming packets 38411290Sgabor.dozsa@arm.com * accordingly. 38511290Sgabor.dozsa@arm.com */ 38611290Sgabor.dozsa@arm.com Tick calcReceiveTick(Tick send_tick, 38711290Sgabor.dozsa@arm.com Tick send_delay, 38811290Sgabor.dozsa@arm.com Tick prev_recv_tick); 38911290Sgabor.dozsa@arm.com 39011290Sgabor.dozsa@arm.com /** 39111290Sgabor.dozsa@arm.com * Flag to set if receive ticks for pending packets need to be 39211290Sgabor.dozsa@arm.com * recalculated due to changed link latencies at a resume 39311290Sgabor.dozsa@arm.com */ 39411290Sgabor.dozsa@arm.com bool ckptRestore; 39510923SN/A 39610923SN/A public: 39710923SN/A /** 39811290Sgabor.dozsa@arm.com * Scheduler for the incoming data packets. 39911290Sgabor.dozsa@arm.com * 40011290Sgabor.dozsa@arm.com * @param em The event manager associated with the simulated Ethernet 40111290Sgabor.dozsa@arm.com * link. 40210923SN/A */ 40311290Sgabor.dozsa@arm.com RecvScheduler(EventManager *em) : 40411290Sgabor.dozsa@arm.com prevRecvTick(0), recvDone(nullptr), linkDelay(0), 40511290Sgabor.dozsa@arm.com eventManager(em), ckptRestore(false) {} 40610923SN/A 40710923SN/A /** 40811290Sgabor.dozsa@arm.com * Initialize network link parameters. 40910923SN/A * 41011290Sgabor.dozsa@arm.com * @note This method is called from the receiver thread (see 41111290Sgabor.dozsa@arm.com * recvThreadFunc()). 41211290Sgabor.dozsa@arm.com */ 41311290Sgabor.dozsa@arm.com void init(Event *recv_done, Tick link_delay); 41411290Sgabor.dozsa@arm.com /** 41511290Sgabor.dozsa@arm.com * Fetch the next packet that is to be received by the simulated network 41611290Sgabor.dozsa@arm.com * link. 41710923SN/A * 41811290Sgabor.dozsa@arm.com * @note This method is called from the process() method of the receive 41911290Sgabor.dozsa@arm.com * done event associated with the network link. 42010923SN/A */ 42111290Sgabor.dozsa@arm.com EthPacketPtr popPacket(); 42210923SN/A /** 42311290Sgabor.dozsa@arm.com * Push a newly arrived packet into the desc queue. 42411290Sgabor.dozsa@arm.com */ 42511290Sgabor.dozsa@arm.com void pushPacket(EthPacketPtr new_packet, 42611290Sgabor.dozsa@arm.com Tick send_tick, 42711290Sgabor.dozsa@arm.com Tick send_delay); 42811290Sgabor.dozsa@arm.com 42911290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 43011290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 43111290Sgabor.dozsa@arm.com /** 43211290Sgabor.dozsa@arm.com * Adjust receive ticks for pending packets when restoring from a 43311290Sgabor.dozsa@arm.com * checkpoint 43410923SN/A * 43511290Sgabor.dozsa@arm.com * @note Link speed and delay parameters may change at resume. 43610923SN/A */ 43711290Sgabor.dozsa@arm.com void resumeRecvTicks(); 43810923SN/A }; 43910923SN/A /** 44011290Sgabor.dozsa@arm.com * Tick to schedule the first dist sync event. 44111290Sgabor.dozsa@arm.com * This is just as optimization : we do not need any dist sync 44210923SN/A * event until the simulated NIC is brought up by the OS. 44310923SN/A */ 44410923SN/A Tick syncStart; 44510923SN/A /** 44611290Sgabor.dozsa@arm.com * Frequency of dist sync events in ticks. 44710923SN/A */ 44810923SN/A Tick syncRepeat; 44910923SN/A /** 45010923SN/A * Receiver thread pointer. 45111290Sgabor.dozsa@arm.com * Each DistIface object must have exactly one receiver thread. 45210923SN/A */ 45310923SN/A std::thread *recvThread; 45410923SN/A /** 45511290Sgabor.dozsa@arm.com * Meta information about data packets received. 45610923SN/A */ 45711290Sgabor.dozsa@arm.com RecvScheduler recvScheduler; 45811703Smichael.lebeane@amd.com /** 45911703Smichael.lebeane@amd.com * Use pseudoOp to start synchronization. 46011703Smichael.lebeane@amd.com */ 46111703Smichael.lebeane@amd.com bool syncStartOnPseudoOp; 46210923SN/A 46311290Sgabor.dozsa@arm.com protected: 46410923SN/A /** 46510923SN/A * The rank of this process among the gem5 peers. 46610923SN/A */ 46710923SN/A unsigned rank; 46810923SN/A /** 46911290Sgabor.dozsa@arm.com * The number of gem5 processes comprising this dist simulation. 47011290Sgabor.dozsa@arm.com */ 47111290Sgabor.dozsa@arm.com unsigned size; 47211290Sgabor.dozsa@arm.com /** 47311290Sgabor.dozsa@arm.com * Number of DistIface objects (i.e. dist links in this gem5 process) 47411290Sgabor.dozsa@arm.com */ 47511290Sgabor.dozsa@arm.com static unsigned distIfaceNum; 47611290Sgabor.dozsa@arm.com /** 47711290Sgabor.dozsa@arm.com * Unique id for the dist link 47811290Sgabor.dozsa@arm.com */ 47911290Sgabor.dozsa@arm.com unsigned distIfaceId; 48011290Sgabor.dozsa@arm.com 48111290Sgabor.dozsa@arm.com bool isMaster; 48211290Sgabor.dozsa@arm.com 48311290Sgabor.dozsa@arm.com private: 48411290Sgabor.dozsa@arm.com /** 48511290Sgabor.dozsa@arm.com * Number of receiver threads (in this gem5 process) 48610923SN/A */ 48710923SN/A static unsigned recvThreadsNum; 48810923SN/A /** 48911290Sgabor.dozsa@arm.com * The singleton Sync object to perform dist synchronisation. 49011290Sgabor.dozsa@arm.com */ 49111290Sgabor.dozsa@arm.com static Sync *sync; 49211290Sgabor.dozsa@arm.com /** 49311290Sgabor.dozsa@arm.com * The singleton SyncEvent object to schedule periodic dist sync. 49411290Sgabor.dozsa@arm.com */ 49511290Sgabor.dozsa@arm.com static SyncEvent *syncEvent; 49611290Sgabor.dozsa@arm.com /** 49711290Sgabor.dozsa@arm.com * The very first DistIface object created becomes the master. We need 49810923SN/A * a master to co-ordinate the global synchronisation. 49910923SN/A */ 50011290Sgabor.dozsa@arm.com static DistIface *master; 50111703Smichael.lebeane@amd.com /** 50211703Smichael.lebeane@amd.com * System pointer used to wakeup sleeping threads when stopping sync. 50311703Smichael.lebeane@amd.com */ 50411703Smichael.lebeane@amd.com static System *sys; 50511703Smichael.lebeane@amd.com /** 50611703Smichael.lebeane@amd.com * Is this node a switch? 50711703Smichael.lebeane@amd.com */ 50811703Smichael.lebeane@amd.com static bool isSwitch; 50910923SN/A 51011290Sgabor.dozsa@arm.com private: 51110923SN/A /** 51211290Sgabor.dozsa@arm.com * Send out a data packet to the remote end. 51311290Sgabor.dozsa@arm.com * @param header Meta info about the packet (which needs to be transferred 51411290Sgabor.dozsa@arm.com * to the destination alongside the packet). 51511290Sgabor.dozsa@arm.com * @param packet Pointer to the packet to send. 51610923SN/A */ 51711290Sgabor.dozsa@arm.com virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0; 51810923SN/A /** 51911290Sgabor.dozsa@arm.com * Send out a control command to the remote end. 52011290Sgabor.dozsa@arm.com * @param header Meta info describing the command (e.g. sync request) 52110923SN/A */ 52211290Sgabor.dozsa@arm.com virtual void sendCmd(const Header &header) = 0; 52310923SN/A /** 52411290Sgabor.dozsa@arm.com * Receive a header (i.e. meta info describing a data packet or a control command) 52511290Sgabor.dozsa@arm.com * from the remote end. 52611290Sgabor.dozsa@arm.com * @param header The meta info structure to store the incoming header. 52710923SN/A */ 52811290Sgabor.dozsa@arm.com virtual bool recvHeader(Header &header) = 0; 52911290Sgabor.dozsa@arm.com /** 53011290Sgabor.dozsa@arm.com * Receive a packet from the remote end. 53111290Sgabor.dozsa@arm.com * @param header Meta info about the incoming packet (obtanied by a previous 53211290Sgabor.dozsa@arm.com * call to the recvHedaer() method). 53311290Sgabor.dozsa@arm.com * @param Pointer to packet received. 53411290Sgabor.dozsa@arm.com */ 53511290Sgabor.dozsa@arm.com virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0; 53611290Sgabor.dozsa@arm.com /** 53711290Sgabor.dozsa@arm.com * Init hook for the underlaying transport 53811290Sgabor.dozsa@arm.com */ 53911290Sgabor.dozsa@arm.com virtual void initTransport() = 0; 54011290Sgabor.dozsa@arm.com /** 54111290Sgabor.dozsa@arm.com * spawn the receiver thread. 54211290Sgabor.dozsa@arm.com * @param recv_done The receive done event associated with the simulated 54311290Sgabor.dozsa@arm.com * Ethernet link. 54411290Sgabor.dozsa@arm.com * @param link_delay The link delay for the simulated Ethernet link. 54511290Sgabor.dozsa@arm.com */ 54611290Sgabor.dozsa@arm.com void spawnRecvThread(const Event *recv_done, Tick link_delay); 54710923SN/A /** 54810923SN/A * The function executed by a receiver thread. 54910923SN/A */ 55011290Sgabor.dozsa@arm.com void recvThreadFunc(Event *recv_done, Tick link_delay); 55110923SN/A 55210923SN/A public: 55310923SN/A 55410923SN/A /** 55510923SN/A * ctor 55611290Sgabor.dozsa@arm.com * @param dist_rank Rank of this gem5 process within the dist run 55711290Sgabor.dozsa@arm.com * @param sync_start Start tick for dist synchronisation 55811290Sgabor.dozsa@arm.com * @param sync_repeat Frequency for dist synchronisation 55910923SN/A * @param em The event manager associated with the simulated Ethernet link 56010923SN/A */ 56111290Sgabor.dozsa@arm.com DistIface(unsigned dist_rank, 56211290Sgabor.dozsa@arm.com unsigned dist_size, 56311290Sgabor.dozsa@arm.com Tick sync_start, 56411290Sgabor.dozsa@arm.com Tick sync_repeat, 56511290Sgabor.dozsa@arm.com EventManager *em, 56611703Smichael.lebeane@amd.com bool use_pseudo_op, 56711290Sgabor.dozsa@arm.com bool is_switch, 56811290Sgabor.dozsa@arm.com int num_nodes); 56910923SN/A 57011290Sgabor.dozsa@arm.com virtual ~DistIface(); 57110923SN/A /** 57210923SN/A * Send out an Ethernet packet. 57310923SN/A * @param pkt The Ethernet packet to send. 57410923SN/A * @param send_delay The delay in ticks for the send completion event. 57510923SN/A */ 57610923SN/A void packetOut(EthPacketPtr pkt, Tick send_delay); 57710923SN/A /** 57811290Sgabor.dozsa@arm.com * Fetch the packet scheduled to be received next by the simulated 57911290Sgabor.dozsa@arm.com * network link. 58011290Sgabor.dozsa@arm.com * 58111290Sgabor.dozsa@arm.com * @note This method is called within the process() method of the link 58211290Sgabor.dozsa@arm.com * receive done event. It also schedules the next receive event if the 58311290Sgabor.dozsa@arm.com * receive queue is not empty. 58410923SN/A */ 58511290Sgabor.dozsa@arm.com EthPacketPtr packetIn() { return recvScheduler.popPacket(); } 58610923SN/A 58711168SN/A DrainState drain() override; 58811290Sgabor.dozsa@arm.com void drainResume() override; 58911290Sgabor.dozsa@arm.com void init(const Event *e, Tick link_delay); 59011290Sgabor.dozsa@arm.com void startup(); 59110923SN/A 59211290Sgabor.dozsa@arm.com void serialize(CheckpointOut &cp) const override; 59311290Sgabor.dozsa@arm.com void unserialize(CheckpointIn &cp) override; 59410923SN/A /** 59511290Sgabor.dozsa@arm.com * Initiate the exit from the simulation. 59611290Sgabor.dozsa@arm.com * @param delay Delay param from the m5 exit command. If Delay is zero 59711290Sgabor.dozsa@arm.com * then a collaborative exit is requested (i.e. all nodes have to call 59811290Sgabor.dozsa@arm.com * this method before the distributed simulation can exit). If Delay is 59911290Sgabor.dozsa@arm.com * not zero then exit is requested asap (and it will happen at the next 60011290Sgabor.dozsa@arm.com * sync tick). 60111290Sgabor.dozsa@arm.com * @return False if we are in distributed mode (i.e. exit can happen only 60211290Sgabor.dozsa@arm.com * at sync), True otherwise. 60310923SN/A */ 60411290Sgabor.dozsa@arm.com static bool readyToExit(Tick delay); 60511290Sgabor.dozsa@arm.com /** 60611290Sgabor.dozsa@arm.com * Initiate taking a checkpoint 60711290Sgabor.dozsa@arm.com * @param delay Delay param from the m5 checkpoint command. If Delay is 60811290Sgabor.dozsa@arm.com * zero then a collaborative checkpoint is requested (i.e. all nodes have 60911290Sgabor.dozsa@arm.com * to call this method before the checkpoint can be taken). If Delay is 61011290Sgabor.dozsa@arm.com * not zero then a checkpoint is requested asap (and it will happen at the 61111290Sgabor.dozsa@arm.com * next sync tick). 61211290Sgabor.dozsa@arm.com * @return False if we are in dist mode (i.e. exit can happen only at 61311290Sgabor.dozsa@arm.com * sync), True otherwise. 61411290Sgabor.dozsa@arm.com */ 61511290Sgabor.dozsa@arm.com static bool readyToCkpt(Tick delay, Tick period); 61611290Sgabor.dozsa@arm.com /** 61711290Sgabor.dozsa@arm.com * Getter for the dist rank param. 61811290Sgabor.dozsa@arm.com */ 61911290Sgabor.dozsa@arm.com static uint64_t rankParam(); 62011290Sgabor.dozsa@arm.com /** 62111290Sgabor.dozsa@arm.com * Getter for the dist size param. 62211290Sgabor.dozsa@arm.com */ 62311290Sgabor.dozsa@arm.com static uint64_t sizeParam(); 62411703Smichael.lebeane@amd.com /** 62511703Smichael.lebeane@amd.com * Trigger the master to start/stop synchronization. 62611703Smichael.lebeane@amd.com */ 62711703Smichael.lebeane@amd.com static void toggleSync(ThreadContext *tc); 62811290Sgabor.dozsa@arm.com }; 62910923SN/A 63011290Sgabor.dozsa@arm.com#endif 631