19342SAndreas.Sandberg@arm.com/* 211937Sandreas.sandberg@arm.com * Copyright (c) 2012, 2015, 2017 ARM Limited 39342SAndreas.Sandberg@arm.com * All rights reserved 49342SAndreas.Sandberg@arm.com * 59342SAndreas.Sandberg@arm.com * The license below extends only to copyright in the software and shall 69342SAndreas.Sandberg@arm.com * not be construed as granting a license to any other intellectual 79342SAndreas.Sandberg@arm.com * property including but not limited to intellectual property relating 89342SAndreas.Sandberg@arm.com * to a hardware implementation of the functionality of the software 99342SAndreas.Sandberg@arm.com * licensed hereunder. You may use the software subject to the license 109342SAndreas.Sandberg@arm.com * terms below provided that you ensure that this notice is replicated 119342SAndreas.Sandberg@arm.com * unmodified and in its entirety in all distributions of the software, 129342SAndreas.Sandberg@arm.com * modified or unmodified, in source code or in binary form. 139342SAndreas.Sandberg@arm.com * 149342SAndreas.Sandberg@arm.com * Redistribution and use in source and binary forms, with or without 159342SAndreas.Sandberg@arm.com * modification, are permitted provided that the following conditions are 169342SAndreas.Sandberg@arm.com * met: redistributions of source code must retain the above copyright 179342SAndreas.Sandberg@arm.com * notice, this list of conditions and the following disclaimer; 189342SAndreas.Sandberg@arm.com * redistributions in binary form must reproduce the above copyright 199342SAndreas.Sandberg@arm.com * notice, this list of conditions and the following disclaimer in the 209342SAndreas.Sandberg@arm.com * documentation and/or other materials provided with the distribution; 219342SAndreas.Sandberg@arm.com * neither the name of the copyright holders nor the names of its 229342SAndreas.Sandberg@arm.com * contributors may be used to endorse or promote products derived from 239342SAndreas.Sandberg@arm.com * this software without specific prior written permission. 249342SAndreas.Sandberg@arm.com * 259342SAndreas.Sandberg@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 269342SAndreas.Sandberg@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 279342SAndreas.Sandberg@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 289342SAndreas.Sandberg@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 299342SAndreas.Sandberg@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 309342SAndreas.Sandberg@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 319342SAndreas.Sandberg@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 329342SAndreas.Sandberg@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 339342SAndreas.Sandberg@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 349342SAndreas.Sandberg@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 359342SAndreas.Sandberg@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 369342SAndreas.Sandberg@arm.com * 379342SAndreas.Sandberg@arm.com * Authors: Andreas Sandberg 389342SAndreas.Sandberg@arm.com */ 399342SAndreas.Sandberg@arm.com 409342SAndreas.Sandberg@arm.com#ifndef __SIM_DRAIN_HH__ 419342SAndreas.Sandberg@arm.com#define __SIM_DRAIN_HH__ 429342SAndreas.Sandberg@arm.com 4310912Sandreas.sandberg@arm.com#include <atomic> 4410912Sandreas.sandberg@arm.com#include <mutex> 4511859Sandreas.hansson@arm.com#include <vector> 469342SAndreas.Sandberg@arm.com 4710910Sandreas.sandberg@arm.comclass Drainable; 4810910Sandreas.sandberg@arm.com 4910910Sandreas.sandberg@arm.com/** 5010910Sandreas.sandberg@arm.com * Object drain/handover states 5110910Sandreas.sandberg@arm.com * 5210910Sandreas.sandberg@arm.com * An object starts out in the Running state. When the simulator 5310910Sandreas.sandberg@arm.com * prepares to take a snapshot or prepares a CPU for handover, it 5410910Sandreas.sandberg@arm.com * calls the drain() method to transfer the object into the Draining 5510910Sandreas.sandberg@arm.com * or Drained state. If any object enters the Draining state 5610910Sandreas.sandberg@arm.com * (Drainable::drain() returning >0), simulation continues until it 5710910Sandreas.sandberg@arm.com * all objects have entered the Drained state. 5810910Sandreas.sandberg@arm.com * 5910910Sandreas.sandberg@arm.com * Before resuming simulation, the simulator calls resume() to 6011937Sandreas.sandberg@arm.com * transfer the object to the Running state. This in turn results in a 6111937Sandreas.sandberg@arm.com * call to drainResume() for all Drainable objects in the 6211937Sandreas.sandberg@arm.com * simulator. New Drainable objects may be created while resuming. In 6311937Sandreas.sandberg@arm.com * such cases, the new objects will be created in the Resuming state 6411937Sandreas.sandberg@arm.com * and later resumed. 6510910Sandreas.sandberg@arm.com * 6610910Sandreas.sandberg@arm.com * \note Even though the state of an object (visible to the rest of 6710910Sandreas.sandberg@arm.com * the world through Drainable::getState()) could be used to determine 6810910Sandreas.sandberg@arm.com * if all objects have entered the Drained state, the protocol is 6910910Sandreas.sandberg@arm.com * actually a bit more elaborate. See Drainable::drain() for details. 7010910Sandreas.sandberg@arm.com */ 7110910Sandreas.sandberg@arm.comenum class DrainState { 7210910Sandreas.sandberg@arm.com Running, /** Running normally */ 7310910Sandreas.sandberg@arm.com Draining, /** Draining buffers pending serialization/handover */ 7411937Sandreas.sandberg@arm.com Drained, /** Buffers drained, ready for serialization/handover */ 7511937Sandreas.sandberg@arm.com Resuming, /** Transient state while the simulator is resuming */ 7610910Sandreas.sandberg@arm.com}; 779342SAndreas.Sandberg@arm.com 789342SAndreas.Sandberg@arm.com/** 799342SAndreas.Sandberg@arm.com * This class coordinates draining of a System. 809342SAndreas.Sandberg@arm.com * 8110912Sandreas.sandberg@arm.com * When draining the simulator, we need to make sure that all 8210912Sandreas.sandberg@arm.com * Drainable objects within the system have ended up in the drained 8310912Sandreas.sandberg@arm.com * state before declaring the operation to be successful. This class 8410912Sandreas.sandberg@arm.com * keeps track of how many objects are still in the process of 8510912Sandreas.sandberg@arm.com * draining. Once it determines that all objects have drained their 8610912Sandreas.sandberg@arm.com * state, it exits the simulation loop. 879342SAndreas.Sandberg@arm.com * 889342SAndreas.Sandberg@arm.com * @note A System might not be completely drained even though the 899342SAndreas.Sandberg@arm.com * DrainManager has caused the simulation loop to exit. Draining needs 909342SAndreas.Sandberg@arm.com * to be restarted until all Drainable objects declare that they don't 919342SAndreas.Sandberg@arm.com * need further simulation to be completely drained. See Drainable for 929342SAndreas.Sandberg@arm.com * more information. 939342SAndreas.Sandberg@arm.com */ 949342SAndreas.Sandberg@arm.comclass DrainManager 959342SAndreas.Sandberg@arm.com{ 9610912Sandreas.sandberg@arm.com private: 9710912Sandreas.sandberg@arm.com DrainManager(); 9810912Sandreas.sandberg@arm.com DrainManager(DrainManager &) = delete; 9910912Sandreas.sandberg@arm.com ~DrainManager(); 10010912Sandreas.sandberg@arm.com 1019342SAndreas.Sandberg@arm.com public: 10210912Sandreas.sandberg@arm.com /** Get the singleton DrainManager instance */ 10310912Sandreas.sandberg@arm.com static DrainManager &instance() { return _instance; } 1049342SAndreas.Sandberg@arm.com 1059342SAndreas.Sandberg@arm.com /** 10610912Sandreas.sandberg@arm.com * Try to drain the system. 1079342SAndreas.Sandberg@arm.com * 10810912Sandreas.sandberg@arm.com * Try to drain the system and return true if all objects are in a 10910912Sandreas.sandberg@arm.com * the Drained state at which point the whole simulator is in a 11010912Sandreas.sandberg@arm.com * consistent state and ready for checkpointing or CPU 11110912Sandreas.sandberg@arm.com * handover. The simulation script must continue simulating until 11210912Sandreas.sandberg@arm.com * the simulation loop returns "Finished drain", at which point 11310912Sandreas.sandberg@arm.com * this method should be called again. This cycle should continue 11410912Sandreas.sandberg@arm.com * until this method returns true. 11510912Sandreas.sandberg@arm.com * 11610912Sandreas.sandberg@arm.com * @return true if all objects were drained successfully, false if 11710912Sandreas.sandberg@arm.com * more simulation is needed. 1189342SAndreas.Sandberg@arm.com */ 11910912Sandreas.sandberg@arm.com bool tryDrain(); 1209342SAndreas.Sandberg@arm.com 12110912Sandreas.sandberg@arm.com /** 12210912Sandreas.sandberg@arm.com * Resume normal simulation in a Drained system. 12310912Sandreas.sandberg@arm.com */ 12410912Sandreas.sandberg@arm.com void resume(); 12510912Sandreas.sandberg@arm.com 12610912Sandreas.sandberg@arm.com /** 12710912Sandreas.sandberg@arm.com * Run state fixups before a checkpoint restore operation 12810912Sandreas.sandberg@arm.com * 12910912Sandreas.sandberg@arm.com * The drain state of an object isn't stored in a checkpoint since 13010912Sandreas.sandberg@arm.com * the whole system is always going to be in the Drained state 13110912Sandreas.sandberg@arm.com * when the checkpoint is created. When the checkpoint is restored 13210912Sandreas.sandberg@arm.com * at a later stage, recreated objects will be in the Running 13310912Sandreas.sandberg@arm.com * state since the state isn't stored in checkpoints. This method 13410912Sandreas.sandberg@arm.com * performs state fixups on all Drainable objects and the 13510912Sandreas.sandberg@arm.com * DrainManager itself. 13610912Sandreas.sandberg@arm.com */ 13710912Sandreas.sandberg@arm.com void preCheckpointRestore(); 13810912Sandreas.sandberg@arm.com 13910912Sandreas.sandberg@arm.com /** Check if the system is drained */ 14011002Sandreas.sandberg@arm.com bool isDrained() const { return _state == DrainState::Drained; } 14110912Sandreas.sandberg@arm.com 14210912Sandreas.sandberg@arm.com /** Get the simulators global drain state */ 14311002Sandreas.sandberg@arm.com DrainState state() const { return _state; } 1449342SAndreas.Sandberg@arm.com 1459342SAndreas.Sandberg@arm.com /** 1469342SAndreas.Sandberg@arm.com * Notify the DrainManager that a Drainable object has finished 1479342SAndreas.Sandberg@arm.com * draining. 1489342SAndreas.Sandberg@arm.com */ 14910912Sandreas.sandberg@arm.com void signalDrainDone(); 1509342SAndreas.Sandberg@arm.com 15110912Sandreas.sandberg@arm.com public: 15210912Sandreas.sandberg@arm.com void registerDrainable(Drainable *obj); 15310912Sandreas.sandberg@arm.com void unregisterDrainable(Drainable *obj); 15410912Sandreas.sandberg@arm.com 15510912Sandreas.sandberg@arm.com private: 1569342SAndreas.Sandberg@arm.com /** 15711937Sandreas.sandberg@arm.com * Helper function to check if all Drainable objects are in a 15811937Sandreas.sandberg@arm.com * specific state. 15911937Sandreas.sandberg@arm.com */ 16011937Sandreas.sandberg@arm.com bool allInState(DrainState state) const; 16111937Sandreas.sandberg@arm.com 16211937Sandreas.sandberg@arm.com /** 16310912Sandreas.sandberg@arm.com * Thread-safe helper function to get the number of Drainable 16410912Sandreas.sandberg@arm.com * objects in a system. 1659342SAndreas.Sandberg@arm.com */ 16610912Sandreas.sandberg@arm.com size_t drainableCount() const; 1679342SAndreas.Sandberg@arm.com 16810912Sandreas.sandberg@arm.com /** Lock protecting the set of drainable objects */ 16910912Sandreas.sandberg@arm.com mutable std::mutex globalLock; 17010912Sandreas.sandberg@arm.com 17110912Sandreas.sandberg@arm.com /** Set of all drainable objects */ 17211859Sandreas.hansson@arm.com std::vector<Drainable *> _allDrainable; 17310912Sandreas.sandberg@arm.com 17410912Sandreas.sandberg@arm.com /** 17510912Sandreas.sandberg@arm.com * Number of objects still draining. This is flagged atomic since 17610912Sandreas.sandberg@arm.com * it can be manipulated by SimObjects living in different 17710912Sandreas.sandberg@arm.com * threads. 17810912Sandreas.sandberg@arm.com */ 17910912Sandreas.sandberg@arm.com std::atomic_uint _count; 18010912Sandreas.sandberg@arm.com 18110912Sandreas.sandberg@arm.com /** Global simulator drain state */ 18210912Sandreas.sandberg@arm.com DrainState _state; 18310912Sandreas.sandberg@arm.com 18410912Sandreas.sandberg@arm.com /** Singleton instance of the drain manager */ 18510912Sandreas.sandberg@arm.com static DrainManager _instance; 1869342SAndreas.Sandberg@arm.com}; 1879342SAndreas.Sandberg@arm.com 1889342SAndreas.Sandberg@arm.com/** 1899342SAndreas.Sandberg@arm.com * Interface for objects that might require draining before 1909342SAndreas.Sandberg@arm.com * checkpointing. 1919342SAndreas.Sandberg@arm.com * 1929342SAndreas.Sandberg@arm.com * An object's internal state needs to be drained when creating a 1939342SAndreas.Sandberg@arm.com * checkpoint, switching between CPU models, or switching between 1949342SAndreas.Sandberg@arm.com * timing models. Once the internal state has been drained from 19510912Sandreas.sandberg@arm.com * <i>all</i> objects in the simulator, the objects are serialized to 1969342SAndreas.Sandberg@arm.com * disc or the configuration change takes place. The process works as 1979342SAndreas.Sandberg@arm.com * follows (see simulate.py for details): 1989342SAndreas.Sandberg@arm.com * 1999342SAndreas.Sandberg@arm.com * <ol> 20010913Sandreas.sandberg@arm.com * <li>DrainManager::tryDrain() calls Drainable::drain() for every 20110913Sandreas.sandberg@arm.com * object in the system. Draining has completed if all of them 20210913Sandreas.sandberg@arm.com * return true. Otherwise, the drain manager keeps track of the 20310913Sandreas.sandberg@arm.com * objects that requested draining and waits for them to signal 20410913Sandreas.sandberg@arm.com * that they are done draining using the signalDrainDone() method. 2059342SAndreas.Sandberg@arm.com * 2069342SAndreas.Sandberg@arm.com * <li>Continue simulation. When an object has finished draining its 20710912Sandreas.sandberg@arm.com * internal state, it calls DrainManager::signalDrainDone() on the 20810913Sandreas.sandberg@arm.com * manager. The drain manager keeps track of the objects that 20910913Sandreas.sandberg@arm.com * haven't drained yet, simulation stops when the set of 21010913Sandreas.sandberg@arm.com * non-drained objects becomes empty. 2119342SAndreas.Sandberg@arm.com * 21210913Sandreas.sandberg@arm.com * <li>Check if any object still needs draining 21310913Sandreas.sandberg@arm.com * (DrainManager::tryDrain()), if so repeat the process above. 2149342SAndreas.Sandberg@arm.com * 2159342SAndreas.Sandberg@arm.com * <li>Serialize objects, switch CPU model, or change timing model. 2169342SAndreas.Sandberg@arm.com * 21711002Sandreas.sandberg@arm.com * <li>Call DrainManager::resume(), which in turn calls 21811002Sandreas.sandberg@arm.com * Drainable::drainResume() for all objects, and then continue the 21910913Sandreas.sandberg@arm.com * simulation. 2209342SAndreas.Sandberg@arm.com * </ol> 2219342SAndreas.Sandberg@arm.com * 2229342SAndreas.Sandberg@arm.com */ 2239342SAndreas.Sandberg@arm.comclass Drainable 2249342SAndreas.Sandberg@arm.com{ 22510912Sandreas.sandberg@arm.com friend class DrainManager; 22610912Sandreas.sandberg@arm.com 22710913Sandreas.sandberg@arm.com protected: 2289342SAndreas.Sandberg@arm.com Drainable(); 2299342SAndreas.Sandberg@arm.com virtual ~Drainable(); 2309342SAndreas.Sandberg@arm.com 2319342SAndreas.Sandberg@arm.com /** 23211002Sandreas.sandberg@arm.com * Notify an object that it needs to drain its state. 2339342SAndreas.Sandberg@arm.com * 23410913Sandreas.sandberg@arm.com * If the object does not need further simulation to drain 23511002Sandreas.sandberg@arm.com * internal buffers, it returns DrainState::Drained and 23611002Sandreas.sandberg@arm.com * automatically switches to the Drained state. If the object 23711002Sandreas.sandberg@arm.com * needs more simulation, it returns DrainState::Draining and 23811002Sandreas.sandberg@arm.com * automatically enters the Draining state. Other return values 23911002Sandreas.sandberg@arm.com * are invalid. 2409342SAndreas.Sandberg@arm.com * 2419342SAndreas.Sandberg@arm.com * @note An object that has entered the Drained state can be 24211002Sandreas.sandberg@arm.com * disturbed by other objects in the system and consequently stop 24311002Sandreas.sandberg@arm.com * being drained. These perturbations are not visible in the drain 24411002Sandreas.sandberg@arm.com * state. The simulator therefore repeats the draining process 24511002Sandreas.sandberg@arm.com * until all objects return DrainState::Drained on the first call 24611002Sandreas.sandberg@arm.com * to drain(). 2479342SAndreas.Sandberg@arm.com * 24811002Sandreas.sandberg@arm.com * @return DrainState::Drained if the object is drained at this 24911002Sandreas.sandberg@arm.com * point in time, DrainState::Draining if it needs further 25010913Sandreas.sandberg@arm.com * simulation. 2519342SAndreas.Sandberg@arm.com */ 25210913Sandreas.sandberg@arm.com virtual DrainState drain() = 0; 2539342SAndreas.Sandberg@arm.com 2549342SAndreas.Sandberg@arm.com /** 2559342SAndreas.Sandberg@arm.com * Resume execution after a successful drain. 25610913Sandreas.sandberg@arm.com */ 25710913Sandreas.sandberg@arm.com virtual void drainResume() {}; 25810913Sandreas.sandberg@arm.com 25910913Sandreas.sandberg@arm.com /** 26010913Sandreas.sandberg@arm.com * Signal that an object is drained 2619342SAndreas.Sandberg@arm.com * 26210913Sandreas.sandberg@arm.com * This method is designed to be called whenever an object enters 26310913Sandreas.sandberg@arm.com * into a state where it is ready to be drained. The method is 26410913Sandreas.sandberg@arm.com * safe to call multiple times and there is no need to check that 26510913Sandreas.sandberg@arm.com * draining has been requested before calling this method. 2669342SAndreas.Sandberg@arm.com */ 26710913Sandreas.sandberg@arm.com void signalDrainDone() const { 26810913Sandreas.sandberg@arm.com switch (_drainState) { 26910913Sandreas.sandberg@arm.com case DrainState::Running: 27010913Sandreas.sandberg@arm.com case DrainState::Drained: 27111937Sandreas.sandberg@arm.com case DrainState::Resuming: 27210913Sandreas.sandberg@arm.com return; 27310913Sandreas.sandberg@arm.com case DrainState::Draining: 27410913Sandreas.sandberg@arm.com _drainState = DrainState::Drained; 27510913Sandreas.sandberg@arm.com _drainManager.signalDrainDone(); 27610913Sandreas.sandberg@arm.com return; 27710913Sandreas.sandberg@arm.com } 27810913Sandreas.sandberg@arm.com } 2799342SAndreas.Sandberg@arm.com 28010913Sandreas.sandberg@arm.com public: 28110913Sandreas.sandberg@arm.com /** Return the current drain state of an object. */ 28210913Sandreas.sandberg@arm.com DrainState drainState() const { return _drainState; } 2839342SAndreas.Sandberg@arm.com 28411360Sandreas@sandberg.pp.se /** 28511360Sandreas@sandberg.pp.se * Notify a child process of a fork. 28611360Sandreas@sandberg.pp.se * 28711360Sandreas@sandberg.pp.se * When calling fork in gem5, we need to ensure that resources 28811360Sandreas@sandberg.pp.se * shared between the parent and the child are consistent. This 28911360Sandreas@sandberg.pp.se * method is intended to be overloaded to handle that. For 29011360Sandreas@sandberg.pp.se * example, an object could use this method to re-open input files 29111360Sandreas@sandberg.pp.se * to get a separate file description with a private file offset. 29211360Sandreas@sandberg.pp.se * 29311360Sandreas@sandberg.pp.se * This method is only called in the child of the fork. The call 29411360Sandreas@sandberg.pp.se * takes place in a drained system. 29511360Sandreas@sandberg.pp.se */ 29611360Sandreas@sandberg.pp.se virtual void notifyFork() {}; 29711360Sandreas@sandberg.pp.se 2989342SAndreas.Sandberg@arm.com private: 29910913Sandreas.sandberg@arm.com /** DrainManager interface to request a drain operation */ 30010913Sandreas.sandberg@arm.com DrainState dmDrain(); 30110913Sandreas.sandberg@arm.com /** DrainManager interface to request a resume operation */ 30210913Sandreas.sandberg@arm.com void dmDrainResume(); 30310913Sandreas.sandberg@arm.com 30410913Sandreas.sandberg@arm.com /** Convenience reference to the drain manager */ 30510912Sandreas.sandberg@arm.com DrainManager &_drainManager; 30610913Sandreas.sandberg@arm.com 30710913Sandreas.sandberg@arm.com /** 30810913Sandreas.sandberg@arm.com * Current drain state of the object. Needs to be mutable since 30910913Sandreas.sandberg@arm.com * objects need to be able to signal that they have transitioned 31010913Sandreas.sandberg@arm.com * into a Drained state even if the calling method is const. 31110913Sandreas.sandberg@arm.com */ 31210913Sandreas.sandberg@arm.com mutable DrainState _drainState; 3139342SAndreas.Sandberg@arm.com}; 3149342SAndreas.Sandberg@arm.com 3159342SAndreas.Sandberg@arm.com#endif 316