#! /bin/bash # # Copyright (c) 2015 ARM Limited # All rights reserved # # The license below extends only to copyright in the software and shall # not be construed as granting a license to any other intellectual # property including but not limited to intellectual property relating # to a hardware implementation of the functionality of the software # licensed hereunder. You may use the software subject to the license # terms below provided that you ensure that this notice is replicated # unmodified and in its entirety in all distributions of the software, # modified or unmodified, in source code or in binary form. # # Copyright (c) 2015 University of Illinois Urbana Champaign # All rights reserved # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer; # redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution; # neither the name of the copyright holders nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Authors: Gabor Dozsa # Mohammad Alian # This is a wrapper script to run a dist gem5 simulations. # See the usage_func() below for hints on how to use it. Also, # there are some examples in the util/dist directory (e.g. # see util/dist/test-2nodes-AArch64.sh) # # # Allocated hosts/cores are assumed to be listed in the LSB_MCPU_HOSTS # environment variable (which is what LSF does by default). # E.g. LSB_MCPU_HOSTS=\"hname1 2 hname2 4\" means we have altogether 6 slots # allocated to launch the gem5 processes, 2 of them are on host hname1 # and 4 of them are on host hname2. # If LSB_MCPU_HOSTS environment variable is not defined then we launch all # processes on the localhost. # # Each gem5 process are passed in a unique rank ID [0..N-1] via the kernel # boot params. The total number of gem5 processes is also passed in. # These values can be used in the boot script to configure the MAC/IP # addresses - among other things (see util/dist/bootscript.rcS). # # Each gem5 process will create an m5out.$GEM5_RANK directory for # the usual output files. Furthermore, there will be a separate log file # for each ssh session (we use ssh to start gem5 processes) and one for # the server. These are called log.$GEM5_RANK and log.switch. # # print help usage_func () { echo "Usage:$0 [-debug] [-n nnodes] [-r rundir] [-c ckptdir] [-p port] [-sw switch] [--sw-args sw_args] [-fs fullsystem] [--fs-args fs_args] [--cf-args conf_args] [--m5-args m5_args] -x gem5_exe " echo " -debug : debug mode (start gem5 in gdb)" echo " nnodes : number of gem5 processes" echo " rundir : run simulation under this path. If not specified, current dir will be used" echo " ckptdir : dump/restore checkpoints to/from this path. If not specified, current dir will be used" echo " fullsystem: fullsystem config file" echo " fs_args : fullsystem config specific argument list: arg1 arg2 ..." echo " port : switch listen port" echo " switch : switch config file" echo " sw_args : switch config specific argument list: arg1 arg2 ..." echo " conf_args : common (for both fullsystem and switch) config argument list: arg1 arg2 ..." echo " gem5_exe : gem5 executable (full path required)" echo " m5_args : common m5 argument list (e.g. debug flags): arg1 arg2 ..." echo "Note: if no LSF slots allocation is found all proceses are launched on the localhost." } # Process (optional) command line options FS_ARGS=" " SW_ARGS=" " CF_ARGS=" " M5_ARGS=" " while (($# > 0)) do case "x$1" in x-debug) GEM5_DEBUG="-debug" shift 1 ;; x-n|x-nodes) NNODES=$2 shift 2 ;; x-r|x-rundir) RUN_DIR=$2 shift 2 ;; x-c|x-ckptdir) CKPT_DIR=$2 shift 2 ;; x-p|x-port) SW_PORT=$2 shift 2 ;; x-s|x-switch) SW_CONFIG=$2 shift 2 ;; x--sw-args) CUR_ARGS="SW_ARGS" shift 1 ;; x-f|x-fullsystem) FS_CONFIG=$2 shift 2 ;; x--fs-args) CUR_ARGS="FS_ARGS" shift 1 ;; x--cf-args) CUR_ARGS="CF_ARGS" shift 1 ;; x--m5-args) CUR_ARGS="M5_ARGS" shift 1 ;; x-x) GEM5_EXE=$2 shift 2 ;; x-*) [ -n "$CUR_ARGS" ] || { echo "Unexpected arg: $1"; usage_func; exit -1; } case "x$2" in x-*|x) eval $CUR_ARGS=\"${!CUR_ARGS} $1\" shift 1 ;; *) eval $CUR_ARGS=\"${!CUR_ARGS} $1 $2\" shift 2 ;; esac ;; *) echo "Unknown arg: $1" usage_func exit 1 ;; esac done # Default values to use (in case they are not defined as command line options) DEFAULT_FS_CONFIG=$M5_PATH/configs/example/fs.py DEFAULT_SW_CONFIG=$M5_PATH/configs/example/sw.py DEFAULT_SW_PORT=2200 [ -z "$FS_CONFIG" ] && FS_CONFIG=$DEFAULT_FS_CONFIG [ -z "$SW_CONFIG" ] && SW_CONFIG=$DEFAULT_SW_CONFIG [ -z "$SW_PORT" ] && SW_PORT=$DEFAULT_SW_PORT [ -z "$NNODES" ] && NNODES=2 [ -z "$RUN_DIR" ] && RUN_DIR=$(pwd) [ -z "$CKPT_DIR" ] && CKPT_DIR=$(pwd) # Check if all the executables we need exist [ -f "$FS_CONFIG" ] || { echo "FS config ${FS_CONFIG} not found"; exit 1; } [ -f "$SW_CONFIG" ] || { echo "Switch config ${SW_CONFIG} not found"; exit 1; } [ -x "$GEM5_EXE" ] || { echo "Executable ${GEM5_EXE} not found"; exit 1; } # make sure that RUN_DIR exists mkdir -p $RUN_DIR > /dev/null 2>&1 declare -a SSH_PIDS declare -a HOSTS declare -a NCORES # Find out which cluster hosts/slots are allocated or # use localhost if there is no LSF allocation. # We assume that allocated slots are listed in the LSB_MCPU_HOSTS # environment variable in the form: # host1 nslots1 host2 nslots2 ... # (This is what LSF does by default.) NH=0 [ "x$LSB_MCPU_HOSTS" != "x" ] || LSB_MCPU_HOSTS="127.0.0.1 $NNODES" host="" for hc in $LSB_MCPU_HOSTS do if [ "x$host" == "x" ] then host=$hc HOSTS+=($hc) else NCORES+=($hc) ((NH+=hc)) host="" fi done ((NNODES==NH)) || { echo "(E) Number of cluster slots ($NH) and gem5 instances ($N) differ"; exit -1; } # function to clean up and abort if something goes wrong abort_func () { echo echo "KILLED $(date)" # Try to Kill the server first. That should trigger an exit for all connected # gem5 processes. [ "x$SW_PID" != "x" ] && kill $SW_PID 2>/dev/null sleep 20 # (try to) kill gem5 processes - just in case something went wrong with the # server triggered exit bname=$(basename $GEM5_EXE) killall -q -s SIGKILL $bname for h in ${HOSTS[@]} do ssh $h killall -q -s SIGKILL $bname done sleep 5 # kill the watchdog [ "x$WATCHDOG_PID" != "x" ] && kill $WATCHDOG_PID 2>/dev/null exit -1 } # We need a watchdog to trigger full clean up if a gem5 process dies watchdog_func () { while true do sleep 30 ((NDEAD=0)) for p in ${SSH_PIDS[*]} do kill -0 $p 2>/dev/null || ((NDEAD+=1)) done kill -0 $SW_PID || ((NDEAD+=1)) if ((NDEAD>0)) then # we may be in the middle of an orderly termination, # give it some time to complete before reporting abort sleep 60 echo -n "(I) (some) gem5 process(es) exited" abort_func fi done } # This function launches the gem5 processes. The only purpose is to enable # launching gem5 processes under gdb control for debugging start_func () { local N=$1 local HOST=$2 local ENV_ARGS=$3 shift 3 if [ "x$GEM5_DEBUG" != "x" ] then echo "DEBUG starting terminal..." MY_ARGS="$@" xterm -e "gdb --args $MY_ARGS" & else ssh $HOST $ENV_ARGS "$@" &> $RUN_DIR/log.$N & fi } # block till the gem5 process starts connected () { FILE=$1 STRING=$2 echo -n "waiting for $3 to start " while : ; do kill -0 $4 || { echo "Failed to start $3"; exit -1; } [[ -f "$FILE" ]] && \ grep -q "$STRING" "$FILE" && \ echo -e "\nnode #$3 started" && \ break sleep 2 echo -n "." done } # Trigger full clean up in case we are being killed by external signal trap 'abort_func' INT TERM # env args to be passed explicitly to gem5 processes started via ssh ENV_ARGS="LD_LIBRARY_PATH=$LD_LIBRARY_PATH M5_PATH=$M5_PATH" #cleanup log files before starting gem5 processes rm $RUN_DIR/log.switch > /dev/null 2>&1 # make sure that CKPT_DIR exists mkdir -p $CKPT_DIR/m5out.switch > /dev/null 2>&1 # launch switch gem5 SW_HOST=${HOSTS[0]} echo "launch switch gem5 process on $SW_HOST ..." start_func "switch" $SW_HOST "$ENV_ARGS" $GEM5_EXE -d $RUN_DIR/m5out.switch \ $M5_ARGS \ $SW_CONFIG \ $SW_ARGS \ $CF_ARGS \ --checkpoint-dir=$CKPT_DIR/m5out.switch \ --is-switch \ --dist-size=$NNODES \ --dist-server-port=$SW_PORT SW_PID=$! # block here till switch process starts connected $RUN_DIR/log.switch "tcp_iface listening on port" "switch" $SW_PID LINE=$(grep -r "tcp_iface listening on port" $RUN_DIR/log.switch) IFS=' ' read -ra ADDR <<< "$LINE" # actual port that switch is listening on may be different # from what we specified if the port was busy SW_PORT=${ADDR[5]} # Now launch all the gem5 processes with ssh. echo "START $(date)" n=0 for ((i=0; i < ${#HOSTS[@]}; i++)) do h=${HOSTS[$i]} for ((j=0; j < ${NCORES[i]}; j++)) do #cleanup log files before starting gem5 processes rm $RUN_DIR/log.$n > /dev/null 2>&1 # make sure that CKPT_DIR exists mkdir -p $CKPT_DIR/m5out.$n > /dev/null 2>&1 echo "starting gem5 on $h ..." start_func $n $h "$ENV_ARGS" $GEM5_EXE -d $RUN_DIR/m5out.$n \ $M5_ARGS \ $FS_CONFIG \ $FS_ARGS \ $CF_ARGS \ --checkpoint-dir=$CKPT_DIR/m5out.$n \ --dist \ --dist-rank=$n \ --dist-size=$NNODES \ --dist-server-name=${HOSTS[0]} \ --dist-server-port=$SW_PORT SSH_PIDS[$n]=$! ((n+=1)) done done # Wait here if it is a debug session [ "x$GEM5_DEBUG" == "x" ] || { echo "DEBUG session"; wait $SW_PID; exit -1; } # start watchdog to trigger complete abort (after a grace period) if any # gem5 process dies watchdog_func & WATCHDOG_PID=$! # wait for exit statuses ((NFAIL=0)) for p in ${SSH_PIDS[*]} do wait $p || ((NFAIL+=1)) done wait $SW_PID || ((NFAIL+=1)) # all done, let's terminate the watchdog kill $WATCHDOG_PID 2>/dev/null if ((NFAIL==0)) then echo "EXIT $(date)" else echo "ABORT $(date)" fi