1# Copyright (c) 2015 Advanced Micro Devices, Inc.
2# All rights reserved.
3#
4# For use for simulation and test purposes only
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9# 1. Redistributions of source code must retain the above copyright notice,
10# this list of conditions and the following disclaimer.
11#
12# 2. Redistributions in binary form must reproduce the above copyright notice,
13# this list of conditions and the following disclaimer in the documentation
14# and/or other materials provided with the distribution.
15#
16# 3. Neither the name of the copyright holder nor the names of its
17# contributors may be used to endorse or promote products derived from this
18# software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
24# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30# POSSIBILITY OF SUCH DAMAGE.
31#
32# Authors: Sooraj Puthoor
33
34from __future__ import print_function
35from __future__ import absolute_import
36
37import optparse, os, re
38import math
39import glob
40import inspect
41
42import m5
43from m5.objects import *
44from m5.util import addToPath
45
46addToPath('../')
47
48from ruby import Ruby
49
50from common import Options
51from common import Simulation
52from common import GPUTLBOptions, GPUTLBConfig
53
54########################## Script Options ########################
55def setOption(parser, opt_str, value = 1):
56    # check to make sure the option actually exists
57    if not parser.has_option(opt_str):
58        raise Exception("cannot find %s in list of possible options" % opt_str)
59
60    opt = parser.get_option(opt_str)
61    # set the value
62    exec("parser.values.%s = %s" % (opt.dest, value))
63
64def getOption(parser, opt_str):
65    # check to make sure the option actually exists
66    if not parser.has_option(opt_str):
67        raise Exception("cannot find %s in list of possible options" % opt_str)
68
69    opt = parser.get_option(opt_str)
70    # get the value
71    exec("return_value = parser.values.%s" % opt.dest)
72    return return_value
73
74# Adding script options
75parser = optparse.OptionParser()
76Options.addCommonOptions(parser)
77Options.addSEOptions(parser)
78
79parser.add_option("--cpu-only-mode", action="store_true", default=False,
80                  help="APU mode. Used to take care of problems in "\
81                       "Ruby.py while running APU protocols")
82parser.add_option("-k", "--kernel-files",
83                  help="file(s) containing GPU kernel code (colon separated)")
84parser.add_option("-u", "--num-compute-units", type="int", default=1,
85                  help="number of GPU compute units"),
86parser.add_option("--num-cp", type="int", default=0,
87                  help="Number of GPU Command Processors (CP)")
88parser.add_option("--benchmark-root", help="Root of benchmark directory tree")
89
90# not super important now, but to avoid putting the number 4 everywhere, make
91# it an option/knob
92parser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \
93                  "sharing an SQC (icache, and thus icache TLB)")
94parser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \
95                  "per CU")
96parser.add_option("--wf-size", type="int", default=64,
97                  help="Wavefront size(in workitems)")
98parser.add_option("--sp-bypass-path-length", type="int", default=4, \
99                  help="Number of stages of bypass path in vector ALU for Single Precision ops")
100parser.add_option("--dp-bypass-path-length", type="int", default=4, \
101                  help="Number of stages of bypass path in vector ALU for Double Precision ops")
102# issue period per SIMD unit: number of cycles before issuing another vector
103parser.add_option("--issue-period", type="int", default=4, \
104                  help="Number of cycles per vector instruction issue period")
105parser.add_option("--glbmem-wr-bus-width", type="int", default=32, \
106                  help="VGPR to Coalescer (Global Memory) data bus width in bytes")
107parser.add_option("--glbmem-rd-bus-width", type="int", default=32, \
108                  help="Coalescer to VGPR (Global Memory) data bus width in bytes")
109# Currently we only support 1 local memory pipe
110parser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \
111                  help="Number of Shared Memory pipelines per CU")
112# Currently we only support 1 global memory pipe
113parser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \
114                  help="Number of Global Memory pipelines per CU")
115parser.add_option("--wfs-per-simd", type="int", default=10, help="Number of " \
116                  "WF slots per SIMD")
117
118parser.add_option("--vreg-file-size", type="int", default=2048,
119                  help="number of physical vector registers per SIMD")
120parser.add_option("--bw-scalor", type="int", default=0,
121                  help="bandwidth scalor for scalability analysis")
122parser.add_option("--CPUClock", type="string", default="2GHz",
123                  help="CPU clock")
124parser.add_option("--GPUClock", type="string", default="1GHz",
125                  help="GPU clock")
126parser.add_option("--cpu-voltage", action="store", type="string",
127                  default='1.0V',
128                  help = """CPU  voltage domain""")
129parser.add_option("--gpu-voltage", action="store", type="string",
130                  default='1.0V',
131                  help = """CPU  voltage domain""")
132parser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST",
133                  help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)")
134parser.add_option("--xact-cas-mode", action="store_true",
135                  help="enable load_compare mode (transactional CAS)")
136parser.add_option("--SegFaultDebug",action="store_true",
137                 help="checks for GPU seg fault before TLB access")
138parser.add_option("--FunctionalTLB",action="store_true",
139                 help="Assumes TLB has no latency")
140parser.add_option("--LocalMemBarrier",action="store_true",
141                 help="Barrier does not wait for writethroughs to complete")
142parser.add_option("--countPages", action="store_true",
143                 help="Count Page Accesses and output in per-CU output files")
144parser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\
145                  "TLBs")
146parser.add_option("--pf-type", type="string", help="type of prefetch: "\
147                  "PF_CU, PF_WF, PF_PHASE, PF_STRIDE")
148parser.add_option("--pf-stride", type="int", help="set prefetch stride")
149parser.add_option("--numLdsBanks", type="int", default=32,
150                  help="number of physical banks per LDS module")
151parser.add_option("--ldsBankConflictPenalty", type="int", default=1,
152                  help="number of cycles per LDS bank conflict")
153parser.add_option('--fast-forward-pseudo-op', action='store_true',
154                  help = 'fast forward using kvm until the m5_switchcpu'
155                  ' pseudo-op is encountered, then switch cpus. subsequent'
156                  ' m5_switchcpu pseudo-ops will toggle back and forth')
157parser.add_option('--outOfOrderDataDelivery', action='store_true',
158                  default=False, help='enable OoO data delivery in the GM'
159                  ' pipeline')
160
161Ruby.define_options(parser)
162
163#add TLB options to the parser
164GPUTLBOptions.tlb_options(parser)
165
166(options, args) = parser.parse_args()
167
168# The GPU cache coherence protocols only work with the backing store
169setOption(parser, "--access-backing-store")
170
171# if benchmark root is specified explicitly, that overrides the search path
172if options.benchmark_root:
173    benchmark_path = [options.benchmark_root]
174else:
175    # Set default benchmark search path to current dir
176    benchmark_path = ['.']
177
178########################## Sanity Check ########################
179
180# Currently the gpu model requires ruby
181if buildEnv['PROTOCOL'] == 'None':
182    fatal("GPU model requires ruby")
183
184# Currently the gpu model requires only timing or detailed CPU
185if not (options.cpu_type == "TimingSimpleCPU" or
186   options.cpu_type == "DerivO3CPU"):
187    fatal("GPU model requires TimingSimpleCPU or DerivO3CPU")
188
189# This file can support multiple compute units
190assert(options.num_compute_units >= 1)
191
192# Currently, the sqc (I-Cache of GPU) is shared by
193# multiple compute units(CUs). The protocol works just fine
194# even if sqc is not shared. Overriding this option here
195# so that the user need not explicitly set this (assuming
196# sharing sqc is the common usage)
197n_cu = options.num_compute_units
198num_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc))
199options.num_sqc = num_sqc # pass this to Ruby
200
201########################## Creating the GPU system ########################
202# shader is the GPU
203shader = Shader(n_wf = options.wfs_per_simd,
204                clk_domain = SrcClockDomain(
205                    clock = options.GPUClock,
206                    voltage_domain = VoltageDomain(
207                        voltage = options.gpu_voltage)))
208
209# GPU_RfO(Read For Ownership) implements SC/TSO memory model.
210# Other GPU protocols implement release consistency at GPU side.
211# So, all GPU protocols other than GPU_RfO should make their writes
212# visible to the global memory and should read from global memory
213# during kernal boundary. The pipeline initiates(or do not initiate)
214# the acquire/release operation depending on this impl_kern_boundary_sync
215# flag. This flag=true means pipeline initiates a acquire/release operation
216# at kernel boundary.
217if buildEnv['PROTOCOL'] == 'GPU_RfO':
218    shader.impl_kern_boundary_sync = False
219else:
220    shader.impl_kern_boundary_sync = True
221
222# Switching off per-lane TLB by default
223per_lane = False
224if options.TLB_config == "perLane":
225    per_lane = True
226
227# List of compute units; one GPU can have multiple compute units
228compute_units = []
229for i in range(n_cu):
230    compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane,
231                                     num_SIMDs = options.simds_per_cu,
232                                     wfSize = options.wf_size,
233                                     spbypass_pipe_length = options.sp_bypass_path_length,
234                                     dpbypass_pipe_length = options.dp_bypass_path_length,
235                                     issue_period = options.issue_period,
236                                     coalescer_to_vrf_bus_width = \
237                                     options.glbmem_rd_bus_width,
238                                     vrf_to_coalescer_bus_width = \
239                                     options.glbmem_wr_bus_width,
240                                     num_global_mem_pipes = \
241                                     options.glb_mem_pipes_per_cu,
242                                     num_shared_mem_pipes = \
243                                     options.shr_mem_pipes_per_cu,
244                                     n_wf = options.wfs_per_simd,
245                                     execPolicy = options.CUExecPolicy,
246                                     xactCasMode = options.xact_cas_mode,
247                                     debugSegFault = options.SegFaultDebug,
248                                     functionalTLB = options.FunctionalTLB,
249                                     localMemBarrier = options.LocalMemBarrier,
250                                     countPages = options.countPages,
251                                     localDataStore = \
252                                     LdsState(banks = options.numLdsBanks,
253                                              bankConflictPenalty = \
254                                              options.ldsBankConflictPenalty),
255                                     out_of_order_data_delivery =
256                                             options.outOfOrderDataDelivery))
257    wavefronts = []
258    vrfs = []
259    for j in range(options.simds_per_cu):
260        for k in range(shader.n_wf):
261            wavefronts.append(Wavefront(simdId = j, wf_slot_id = k,
262                                        wfSize = options.wf_size))
263        vrfs.append(VectorRegisterFile(simd_id=j,
264                              num_regs_per_simd=options.vreg_file_size))
265    compute_units[-1].wavefronts = wavefronts
266    compute_units[-1].vector_register_file = vrfs
267    if options.TLB_prefetch:
268        compute_units[-1].prefetch_depth = options.TLB_prefetch
269        compute_units[-1].prefetch_prev_type = options.pf_type
270
271    # attach the LDS and the CU to the bus (actually a Bridge)
272    compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave
273    compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort
274
275# Attach compute units to GPU
276shader.CUs = compute_units
277
278########################## Creating the CPU system ########################
279options.num_cpus = options.num_cpus
280
281# The shader core will be whatever is after the CPU cores are accounted for
282shader_idx = options.num_cpus
283
284# The command processor will be whatever is after the shader is accounted for
285cp_idx = shader_idx + 1
286cp_list = []
287
288# List of CPUs
289cpu_list = []
290
291CpuClass, mem_mode = Simulation.getCPUClass(options.cpu_type)
292if CpuClass == AtomicSimpleCPU:
293    fatal("AtomicSimpleCPU is not supported")
294if mem_mode != 'timing':
295    fatal("Only the timing memory mode is supported")
296shader.timing = True
297
298if options.fast_forward and options.fast_forward_pseudo_op:
299    fatal("Cannot fast-forward based both on the number of instructions and"
300          " on pseudo-ops")
301fast_forward = options.fast_forward or options.fast_forward_pseudo_op
302
303if fast_forward:
304    FutureCpuClass, future_mem_mode = CpuClass, mem_mode
305
306    CpuClass = X86KvmCPU
307    mem_mode = 'atomic_noncaching'
308    # Leave shader.timing untouched, because its value only matters at the
309    # start of the simulation and because we require switching cpus
310    # *before* the first kernel launch.
311
312    future_cpu_list = []
313
314    # Initial CPUs to be used during fast-forwarding.
315    for i in range(options.num_cpus):
316        cpu = CpuClass(cpu_id = i,
317                       clk_domain = SrcClockDomain(
318                           clock = options.CPUClock,
319                           voltage_domain = VoltageDomain(
320                               voltage = options.cpu_voltage)))
321        cpu_list.append(cpu)
322
323        if options.fast_forward:
324            cpu.max_insts_any_thread = int(options.fast_forward)
325
326if fast_forward:
327    MainCpuClass = FutureCpuClass
328else:
329    MainCpuClass = CpuClass
330
331# CPs to be used throughout the simulation.
332for i in range(options.num_cp):
333    cp = MainCpuClass(cpu_id = options.num_cpus + i,
334                      clk_domain = SrcClockDomain(
335                          clock = options.CPUClock,
336                          voltage_domain = VoltageDomain(
337                              voltage = options.cpu_voltage)))
338    cp_list.append(cp)
339
340# Main CPUs (to be used after fast-forwarding if fast-forwarding is specified).
341for i in range(options.num_cpus):
342    cpu = MainCpuClass(cpu_id = i,
343                       clk_domain = SrcClockDomain(
344                           clock = options.CPUClock,
345                           voltage_domain = VoltageDomain(
346                               voltage = options.cpu_voltage)))
347    if fast_forward:
348        cpu.switched_out = True
349        future_cpu_list.append(cpu)
350    else:
351        cpu_list.append(cpu)
352
353########################## Creating the GPU dispatcher ########################
354# Dispatcher dispatches work from host CPU to GPU
355host_cpu = cpu_list[0]
356dispatcher = GpuDispatcher()
357
358########################## Create and assign the workload ########################
359# Check for rel_path in elements of base_list using test, returning
360# the first full path that satisfies test
361def find_path(base_list, rel_path, test):
362    for base in base_list:
363        if not base:
364            # base could be None if environment var not set
365            continue
366        full_path = os.path.join(base, rel_path)
367        if test(full_path):
368            return full_path
369    fatal("%s not found in %s" % (rel_path, base_list))
370
371def find_file(base_list, rel_path):
372    return find_path(base_list, rel_path, os.path.isfile)
373
374executable = find_path(benchmark_path, options.cmd, os.path.exists)
375# it's common for a benchmark to be in a directory with the same
376# name as the executable, so we handle that automatically
377if os.path.isdir(executable):
378    benchmark_path = [executable]
379    executable = find_file(benchmark_path, options.cmd)
380if options.kernel_files:
381    kernel_files = [find_file(benchmark_path, f)
382                    for f in options.kernel_files.split(':')]
383else:
384    # if kernel_files is not set, see if there's a unique .asm file
385    # in the same directory as the executable
386    kernel_path = os.path.dirname(executable)
387    kernel_files = glob.glob(os.path.join(kernel_path, '*.asm'))
388    if kernel_files:
389        print("Using GPU kernel code file(s)", ",".join(kernel_files))
390    else:
391        fatal("Can't locate kernel code (.asm) in " + kernel_path)
392
393# OpenCL driver
394driver = ClDriver(filename="hsa", codefile=kernel_files)
395for cpu in cpu_list:
396    cpu.createThreads()
397    cpu.workload = Process(executable = executable,
398                           cmd = [options.cmd] + options.options.split(),
399                           drivers = [driver])
400for cp in cp_list:
401    cp.workload = host_cpu.workload
402
403if fast_forward:
404    for i in range(len(future_cpu_list)):
405        future_cpu_list[i].workload = cpu_list[i].workload
406        future_cpu_list[i].createThreads()
407
408########################## Create the overall system ########################
409# List of CPUs that must be switched when moving between KVM and simulation
410if fast_forward:
411    switch_cpu_list = \
412        [(cpu_list[i], future_cpu_list[i]) for i in range(options.num_cpus)]
413
414# Full list of processing cores in the system. Note that
415# dispatcher is also added to cpu_list although it is
416# not a processing element
417cpu_list = cpu_list + [shader] + cp_list + [dispatcher]
418
419# creating the overall system
420# notice the cpu list is explicitly added as a parameter to System
421system = System(cpu = cpu_list,
422                mem_ranges = [AddrRange(options.mem_size)],
423                cache_line_size = options.cacheline_size,
424                mem_mode = mem_mode)
425if fast_forward:
426    system.future_cpu = future_cpu_list
427system.voltage_domain = VoltageDomain(voltage = options.sys_voltage)
428system.clk_domain = SrcClockDomain(clock =  options.sys_clock,
429                                   voltage_domain = system.voltage_domain)
430
431if fast_forward:
432    have_kvm_support = 'BaseKvmCPU' in globals()
433    if have_kvm_support and buildEnv['TARGET_ISA'] == "x86":
434        system.vm = KvmVM()
435        for i in range(len(host_cpu.workload)):
436            host_cpu.workload[i].useArchPT = True
437            host_cpu.workload[i].kvmInSE = True
438    else:
439        fatal("KvmCPU can only be used in SE mode with x86")
440
441# configure the TLB hierarchy
442GPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx)
443
444# create Ruby system
445system.piobus = IOXBar(width=32, response_latency=0,
446                       frontend_latency=0, forward_latency=0)
447Ruby.create_system(options, None, system)
448system.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,
449                                    voltage_domain = system.voltage_domain)
450
451# attach the CPU ports to Ruby
452for i in range(options.num_cpus):
453    ruby_port = system.ruby._cpu_ports[i]
454
455    # Create interrupt controller
456    system.cpu[i].createInterruptController()
457
458    # Connect cache port's to ruby
459    system.cpu[i].icache_port = ruby_port.slave
460    system.cpu[i].dcache_port = ruby_port.slave
461
462    ruby_port.mem_master_port = system.piobus.slave
463    if buildEnv['TARGET_ISA'] == "x86":
464        system.cpu[i].interrupts[0].pio = system.piobus.master
465        system.cpu[i].interrupts[0].int_master = system.piobus.slave
466        system.cpu[i].interrupts[0].int_slave = system.piobus.master
467        if fast_forward:
468            system.cpu[i].itb.walker.port = ruby_port.slave
469            system.cpu[i].dtb.walker.port = ruby_port.slave
470
471# attach CU ports to Ruby
472# Because of the peculiarities of the CP core, you may have 1 CPU but 2
473# sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be
474# hooked up until after the CP. To make this script generic, figure out
475# the index as below, but note that this assumes there is one sequencer
476# per compute unit and one sequencer per SQC for the math to work out
477# correctly.
478gpu_port_idx = len(system.ruby._cpu_ports) \
479               - options.num_compute_units - options.num_sqc
480gpu_port_idx = gpu_port_idx - options.num_cp * 2
481
482wavefront_size = options.wf_size
483for i in range(n_cu):
484    # The pipeline issues wavefront_size number of uncoalesced requests
485    # in one GPU issue cycle. Hence wavefront_size mem ports.
486    for j in range(wavefront_size):
487        system.cpu[shader_idx].CUs[i].memory_port[j] = \
488                  system.ruby._cpu_ports[gpu_port_idx].slave[j]
489    gpu_port_idx += 1
490
491for i in range(n_cu):
492    if i > 0 and not i % options.cu_per_sqc:
493        print("incrementing idx on ", i)
494        gpu_port_idx += 1
495    system.cpu[shader_idx].CUs[i].sqc_port = \
496            system.ruby._cpu_ports[gpu_port_idx].slave
497gpu_port_idx = gpu_port_idx + 1
498
499# attach CP ports to Ruby
500for i in range(options.num_cp):
501    system.cpu[cp_idx].createInterruptController()
502    system.cpu[cp_idx].dcache_port = \
503                system.ruby._cpu_ports[gpu_port_idx + i * 2].slave
504    system.cpu[cp_idx].icache_port = \
505                system.ruby._cpu_ports[gpu_port_idx + i * 2 + 1].slave
506    system.cpu[cp_idx].interrupts[0].pio = system.piobus.master
507    system.cpu[cp_idx].interrupts[0].int_master = system.piobus.slave
508    system.cpu[cp_idx].interrupts[0].int_slave = system.piobus.master
509    cp_idx = cp_idx + 1
510
511# connect dispatcher to the system.piobus
512dispatcher.pio = system.piobus.master
513dispatcher.dma = system.piobus.slave
514
515################# Connect the CPU and GPU via GPU Dispatcher ###################
516# CPU rings the GPU doorbell to notify a pending task
517# using this interface.
518# And GPU uses this interface to notify the CPU of task completion
519# The communcation happens through emulated driver.
520
521# Note this implicit setting of the cpu_pointer, shader_pointer and tlb array
522# parameters must be after the explicit setting of the System cpu list
523if fast_forward:
524    shader.cpu_pointer = future_cpu_list[0]
525    dispatcher.cpu = future_cpu_list[0]
526else:
527    shader.cpu_pointer = host_cpu
528    dispatcher.cpu = host_cpu
529dispatcher.shader_pointer = shader
530dispatcher.cl_driver = driver
531
532########################## Start simulation ########################
533
534root = Root(system=system, full_system=False)
535m5.ticks.setGlobalFrequency('1THz')
536if options.abs_max_tick:
537    maxtick = options.abs_max_tick
538else:
539    maxtick = m5.MaxTick
540
541# Benchmarks support work item annotations
542Simulation.setWorkCountOptions(system, options)
543
544# Checkpointing is not supported by APU model
545if (options.checkpoint_dir != None or
546    options.checkpoint_restore != None):
547    fatal("Checkpointing not supported by apu model")
548
549checkpoint_dir = None
550m5.instantiate(checkpoint_dir)
551
552# Map workload to this address space
553host_cpu.workload[0].map(0x10000000, 0x200000000, 4096)
554
555if options.fast_forward:
556    print("Switch at instruction count: %d" % cpu_list[0].max_insts_any_thread)
557
558exit_event = m5.simulate(maxtick)
559
560if options.fast_forward:
561    if exit_event.getCause() == "a thread reached the max instruction count":
562        m5.switchCpus(system, switch_cpu_list)
563        print("Switched CPUS @ tick %s" % (m5.curTick()))
564        m5.stats.reset()
565        exit_event = m5.simulate(maxtick - m5.curTick())
566elif options.fast_forward_pseudo_op:
567    while exit_event.getCause() == "switchcpu":
568        # If we are switching *to* kvm, then the current stats are meaningful
569        # Note that we don't do any warmup by default
570        if type(switch_cpu_list[0][0]) == FutureCpuClass:
571            print("Dumping stats...")
572            m5.stats.dump()
573        m5.switchCpus(system, switch_cpu_list)
574        print("Switched CPUS @ tick %s" % (m5.curTick()))
575        m5.stats.reset()
576        # This lets us switch back and forth without keeping a counter
577        switch_cpu_list = [(x[1], x[0]) for x in switch_cpu_list]
578        exit_event = m5.simulate(maxtick - m5.curTick())
579
580print("Ticks:", m5.curTick())
581print('Exiting because ', exit_event.getCause())
582sys.exit(exit_event.getCode())
583