35 36from __future__ import print_function 37 38import optparse, os, re 39import math 40import glob 41import inspect 42 43import m5 44from m5.objects import * 45from m5.util import addToPath 46 47addToPath('../') 48 49from ruby import Ruby 50 51from common import Options 52from common import Simulation 53from common import GPUTLBOptions, GPUTLBConfig 54 55########################## Script Options ######################## 56def setOption(parser, opt_str, value = 1): 57 # check to make sure the option actually exists 58 if not parser.has_option(opt_str): 59 raise Exception("cannot find %s in list of possible options" % opt_str) 60 61 opt = parser.get_option(opt_str) 62 # set the value 63 exec("parser.values.%s = %s" % (opt.dest, value)) 64 65def getOption(parser, opt_str): 66 # check to make sure the option actually exists 67 if not parser.has_option(opt_str): 68 raise Exception("cannot find %s in list of possible options" % opt_str) 69 70 opt = parser.get_option(opt_str) 71 # get the value 72 exec("return_value = parser.values.%s" % opt.dest) 73 return return_value 74 75# Adding script options 76parser = optparse.OptionParser() 77Options.addCommonOptions(parser) 78Options.addSEOptions(parser) 79 80parser.add_option("--cpu-only-mode", action="store_true", default=False, 81 help="APU mode. Used to take care of problems in "\ 82 "Ruby.py while running APU protocols") 83parser.add_option("-k", "--kernel-files", 84 help="file(s) containing GPU kernel code (colon separated)") 85parser.add_option("-u", "--num-compute-units", type="int", default=1, 86 help="number of GPU compute units"), 87parser.add_option("--num-cp", type="int", default=0, 88 help="Number of GPU Command Processors (CP)") 89parser.add_option("--benchmark-root", help="Root of benchmark directory tree") 90 91# not super important now, but to avoid putting the number 4 everywhere, make 92# it an option/knob 93parser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \ 94 "sharing an SQC (icache, and thus icache TLB)") 95parser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \ 96 "per CU") 97parser.add_option("--wf-size", type="int", default=64, 98 help="Wavefront size(in workitems)") 99parser.add_option("--sp-bypass-path-length", type="int", default=4, \ 100 help="Number of stages of bypass path in vector ALU for Single Precision ops") 101parser.add_option("--dp-bypass-path-length", type="int", default=4, \ 102 help="Number of stages of bypass path in vector ALU for Double Precision ops") 103# issue period per SIMD unit: number of cycles before issuing another vector 104parser.add_option("--issue-period", type="int", default=4, \ 105 help="Number of cycles per vector instruction issue period") 106parser.add_option("--glbmem-wr-bus-width", type="int", default=32, \ 107 help="VGPR to Coalescer (Global Memory) data bus width in bytes") 108parser.add_option("--glbmem-rd-bus-width", type="int", default=32, \ 109 help="Coalescer to VGPR (Global Memory) data bus width in bytes") 110# Currently we only support 1 local memory pipe 111parser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \ 112 help="Number of Shared Memory pipelines per CU") 113# Currently we only support 1 global memory pipe 114parser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \ 115 help="Number of Global Memory pipelines per CU") 116parser.add_option("--wfs-per-simd", type="int", default=10, help="Number of " \ 117 "WF slots per SIMD") 118 119parser.add_option("--vreg-file-size", type="int", default=2048, 120 help="number of physical vector registers per SIMD") 121parser.add_option("--bw-scalor", type="int", default=0, 122 help="bandwidth scalor for scalability analysis") 123parser.add_option("--CPUClock", type="string", default="2GHz", 124 help="CPU clock") 125parser.add_option("--GPUClock", type="string", default="1GHz", 126 help="GPU clock") 127parser.add_option("--cpu-voltage", action="store", type="string", 128 default='1.0V', 129 help = """CPU voltage domain""") 130parser.add_option("--gpu-voltage", action="store", type="string", 131 default='1.0V', 132 help = """CPU voltage domain""") 133parser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST", 134 help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)") 135parser.add_option("--xact-cas-mode", action="store_true", 136 help="enable load_compare mode (transactional CAS)") 137parser.add_option("--SegFaultDebug",action="store_true", 138 help="checks for GPU seg fault before TLB access") 139parser.add_option("--FunctionalTLB",action="store_true", 140 help="Assumes TLB has no latency") 141parser.add_option("--LocalMemBarrier",action="store_true", 142 help="Barrier does not wait for writethroughs to complete") 143parser.add_option("--countPages", action="store_true", 144 help="Count Page Accesses and output in per-CU output files") 145parser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\ 146 "TLBs") 147parser.add_option("--pf-type", type="string", help="type of prefetch: "\ 148 "PF_CU, PF_WF, PF_PHASE, PF_STRIDE") 149parser.add_option("--pf-stride", type="int", help="set prefetch stride") 150parser.add_option("--numLdsBanks", type="int", default=32, 151 help="number of physical banks per LDS module") 152parser.add_option("--ldsBankConflictPenalty", type="int", default=1, 153 help="number of cycles per LDS bank conflict") 154parser.add_option('--fast-forward-pseudo-op', action='store_true', 155 help = 'fast forward using kvm until the m5_switchcpu' 156 ' pseudo-op is encountered, then switch cpus. subsequent' 157 ' m5_switchcpu pseudo-ops will toggle back and forth') 158parser.add_option('--outOfOrderDataDelivery', action='store_true', 159 default=False, help='enable OoO data delivery in the GM' 160 ' pipeline') 161 162Ruby.define_options(parser) 163 164#add TLB options to the parser 165GPUTLBOptions.tlb_options(parser) 166 167(options, args) = parser.parse_args() 168 169# The GPU cache coherence protocols only work with the backing store 170setOption(parser, "--access-backing-store") 171 172# if benchmark root is specified explicitly, that overrides the search path 173if options.benchmark_root: 174 benchmark_path = [options.benchmark_root] 175else: 176 # Set default benchmark search path to current dir 177 benchmark_path = ['.'] 178 179########################## Sanity Check ######################## 180 181# Currently the gpu model requires ruby 182if buildEnv['PROTOCOL'] == 'None': 183 fatal("GPU model requires ruby") 184 185# Currently the gpu model requires only timing or detailed CPU 186if not (options.cpu_type == "TimingSimpleCPU" or 187 options.cpu_type == "DerivO3CPU"): 188 fatal("GPU model requires TimingSimpleCPU or DerivO3CPU") 189 190# This file can support multiple compute units 191assert(options.num_compute_units >= 1) 192 193# Currently, the sqc (I-Cache of GPU) is shared by 194# multiple compute units(CUs). The protocol works just fine 195# even if sqc is not shared. Overriding this option here 196# so that the user need not explicitly set this (assuming 197# sharing sqc is the common usage) 198n_cu = options.num_compute_units 199num_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc)) 200options.num_sqc = num_sqc # pass this to Ruby 201 202########################## Creating the GPU system ######################## 203# shader is the GPU 204shader = Shader(n_wf = options.wfs_per_simd, 205 clk_domain = SrcClockDomain( 206 clock = options.GPUClock, 207 voltage_domain = VoltageDomain( 208 voltage = options.gpu_voltage))) 209 210# GPU_RfO(Read For Ownership) implements SC/TSO memory model. 211# Other GPU protocols implement release consistency at GPU side. 212# So, all GPU protocols other than GPU_RfO should make their writes 213# visible to the global memory and should read from global memory 214# during kernal boundary. The pipeline initiates(or do not initiate) 215# the acquire/release operation depending on this impl_kern_boundary_sync 216# flag. This flag=true means pipeline initiates a acquire/release operation 217# at kernel boundary. 218if buildEnv['PROTOCOL'] == 'GPU_RfO': 219 shader.impl_kern_boundary_sync = False 220else: 221 shader.impl_kern_boundary_sync = True 222 223# Switching off per-lane TLB by default 224per_lane = False 225if options.TLB_config == "perLane": 226 per_lane = True 227 228# List of compute units; one GPU can have multiple compute units 229compute_units = [] 230for i in xrange(n_cu): 231 compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane, 232 num_SIMDs = options.simds_per_cu, 233 wfSize = options.wf_size, 234 spbypass_pipe_length = options.sp_bypass_path_length, 235 dpbypass_pipe_length = options.dp_bypass_path_length, 236 issue_period = options.issue_period, 237 coalescer_to_vrf_bus_width = \ 238 options.glbmem_rd_bus_width, 239 vrf_to_coalescer_bus_width = \ 240 options.glbmem_wr_bus_width, 241 num_global_mem_pipes = \ 242 options.glb_mem_pipes_per_cu, 243 num_shared_mem_pipes = \ 244 options.shr_mem_pipes_per_cu, 245 n_wf = options.wfs_per_simd, 246 execPolicy = options.CUExecPolicy, 247 xactCasMode = options.xact_cas_mode, 248 debugSegFault = options.SegFaultDebug, 249 functionalTLB = options.FunctionalTLB, 250 localMemBarrier = options.LocalMemBarrier, 251 countPages = options.countPages, 252 localDataStore = \ 253 LdsState(banks = options.numLdsBanks, 254 bankConflictPenalty = \ 255 options.ldsBankConflictPenalty), 256 out_of_order_data_delivery = 257 options.outOfOrderDataDelivery)) 258 wavefronts = [] 259 vrfs = [] 260 for j in xrange(options.simds_per_cu): 261 for k in xrange(shader.n_wf): 262 wavefronts.append(Wavefront(simdId = j, wf_slot_id = k, 263 wfSize = options.wf_size)) 264 vrfs.append(VectorRegisterFile(simd_id=j, 265 num_regs_per_simd=options.vreg_file_size)) 266 compute_units[-1].wavefronts = wavefronts 267 compute_units[-1].vector_register_file = vrfs 268 if options.TLB_prefetch: 269 compute_units[-1].prefetch_depth = options.TLB_prefetch 270 compute_units[-1].prefetch_prev_type = options.pf_type 271 272 # attach the LDS and the CU to the bus (actually a Bridge) 273 compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave 274 compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort 275 276# Attach compute units to GPU 277shader.CUs = compute_units 278 279########################## Creating the CPU system ######################## 280options.num_cpus = options.num_cpus 281 282# The shader core will be whatever is after the CPU cores are accounted for 283shader_idx = options.num_cpus 284 285# The command processor will be whatever is after the shader is accounted for 286cp_idx = shader_idx + 1 287cp_list = [] 288 289# List of CPUs 290cpu_list = [] 291 292CpuClass, mem_mode = Simulation.getCPUClass(options.cpu_type) 293if CpuClass == AtomicSimpleCPU: 294 fatal("AtomicSimpleCPU is not supported") 295if mem_mode != 'timing': 296 fatal("Only the timing memory mode is supported") 297shader.timing = True 298 299if options.fast_forward and options.fast_forward_pseudo_op: 300 fatal("Cannot fast-forward based both on the number of instructions and" 301 " on pseudo-ops") 302fast_forward = options.fast_forward or options.fast_forward_pseudo_op 303 304if fast_forward: 305 FutureCpuClass, future_mem_mode = CpuClass, mem_mode 306 307 CpuClass = X86KvmCPU 308 mem_mode = 'atomic_noncaching' 309 # Leave shader.timing untouched, because its value only matters at the 310 # start of the simulation and because we require switching cpus 311 # *before* the first kernel launch. 312 313 future_cpu_list = [] 314 315 # Initial CPUs to be used during fast-forwarding. 316 for i in xrange(options.num_cpus): 317 cpu = CpuClass(cpu_id = i, 318 clk_domain = SrcClockDomain( 319 clock = options.CPUClock, 320 voltage_domain = VoltageDomain( 321 voltage = options.cpu_voltage))) 322 cpu_list.append(cpu) 323 324 if options.fast_forward: 325 cpu.max_insts_any_thread = int(options.fast_forward) 326 327if fast_forward: 328 MainCpuClass = FutureCpuClass 329else: 330 MainCpuClass = CpuClass 331 332# CPs to be used throughout the simulation. 333for i in xrange(options.num_cp): 334 cp = MainCpuClass(cpu_id = options.num_cpus + i, 335 clk_domain = SrcClockDomain( 336 clock = options.CPUClock, 337 voltage_domain = VoltageDomain( 338 voltage = options.cpu_voltage))) 339 cp_list.append(cp) 340 341# Main CPUs (to be used after fast-forwarding if fast-forwarding is specified). 342for i in xrange(options.num_cpus): 343 cpu = MainCpuClass(cpu_id = i, 344 clk_domain = SrcClockDomain( 345 clock = options.CPUClock, 346 voltage_domain = VoltageDomain( 347 voltage = options.cpu_voltage))) 348 if fast_forward: 349 cpu.switched_out = True 350 future_cpu_list.append(cpu) 351 else: 352 cpu_list.append(cpu) 353 354########################## Creating the GPU dispatcher ######################## 355# Dispatcher dispatches work from host CPU to GPU 356host_cpu = cpu_list[0] 357dispatcher = GpuDispatcher() 358 359########################## Create and assign the workload ######################## 360# Check for rel_path in elements of base_list using test, returning 361# the first full path that satisfies test 362def find_path(base_list, rel_path, test): 363 for base in base_list: 364 if not base: 365 # base could be None if environment var not set 366 continue 367 full_path = os.path.join(base, rel_path) 368 if test(full_path): 369 return full_path 370 fatal("%s not found in %s" % (rel_path, base_list)) 371 372def find_file(base_list, rel_path): 373 return find_path(base_list, rel_path, os.path.isfile) 374 375executable = find_path(benchmark_path, options.cmd, os.path.exists) 376# it's common for a benchmark to be in a directory with the same 377# name as the executable, so we handle that automatically 378if os.path.isdir(executable): 379 benchmark_path = [executable] 380 executable = find_file(benchmark_path, options.cmd) 381if options.kernel_files: 382 kernel_files = [find_file(benchmark_path, f) 383 for f in options.kernel_files.split(':')] 384else: 385 # if kernel_files is not set, see if there's a unique .asm file 386 # in the same directory as the executable 387 kernel_path = os.path.dirname(executable) 388 kernel_files = glob.glob(os.path.join(kernel_path, '*.asm')) 389 if kernel_files: 390 print("Using GPU kernel code file(s)", ",".join(kernel_files)) 391 else: 392 fatal("Can't locate kernel code (.asm) in " + kernel_path) 393 394# OpenCL driver 395driver = ClDriver(filename="hsa", codefile=kernel_files) 396for cpu in cpu_list: 397 cpu.createThreads() 398 cpu.workload = Process(executable = executable, 399 cmd = [options.cmd] + options.options.split(), 400 drivers = [driver]) 401for cp in cp_list: 402 cp.workload = host_cpu.workload 403 404if fast_forward: 405 for i in xrange(len(future_cpu_list)): 406 future_cpu_list[i].workload = cpu_list[i].workload 407 future_cpu_list[i].createThreads() 408 409########################## Create the overall system ######################## 410# List of CPUs that must be switched when moving between KVM and simulation 411if fast_forward: 412 switch_cpu_list = \ 413 [(cpu_list[i], future_cpu_list[i]) for i in xrange(options.num_cpus)] 414 415# Full list of processing cores in the system. Note that 416# dispatcher is also added to cpu_list although it is 417# not a processing element 418cpu_list = cpu_list + [shader] + cp_list + [dispatcher] 419 420# creating the overall system 421# notice the cpu list is explicitly added as a parameter to System 422system = System(cpu = cpu_list, 423 mem_ranges = [AddrRange(options.mem_size)], 424 cache_line_size = options.cacheline_size, 425 mem_mode = mem_mode) 426if fast_forward: 427 system.future_cpu = future_cpu_list 428system.voltage_domain = VoltageDomain(voltage = options.sys_voltage) 429system.clk_domain = SrcClockDomain(clock = options.sys_clock, 430 voltage_domain = system.voltage_domain) 431 432if fast_forward: 433 have_kvm_support = 'BaseKvmCPU' in globals() 434 if have_kvm_support and buildEnv['TARGET_ISA'] == "x86": 435 system.vm = KvmVM() 436 for i in xrange(len(host_cpu.workload)): 437 host_cpu.workload[i].useArchPT = True 438 host_cpu.workload[i].kvmInSE = True 439 else: 440 fatal("KvmCPU can only be used in SE mode with x86") 441 442# configure the TLB hierarchy 443GPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx) 444 445# create Ruby system 446system.piobus = IOXBar(width=32, response_latency=0, 447 frontend_latency=0, forward_latency=0) 448Ruby.create_system(options, None, system) 449system.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock, 450 voltage_domain = system.voltage_domain) 451 452# attach the CPU ports to Ruby 453for i in range(options.num_cpus): 454 ruby_port = system.ruby._cpu_ports[i] 455 456 # Create interrupt controller 457 system.cpu[i].createInterruptController() 458 459 # Connect cache port's to ruby 460 system.cpu[i].icache_port = ruby_port.slave 461 system.cpu[i].dcache_port = ruby_port.slave 462 463 ruby_port.mem_master_port = system.piobus.slave 464 if buildEnv['TARGET_ISA'] == "x86": 465 system.cpu[i].interrupts[0].pio = system.piobus.master 466 system.cpu[i].interrupts[0].int_master = system.piobus.slave 467 system.cpu[i].interrupts[0].int_slave = system.piobus.master 468 if fast_forward: 469 system.cpu[i].itb.walker.port = ruby_port.slave 470 system.cpu[i].dtb.walker.port = ruby_port.slave 471 472# attach CU ports to Ruby 473# Because of the peculiarities of the CP core, you may have 1 CPU but 2 474# sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be 475# hooked up until after the CP. To make this script generic, figure out 476# the index as below, but note that this assumes there is one sequencer 477# per compute unit and one sequencer per SQC for the math to work out 478# correctly. 479gpu_port_idx = len(system.ruby._cpu_ports) \ 480 - options.num_compute_units - options.num_sqc 481gpu_port_idx = gpu_port_idx - options.num_cp * 2 482 483wavefront_size = options.wf_size 484for i in xrange(n_cu): 485 # The pipeline issues wavefront_size number of uncoalesced requests 486 # in one GPU issue cycle. Hence wavefront_size mem ports. 487 for j in xrange(wavefront_size): 488 system.cpu[shader_idx].CUs[i].memory_port[j] = \ 489 system.ruby._cpu_ports[gpu_port_idx].slave[j] 490 gpu_port_idx += 1 491 492for i in xrange(n_cu): 493 if i > 0 and not i % options.cu_per_sqc: 494 print("incrementing idx on ", i) 495 gpu_port_idx += 1 496 system.cpu[shader_idx].CUs[i].sqc_port = \ 497 system.ruby._cpu_ports[gpu_port_idx].slave 498gpu_port_idx = gpu_port_idx + 1 499 500# attach CP ports to Ruby 501for i in xrange(options.num_cp): 502 system.cpu[cp_idx].createInterruptController() 503 system.cpu[cp_idx].dcache_port = \ 504 system.ruby._cpu_ports[gpu_port_idx + i * 2].slave 505 system.cpu[cp_idx].icache_port = \ 506 system.ruby._cpu_ports[gpu_port_idx + i * 2 + 1].slave 507 system.cpu[cp_idx].interrupts[0].pio = system.piobus.master 508 system.cpu[cp_idx].interrupts[0].int_master = system.piobus.slave 509 system.cpu[cp_idx].interrupts[0].int_slave = system.piobus.master 510 cp_idx = cp_idx + 1 511 512# connect dispatcher to the system.piobus 513dispatcher.pio = system.piobus.master 514dispatcher.dma = system.piobus.slave 515 516################# Connect the CPU and GPU via GPU Dispatcher ################### 517# CPU rings the GPU doorbell to notify a pending task 518# using this interface. 519# And GPU uses this interface to notify the CPU of task completion 520# The communcation happens through emulated driver. 521 522# Note this implicit setting of the cpu_pointer, shader_pointer and tlb array 523# parameters must be after the explicit setting of the System cpu list 524if fast_forward: 525 shader.cpu_pointer = future_cpu_list[0] 526 dispatcher.cpu = future_cpu_list[0] 527else: 528 shader.cpu_pointer = host_cpu 529 dispatcher.cpu = host_cpu 530dispatcher.shader_pointer = shader 531dispatcher.cl_driver = driver 532 533########################## Start simulation ######################## 534 535root = Root(system=system, full_system=False) 536m5.ticks.setGlobalFrequency('1THz') 537if options.abs_max_tick: 538 maxtick = options.abs_max_tick 539else: 540 maxtick = m5.MaxTick 541 542# Benchmarks support work item annotations 543Simulation.setWorkCountOptions(system, options) 544 545# Checkpointing is not supported by APU model 546if (options.checkpoint_dir != None or 547 options.checkpoint_restore != None): 548 fatal("Checkpointing not supported by apu model") 549 550checkpoint_dir = None 551m5.instantiate(checkpoint_dir) 552 553# Map workload to this address space 554host_cpu.workload[0].map(0x10000000, 0x200000000, 4096) 555 556if options.fast_forward: 557 print("Switch at instruction count: %d" % cpu_list[0].max_insts_any_thread) 558 559exit_event = m5.simulate(maxtick) 560 561if options.fast_forward: 562 if exit_event.getCause() == "a thread reached the max instruction count": 563 m5.switchCpus(system, switch_cpu_list) 564 print("Switched CPUS @ tick %s" % (m5.curTick())) 565 m5.stats.reset() 566 exit_event = m5.simulate(maxtick - m5.curTick()) 567elif options.fast_forward_pseudo_op: 568 while exit_event.getCause() == "switchcpu": 569 # If we are switching *to* kvm, then the current stats are meaningful 570 # Note that we don't do any warmup by default 571 if type(switch_cpu_list[0][0]) == FutureCpuClass: 572 print("Dumping stats...") 573 m5.stats.dump() 574 m5.switchCpus(system, switch_cpu_list) 575 print("Switched CPUS @ tick %s" % (m5.curTick())) 576 m5.stats.reset() 577 # This lets us switch back and forth without keeping a counter 578 switch_cpu_list = [(x[1], x[0]) for x in switch_cpu_list] 579 exit_event = m5.simulate(maxtick - m5.curTick()) 580 581print("Ticks:", m5.curTick()) 582print('Exiting because ', exit_event.getCause()) 583sys.exit(exit_event.getCode())
| 33 34from __future__ import print_function 35 36import optparse, os, re 37import math 38import glob 39import inspect 40 41import m5 42from m5.objects import * 43from m5.util import addToPath 44 45addToPath('../') 46 47from ruby import Ruby 48 49from common import Options 50from common import Simulation 51from common import GPUTLBOptions, GPUTLBConfig 52 53########################## Script Options ######################## 54def setOption(parser, opt_str, value = 1): 55 # check to make sure the option actually exists 56 if not parser.has_option(opt_str): 57 raise Exception("cannot find %s in list of possible options" % opt_str) 58 59 opt = parser.get_option(opt_str) 60 # set the value 61 exec("parser.values.%s = %s" % (opt.dest, value)) 62 63def getOption(parser, opt_str): 64 # check to make sure the option actually exists 65 if not parser.has_option(opt_str): 66 raise Exception("cannot find %s in list of possible options" % opt_str) 67 68 opt = parser.get_option(opt_str) 69 # get the value 70 exec("return_value = parser.values.%s" % opt.dest) 71 return return_value 72 73# Adding script options 74parser = optparse.OptionParser() 75Options.addCommonOptions(parser) 76Options.addSEOptions(parser) 77 78parser.add_option("--cpu-only-mode", action="store_true", default=False, 79 help="APU mode. Used to take care of problems in "\ 80 "Ruby.py while running APU protocols") 81parser.add_option("-k", "--kernel-files", 82 help="file(s) containing GPU kernel code (colon separated)") 83parser.add_option("-u", "--num-compute-units", type="int", default=1, 84 help="number of GPU compute units"), 85parser.add_option("--num-cp", type="int", default=0, 86 help="Number of GPU Command Processors (CP)") 87parser.add_option("--benchmark-root", help="Root of benchmark directory tree") 88 89# not super important now, but to avoid putting the number 4 everywhere, make 90# it an option/knob 91parser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \ 92 "sharing an SQC (icache, and thus icache TLB)") 93parser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \ 94 "per CU") 95parser.add_option("--wf-size", type="int", default=64, 96 help="Wavefront size(in workitems)") 97parser.add_option("--sp-bypass-path-length", type="int", default=4, \ 98 help="Number of stages of bypass path in vector ALU for Single Precision ops") 99parser.add_option("--dp-bypass-path-length", type="int", default=4, \ 100 help="Number of stages of bypass path in vector ALU for Double Precision ops") 101# issue period per SIMD unit: number of cycles before issuing another vector 102parser.add_option("--issue-period", type="int", default=4, \ 103 help="Number of cycles per vector instruction issue period") 104parser.add_option("--glbmem-wr-bus-width", type="int", default=32, \ 105 help="VGPR to Coalescer (Global Memory) data bus width in bytes") 106parser.add_option("--glbmem-rd-bus-width", type="int", default=32, \ 107 help="Coalescer to VGPR (Global Memory) data bus width in bytes") 108# Currently we only support 1 local memory pipe 109parser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \ 110 help="Number of Shared Memory pipelines per CU") 111# Currently we only support 1 global memory pipe 112parser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \ 113 help="Number of Global Memory pipelines per CU") 114parser.add_option("--wfs-per-simd", type="int", default=10, help="Number of " \ 115 "WF slots per SIMD") 116 117parser.add_option("--vreg-file-size", type="int", default=2048, 118 help="number of physical vector registers per SIMD") 119parser.add_option("--bw-scalor", type="int", default=0, 120 help="bandwidth scalor for scalability analysis") 121parser.add_option("--CPUClock", type="string", default="2GHz", 122 help="CPU clock") 123parser.add_option("--GPUClock", type="string", default="1GHz", 124 help="GPU clock") 125parser.add_option("--cpu-voltage", action="store", type="string", 126 default='1.0V', 127 help = """CPU voltage domain""") 128parser.add_option("--gpu-voltage", action="store", type="string", 129 default='1.0V', 130 help = """CPU voltage domain""") 131parser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST", 132 help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)") 133parser.add_option("--xact-cas-mode", action="store_true", 134 help="enable load_compare mode (transactional CAS)") 135parser.add_option("--SegFaultDebug",action="store_true", 136 help="checks for GPU seg fault before TLB access") 137parser.add_option("--FunctionalTLB",action="store_true", 138 help="Assumes TLB has no latency") 139parser.add_option("--LocalMemBarrier",action="store_true", 140 help="Barrier does not wait for writethroughs to complete") 141parser.add_option("--countPages", action="store_true", 142 help="Count Page Accesses and output in per-CU output files") 143parser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\ 144 "TLBs") 145parser.add_option("--pf-type", type="string", help="type of prefetch: "\ 146 "PF_CU, PF_WF, PF_PHASE, PF_STRIDE") 147parser.add_option("--pf-stride", type="int", help="set prefetch stride") 148parser.add_option("--numLdsBanks", type="int", default=32, 149 help="number of physical banks per LDS module") 150parser.add_option("--ldsBankConflictPenalty", type="int", default=1, 151 help="number of cycles per LDS bank conflict") 152parser.add_option('--fast-forward-pseudo-op', action='store_true', 153 help = 'fast forward using kvm until the m5_switchcpu' 154 ' pseudo-op is encountered, then switch cpus. subsequent' 155 ' m5_switchcpu pseudo-ops will toggle back and forth') 156parser.add_option('--outOfOrderDataDelivery', action='store_true', 157 default=False, help='enable OoO data delivery in the GM' 158 ' pipeline') 159 160Ruby.define_options(parser) 161 162#add TLB options to the parser 163GPUTLBOptions.tlb_options(parser) 164 165(options, args) = parser.parse_args() 166 167# The GPU cache coherence protocols only work with the backing store 168setOption(parser, "--access-backing-store") 169 170# if benchmark root is specified explicitly, that overrides the search path 171if options.benchmark_root: 172 benchmark_path = [options.benchmark_root] 173else: 174 # Set default benchmark search path to current dir 175 benchmark_path = ['.'] 176 177########################## Sanity Check ######################## 178 179# Currently the gpu model requires ruby 180if buildEnv['PROTOCOL'] == 'None': 181 fatal("GPU model requires ruby") 182 183# Currently the gpu model requires only timing or detailed CPU 184if not (options.cpu_type == "TimingSimpleCPU" or 185 options.cpu_type == "DerivO3CPU"): 186 fatal("GPU model requires TimingSimpleCPU or DerivO3CPU") 187 188# This file can support multiple compute units 189assert(options.num_compute_units >= 1) 190 191# Currently, the sqc (I-Cache of GPU) is shared by 192# multiple compute units(CUs). The protocol works just fine 193# even if sqc is not shared. Overriding this option here 194# so that the user need not explicitly set this (assuming 195# sharing sqc is the common usage) 196n_cu = options.num_compute_units 197num_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc)) 198options.num_sqc = num_sqc # pass this to Ruby 199 200########################## Creating the GPU system ######################## 201# shader is the GPU 202shader = Shader(n_wf = options.wfs_per_simd, 203 clk_domain = SrcClockDomain( 204 clock = options.GPUClock, 205 voltage_domain = VoltageDomain( 206 voltage = options.gpu_voltage))) 207 208# GPU_RfO(Read For Ownership) implements SC/TSO memory model. 209# Other GPU protocols implement release consistency at GPU side. 210# So, all GPU protocols other than GPU_RfO should make their writes 211# visible to the global memory and should read from global memory 212# during kernal boundary. The pipeline initiates(or do not initiate) 213# the acquire/release operation depending on this impl_kern_boundary_sync 214# flag. This flag=true means pipeline initiates a acquire/release operation 215# at kernel boundary. 216if buildEnv['PROTOCOL'] == 'GPU_RfO': 217 shader.impl_kern_boundary_sync = False 218else: 219 shader.impl_kern_boundary_sync = True 220 221# Switching off per-lane TLB by default 222per_lane = False 223if options.TLB_config == "perLane": 224 per_lane = True 225 226# List of compute units; one GPU can have multiple compute units 227compute_units = [] 228for i in xrange(n_cu): 229 compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane, 230 num_SIMDs = options.simds_per_cu, 231 wfSize = options.wf_size, 232 spbypass_pipe_length = options.sp_bypass_path_length, 233 dpbypass_pipe_length = options.dp_bypass_path_length, 234 issue_period = options.issue_period, 235 coalescer_to_vrf_bus_width = \ 236 options.glbmem_rd_bus_width, 237 vrf_to_coalescer_bus_width = \ 238 options.glbmem_wr_bus_width, 239 num_global_mem_pipes = \ 240 options.glb_mem_pipes_per_cu, 241 num_shared_mem_pipes = \ 242 options.shr_mem_pipes_per_cu, 243 n_wf = options.wfs_per_simd, 244 execPolicy = options.CUExecPolicy, 245 xactCasMode = options.xact_cas_mode, 246 debugSegFault = options.SegFaultDebug, 247 functionalTLB = options.FunctionalTLB, 248 localMemBarrier = options.LocalMemBarrier, 249 countPages = options.countPages, 250 localDataStore = \ 251 LdsState(banks = options.numLdsBanks, 252 bankConflictPenalty = \ 253 options.ldsBankConflictPenalty), 254 out_of_order_data_delivery = 255 options.outOfOrderDataDelivery)) 256 wavefronts = [] 257 vrfs = [] 258 for j in xrange(options.simds_per_cu): 259 for k in xrange(shader.n_wf): 260 wavefronts.append(Wavefront(simdId = j, wf_slot_id = k, 261 wfSize = options.wf_size)) 262 vrfs.append(VectorRegisterFile(simd_id=j, 263 num_regs_per_simd=options.vreg_file_size)) 264 compute_units[-1].wavefronts = wavefronts 265 compute_units[-1].vector_register_file = vrfs 266 if options.TLB_prefetch: 267 compute_units[-1].prefetch_depth = options.TLB_prefetch 268 compute_units[-1].prefetch_prev_type = options.pf_type 269 270 # attach the LDS and the CU to the bus (actually a Bridge) 271 compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave 272 compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort 273 274# Attach compute units to GPU 275shader.CUs = compute_units 276 277########################## Creating the CPU system ######################## 278options.num_cpus = options.num_cpus 279 280# The shader core will be whatever is after the CPU cores are accounted for 281shader_idx = options.num_cpus 282 283# The command processor will be whatever is after the shader is accounted for 284cp_idx = shader_idx + 1 285cp_list = [] 286 287# List of CPUs 288cpu_list = [] 289 290CpuClass, mem_mode = Simulation.getCPUClass(options.cpu_type) 291if CpuClass == AtomicSimpleCPU: 292 fatal("AtomicSimpleCPU is not supported") 293if mem_mode != 'timing': 294 fatal("Only the timing memory mode is supported") 295shader.timing = True 296 297if options.fast_forward and options.fast_forward_pseudo_op: 298 fatal("Cannot fast-forward based both on the number of instructions and" 299 " on pseudo-ops") 300fast_forward = options.fast_forward or options.fast_forward_pseudo_op 301 302if fast_forward: 303 FutureCpuClass, future_mem_mode = CpuClass, mem_mode 304 305 CpuClass = X86KvmCPU 306 mem_mode = 'atomic_noncaching' 307 # Leave shader.timing untouched, because its value only matters at the 308 # start of the simulation and because we require switching cpus 309 # *before* the first kernel launch. 310 311 future_cpu_list = [] 312 313 # Initial CPUs to be used during fast-forwarding. 314 for i in xrange(options.num_cpus): 315 cpu = CpuClass(cpu_id = i, 316 clk_domain = SrcClockDomain( 317 clock = options.CPUClock, 318 voltage_domain = VoltageDomain( 319 voltage = options.cpu_voltage))) 320 cpu_list.append(cpu) 321 322 if options.fast_forward: 323 cpu.max_insts_any_thread = int(options.fast_forward) 324 325if fast_forward: 326 MainCpuClass = FutureCpuClass 327else: 328 MainCpuClass = CpuClass 329 330# CPs to be used throughout the simulation. 331for i in xrange(options.num_cp): 332 cp = MainCpuClass(cpu_id = options.num_cpus + i, 333 clk_domain = SrcClockDomain( 334 clock = options.CPUClock, 335 voltage_domain = VoltageDomain( 336 voltage = options.cpu_voltage))) 337 cp_list.append(cp) 338 339# Main CPUs (to be used after fast-forwarding if fast-forwarding is specified). 340for i in xrange(options.num_cpus): 341 cpu = MainCpuClass(cpu_id = i, 342 clk_domain = SrcClockDomain( 343 clock = options.CPUClock, 344 voltage_domain = VoltageDomain( 345 voltage = options.cpu_voltage))) 346 if fast_forward: 347 cpu.switched_out = True 348 future_cpu_list.append(cpu) 349 else: 350 cpu_list.append(cpu) 351 352########################## Creating the GPU dispatcher ######################## 353# Dispatcher dispatches work from host CPU to GPU 354host_cpu = cpu_list[0] 355dispatcher = GpuDispatcher() 356 357########################## Create and assign the workload ######################## 358# Check for rel_path in elements of base_list using test, returning 359# the first full path that satisfies test 360def find_path(base_list, rel_path, test): 361 for base in base_list: 362 if not base: 363 # base could be None if environment var not set 364 continue 365 full_path = os.path.join(base, rel_path) 366 if test(full_path): 367 return full_path 368 fatal("%s not found in %s" % (rel_path, base_list)) 369 370def find_file(base_list, rel_path): 371 return find_path(base_list, rel_path, os.path.isfile) 372 373executable = find_path(benchmark_path, options.cmd, os.path.exists) 374# it's common for a benchmark to be in a directory with the same 375# name as the executable, so we handle that automatically 376if os.path.isdir(executable): 377 benchmark_path = [executable] 378 executable = find_file(benchmark_path, options.cmd) 379if options.kernel_files: 380 kernel_files = [find_file(benchmark_path, f) 381 for f in options.kernel_files.split(':')] 382else: 383 # if kernel_files is not set, see if there's a unique .asm file 384 # in the same directory as the executable 385 kernel_path = os.path.dirname(executable) 386 kernel_files = glob.glob(os.path.join(kernel_path, '*.asm')) 387 if kernel_files: 388 print("Using GPU kernel code file(s)", ",".join(kernel_files)) 389 else: 390 fatal("Can't locate kernel code (.asm) in " + kernel_path) 391 392# OpenCL driver 393driver = ClDriver(filename="hsa", codefile=kernel_files) 394for cpu in cpu_list: 395 cpu.createThreads() 396 cpu.workload = Process(executable = executable, 397 cmd = [options.cmd] + options.options.split(), 398 drivers = [driver]) 399for cp in cp_list: 400 cp.workload = host_cpu.workload 401 402if fast_forward: 403 for i in xrange(len(future_cpu_list)): 404 future_cpu_list[i].workload = cpu_list[i].workload 405 future_cpu_list[i].createThreads() 406 407########################## Create the overall system ######################## 408# List of CPUs that must be switched when moving between KVM and simulation 409if fast_forward: 410 switch_cpu_list = \ 411 [(cpu_list[i], future_cpu_list[i]) for i in xrange(options.num_cpus)] 412 413# Full list of processing cores in the system. Note that 414# dispatcher is also added to cpu_list although it is 415# not a processing element 416cpu_list = cpu_list + [shader] + cp_list + [dispatcher] 417 418# creating the overall system 419# notice the cpu list is explicitly added as a parameter to System 420system = System(cpu = cpu_list, 421 mem_ranges = [AddrRange(options.mem_size)], 422 cache_line_size = options.cacheline_size, 423 mem_mode = mem_mode) 424if fast_forward: 425 system.future_cpu = future_cpu_list 426system.voltage_domain = VoltageDomain(voltage = options.sys_voltage) 427system.clk_domain = SrcClockDomain(clock = options.sys_clock, 428 voltage_domain = system.voltage_domain) 429 430if fast_forward: 431 have_kvm_support = 'BaseKvmCPU' in globals() 432 if have_kvm_support and buildEnv['TARGET_ISA'] == "x86": 433 system.vm = KvmVM() 434 for i in xrange(len(host_cpu.workload)): 435 host_cpu.workload[i].useArchPT = True 436 host_cpu.workload[i].kvmInSE = True 437 else: 438 fatal("KvmCPU can only be used in SE mode with x86") 439 440# configure the TLB hierarchy 441GPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx) 442 443# create Ruby system 444system.piobus = IOXBar(width=32, response_latency=0, 445 frontend_latency=0, forward_latency=0) 446Ruby.create_system(options, None, system) 447system.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock, 448 voltage_domain = system.voltage_domain) 449 450# attach the CPU ports to Ruby 451for i in range(options.num_cpus): 452 ruby_port = system.ruby._cpu_ports[i] 453 454 # Create interrupt controller 455 system.cpu[i].createInterruptController() 456 457 # Connect cache port's to ruby 458 system.cpu[i].icache_port = ruby_port.slave 459 system.cpu[i].dcache_port = ruby_port.slave 460 461 ruby_port.mem_master_port = system.piobus.slave 462 if buildEnv['TARGET_ISA'] == "x86": 463 system.cpu[i].interrupts[0].pio = system.piobus.master 464 system.cpu[i].interrupts[0].int_master = system.piobus.slave 465 system.cpu[i].interrupts[0].int_slave = system.piobus.master 466 if fast_forward: 467 system.cpu[i].itb.walker.port = ruby_port.slave 468 system.cpu[i].dtb.walker.port = ruby_port.slave 469 470# attach CU ports to Ruby 471# Because of the peculiarities of the CP core, you may have 1 CPU but 2 472# sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be 473# hooked up until after the CP. To make this script generic, figure out 474# the index as below, but note that this assumes there is one sequencer 475# per compute unit and one sequencer per SQC for the math to work out 476# correctly. 477gpu_port_idx = len(system.ruby._cpu_ports) \ 478 - options.num_compute_units - options.num_sqc 479gpu_port_idx = gpu_port_idx - options.num_cp * 2 480 481wavefront_size = options.wf_size 482for i in xrange(n_cu): 483 # The pipeline issues wavefront_size number of uncoalesced requests 484 # in one GPU issue cycle. Hence wavefront_size mem ports. 485 for j in xrange(wavefront_size): 486 system.cpu[shader_idx].CUs[i].memory_port[j] = \ 487 system.ruby._cpu_ports[gpu_port_idx].slave[j] 488 gpu_port_idx += 1 489 490for i in xrange(n_cu): 491 if i > 0 and not i % options.cu_per_sqc: 492 print("incrementing idx on ", i) 493 gpu_port_idx += 1 494 system.cpu[shader_idx].CUs[i].sqc_port = \ 495 system.ruby._cpu_ports[gpu_port_idx].slave 496gpu_port_idx = gpu_port_idx + 1 497 498# attach CP ports to Ruby 499for i in xrange(options.num_cp): 500 system.cpu[cp_idx].createInterruptController() 501 system.cpu[cp_idx].dcache_port = \ 502 system.ruby._cpu_ports[gpu_port_idx + i * 2].slave 503 system.cpu[cp_idx].icache_port = \ 504 system.ruby._cpu_ports[gpu_port_idx + i * 2 + 1].slave 505 system.cpu[cp_idx].interrupts[0].pio = system.piobus.master 506 system.cpu[cp_idx].interrupts[0].int_master = system.piobus.slave 507 system.cpu[cp_idx].interrupts[0].int_slave = system.piobus.master 508 cp_idx = cp_idx + 1 509 510# connect dispatcher to the system.piobus 511dispatcher.pio = system.piobus.master 512dispatcher.dma = system.piobus.slave 513 514################# Connect the CPU and GPU via GPU Dispatcher ################### 515# CPU rings the GPU doorbell to notify a pending task 516# using this interface. 517# And GPU uses this interface to notify the CPU of task completion 518# The communcation happens through emulated driver. 519 520# Note this implicit setting of the cpu_pointer, shader_pointer and tlb array 521# parameters must be after the explicit setting of the System cpu list 522if fast_forward: 523 shader.cpu_pointer = future_cpu_list[0] 524 dispatcher.cpu = future_cpu_list[0] 525else: 526 shader.cpu_pointer = host_cpu 527 dispatcher.cpu = host_cpu 528dispatcher.shader_pointer = shader 529dispatcher.cl_driver = driver 530 531########################## Start simulation ######################## 532 533root = Root(system=system, full_system=False) 534m5.ticks.setGlobalFrequency('1THz') 535if options.abs_max_tick: 536 maxtick = options.abs_max_tick 537else: 538 maxtick = m5.MaxTick 539 540# Benchmarks support work item annotations 541Simulation.setWorkCountOptions(system, options) 542 543# Checkpointing is not supported by APU model 544if (options.checkpoint_dir != None or 545 options.checkpoint_restore != None): 546 fatal("Checkpointing not supported by apu model") 547 548checkpoint_dir = None 549m5.instantiate(checkpoint_dir) 550 551# Map workload to this address space 552host_cpu.workload[0].map(0x10000000, 0x200000000, 4096) 553 554if options.fast_forward: 555 print("Switch at instruction count: %d" % cpu_list[0].max_insts_any_thread) 556 557exit_event = m5.simulate(maxtick) 558 559if options.fast_forward: 560 if exit_event.getCause() == "a thread reached the max instruction count": 561 m5.switchCpus(system, switch_cpu_list) 562 print("Switched CPUS @ tick %s" % (m5.curTick())) 563 m5.stats.reset() 564 exit_event = m5.simulate(maxtick - m5.curTick()) 565elif options.fast_forward_pseudo_op: 566 while exit_event.getCause() == "switchcpu": 567 # If we are switching *to* kvm, then the current stats are meaningful 568 # Note that we don't do any warmup by default 569 if type(switch_cpu_list[0][0]) == FutureCpuClass: 570 print("Dumping stats...") 571 m5.stats.dump() 572 m5.switchCpus(system, switch_cpu_list) 573 print("Switched CPUS @ tick %s" % (m5.curTick())) 574 m5.stats.reset() 575 # This lets us switch back and forth without keeping a counter 576 switch_cpu_list = [(x[1], x[0]) for x in switch_cpu_list] 577 exit_event = m5.simulate(maxtick - m5.curTick()) 578 579print("Ticks:", m5.curTick()) 580print('Exiting because ', exit_event.getCause()) 581sys.exit(exit_event.getCode())
|