apu_se.py revision 11851
17139Sgblack@eecs.umich.edu# 27139Sgblack@eecs.umich.edu# Copyright (c) 2015 Advanced Micro Devices, Inc. 37139Sgblack@eecs.umich.edu# All rights reserved. 47139Sgblack@eecs.umich.edu# 57139Sgblack@eecs.umich.edu# For use for simulation and test purposes only 67139Sgblack@eecs.umich.edu# 77139Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 87139Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are met: 97139Sgblack@eecs.umich.edu# 107139Sgblack@eecs.umich.edu# 1. Redistributions of source code must retain the above copyright notice, 117139Sgblack@eecs.umich.edu# this list of conditions and the following disclaimer. 127139Sgblack@eecs.umich.edu# 137139Sgblack@eecs.umich.edu# 2. Redistributions in binary form must reproduce the above copyright notice, 147139Sgblack@eecs.umich.edu# this list of conditions and the following disclaimer in the documentation 157139Sgblack@eecs.umich.edu# and/or other materials provided with the distribution. 167139Sgblack@eecs.umich.edu# 177139Sgblack@eecs.umich.edu# 3. Neither the name of the copyright holder nor the names of its contributors 187139Sgblack@eecs.umich.edu# may be used to endorse or promote products derived from this software 197139Sgblack@eecs.umich.edu# without specific prior written permission. 207139Sgblack@eecs.umich.edu# 217139Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 227139Sgblack@eecs.umich.edu# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 237139Sgblack@eecs.umich.edu# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 247139Sgblack@eecs.umich.edu# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 257139Sgblack@eecs.umich.edu# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 267139Sgblack@eecs.umich.edu# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 277139Sgblack@eecs.umich.edu# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 287139Sgblack@eecs.umich.edu# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 297139Sgblack@eecs.umich.edu# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 307139Sgblack@eecs.umich.edu# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 317139Sgblack@eecs.umich.edu# POSSIBILITY OF SUCH DAMAGE. 327139Sgblack@eecs.umich.edu# 337139Sgblack@eecs.umich.edu# Author: Sooraj Puthoor 347139Sgblack@eecs.umich.edu# 357139Sgblack@eecs.umich.edu 367139Sgblack@eecs.umich.eduimport optparse, os, re 377139Sgblack@eecs.umich.eduimport math 387255Sgblack@eecs.umich.eduimport glob 397243Sgblack@eecs.umich.eduimport inspect 407243Sgblack@eecs.umich.edu 417255Sgblack@eecs.umich.eduimport m5 427255Sgblack@eecs.umich.edufrom m5.objects import * 437243Sgblack@eecs.umich.edufrom m5.util import addToPath 447243Sgblack@eecs.umich.edu 457255Sgblack@eecs.umich.eduaddToPath('../') 467255Sgblack@eecs.umich.edu 477255Sgblack@eecs.umich.edufrom ruby import Ruby 487255Sgblack@eecs.umich.edu 497255Sgblack@eecs.umich.edufrom common import Options 507255Sgblack@eecs.umich.edufrom common import Simulation 517255Sgblack@eecs.umich.edufrom common import GPUTLBOptions, GPUTLBConfig 527255Sgblack@eecs.umich.edu 537255Sgblack@eecs.umich.edu########################## Script Options ######################## 547255Sgblack@eecs.umich.edudef setOption(parser, opt_str, value = 1): 557255Sgblack@eecs.umich.edu # check to make sure the option actually exists 567256Sgblack@eecs.umich.edu if not parser.has_option(opt_str): 577256Sgblack@eecs.umich.edu raise Exception("cannot find %s in list of possible options" % opt_str) 587255Sgblack@eecs.umich.edu 597256Sgblack@eecs.umich.edu opt = parser.get_option(opt_str) 607255Sgblack@eecs.umich.edu # set the value 617256Sgblack@eecs.umich.edu exec("parser.values.%s = %s" % (opt.dest, value)) 627255Sgblack@eecs.umich.edu 637255Sgblack@eecs.umich.edudef getOption(parser, opt_str): 647258Sgblack@eecs.umich.edu # check to make sure the option actually exists 657258Sgblack@eecs.umich.edu if not parser.has_option(opt_str): 667255Sgblack@eecs.umich.edu raise Exception("cannot find %s in list of possible options" % opt_str) 677258Sgblack@eecs.umich.edu 687255Sgblack@eecs.umich.edu opt = parser.get_option(opt_str) 697258Sgblack@eecs.umich.edu # get the value 707255Sgblack@eecs.umich.edu exec("return_value = parser.values.%s" % opt.dest) 717243Sgblack@eecs.umich.edu return return_value 727255Sgblack@eecs.umich.edu 737243Sgblack@eecs.umich.edu# Adding script options 747243Sgblack@eecs.umich.eduparser = optparse.OptionParser() 757243Sgblack@eecs.umich.eduOptions.addCommonOptions(parser) 767243Sgblack@eecs.umich.eduOptions.addSEOptions(parser) 777139Sgblack@eecs.umich.edu 787188Sgblack@eecs.umich.eduparser.add_option("--cpu-only-mode", action="store_true", default=False, 797188Sgblack@eecs.umich.edu help="APU mode. Used to take care of problems in "\ 807188Sgblack@eecs.umich.edu "Ruby.py while running APU protocols") 817188Sgblack@eecs.umich.eduparser.add_option("-k", "--kernel-files", 827188Sgblack@eecs.umich.edu help="file(s) containing GPU kernel code (colon separated)") 837139Sgblack@eecs.umich.eduparser.add_option("-u", "--num-compute-units", type="int", default=1, 847139Sgblack@eecs.umich.edu help="number of GPU compute units"), 857139Sgblack@eecs.umich.eduparser.add_option("--num-cp", type="int", default=0, 867139Sgblack@eecs.umich.edu help="Number of GPU Command Processors (CP)") 877188Sgblack@eecs.umich.eduparser.add_option("--benchmark-root", help="Root of benchmark directory tree") 887188Sgblack@eecs.umich.edu 897188Sgblack@eecs.umich.edu# not super important now, but to avoid putting the number 4 everywhere, make 907188Sgblack@eecs.umich.edu# it an option/knob 917188Sgblack@eecs.umich.eduparser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \ 927188Sgblack@eecs.umich.edu "sharing an SQC (icache, and thus icache TLB)") 937139Sgblack@eecs.umich.eduparser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \ 947146Sgblack@eecs.umich.edu "per CU") 957141Sgblack@eecs.umich.eduparser.add_option("--wf-size", type="int", default=64, 967139Sgblack@eecs.umich.edu help="Wavefront size(in workitems)") 977139Sgblack@eecs.umich.eduparser.add_option("--sp-bypass-path-length", type="int", default=4, \ 987139Sgblack@eecs.umich.edu help="Number of stages of bypass path in vector ALU for Single Precision ops") 997146Sgblack@eecs.umich.eduparser.add_option("--dp-bypass-path-length", type="int", default=4, \ 1007141Sgblack@eecs.umich.edu help="Number of stages of bypass path in vector ALU for Double Precision ops") 1017139Sgblack@eecs.umich.edu# issue period per SIMD unit: number of cycles before issuing another vector 1027146Sgblack@eecs.umich.eduparser.add_option("--issue-period", type="int", default=4, \ 1037141Sgblack@eecs.umich.edu help="Number of cycles per vector instruction issue period") 1047139Sgblack@eecs.umich.eduparser.add_option("--glbmem-wr-bus-width", type="int", default=32, \ 1057139Sgblack@eecs.umich.edu help="VGPR to Coalescer (Global Memory) data bus width in bytes") 1067139Sgblack@eecs.umich.eduparser.add_option("--glbmem-rd-bus-width", type="int", default=32, \ 1077139Sgblack@eecs.umich.edu help="Coalescer to VGPR (Global Memory) data bus width in bytes") 1087139Sgblack@eecs.umich.edu# Currently we only support 1 local memory pipe 1097188Sgblack@eecs.umich.eduparser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \ 1107188Sgblack@eecs.umich.edu help="Number of Shared Memory pipelines per CU") 1117188Sgblack@eecs.umich.edu# Currently we only support 1 global memory pipe 1127188Sgblack@eecs.umich.eduparser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \ 1137188Sgblack@eecs.umich.edu help="Number of Global Memory pipelines per CU") 1147188Sgblack@eecs.umich.eduparser.add_option("--wfs-per-simd", type="int", default=10, help="Number of " \ 1157188Sgblack@eecs.umich.edu "WF slots per SIMD") 1167188Sgblack@eecs.umich.edu 1177188Sgblack@eecs.umich.eduparser.add_option("--vreg-file-size", type="int", default=2048, 1187188Sgblack@eecs.umich.edu help="number of physical vector registers per SIMD") 1197188Sgblack@eecs.umich.eduparser.add_option("--bw-scalor", type="int", default=0, 1207188Sgblack@eecs.umich.edu help="bandwidth scalor for scalability analysis") 1217188Sgblack@eecs.umich.eduparser.add_option("--CPUClock", type="string", default="2GHz", 1227188Sgblack@eecs.umich.edu help="CPU clock") 1237188Sgblack@eecs.umich.eduparser.add_option("--GPUClock", type="string", default="1GHz", 1247188Sgblack@eecs.umich.edu help="GPU clock") 1257188Sgblack@eecs.umich.eduparser.add_option("--cpu-voltage", action="store", type="string", 1267188Sgblack@eecs.umich.edu default='1.0V', 1277188Sgblack@eecs.umich.edu help = """CPU voltage domain""") 1287188Sgblack@eecs.umich.eduparser.add_option("--gpu-voltage", action="store", type="string", 1297139Sgblack@eecs.umich.edu default='1.0V', 1307139Sgblack@eecs.umich.edu help = """CPU voltage domain""") 1317139Sgblack@eecs.umich.eduparser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST", 1327139Sgblack@eecs.umich.edu help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)") 1337139Sgblack@eecs.umich.eduparser.add_option("--xact-cas-mode", action="store_true", 1347139Sgblack@eecs.umich.edu help="enable load_compare mode (transactional CAS)") 1357139Sgblack@eecs.umich.eduparser.add_option("--SegFaultDebug",action="store_true", 1367139Sgblack@eecs.umich.edu help="checks for GPU seg fault before TLB access") 1377139Sgblack@eecs.umich.eduparser.add_option("--FunctionalTLB",action="store_true", 1387139Sgblack@eecs.umich.edu help="Assumes TLB has no latency") 1397139Sgblack@eecs.umich.eduparser.add_option("--LocalMemBarrier",action="store_true", 1407139Sgblack@eecs.umich.edu help="Barrier does not wait for writethroughs to complete") 1417139Sgblack@eecs.umich.eduparser.add_option("--countPages", action="store_true", 1427139Sgblack@eecs.umich.edu help="Count Page Accesses and output in per-CU output files") 1437139Sgblack@eecs.umich.eduparser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\ 1447139Sgblack@eecs.umich.edu "TLBs") 1457139Sgblack@eecs.umich.eduparser.add_option("--pf-type", type="string", help="type of prefetch: "\ 1467139Sgblack@eecs.umich.edu "PF_CU, PF_WF, PF_PHASE, PF_STRIDE") 1477139Sgblack@eecs.umich.eduparser.add_option("--pf-stride", type="int", help="set prefetch stride") 1487139Sgblack@eecs.umich.eduparser.add_option("--numLdsBanks", type="int", default=32, 1497139Sgblack@eecs.umich.edu help="number of physical banks per LDS module") 1507188Sgblack@eecs.umich.eduparser.add_option("--ldsBankConflictPenalty", type="int", default=1, 1517188Sgblack@eecs.umich.edu help="number of cycles per LDS bank conflict") 1527188Sgblack@eecs.umich.eduparser.add_option('--fast-forward-pseudo-op', action='store_true', 1537188Sgblack@eecs.umich.edu help = 'fast forward using kvm until the m5_switchcpu' 1547139Sgblack@eecs.umich.edu ' pseudo-op is encountered, then switch cpus. subsequent' 1557188Sgblack@eecs.umich.edu ' m5_switchcpu pseudo-ops will toggle back and forth') 1567139Sgblack@eecs.umich.eduparser.add_option('--outOfOrderDataDelivery', action='store_true', 1577188Sgblack@eecs.umich.edu default=False, help='enable OoO data delivery in the GM' 1587139Sgblack@eecs.umich.edu ' pipeline') 1597139Sgblack@eecs.umich.edu 1607139Sgblack@eecs.umich.eduRuby.define_options(parser) 1617139Sgblack@eecs.umich.edu 1627139Sgblack@eecs.umich.edu#add TLB options to the parser 1637139Sgblack@eecs.umich.eduGPUTLBOptions.tlb_options(parser) 1647139Sgblack@eecs.umich.edu 1657139Sgblack@eecs.umich.edu(options, args) = parser.parse_args() 1667210Sgblack@eecs.umich.edu 1677210Sgblack@eecs.umich.edu# The GPU cache coherence protocols only work with the backing store 1687210Sgblack@eecs.umich.edusetOption(parser, "--access-backing-store") 1697210Sgblack@eecs.umich.edu 1707210Sgblack@eecs.umich.edu# if benchmark root is specified explicitly, that overrides the search path 1717210Sgblack@eecs.umich.eduif options.benchmark_root: 1727210Sgblack@eecs.umich.edu benchmark_path = [options.benchmark_root] 1737227Sgblack@eecs.umich.eduelse: 1747227Sgblack@eecs.umich.edu # Set default benchmark search path to current dir 1757227Sgblack@eecs.umich.edu benchmark_path = ['.'] 1767227Sgblack@eecs.umich.edu 1777227Sgblack@eecs.umich.edu########################## Sanity Check ######################## 1787227Sgblack@eecs.umich.edu 1797227Sgblack@eecs.umich.edu# Currently the gpu model requires ruby 1807227Sgblack@eecs.umich.eduif buildEnv['PROTOCOL'] == 'None': 1817210Sgblack@eecs.umich.edu fatal("GPU model requires ruby") 1827237Sgblack@eecs.umich.edu 1837237Sgblack@eecs.umich.edu# Currently the gpu model requires only timing or detailed CPU 1847237Sgblack@eecs.umich.eduif not (options.cpu_type == "timing" or 1857237Sgblack@eecs.umich.edu options.cpu_type == "detailed"): 1867237Sgblack@eecs.umich.edu fatal("GPU model requires timing or detailed CPU") 1877237Sgblack@eecs.umich.edu 1887237Sgblack@eecs.umich.edu# This file can support multiple compute units 1897210Sgblack@eecs.umich.eduassert(options.num_compute_units >= 1) 1907227Sgblack@eecs.umich.edu 1917210Sgblack@eecs.umich.edu# Currently, the sqc (I-Cache of GPU) is shared by 1927227Sgblack@eecs.umich.edu# multiple compute units(CUs). The protocol works just fine 1937210Sgblack@eecs.umich.edu# even if sqc is not shared. Overriding this option here 1947210Sgblack@eecs.umich.edu# so that the user need not explicitly set this (assuming 1957210Sgblack@eecs.umich.edu# sharing sqc is the common usage) 1967210Sgblack@eecs.umich.edun_cu = options.num_compute_units 1977210Sgblack@eecs.umich.edunum_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc)) 1987240Sgblack@eecs.umich.eduoptions.num_sqc = num_sqc # pass this to Ruby 1997235Sgblack@eecs.umich.edu 2007235Sgblack@eecs.umich.edu########################## Creating the GPU system ######################## 2017235Sgblack@eecs.umich.edu# shader is the GPU 2027235Sgblack@eecs.umich.edushader = Shader(n_wf = options.wfs_per_simd, 2037235Sgblack@eecs.umich.edu clk_domain = SrcClockDomain( 2047235Sgblack@eecs.umich.edu clock = options.GPUClock, 2057240Sgblack@eecs.umich.edu voltage_domain = VoltageDomain( 2067240Sgblack@eecs.umich.edu voltage = options.gpu_voltage))) 2077240Sgblack@eecs.umich.edu 2087240Sgblack@eecs.umich.edu# GPU_RfO(Read For Ownership) implements SC/TSO memory model. 2097240Sgblack@eecs.umich.edu# Other GPU protocols implement release consistency at GPU side. 2107240Sgblack@eecs.umich.edu# So, all GPU protocols other than GPU_RfO should make their writes 2117240Sgblack@eecs.umich.edu# visible to the global memory and should read from global memory 2127240Sgblack@eecs.umich.edu# during kernal boundary. The pipeline initiates(or do not initiate) 2137240Sgblack@eecs.umich.edu# the acquire/release operation depending on this impl_kern_boundary_sync 2147240Sgblack@eecs.umich.edu# flag. This flag=true means pipeline initiates a acquire/release operation 2157210Sgblack@eecs.umich.edu# at kernel boundary. 2167210Sgblack@eecs.umich.eduif buildEnv['PROTOCOL'] == 'GPU_RfO': 2177210Sgblack@eecs.umich.edu shader.impl_kern_boundary_sync = False 2187210Sgblack@eecs.umich.eduelse: 2197210Sgblack@eecs.umich.edu shader.impl_kern_boundary_sync = True 2207227Sgblack@eecs.umich.edu 2217227Sgblack@eecs.umich.edu# Switching off per-lane TLB by default 2227227Sgblack@eecs.umich.eduper_lane = False 2237227Sgblack@eecs.umich.eduif options.TLB_config == "perLane": 2247227Sgblack@eecs.umich.edu per_lane = True 2257227Sgblack@eecs.umich.edu 2267210Sgblack@eecs.umich.edu# List of compute units; one GPU can have multiple compute units 2277235Sgblack@eecs.umich.educompute_units = [] 2287235Sgblack@eecs.umich.edufor i in xrange(n_cu): 2297235Sgblack@eecs.umich.edu compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane, 2307235Sgblack@eecs.umich.edu num_SIMDs = options.simds_per_cu, 2317235Sgblack@eecs.umich.edu wfSize = options.wf_size, 2327235Sgblack@eecs.umich.edu spbypass_pipe_length = options.sp_bypass_path_length, 2337235Sgblack@eecs.umich.edu dpbypass_pipe_length = options.dp_bypass_path_length, 2347235Sgblack@eecs.umich.edu issue_period = options.issue_period, 2357210Sgblack@eecs.umich.edu coalescer_to_vrf_bus_width = \ 2367235Sgblack@eecs.umich.edu options.glbmem_rd_bus_width, 2377210Sgblack@eecs.umich.edu vrf_to_coalescer_bus_width = \ 2387235Sgblack@eecs.umich.edu options.glbmem_wr_bus_width, 2397210Sgblack@eecs.umich.edu num_global_mem_pipes = \ 2407210Sgblack@eecs.umich.edu options.glb_mem_pipes_per_cu, 2417210Sgblack@eecs.umich.edu num_shared_mem_pipes = \ 2427210Sgblack@eecs.umich.edu options.shr_mem_pipes_per_cu, 2437210Sgblack@eecs.umich.edu n_wf = options.wfs_per_simd, 2447211Sgblack@eecs.umich.edu execPolicy = options.CUExecPolicy, 2457211Sgblack@eecs.umich.edu xactCasMode = options.xact_cas_mode, 2467211Sgblack@eecs.umich.edu debugSegFault = options.SegFaultDebug, 2477210Sgblack@eecs.umich.edu functionalTLB = options.FunctionalTLB, 2487235Sgblack@eecs.umich.edu localMemBarrier = options.LocalMemBarrier, 2497235Sgblack@eecs.umich.edu countPages = options.countPages, 2507235Sgblack@eecs.umich.edu localDataStore = \ 2517235Sgblack@eecs.umich.edu LdsState(banks = options.numLdsBanks, 2527235Sgblack@eecs.umich.edu bankConflictPenalty = \ 2537235Sgblack@eecs.umich.edu options.ldsBankConflictPenalty), 2547235Sgblack@eecs.umich.edu out_of_order_data_delivery = 2557235Sgblack@eecs.umich.edu options.outOfOrderDataDelivery)) 2567210Sgblack@eecs.umich.edu wavefronts = [] 2577235Sgblack@eecs.umich.edu vrfs = [] 2587210Sgblack@eecs.umich.edu for j in xrange(options.simds_per_cu): 2597235Sgblack@eecs.umich.edu for k in xrange(shader.n_wf): 2607210Sgblack@eecs.umich.edu wavefronts.append(Wavefront(simdId = j, wf_slot_id = k, 2617210Sgblack@eecs.umich.edu wfSize = options.wf_size)) 2627211Sgblack@eecs.umich.edu vrfs.append(VectorRegisterFile(simd_id=j, 2637211Sgblack@eecs.umich.edu num_regs_per_simd=options.vreg_file_size)) 2647211Sgblack@eecs.umich.edu compute_units[-1].wavefronts = wavefronts 2657210Sgblack@eecs.umich.edu compute_units[-1].vector_register_file = vrfs 2667210Sgblack@eecs.umich.edu if options.TLB_prefetch: 2677210Sgblack@eecs.umich.edu compute_units[-1].prefetch_depth = options.TLB_prefetch 2687210Sgblack@eecs.umich.edu compute_units[-1].prefetch_prev_type = options.pf_type 2697235Sgblack@eecs.umich.edu 2707235Sgblack@eecs.umich.edu # attach the LDS and the CU to the bus (actually a Bridge) 2717235Sgblack@eecs.umich.edu compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave 2727235Sgblack@eecs.umich.edu compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort 2737235Sgblack@eecs.umich.edu 2747235Sgblack@eecs.umich.edu# Attach compute units to GPU 2757235Sgblack@eecs.umich.edushader.CUs = compute_units 2767235Sgblack@eecs.umich.edu 2777210Sgblack@eecs.umich.edu########################## Creating the CPU system ######################## 2787235Sgblack@eecs.umich.eduoptions.num_cpus = options.num_cpus 2797210Sgblack@eecs.umich.edu 2807235Sgblack@eecs.umich.edu# The shader core will be whatever is after the CPU cores are accounted for 2817210Sgblack@eecs.umich.edushader_idx = options.num_cpus 2827210Sgblack@eecs.umich.edu 2837210Sgblack@eecs.umich.edu# The command processor will be whatever is after the shader is accounted for 2847210Sgblack@eecs.umich.educp_idx = shader_idx + 1 2857210Sgblack@eecs.umich.educp_list = [] 2867227Sgblack@eecs.umich.edu 2877227Sgblack@eecs.umich.edu# List of CPUs 2887227Sgblack@eecs.umich.educpu_list = [] 2897227Sgblack@eecs.umich.edu 2907227Sgblack@eecs.umich.eduCpuClass, mem_mode = Simulation.getCPUClass(options.cpu_type) 2917227Sgblack@eecs.umich.eduif CpuClass == AtomicSimpleCPU: 2927210Sgblack@eecs.umich.edu fatal("AtomicSimpleCPU is not supported") 2937235Sgblack@eecs.umich.eduif mem_mode != 'timing': 2947235Sgblack@eecs.umich.edu fatal("Only the timing memory mode is supported") 2957235Sgblack@eecs.umich.edushader.timing = True 2967235Sgblack@eecs.umich.edu 2977235Sgblack@eecs.umich.eduif options.fast_forward and options.fast_forward_pseudo_op: 2987235Sgblack@eecs.umich.edu fatal("Cannot fast-forward based both on the number of instructions and" 2997235Sgblack@eecs.umich.edu " on pseudo-ops") 3007235Sgblack@eecs.umich.edufast_forward = options.fast_forward or options.fast_forward_pseudo_op 3017210Sgblack@eecs.umich.edu 3027235Sgblack@eecs.umich.eduif fast_forward: 3037210Sgblack@eecs.umich.edu FutureCpuClass, future_mem_mode = CpuClass, mem_mode 3047235Sgblack@eecs.umich.edu 3057210Sgblack@eecs.umich.edu CpuClass = X86KvmCPU 3067210Sgblack@eecs.umich.edu mem_mode = 'atomic_noncaching' 3077210Sgblack@eecs.umich.edu # Leave shader.timing untouched, because its value only matters at the 3087210Sgblack@eecs.umich.edu # start of the simulation and because we require switching cpus 3097250Sgblack@eecs.umich.edu # *before* the first kernel launch. 3107235Sgblack@eecs.umich.edu 3117235Sgblack@eecs.umich.edu future_cpu_list = [] 3127235Sgblack@eecs.umich.edu 3137235Sgblack@eecs.umich.edu # Initial CPUs to be used during fast-forwarding. 3147235Sgblack@eecs.umich.edu for i in xrange(options.num_cpus): 3157235Sgblack@eecs.umich.edu cpu = CpuClass(cpu_id = i, 3167250Sgblack@eecs.umich.edu clk_domain = SrcClockDomain( 3177250Sgblack@eecs.umich.edu clock = options.CPUClock, 3187250Sgblack@eecs.umich.edu voltage_domain = VoltageDomain( 3197250Sgblack@eecs.umich.edu voltage = options.cpu_voltage))) 3207250Sgblack@eecs.umich.edu cpu_list.append(cpu) 3217250Sgblack@eecs.umich.edu 3227250Sgblack@eecs.umich.edu if options.fast_forward: 3237250Sgblack@eecs.umich.edu cpu.max_insts_any_thread = int(options.fast_forward) 3247250Sgblack@eecs.umich.edu 3257250Sgblack@eecs.umich.eduif fast_forward: 3267250Sgblack@eecs.umich.edu MainCpuClass = FutureCpuClass 3277250Sgblack@eecs.umich.eduelse: 3287210Sgblack@eecs.umich.edu MainCpuClass = CpuClass 3297210Sgblack@eecs.umich.edu 3307210Sgblack@eecs.umich.edu# CPs to be used throughout the simulation. 3317210Sgblack@eecs.umich.edufor i in xrange(options.num_cp): 3327210Sgblack@eecs.umich.edu cp = MainCpuClass(cpu_id = options.num_cpus + i, 3337210Sgblack@eecs.umich.edu clk_domain = SrcClockDomain( 3347210Sgblack@eecs.umich.edu clock = options.CPUClock, 3357210Sgblack@eecs.umich.edu voltage_domain = VoltageDomain( 3367210Sgblack@eecs.umich.edu voltage = options.cpu_voltage))) 3377194Sgblack@eecs.umich.edu cp_list.append(cp) 3387194Sgblack@eecs.umich.edu 3397194Sgblack@eecs.umich.edu# Main CPUs (to be used after fast-forwarding if fast-forwarding is specified). 3407194Sgblack@eecs.umich.edufor i in xrange(options.num_cpus): 3417194Sgblack@eecs.umich.edu cpu = MainCpuClass(cpu_id = i, 3427194Sgblack@eecs.umich.edu clk_domain = SrcClockDomain( 3437194Sgblack@eecs.umich.edu clock = options.CPUClock, 3447194Sgblack@eecs.umich.edu voltage_domain = VoltageDomain( 3457194Sgblack@eecs.umich.edu voltage = options.cpu_voltage))) 3467194Sgblack@eecs.umich.edu if fast_forward: 3477194Sgblack@eecs.umich.edu cpu.switched_out = True 3487194Sgblack@eecs.umich.edu future_cpu_list.append(cpu) 3497194Sgblack@eecs.umich.edu else: 3507216Sgblack@eecs.umich.edu cpu_list.append(cpu) 3517194Sgblack@eecs.umich.edu 3527224Sgblack@eecs.umich.edu########################## Creating the GPU dispatcher ######################## 3537194Sgblack@eecs.umich.edu# Dispatcher dispatches work from host CPU to GPU 3547224Sgblack@eecs.umich.eduhost_cpu = cpu_list[0] 3557194Sgblack@eecs.umich.edudispatcher = GpuDispatcher() 3567218Sgblack@eecs.umich.edu 3577194Sgblack@eecs.umich.edu########################## Create and assign the workload ######################## 3587216Sgblack@eecs.umich.edu# Check for rel_path in elements of base_list using test, returning 3597194Sgblack@eecs.umich.edu# the first full path that satisfies test 3607218Sgblack@eecs.umich.edudef find_path(base_list, rel_path, test): 3617194Sgblack@eecs.umich.edu for base in base_list: 3627194Sgblack@eecs.umich.edu if not base: 3637194Sgblack@eecs.umich.edu # base could be None if environment var not set 3647194Sgblack@eecs.umich.edu continue 3657194Sgblack@eecs.umich.edu full_path = os.path.join(base, rel_path) 3667194Sgblack@eecs.umich.edu if test(full_path): 3677194Sgblack@eecs.umich.edu return full_path 3687194Sgblack@eecs.umich.edu fatal("%s not found in %s" % (rel_path, base_list)) 3697194Sgblack@eecs.umich.edu 3707194Sgblack@eecs.umich.edudef find_file(base_list, rel_path): 3717194Sgblack@eecs.umich.edu return find_path(base_list, rel_path, os.path.isfile) 3727194Sgblack@eecs.umich.edu 3737194Sgblack@eecs.umich.eduexecutable = find_path(benchmark_path, options.cmd, os.path.exists) 3747194Sgblack@eecs.umich.edu# it's common for a benchmark to be in a directory with the same 3757194Sgblack@eecs.umich.edu# name as the executable, so we handle that automatically 3767194Sgblack@eecs.umich.eduif os.path.isdir(executable): 3777194Sgblack@eecs.umich.edu benchmark_path = [executable] 3787194Sgblack@eecs.umich.edu executable = find_file(benchmark_path, options.cmd) 3797194Sgblack@eecs.umich.eduif options.kernel_files: 3807194Sgblack@eecs.umich.edu kernel_files = [find_file(benchmark_path, f) 3817194Sgblack@eecs.umich.edu for f in options.kernel_files.split(':')] 3827231Sgblack@eecs.umich.eduelse: 3837194Sgblack@eecs.umich.edu # if kernel_files is not set, see if there's a unique .asm file 3847231Sgblack@eecs.umich.edu # in the same directory as the executable 3857194Sgblack@eecs.umich.edu kernel_path = os.path.dirname(executable) 3867231Sgblack@eecs.umich.edu kernel_files = glob.glob(os.path.join(kernel_path, '*.asm')) 3877194Sgblack@eecs.umich.edu if kernel_files: 3887231Sgblack@eecs.umich.edu print "Using GPU kernel code file(s)", ",".join(kernel_files) 3897194Sgblack@eecs.umich.edu else: 3907231Sgblack@eecs.umich.edu fatal("Can't locate kernel code (.asm) in " + kernel_path) 3917194Sgblack@eecs.umich.edu 3927231Sgblack@eecs.umich.edu# OpenCL driver 3937194Sgblack@eecs.umich.edudriver = ClDriver(filename="hsa", codefile=kernel_files) 3947194Sgblack@eecs.umich.edufor cpu in cpu_list: 3957194Sgblack@eecs.umich.edu cpu.workload = Process(executable = executable, 3967194Sgblack@eecs.umich.edu cmd = [options.cmd] + options.options.split(), 3977194Sgblack@eecs.umich.edu drivers = [driver]) 3987194Sgblack@eecs.umich.edufor cp in cp_list: 3997194Sgblack@eecs.umich.edu cp.workload = host_cpu.workload 4007194Sgblack@eecs.umich.edu 4017222Sgblack@eecs.umich.eduif fast_forward: 4027194Sgblack@eecs.umich.edu for i in xrange(len(future_cpu_list)): 4037222Sgblack@eecs.umich.edu future_cpu_list[i].workload = cpu_list[i].workload 4047194Sgblack@eecs.umich.edu 4057222Sgblack@eecs.umich.edu########################## Create the overall system ######################## 4067194Sgblack@eecs.umich.edu# List of CPUs that must be switched when moving between KVM and simulation 4077222Sgblack@eecs.umich.eduif fast_forward: 4087194Sgblack@eecs.umich.edu switch_cpu_list = \ 4097222Sgblack@eecs.umich.edu [(cpu_list[i], future_cpu_list[i]) for i in xrange(options.num_cpus)] 4107194Sgblack@eecs.umich.edu 4117222Sgblack@eecs.umich.edu# Full list of processing cores in the system. Note that 4127194Sgblack@eecs.umich.edu# dispatcher is also added to cpu_list although it is 4137194Sgblack@eecs.umich.edu# not a processing element 4147194Sgblack@eecs.umich.educpu_list = cpu_list + [shader] + cp_list + [dispatcher] 4157194Sgblack@eecs.umich.edu 4167194Sgblack@eecs.umich.edu# creating the overall system 4177220Sgblack@eecs.umich.edu# notice the cpu list is explicitly added as a parameter to System 4187194Sgblack@eecs.umich.edusystem = System(cpu = cpu_list, 4197220Sgblack@eecs.umich.edu mem_ranges = [AddrRange(options.mem_size)], 4207194Sgblack@eecs.umich.edu cache_line_size = options.cacheline_size, 4217220Sgblack@eecs.umich.edu mem_mode = mem_mode) 4227194Sgblack@eecs.umich.eduif fast_forward: 4237220Sgblack@eecs.umich.edu system.future_cpu = future_cpu_list 4247194Sgblack@eecs.umich.edusystem.voltage_domain = VoltageDomain(voltage = options.sys_voltage) 4257220Sgblack@eecs.umich.edusystem.clk_domain = SrcClockDomain(clock = options.sys_clock, 4267194Sgblack@eecs.umich.edu voltage_domain = system.voltage_domain) 4277220Sgblack@eecs.umich.edu 4287194Sgblack@eecs.umich.eduif fast_forward: 4297194Sgblack@eecs.umich.edu have_kvm_support = 'BaseKvmCPU' in globals() 4307194Sgblack@eecs.umich.edu if have_kvm_support and buildEnv['TARGET_ISA'] == "x86": 4317194Sgblack@eecs.umich.edu system.vm = KvmVM() 4327194Sgblack@eecs.umich.edu for i in xrange(len(host_cpu.workload)): 4337231Sgblack@eecs.umich.edu host_cpu.workload[i].useArchPT = True 4347194Sgblack@eecs.umich.edu host_cpu.workload[i].kvmInSE = True 4357231Sgblack@eecs.umich.edu else: 4367194Sgblack@eecs.umich.edu fatal("KvmCPU can only be used in SE mode with x86") 4377231Sgblack@eecs.umich.edu 4387194Sgblack@eecs.umich.edu# configure the TLB hierarchy 4397231Sgblack@eecs.umich.eduGPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx) 4407194Sgblack@eecs.umich.edu 4417231Sgblack@eecs.umich.edu# create Ruby system 4427194Sgblack@eecs.umich.edusystem.piobus = IOXBar(width=32, response_latency=0, 4437231Sgblack@eecs.umich.edu frontend_latency=0, forward_latency=0) 4447194Sgblack@eecs.umich.eduRuby.create_system(options, None, system) 4457194Sgblack@eecs.umich.edusystem.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock, 4467194Sgblack@eecs.umich.edu voltage_domain = system.voltage_domain) 4477194Sgblack@eecs.umich.edu 4487194Sgblack@eecs.umich.edu# attach the CPU ports to Ruby 4497194Sgblack@eecs.umich.edufor i in range(options.num_cpus): 4507194Sgblack@eecs.umich.edu ruby_port = system.ruby._cpu_ports[i] 4517194Sgblack@eecs.umich.edu 4527194Sgblack@eecs.umich.edu # Create interrupt controller 4537139Sgblack@eecs.umich.edu system.cpu[i].createInterruptController() 4547188Sgblack@eecs.umich.edu 4557188Sgblack@eecs.umich.edu # Connect cache port's to ruby 4567188Sgblack@eecs.umich.edu system.cpu[i].icache_port = ruby_port.slave 4577188Sgblack@eecs.umich.edu system.cpu[i].dcache_port = ruby_port.slave 4587188Sgblack@eecs.umich.edu 4597188Sgblack@eecs.umich.edu ruby_port.mem_master_port = system.piobus.slave 4607188Sgblack@eecs.umich.edu if buildEnv['TARGET_ISA'] == "x86": 4617188Sgblack@eecs.umich.edu system.cpu[i].interrupts[0].pio = system.piobus.master 4627139Sgblack@eecs.umich.edu system.cpu[i].interrupts[0].int_master = system.piobus.slave 4637188Sgblack@eecs.umich.edu system.cpu[i].interrupts[0].int_slave = system.piobus.master 4647139Sgblack@eecs.umich.edu if fast_forward: 4657188Sgblack@eecs.umich.edu system.cpu[i].itb.walker.port = ruby_port.slave 4667188Sgblack@eecs.umich.edu system.cpu[i].dtb.walker.port = ruby_port.slave 4677188Sgblack@eecs.umich.edu 4687188Sgblack@eecs.umich.edu# attach CU ports to Ruby 4697188Sgblack@eecs.umich.edu# Because of the peculiarities of the CP core, you may have 1 CPU but 2 4707188Sgblack@eecs.umich.edu# sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be 4717139Sgblack@eecs.umich.edu# hooked up until after the CP. To make this script generic, figure out 4727188Sgblack@eecs.umich.edu# the index as below, but note that this assumes there is one sequencer 4737188Sgblack@eecs.umich.edu# per compute unit and one sequencer per SQC for the math to work out 4747188Sgblack@eecs.umich.edu# correctly. 4757188Sgblack@eecs.umich.edugpu_port_idx = len(system.ruby._cpu_ports) \ 4767188Sgblack@eecs.umich.edu - options.num_compute_units - options.num_sqc 4777188Sgblack@eecs.umich.edugpu_port_idx = gpu_port_idx - options.num_cp * 2 4787139Sgblack@eecs.umich.edu 4797139Sgblack@eecs.umich.eduwavefront_size = options.wf_size 4807139Sgblack@eecs.umich.edufor i in xrange(n_cu): 4817139Sgblack@eecs.umich.edu # The pipeline issues wavefront_size number of uncoalesced requests 4827188Sgblack@eecs.umich.edu # in one GPU issue cycle. Hence wavefront_size mem ports. 4837188Sgblack@eecs.umich.edu for j in xrange(wavefront_size): 4847188Sgblack@eecs.umich.edu system.cpu[shader_idx].CUs[i].memory_port[j] = \ 4857188Sgblack@eecs.umich.edu system.ruby._cpu_ports[gpu_port_idx].slave[j] 4867188Sgblack@eecs.umich.edu gpu_port_idx += 1 4877188Sgblack@eecs.umich.edu 4887188Sgblack@eecs.umich.edufor i in xrange(n_cu): 4897188Sgblack@eecs.umich.edu if i > 0 and not i % options.cu_per_sqc: 4907188Sgblack@eecs.umich.edu print "incrementing idx on ", i 4917188Sgblack@eecs.umich.edu gpu_port_idx += 1 4927188Sgblack@eecs.umich.edu system.cpu[shader_idx].CUs[i].sqc_port = \ 4937188Sgblack@eecs.umich.edu system.ruby._cpu_ports[gpu_port_idx].slave 4947188Sgblack@eecs.umich.edugpu_port_idx = gpu_port_idx + 1 4957188Sgblack@eecs.umich.edu 4967188Sgblack@eecs.umich.edu# attach CP ports to Ruby 4977188Sgblack@eecs.umich.edufor i in xrange(options.num_cp): 4987188Sgblack@eecs.umich.edu system.cpu[cp_idx].createInterruptController() 4997188Sgblack@eecs.umich.edu system.cpu[cp_idx].dcache_port = \ 5007188Sgblack@eecs.umich.edu system.ruby._cpu_ports[gpu_port_idx + i * 2].slave 5017188Sgblack@eecs.umich.edu system.cpu[cp_idx].icache_port = \ 5027188Sgblack@eecs.umich.edu system.ruby._cpu_ports[gpu_port_idx + i * 2 + 1].slave 5037188Sgblack@eecs.umich.edu system.cpu[cp_idx].interrupts[0].pio = system.piobus.master 5047188Sgblack@eecs.umich.edu system.cpu[cp_idx].interrupts[0].int_master = system.piobus.slave 5057185Sgblack@eecs.umich.edu system.cpu[cp_idx].interrupts[0].int_slave = system.piobus.master 5067188Sgblack@eecs.umich.edu cp_idx = cp_idx + 1 5077188Sgblack@eecs.umich.edu 5087188Sgblack@eecs.umich.edu# connect dispatcher to the system.piobus 5097188Sgblack@eecs.umich.edudispatcher.pio = system.piobus.master 5107188Sgblack@eecs.umich.edudispatcher.dma = system.piobus.slave 5117188Sgblack@eecs.umich.edu 5127188Sgblack@eecs.umich.edu################# Connect the CPU and GPU via GPU Dispatcher ################### 5137188Sgblack@eecs.umich.edu# CPU rings the GPU doorbell to notify a pending task 5147188Sgblack@eecs.umich.edu# using this interface. 5157188Sgblack@eecs.umich.edu# And GPU uses this interface to notify the CPU of task completion 5167188Sgblack@eecs.umich.edu# The communcation happens through emulated driver. 5177188Sgblack@eecs.umich.edu 5187139Sgblack@eecs.umich.edu# Note this implicit setting of the cpu_pointer, shader_pointer and tlb array 5197139Sgblack@eecs.umich.edu# parameters must be after the explicit setting of the System cpu list 5207139Sgblack@eecs.umich.eduif fast_forward: 5217139Sgblack@eecs.umich.edu shader.cpu_pointer = future_cpu_list[0] 5227139Sgblack@eecs.umich.edu dispatcher.cpu = future_cpu_list[0] 5237139Sgblack@eecs.umich.eduelse: 5247139Sgblack@eecs.umich.edu shader.cpu_pointer = host_cpu 5257139Sgblack@eecs.umich.edu dispatcher.cpu = host_cpu 5267139Sgblack@eecs.umich.edudispatcher.shader_pointer = shader 5277139Sgblack@eecs.umich.edudispatcher.cl_driver = driver 5287139Sgblack@eecs.umich.edu 5297139Sgblack@eecs.umich.edu########################## Start simulation ######################## 5307139Sgblack@eecs.umich.edu 5317139Sgblack@eecs.umich.eduroot = Root(system=system, full_system=False) 5327185Sgblack@eecs.umich.edum5.ticks.setGlobalFrequency('1THz') 5337139Sgblack@eecs.umich.eduif options.abs_max_tick: 5347185Sgblack@eecs.umich.edu maxtick = options.abs_max_tick 5357139Sgblack@eecs.umich.eduelse: 5367139Sgblack@eecs.umich.edu maxtick = m5.MaxTick 5377139Sgblack@eecs.umich.edu 5387188Sgblack@eecs.umich.edu# Benchmarks support work item annotations 5397188Sgblack@eecs.umich.eduSimulation.setWorkCountOptions(system, options) 5407188Sgblack@eecs.umich.edu 5417188Sgblack@eecs.umich.edu# Checkpointing is not supported by APU model 5427139Sgblack@eecs.umich.eduif (options.checkpoint_dir != None or 5437188Sgblack@eecs.umich.edu options.checkpoint_restore != None): 5447139Sgblack@eecs.umich.edu fatal("Checkpointing not supported by apu model") 5457188Sgblack@eecs.umich.edu 5467139Sgblack@eecs.umich.educheckpoint_dir = None 5477139Sgblack@eecs.umich.edum5.instantiate(checkpoint_dir) 5487139Sgblack@eecs.umich.edu 5497139Sgblack@eecs.umich.edu# Map workload to this address space 5507139Sgblack@eecs.umich.eduhost_cpu.workload[0].map(0x10000000, 0x200000000, 4096) 5517139Sgblack@eecs.umich.edu 5527139Sgblack@eecs.umich.eduif options.fast_forward: 5537141Sgblack@eecs.umich.edu print "Switch at instruction count: %d" % \ 5547195Sgblack@eecs.umich.edu cpu_list[0].max_insts_any_thread 5557195Sgblack@eecs.umich.edu 5567195Sgblack@eecs.umich.eduexit_event = m5.simulate(maxtick) 5577195Sgblack@eecs.umich.edu 5587195Sgblack@eecs.umich.eduif options.fast_forward: 5597195Sgblack@eecs.umich.edu if exit_event.getCause() == "a thread reached the max instruction count": 5607195Sgblack@eecs.umich.edu m5.switchCpus(system, switch_cpu_list) 5617195Sgblack@eecs.umich.edu print "Switched CPUS @ tick %s" % (m5.curTick()) 5627195Sgblack@eecs.umich.edu m5.stats.reset() 5637195Sgblack@eecs.umich.edu exit_event = m5.simulate(maxtick - m5.curTick()) 5647195Sgblack@eecs.umich.eduelif options.fast_forward_pseudo_op: 5657195Sgblack@eecs.umich.edu while exit_event.getCause() == "switchcpu": 5667195Sgblack@eecs.umich.edu # If we are switching *to* kvm, then the current stats are meaningful 5677195Sgblack@eecs.umich.edu # Note that we don't do any warmup by default 5687195Sgblack@eecs.umich.edu if type(switch_cpu_list[0][0]) == FutureCpuClass: 5697195Sgblack@eecs.umich.edu print "Dumping stats..." 5707195Sgblack@eecs.umich.edu m5.stats.dump() 5717195Sgblack@eecs.umich.edu m5.switchCpus(system, switch_cpu_list) 5727195Sgblack@eecs.umich.edu print "Switched CPUS @ tick %s" % (m5.curTick()) 5737195Sgblack@eecs.umich.edu m5.stats.reset() 5747195Sgblack@eecs.umich.edu # This lets us switch back and forth without keeping a counter 5757195Sgblack@eecs.umich.edu switch_cpu_list = [(x[1], x[0]) for x in switch_cpu_list] 5767213Sgblack@eecs.umich.edu exit_event = m5.simulate(maxtick - m5.curTick()) 5777213Sgblack@eecs.umich.edu 5787213Sgblack@eecs.umich.eduprint "Ticks:", m5.curTick() 5797213Sgblack@eecs.umich.eduprint 'Exiting because ', exit_event.getCause() 5807213Sgblack@eecs.umich.edusys.exit(exit_event.getCode()) 5817213Sgblack@eecs.umich.edu