1#
2#  Copyright (c) 2015 Advanced Micro Devices, Inc.
3#  All rights reserved.
4#
5#  For use for simulation and test purposes only
6#
7#  Redistribution and use in source and binary forms, with or without
8#  modification, are permitted provided that the following conditions are met:
9#
10#  1. Redistributions of source code must retain the above copyright notice,
11#  this list of conditions and the following disclaimer.
12#
13#  2. Redistributions in binary form must reproduce the above copyright notice,
14#  this list of conditions and the following disclaimer in the documentation
15#  and/or other materials provided with the distribution.
16#
17#  3. Neither the name of the copyright holder nor the names of its contributors
18#  may be used to endorse or promote products derived from this software
19#  without specific prior written permission.
20#
21#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31#  POSSIBILITY OF SUCH DAMAGE.
32#
33#  Author: Brad Beckmann
34#
35
36from __future__ import print_function
37
38import m5
39from m5.objects import *
40from m5.defines import buildEnv
41from m5.util import addToPath
42import os, optparse, sys, math, glob
43
44m5.util.addToPath('../configs/')
45
46from ruby import Ruby
47from common import Options
48from common import GPUTLBOptions, GPUTLBConfig
49
50########################## Script Options ########################
51def setOption(parser, opt_str, value = 1):
52    # check to make sure the option actually exists
53    if not parser.has_option(opt_str):
54        raise Exception("cannot find %s in list of possible options" % opt_str)
55
56    opt = parser.get_option(opt_str)
57    # set the value
58    exec("parser.values.%s = %s" % (opt.dest, value))
59
60def getOption(parser, opt_str):
61    # check to make sure the option actually exists
62    if not parser.has_option(opt_str):
63        raise Exception("cannot find %s in list of possible options" % opt_str)
64
65    opt = parser.get_option(opt_str)
66    # get the value
67    exec("return_value = parser.values.%s" % opt.dest)
68    return return_value
69
70def run_test(root):
71    """gpu test requires a specialized run_test implementation to set up the
72    mmio space."""
73
74    # instantiate configuration
75    m5.instantiate()
76
77    # Now that the system has been constructed, setup the mmio space
78    root.system.cpu[0].workload[0].map(0x10000000, 0x200000000, 4096)
79
80    # simulate until program terminates
81    exit_event = m5.simulate(maxtick)
82    print('Exiting @ tick', m5.curTick(), 'because', exit_event.getCause())
83
84parser = optparse.OptionParser()
85Options.addCommonOptions(parser)
86Options.addSEOptions(parser)
87
88parser.add_option("-k", "--kernel-files",
89                  help="file(s) containing GPU kernel code (colon separated)")
90parser.add_option("-u", "--num-compute-units", type="int", default=2,
91                  help="number of GPU compute units"),
92parser.add_option("--num-cp", type="int", default=0,
93                  help="Number of GPU Command Processors (CP)")
94parser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \
95                  "per CU")
96parser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \
97                  "sharing an SQC (icache, and thus icache TLB)")
98parser.add_option("--wf-size", type="int", default=64,
99                  help="Wavefront size(in workitems)")
100parser.add_option("--wfs-per-simd", type="int", default=8, help="Number of " \
101                  "WF slots per SIMD")
102parser.add_option("--sp-bypass-path-length", type="int", default=4, \
103                  help="Number of stages of bypass path in vector ALU for Single "\
104                  "Precision ops")
105parser.add_option("--dp-bypass-path-length", type="int", default=4, \
106                  help="Number of stages of bypass path in vector ALU for Double "\
107                  "Precision ops")
108parser.add_option("--issue-period", type="int", default=4, \
109                  help="Number of cycles per vector instruction issue period")
110parser.add_option("--glbmem-wr-bus-width", type="int", default=32, \
111                  help="VGPR to Coalescer (Global Memory) data bus width in bytes")
112parser.add_option("--glbmem-rd-bus-width", type="int", default=32, \
113                  help="Coalescer to VGPR (Global Memory) data bus width in bytes")
114parser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \
115                  help="Number of Shared Memory pipelines per CU")
116parser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \
117                  help="Number of Global Memory pipelines per CU")
118parser.add_option("--vreg-file-size", type="int", default=2048,
119                  help="number of physical vector registers per SIMD")
120parser.add_option("--bw-scalor", type="int", default=0,
121                  help="bandwidth scalor for scalability analysis")
122parser.add_option("--CPUClock", type="string", default="2GHz",
123                  help="CPU clock")
124parser.add_option("--GPUClock", type="string", default="1GHz",
125                  help="GPU clock")
126parser.add_option("--cpu-voltage", action="store", type="string",
127                  default='1.0V',
128                  help = """CPU  voltage domain""")
129parser.add_option("--gpu-voltage", action="store", type="string",
130                  default='1.0V',
131                  help = """CPU  voltage domain""")
132parser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST",
133                  help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)")
134parser.add_option("--xact-cas-mode", action="store_true",
135                  help="enable load_compare mode (transactional CAS)")
136parser.add_option("--SegFaultDebug",action="store_true",
137                 help="checks for GPU seg fault before TLB access")
138parser.add_option("--LocalMemBarrier",action="store_true",
139                 help="Barrier does not wait for writethroughs to complete")
140parser.add_option("--countPages", action="store_true",
141                 help="Count Page Accesses and output in per-CU output files")
142parser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\
143                  "TLBs")
144parser.add_option("--pf-type", type="string", help="type of prefetch: "\
145                  "PF_CU, PF_WF, PF_PHASE, PF_STRIDE")
146parser.add_option("--pf-stride", type="int", help="set prefetch stride")
147parser.add_option("--numLdsBanks", type="int", default=32,
148                  help="number of physical banks per LDS module")
149parser.add_option("--ldsBankConflictPenalty", type="int", default=1,
150                  help="number of cycles per LDS bank conflict")
151
152# Add the ruby specific and protocol specific options
153Ruby.define_options(parser)
154
155GPUTLBOptions.tlb_options(parser)
156
157(options, args) = parser.parse_args()
158
159# The GPU cache coherence protocols only work with the backing store
160setOption(parser, "--access-backing-store")
161
162# Currently, the sqc (I-Cache of GPU) is shared by
163# multiple compute units(CUs). The protocol works just fine
164# even if sqc is not shared. Overriding this option here
165# so that the user need not explicitly set this (assuming
166# sharing sqc is the common usage)
167n_cu = options.num_compute_units
168num_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc))
169options.num_sqc = num_sqc # pass this to Ruby
170
171########################## Creating the GPU system ########################
172# shader is the GPU
173shader = Shader(n_wf = options.wfs_per_simd,
174                clk_domain = SrcClockDomain(
175                    clock = options.GPUClock,
176                    voltage_domain = VoltageDomain(
177                        voltage = options.gpu_voltage)),
178                timing = True)
179
180# GPU_RfO(Read For Ownership) implements SC/TSO memory model.
181# Other GPU protocols implement release consistency at GPU side.
182# So, all GPU protocols other than GPU_RfO should make their writes
183# visible to the global memory and should read from global memory
184# during kernal boundary. The pipeline initiates(or do not initiate)
185# the acquire/release operation depending on this impl_kern_boundary_sync
186# flag. This flag=true means pipeline initiates a acquire/release operation
187# at kernel boundary.
188if buildEnv['PROTOCOL'] == 'GPU_RfO':
189    shader.impl_kern_boundary_sync = False
190else:
191    shader.impl_kern_boundary_sync = True
192
193# Switching off per-lane TLB by default
194per_lane = False
195if options.TLB_config == "perLane":
196    per_lane = True
197
198# List of compute units; one GPU can have multiple compute units
199compute_units = []
200for i in range(n_cu):
201    compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane,
202                                     num_SIMDs = options.simds_per_cu,
203                                     wfSize = options.wf_size,
204                                     spbypass_pipe_length = \
205                                     options.sp_bypass_path_length,
206                                     dpbypass_pipe_length = \
207                                     options.dp_bypass_path_length,
208                                     issue_period = options.issue_period,
209                                     coalescer_to_vrf_bus_width = \
210                                     options.glbmem_rd_bus_width,
211                                     vrf_to_coalescer_bus_width = \
212                                     options.glbmem_wr_bus_width,
213                                     num_global_mem_pipes = \
214                                     options.glb_mem_pipes_per_cu,
215                                     num_shared_mem_pipes = \
216                                     options.shr_mem_pipes_per_cu,
217                                     n_wf = options.wfs_per_simd,
218                                     execPolicy = options.CUExecPolicy,
219                                     xactCasMode = options.xact_cas_mode,
220                                     debugSegFault = options.SegFaultDebug,
221                                     functionalTLB = True,
222                                     localMemBarrier = options.LocalMemBarrier,
223                                     countPages = options.countPages,
224                                     localDataStore = \
225                                     LdsState(banks = options.numLdsBanks,
226                                              bankConflictPenalty = \
227                                              options.ldsBankConflictPenalty)))
228    wavefronts = []
229    vrfs = []
230    for j in range(options.simds_per_cu):
231        for k in range(int(shader.n_wf)):
232            wavefronts.append(Wavefront(simdId = j, wf_slot_id = k))
233        vrfs.append(VectorRegisterFile(simd_id=j,
234                              num_regs_per_simd=options.vreg_file_size))
235    compute_units[-1].wavefronts = wavefronts
236    compute_units[-1].vector_register_file = vrfs
237    if options.TLB_prefetch:
238        compute_units[-1].prefetch_depth = options.TLB_prefetch
239        compute_units[-1].prefetch_prev_type = options.pf_type
240
241    # attach the LDS and the CU to the bus (actually a Bridge)
242    compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave
243    compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort
244
245# Attach compute units to GPU
246shader.CUs = compute_units
247
248# this is a uniprocessor only test, thus the shader is the second index in the
249# list of "system.cpus"
250options.num_cpus = 1
251shader_idx = 1
252cpu = TimingSimpleCPU(cpu_id=0)
253
254########################## Creating the GPU dispatcher ########################
255# Dispatcher dispatches work from host CPU to GPU
256host_cpu = cpu
257dispatcher = GpuDispatcher()
258
259# Currently does not test for command processors
260cpu_list = [cpu] + [shader] + [dispatcher]
261
262system = System(cpu = cpu_list,
263                mem_ranges = [AddrRange(options.mem_size)],
264                mem_mode = 'timing')
265
266# Dummy voltage domain for all our clock domains
267system.voltage_domain = VoltageDomain(voltage = options.sys_voltage)
268system.clk_domain = SrcClockDomain(clock = '1GHz',
269                                   voltage_domain = system.voltage_domain)
270
271# Create a seperate clock domain for components that should run at
272# CPUs frequency
273system.cpu[0].clk_domain = SrcClockDomain(clock = '2GHz',
274                                          voltage_domain = \
275                                          system.voltage_domain)
276
277# configure the TLB hierarchy
278GPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx)
279
280# create Ruby system
281system.piobus = IOXBar(width=32, response_latency=0,
282                       frontend_latency=0, forward_latency=0)
283Ruby.create_system(options, None, system)
284
285# Create a separate clock for Ruby
286system.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,
287                                        voltage_domain = system.voltage_domain)
288
289# create the interrupt controller
290cpu.createInterruptController()
291
292#
293# Tie the cpu cache ports to the ruby cpu ports and
294# physmem, respectively
295#
296cpu.connectAllPorts(system.ruby._cpu_ports[0])
297system.ruby._cpu_ports[0].mem_master_port = system.piobus.slave
298
299# attach CU ports to Ruby
300# Because of the peculiarities of the CP core, you may have 1 CPU but 2
301# sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be
302# hooked up until after the CP. To make this script generic, figure out
303# the index as below, but note that this assumes there is one sequencer
304# per compute unit and one sequencer per SQC for the math to work out
305# correctly.
306gpu_port_idx = len(system.ruby._cpu_ports) \
307               - options.num_compute_units - options.num_sqc
308gpu_port_idx = gpu_port_idx - options.num_cp * 2
309
310wavefront_size = options.wf_size
311for i in range(n_cu):
312    # The pipeline issues wavefront_size number of uncoalesced requests
313    # in one GPU issue cycle. Hence wavefront_size mem ports.
314    for j in range(wavefront_size):
315        system.cpu[shader_idx].CUs[i].memory_port[j] = \
316                  system.ruby._cpu_ports[gpu_port_idx].slave[j]
317    gpu_port_idx += 1
318
319for i in range(n_cu):
320    if i > 0 and not i % options.cu_per_sqc:
321        gpu_port_idx += 1
322    system.cpu[shader_idx].CUs[i].sqc_port = \
323            system.ruby._cpu_ports[gpu_port_idx].slave
324gpu_port_idx = gpu_port_idx + 1
325
326# Current regression tests do not support the command processor
327assert(options.num_cp == 0)
328
329# connect dispatcher to the system.piobus
330dispatcher.pio = system.piobus.master
331dispatcher.dma = system.piobus.slave
332
333################# Connect the CPU and GPU via GPU Dispatcher ###################
334# CPU rings the GPU doorbell to notify a pending task
335# using this interface.
336# And GPU uses this interface to notify the CPU of task completion
337# The communcation happens through emulated driver.
338
339# Note this implicit setting of the cpu_pointer, shader_pointer and tlb array
340# parameters must be after the explicit setting of the System cpu list
341shader.cpu_pointer = host_cpu
342dispatcher.cpu = host_cpu
343dispatcher.shader_pointer = shader
344
345# -----------------------
346# run simulation
347# -----------------------
348
349root = Root(full_system = False, system = system)
350m5.ticks.setGlobalFrequency('1THz')
351root.system.mem_mode = 'timing'
352