1#
2# Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3# All rights reserved.
4#
5# For use for simulation and test purposes only
6#
7# Redistribution and use in source and binary forms, with or without
8# modification, are permitted provided that the following conditions are met:
9#
10# 1. Redistributions of source code must retain the above copyright notice,
11# this list of conditions and the following disclaimer.
12#
13# 2. Redistributions in binary form must reproduce the above copyright notice,
14# this list of conditions and the following disclaimer in the documentation
15# and/or other materials provided with the distribution.
16#
17# 3. Neither the name of the copyright holder nor the names of its contributors
18# may be used to endorse or promote products derived from this software
19# without specific prior written permission.
20#
21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31# POSSIBILITY OF SUCH DAMAGE.
32#
33# Author: Lisa Hsu
34#
35
36import math
37import m5
38from m5.objects import *
39from m5.defines import buildEnv
40from Ruby import create_topology
41from Ruby import send_evicts
42
43from Cluster import Cluster
44from Crossbar import Crossbar
45
46class CntrlBase:
47 _seqs = 0
48 @classmethod
49 def seqCount(cls):
50 # Use SeqCount not class since we need global count
51 CntrlBase._seqs += 1
52 return CntrlBase._seqs - 1
53
54 _cntrls = 0
55 @classmethod
56 def cntrlCount(cls):
57 # Use CntlCount not class since we need global count
58 CntrlBase._cntrls += 1
59 return CntrlBase._cntrls - 1
60
61 _version = 0
62 @classmethod
63 def versionCount(cls):
64 cls._version += 1 # Use count for this particular type
65 return cls._version - 1
66
67class TccDirCache(RubyCache):
68 size = "512kB"
69 assoc = 16
70 resourceStalls = False
71 def create(self, options):
72 self.size = MemorySize(options.tcc_size)
73 self.size.value += (options.num_compute_units *
74 (MemorySize(options.tcp_size).value) *
75 options.tcc_dir_factor) / long(options.num_tccs)
76 self.start_index_bit = math.log(options.cacheline_size, 2) + \
77 math.log(options.num_tccs, 2)
78 self.replacement_policy = PseudoLRUReplacementPolicy()
79
80class L1DCache(RubyCache):
81 resourceStalls = False
82 def create(self, options):
83 self.size = MemorySize(options.l1d_size)
84 self.assoc = options.l1d_assoc
85 self.replacement_policy = PseudoLRUReplacementPolicy()
86
87class L1ICache(RubyCache):
88 resourceStalls = False
89 def create(self, options):
90 self.size = MemorySize(options.l1i_size)
91 self.assoc = options.l1i_assoc
92 self.replacement_policy = PseudoLRUReplacementPolicy()
93
94class L2Cache(RubyCache):
95 resourceStalls = False
96 def create(self, options):
97 self.size = MemorySize(options.l2_size)
98 self.assoc = options.l2_assoc
99 self.replacement_policy = PseudoLRUReplacementPolicy()
100
101
102class CPCntrl(CorePair_Controller, CntrlBase):
103
104 def create(self, options, ruby_system, system):
105 self.version = self.versionCount()
106
107 self.L1Icache = L1ICache()
108 self.L1Icache.create(options)
109 self.L1D0cache = L1DCache()
110 self.L1D0cache.create(options)
111 self.L1D1cache = L1DCache()
112 self.L1D1cache.create(options)
113 self.L2cache = L2Cache()
114 self.L2cache.create(options)
115
116 self.sequencer = RubySequencer()
117 self.sequencer.icache_hit_latency = 2
118 self.sequencer.dcache_hit_latency = 2
119 self.sequencer.version = self.seqCount()
120 self.sequencer.icache = self.L1Icache
121 self.sequencer.dcache = self.L1D0cache
122 self.sequencer.ruby_system = ruby_system
123 self.sequencer.coreid = 0
124 self.sequencer.is_cpu_sequencer = True
125
126 self.sequencer1 = RubySequencer()
127 self.sequencer1.version = self.seqCount()
128 self.sequencer1.icache = self.L1Icache
129 self.sequencer1.dcache = self.L1D1cache
130 self.sequencer1.icache_hit_latency = 2
131 self.sequencer1.dcache_hit_latency = 2
132 self.sequencer1.ruby_system = ruby_system
133 self.sequencer1.coreid = 1
134 self.sequencer1.is_cpu_sequencer = True
135
136 self.issue_latency = options.cpu_to_dir_latency
137 self.send_evictions = send_evicts(options)
138
139 self.ruby_system = ruby_system
140
141 if options.recycle_latency:
142 self.recycle_latency = options.recycle_latency
143
144class TCPCache(RubyCache):
145 assoc = 8
146 dataArrayBanks = 16
147 tagArrayBanks = 4
148 dataAccessLatency = 4
149 tagAccessLatency = 1
150 def create(self, options):
151 self.size = MemorySize(options.tcp_size)
152 self.replacement_policy = PseudoLRUReplacementPolicy()
153
154class TCPCntrl(TCP_Controller, CntrlBase):
155
156 def create(self, options, ruby_system, system):
157 self.version = self.versionCount()
158
159 self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
160 self.L1cache.resourceStalls = options.no_resource_stalls
161 self.L1cache.create(options)
162
163 self.coalescer = RubyGPUCoalescer()
164 self.coalescer.version = self.seqCount()
165 self.coalescer.icache = self.L1cache
166 self.coalescer.dcache = self.L1cache
167 self.coalescer.ruby_system = ruby_system
168 self.coalescer.support_inst_reqs = False
169 self.coalescer.is_cpu_sequencer = False
170 self.coalescer.max_outstanding_requests = options.simds_per_cu * \
171 options.wfs_per_simd * \
172 options.wf_size
173
174 self.sequencer = RubySequencer()
175 self.sequencer.version = self.seqCount()
176 self.sequencer.icache = self.L1cache
177 self.sequencer.dcache = self.L1cache
178 self.sequencer.ruby_system = ruby_system
179 self.sequencer.is_cpu_sequencer = True
180
181 self.use_seq_not_coal = False
182
183 self.ruby_system = ruby_system
184
185 if options.recycle_latency:
186 self.recycle_latency = options.recycle_latency
187
188 def createCP(self, options, ruby_system, system):
189 self.version = self.versionCount()
190
191 self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
192 self.L1cache.resourceStalls = options.no_resource_stalls
193 self.L1cache.create(options)
194
195 self.coalescer = RubyGPUCoalescer()
196 self.coalescer.version = self.seqCount()
197 self.coalescer.icache = self.L1cache
198 self.coalescer.dcache = self.L1cache
199 self.coalescer.ruby_system = ruby_system
200 self.coalescer.support_inst_reqs = False
201 self.coalescer.is_cpu_sequencer = False
202
203 self.sequencer = RubySequencer()
204 self.sequencer.version = self.seqCount()
205 self.sequencer.icache = self.L1cache
206 self.sequencer.dcache = self.L1cache
207 self.sequencer.ruby_system = ruby_system
208 self.sequencer.is_cpu_sequencer = True
209
210 self.use_seq_not_coal = True
211
212 self.ruby_system = ruby_system
213
214 if options.recycle_latency:
215 self.recycle_latency = options.recycle_latency
216
217class SQCCache(RubyCache):
218 size = "32kB"
219 assoc = 8
220 dataArrayBanks = 16
221 tagArrayBanks = 4
222 dataAccessLatency = 4
223 tagAccessLatency = 1
224 def create(self, options):
225 self.replacement_policy = PseudoLRUReplacementPolicy()
226
227class SQCCntrl(SQC_Controller, CntrlBase):
228
229 def create(self, options, ruby_system, system):
230 self.version = self.versionCount()
231
232 self.L1cache = SQCCache()
233 self.L1cache.create(options)
234 self.L1cache.resourceStalls = options.no_resource_stalls
235
236 self.sequencer = RubySequencer()
237
238 self.sequencer.version = self.seqCount()
239 self.sequencer.icache = self.L1cache
240 self.sequencer.dcache = self.L1cache
241 self.sequencer.ruby_system = ruby_system
242 self.sequencer.support_data_reqs = False
243 self.sequencer.is_cpu_sequencer = False
244
245 self.ruby_system = ruby_system
246
247 if options.recycle_latency:
248 self.recycle_latency = options.recycle_latency
249
250 def createCP(self, options, ruby_system, system):
251 self.version = self.versionCount()
252
253 self.L1cache = SQCCache()
254 self.L1cache.create(options)
255 self.L1cache.resourceStalls = options.no_resource_stalls
256
257 self.sequencer = RubySequencer()
258
259 self.sequencer.version = self.seqCount()
260 self.sequencer.icache = self.L1cache
261 self.sequencer.dcache = self.L1cache
262 self.sequencer.ruby_system = ruby_system
263 self.sequencer.support_data_reqs = False
264
265 self.ruby_system = ruby_system
266
267 if options.recycle_latency:
268 self.recycle_latency = options.recycle_latency
269
270
271class TCC(RubyCache):
272 assoc = 16
273 dataAccessLatency = 8
274 tagAccessLatency = 2
275 resourceStalls = True
276 def create(self, options):
277 self.size = MemorySize(options.tcc_size)
278 self.size = self.size / options.num_tccs
279 self.dataArrayBanks = 256 / options.num_tccs #number of data banks
280 self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
281 if ((self.size.value / long(self.assoc)) < 128):
282 self.size.value = long(128 * self.assoc)
283 self.start_index_bit = math.log(options.cacheline_size, 2) + \
284 math.log(options.num_tccs, 2)
285 self.replacement_policy = PseudoLRUReplacementPolicy()
286
287class TCCCntrl(TCC_Controller, CntrlBase):
288 def create(self, options, ruby_system, system):
289 self.version = self.versionCount()
290 self.L2cache = TCC()
291 self.L2cache.create(options)
292 self.l2_response_latency = options.TCC_latency
293
294 self.number_of_TBEs = 2048
295
296 self.ruby_system = ruby_system
297
298 if options.recycle_latency:
299 self.recycle_latency = options.recycle_latency
300
301 def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
302 tcc_unblock_to_tccdir, req_to_tcc,
303 probe_to_tcc, resp_to_tcc):
304 self.w_reqToTCCDir = req_to_tccdir
305 self.w_respToTCCDir = resp_to_tccdir
306 self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
307 self.w_reqToTCC = req_to_tcc
308 self.w_probeToTCC = probe_to_tcc
309 self.w_respToTCC = resp_to_tcc
310
311class TCCDirCntrl(TCCdir_Controller, CntrlBase):
312 def create(self, options, ruby_system, system):
313 self.version = self.versionCount()
314
315 self.directory = TccDirCache()
316 self.directory.create(options)
317
318 self.number_of_TBEs = 1024
319
320 self.ruby_system = ruby_system
321
322 if options.recycle_latency:
323 self.recycle_latency = options.recycle_latency
324
325 def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
326 tcc_unblock_to_tccdir, req_to_tcc,
327 probe_to_tcc, resp_to_tcc):
328 self.w_reqToTCCDir = req_to_tccdir
329 self.w_respToTCCDir = resp_to_tccdir
330 self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
331 self.w_reqToTCC = req_to_tcc
332 self.w_probeToTCC = probe_to_tcc
333 self.w_respToTCC = resp_to_tcc
334
335class L3Cache(RubyCache):
336 assoc = 8
337 dataArrayBanks = 256
338 tagArrayBanks = 256
339
340 def create(self, options, ruby_system, system):
341 self.size = MemorySize(options.l3_size)
342 self.size.value /= options.num_dirs
343 self.dataArrayBanks /= options.num_dirs
344 self.tagArrayBanks /= options.num_dirs
345 self.dataArrayBanks /= options.num_dirs
346 self.tagArrayBanks /= options.num_dirs
347 self.dataAccessLatency = options.l3_data_latency
348 self.tagAccessLatency = options.l3_tag_latency
349 self.resourceStalls = options.no_resource_stalls
350 self.replacement_policy = PseudoLRUReplacementPolicy()
351
352class L3Cntrl(L3Cache_Controller, CntrlBase):
353 def create(self, options, ruby_system, system):
354 self.version = self.versionCount()
355 self.L3cache = L3Cache()
356 self.L3cache.create(options, ruby_system, system)
357
358 self.l3_response_latency = max(self.L3cache.dataAccessLatency,
359 self.L3cache.tagAccessLatency)
360 self.ruby_system = ruby_system
361
362 if options.recycle_latency:
363 self.recycle_latency = options.recycle_latency
364
365 def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
366 req_to_l3, probe_to_l3, resp_to_l3):
367 self.reqToDir = req_to_dir
368 self.respToDir = resp_to_dir
369 self.l3UnblockToDir = l3_unblock_to_dir
370 self.reqToL3 = req_to_l3
371 self.probeToL3 = probe_to_l3
372 self.respToL3 = resp_to_l3
373
374class DirMem(RubyDirectoryMemory, CntrlBase):
375 def create(self, options, ruby_system, system):
376 self.version = self.versionCount()
377
378 phys_mem_size = AddrRange(options.mem_size).size()
379 mem_module_size = phys_mem_size / options.num_dirs
380 dir_size = MemorySize('0B')
381 dir_size.value = mem_module_size
382 self.size = dir_size
383
384class DirCntrl(Directory_Controller, CntrlBase):
385 def create(self, options, ruby_system, system):
386 self.version = self.versionCount()
387
388 self.response_latency = 30
389
390 self.directory = DirMem()
391 self.directory.create(options, ruby_system, system)
392
393 self.L3CacheMemory = L3Cache()
394 self.L3CacheMemory.create(options, ruby_system, system)
395
396 self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
397 self.L3CacheMemory.tagAccessLatency)
398
399 self.number_of_TBEs = options.num_tbes
400
401 self.ruby_system = ruby_system
402
403 if options.recycle_latency:
404 self.recycle_latency = options.recycle_latency
405
406 def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
407 req_to_l3, probe_to_l3, resp_to_l3):
408 self.reqToDir = req_to_dir
409 self.respToDir = resp_to_dir
410 self.l3UnblockToDir = l3_unblock_to_dir
411 self.reqToL3 = req_to_l3
412 self.probeToL3 = probe_to_l3
413 self.respToL3 = resp_to_l3
414
415
416
417def define_options(parser):
418 parser.add_option("--num-subcaches", type="int", default=4)
419 parser.add_option("--l3-data-latency", type="int", default=20)
420 parser.add_option("--l3-tag-latency", type="int", default=15)
421 parser.add_option("--cpu-to-dir-latency", type="int", default=15)
422 parser.add_option("--gpu-to-dir-latency", type="int", default=160)
423 parser.add_option("--no-resource-stalls", action="store_false",
424 default=True)
425 parser.add_option("--num-tbes", type="int", default=256)
426 parser.add_option("--l2-latency", type="int", default=50) # load to use
427 parser.add_option("--num-tccs", type="int", default=1,
428 help="number of TCC directories and banks in the GPU")
429 parser.add_option("--TCP_latency", type="int", default=4,
430 help="TCP latency")
431 parser.add_option("--TCC_latency", type="int", default=16,
432 help="TCC latency")
433 parser.add_option("--tcc-size", type='string', default='256kB',
434 help="agregate tcc size")
435 parser.add_option("--tcp-size", type='string', default='16kB',
436 help="tcp size")
437 parser.add_option("--tcc-dir-factor", type='int', default=4,
438 help="TCCdir size = factor *(TCPs + TCC)")
439
440def create_system(options, full_system, system, dma_devices, ruby_system):
441 if buildEnv['PROTOCOL'] != 'GPU_RfO':
442 panic("This script requires the GPU_RfO protocol to be built.")
443
444 cpu_sequencers = []
445
446 #
447 # The ruby network creation expects the list of nodes in the system to be
448 # consistent with the NetDest list. Therefore the l1 controller nodes
449 # must be listed before the directory nodes and directory nodes before
450 # dma nodes, etc.
451 #
452 cp_cntrl_nodes = []
453 tcp_cntrl_nodes = []
454 sqc_cntrl_nodes = []
455 tcc_cntrl_nodes = []
456 tccdir_cntrl_nodes = []
457 dir_cntrl_nodes = []
458 l3_cntrl_nodes = []
459
460 #
461 # Must create the individual controllers before the network to ensure the
462 # controller constructors are called before the network constructor
463 #
464
465 TCC_bits = int(math.log(options.num_tccs, 2))
466
467 # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
468 # Clusters
469 mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
470 for i in xrange(options.num_dirs):
471
472 dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
473 dir_cntrl.create(options, ruby_system, system)
474 dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
475 #Enough TBEs for all TCP TBEs
476
477 # Connect the Directory controller to the ruby network
478 dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
479 dir_cntrl.requestFromCores.slave = ruby_system.network.master
480
481 dir_cntrl.responseFromCores = MessageBuffer()
482 dir_cntrl.responseFromCores.slave = ruby_system.network.master
483
484 dir_cntrl.unblockFromCores = MessageBuffer()
485 dir_cntrl.unblockFromCores.slave = ruby_system.network.master
486
487 dir_cntrl.probeToCore = MessageBuffer()
488 dir_cntrl.probeToCore.master = ruby_system.network.slave
489
490 dir_cntrl.responseToCore = MessageBuffer()
491 dir_cntrl.responseToCore.master = ruby_system.network.slave
492
493 dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
494 dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
495 dir_cntrl.responseFromMemory = MessageBuffer()
496
497 exec("system.dir_cntrl%d = dir_cntrl" % i)
498 dir_cntrl_nodes.append(dir_cntrl)
499
500 mainCluster.add(dir_cntrl)
501
502 # For an odd number of CPUs, still create the right number of controllers
503 cpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
504 for i in xrange((options.num_cpus + 1) / 2):
505
506 cp_cntrl = CPCntrl()
507 cp_cntrl.create(options, ruby_system, system)
508
509 exec("system.cp_cntrl%d = cp_cntrl" % i)
510 #
511 # Add controllers and sequencers to the appropriate lists
512 #
513 cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
514
515 # Connect the CP controllers and the network
516 cp_cntrl.requestFromCore = MessageBuffer()
517 cp_cntrl.requestFromCore.master = ruby_system.network.slave
518
519 cp_cntrl.responseFromCore = MessageBuffer()
520 cp_cntrl.responseFromCore.master = ruby_system.network.slave
521
522 cp_cntrl.unblockFromCore = MessageBuffer()
523 cp_cntrl.unblockFromCore.master = ruby_system.network.slave
524
525 cp_cntrl.probeToCore = MessageBuffer()
526 cp_cntrl.probeToCore.slave = ruby_system.network.master
527
528 cp_cntrl.responseToCore = MessageBuffer()
529 cp_cntrl.responseToCore.slave = ruby_system.network.master
530
531 cp_cntrl.mandatoryQueue = MessageBuffer()
532 cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
533
534 cpuCluster.add(cp_cntrl)
535
536 gpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
537
538 for i in xrange(options.num_compute_units):
539
540 tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
541 number_of_TBEs = 2560) # max outstanding requests
542 tcp_cntrl.create(options, ruby_system, system)
543
544 exec("system.tcp_cntrl%d = tcp_cntrl" % i)
545 #
546 # Add controllers and sequencers to the appropriate lists
547 #
548 cpu_sequencers.append(tcp_cntrl.coalescer)
549 tcp_cntrl_nodes.append(tcp_cntrl)
550
551 # Connect the TCP controller to the ruby network
552 tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
553 tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
554
555 tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
556 tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
557
558 tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
559 tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
560
561 tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
562 tcp_cntrl.probeToTCP.slave = ruby_system.network.master
563
564 tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
565 tcp_cntrl.responseToTCP.slave = ruby_system.network.master
566
567 tcp_cntrl.mandatoryQueue = MessageBuffer()
568
569 gpuCluster.add(tcp_cntrl)
570
571 for i in xrange(options.num_sqc):
572
573 sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
574 sqc_cntrl.create(options, ruby_system, system)
575
576 exec("system.sqc_cntrl%d = sqc_cntrl" % i)
577 #
578 # Add controllers and sequencers to the appropriate lists
579 #
580 cpu_sequencers.append(sqc_cntrl.sequencer)
581
582 # Connect the SQC controller to the ruby network
583 sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
584 sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
585
586 sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
587 sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
588
589 sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
590 sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
591
592 sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
593 sqc_cntrl.probeToSQC.slave = ruby_system.network.master
594
595 sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
596 sqc_cntrl.responseToSQC.slave = ruby_system.network.master
597
598 sqc_cntrl.mandatoryQueue = MessageBuffer()
599
600 # SQC also in GPU cluster
601 gpuCluster.add(sqc_cntrl)
602
| 1#
2# Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3# All rights reserved.
4#
5# For use for simulation and test purposes only
6#
7# Redistribution and use in source and binary forms, with or without
8# modification, are permitted provided that the following conditions are met:
9#
10# 1. Redistributions of source code must retain the above copyright notice,
11# this list of conditions and the following disclaimer.
12#
13# 2. Redistributions in binary form must reproduce the above copyright notice,
14# this list of conditions and the following disclaimer in the documentation
15# and/or other materials provided with the distribution.
16#
17# 3. Neither the name of the copyright holder nor the names of its contributors
18# may be used to endorse or promote products derived from this software
19# without specific prior written permission.
20#
21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31# POSSIBILITY OF SUCH DAMAGE.
32#
33# Author: Lisa Hsu
34#
35
36import math
37import m5
38from m5.objects import *
39from m5.defines import buildEnv
40from Ruby import create_topology
41from Ruby import send_evicts
42
43from Cluster import Cluster
44from Crossbar import Crossbar
45
46class CntrlBase:
47 _seqs = 0
48 @classmethod
49 def seqCount(cls):
50 # Use SeqCount not class since we need global count
51 CntrlBase._seqs += 1
52 return CntrlBase._seqs - 1
53
54 _cntrls = 0
55 @classmethod
56 def cntrlCount(cls):
57 # Use CntlCount not class since we need global count
58 CntrlBase._cntrls += 1
59 return CntrlBase._cntrls - 1
60
61 _version = 0
62 @classmethod
63 def versionCount(cls):
64 cls._version += 1 # Use count for this particular type
65 return cls._version - 1
66
67class TccDirCache(RubyCache):
68 size = "512kB"
69 assoc = 16
70 resourceStalls = False
71 def create(self, options):
72 self.size = MemorySize(options.tcc_size)
73 self.size.value += (options.num_compute_units *
74 (MemorySize(options.tcp_size).value) *
75 options.tcc_dir_factor) / long(options.num_tccs)
76 self.start_index_bit = math.log(options.cacheline_size, 2) + \
77 math.log(options.num_tccs, 2)
78 self.replacement_policy = PseudoLRUReplacementPolicy()
79
80class L1DCache(RubyCache):
81 resourceStalls = False
82 def create(self, options):
83 self.size = MemorySize(options.l1d_size)
84 self.assoc = options.l1d_assoc
85 self.replacement_policy = PseudoLRUReplacementPolicy()
86
87class L1ICache(RubyCache):
88 resourceStalls = False
89 def create(self, options):
90 self.size = MemorySize(options.l1i_size)
91 self.assoc = options.l1i_assoc
92 self.replacement_policy = PseudoLRUReplacementPolicy()
93
94class L2Cache(RubyCache):
95 resourceStalls = False
96 def create(self, options):
97 self.size = MemorySize(options.l2_size)
98 self.assoc = options.l2_assoc
99 self.replacement_policy = PseudoLRUReplacementPolicy()
100
101
102class CPCntrl(CorePair_Controller, CntrlBase):
103
104 def create(self, options, ruby_system, system):
105 self.version = self.versionCount()
106
107 self.L1Icache = L1ICache()
108 self.L1Icache.create(options)
109 self.L1D0cache = L1DCache()
110 self.L1D0cache.create(options)
111 self.L1D1cache = L1DCache()
112 self.L1D1cache.create(options)
113 self.L2cache = L2Cache()
114 self.L2cache.create(options)
115
116 self.sequencer = RubySequencer()
117 self.sequencer.icache_hit_latency = 2
118 self.sequencer.dcache_hit_latency = 2
119 self.sequencer.version = self.seqCount()
120 self.sequencer.icache = self.L1Icache
121 self.sequencer.dcache = self.L1D0cache
122 self.sequencer.ruby_system = ruby_system
123 self.sequencer.coreid = 0
124 self.sequencer.is_cpu_sequencer = True
125
126 self.sequencer1 = RubySequencer()
127 self.sequencer1.version = self.seqCount()
128 self.sequencer1.icache = self.L1Icache
129 self.sequencer1.dcache = self.L1D1cache
130 self.sequencer1.icache_hit_latency = 2
131 self.sequencer1.dcache_hit_latency = 2
132 self.sequencer1.ruby_system = ruby_system
133 self.sequencer1.coreid = 1
134 self.sequencer1.is_cpu_sequencer = True
135
136 self.issue_latency = options.cpu_to_dir_latency
137 self.send_evictions = send_evicts(options)
138
139 self.ruby_system = ruby_system
140
141 if options.recycle_latency:
142 self.recycle_latency = options.recycle_latency
143
144class TCPCache(RubyCache):
145 assoc = 8
146 dataArrayBanks = 16
147 tagArrayBanks = 4
148 dataAccessLatency = 4
149 tagAccessLatency = 1
150 def create(self, options):
151 self.size = MemorySize(options.tcp_size)
152 self.replacement_policy = PseudoLRUReplacementPolicy()
153
154class TCPCntrl(TCP_Controller, CntrlBase):
155
156 def create(self, options, ruby_system, system):
157 self.version = self.versionCount()
158
159 self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
160 self.L1cache.resourceStalls = options.no_resource_stalls
161 self.L1cache.create(options)
162
163 self.coalescer = RubyGPUCoalescer()
164 self.coalescer.version = self.seqCount()
165 self.coalescer.icache = self.L1cache
166 self.coalescer.dcache = self.L1cache
167 self.coalescer.ruby_system = ruby_system
168 self.coalescer.support_inst_reqs = False
169 self.coalescer.is_cpu_sequencer = False
170 self.coalescer.max_outstanding_requests = options.simds_per_cu * \
171 options.wfs_per_simd * \
172 options.wf_size
173
174 self.sequencer = RubySequencer()
175 self.sequencer.version = self.seqCount()
176 self.sequencer.icache = self.L1cache
177 self.sequencer.dcache = self.L1cache
178 self.sequencer.ruby_system = ruby_system
179 self.sequencer.is_cpu_sequencer = True
180
181 self.use_seq_not_coal = False
182
183 self.ruby_system = ruby_system
184
185 if options.recycle_latency:
186 self.recycle_latency = options.recycle_latency
187
188 def createCP(self, options, ruby_system, system):
189 self.version = self.versionCount()
190
191 self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
192 self.L1cache.resourceStalls = options.no_resource_stalls
193 self.L1cache.create(options)
194
195 self.coalescer = RubyGPUCoalescer()
196 self.coalescer.version = self.seqCount()
197 self.coalescer.icache = self.L1cache
198 self.coalescer.dcache = self.L1cache
199 self.coalescer.ruby_system = ruby_system
200 self.coalescer.support_inst_reqs = False
201 self.coalescer.is_cpu_sequencer = False
202
203 self.sequencer = RubySequencer()
204 self.sequencer.version = self.seqCount()
205 self.sequencer.icache = self.L1cache
206 self.sequencer.dcache = self.L1cache
207 self.sequencer.ruby_system = ruby_system
208 self.sequencer.is_cpu_sequencer = True
209
210 self.use_seq_not_coal = True
211
212 self.ruby_system = ruby_system
213
214 if options.recycle_latency:
215 self.recycle_latency = options.recycle_latency
216
217class SQCCache(RubyCache):
218 size = "32kB"
219 assoc = 8
220 dataArrayBanks = 16
221 tagArrayBanks = 4
222 dataAccessLatency = 4
223 tagAccessLatency = 1
224 def create(self, options):
225 self.replacement_policy = PseudoLRUReplacementPolicy()
226
227class SQCCntrl(SQC_Controller, CntrlBase):
228
229 def create(self, options, ruby_system, system):
230 self.version = self.versionCount()
231
232 self.L1cache = SQCCache()
233 self.L1cache.create(options)
234 self.L1cache.resourceStalls = options.no_resource_stalls
235
236 self.sequencer = RubySequencer()
237
238 self.sequencer.version = self.seqCount()
239 self.sequencer.icache = self.L1cache
240 self.sequencer.dcache = self.L1cache
241 self.sequencer.ruby_system = ruby_system
242 self.sequencer.support_data_reqs = False
243 self.sequencer.is_cpu_sequencer = False
244
245 self.ruby_system = ruby_system
246
247 if options.recycle_latency:
248 self.recycle_latency = options.recycle_latency
249
250 def createCP(self, options, ruby_system, system):
251 self.version = self.versionCount()
252
253 self.L1cache = SQCCache()
254 self.L1cache.create(options)
255 self.L1cache.resourceStalls = options.no_resource_stalls
256
257 self.sequencer = RubySequencer()
258
259 self.sequencer.version = self.seqCount()
260 self.sequencer.icache = self.L1cache
261 self.sequencer.dcache = self.L1cache
262 self.sequencer.ruby_system = ruby_system
263 self.sequencer.support_data_reqs = False
264
265 self.ruby_system = ruby_system
266
267 if options.recycle_latency:
268 self.recycle_latency = options.recycle_latency
269
270
271class TCC(RubyCache):
272 assoc = 16
273 dataAccessLatency = 8
274 tagAccessLatency = 2
275 resourceStalls = True
276 def create(self, options):
277 self.size = MemorySize(options.tcc_size)
278 self.size = self.size / options.num_tccs
279 self.dataArrayBanks = 256 / options.num_tccs #number of data banks
280 self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
281 if ((self.size.value / long(self.assoc)) < 128):
282 self.size.value = long(128 * self.assoc)
283 self.start_index_bit = math.log(options.cacheline_size, 2) + \
284 math.log(options.num_tccs, 2)
285 self.replacement_policy = PseudoLRUReplacementPolicy()
286
287class TCCCntrl(TCC_Controller, CntrlBase):
288 def create(self, options, ruby_system, system):
289 self.version = self.versionCount()
290 self.L2cache = TCC()
291 self.L2cache.create(options)
292 self.l2_response_latency = options.TCC_latency
293
294 self.number_of_TBEs = 2048
295
296 self.ruby_system = ruby_system
297
298 if options.recycle_latency:
299 self.recycle_latency = options.recycle_latency
300
301 def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
302 tcc_unblock_to_tccdir, req_to_tcc,
303 probe_to_tcc, resp_to_tcc):
304 self.w_reqToTCCDir = req_to_tccdir
305 self.w_respToTCCDir = resp_to_tccdir
306 self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
307 self.w_reqToTCC = req_to_tcc
308 self.w_probeToTCC = probe_to_tcc
309 self.w_respToTCC = resp_to_tcc
310
311class TCCDirCntrl(TCCdir_Controller, CntrlBase):
312 def create(self, options, ruby_system, system):
313 self.version = self.versionCount()
314
315 self.directory = TccDirCache()
316 self.directory.create(options)
317
318 self.number_of_TBEs = 1024
319
320 self.ruby_system = ruby_system
321
322 if options.recycle_latency:
323 self.recycle_latency = options.recycle_latency
324
325 def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
326 tcc_unblock_to_tccdir, req_to_tcc,
327 probe_to_tcc, resp_to_tcc):
328 self.w_reqToTCCDir = req_to_tccdir
329 self.w_respToTCCDir = resp_to_tccdir
330 self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
331 self.w_reqToTCC = req_to_tcc
332 self.w_probeToTCC = probe_to_tcc
333 self.w_respToTCC = resp_to_tcc
334
335class L3Cache(RubyCache):
336 assoc = 8
337 dataArrayBanks = 256
338 tagArrayBanks = 256
339
340 def create(self, options, ruby_system, system):
341 self.size = MemorySize(options.l3_size)
342 self.size.value /= options.num_dirs
343 self.dataArrayBanks /= options.num_dirs
344 self.tagArrayBanks /= options.num_dirs
345 self.dataArrayBanks /= options.num_dirs
346 self.tagArrayBanks /= options.num_dirs
347 self.dataAccessLatency = options.l3_data_latency
348 self.tagAccessLatency = options.l3_tag_latency
349 self.resourceStalls = options.no_resource_stalls
350 self.replacement_policy = PseudoLRUReplacementPolicy()
351
352class L3Cntrl(L3Cache_Controller, CntrlBase):
353 def create(self, options, ruby_system, system):
354 self.version = self.versionCount()
355 self.L3cache = L3Cache()
356 self.L3cache.create(options, ruby_system, system)
357
358 self.l3_response_latency = max(self.L3cache.dataAccessLatency,
359 self.L3cache.tagAccessLatency)
360 self.ruby_system = ruby_system
361
362 if options.recycle_latency:
363 self.recycle_latency = options.recycle_latency
364
365 def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
366 req_to_l3, probe_to_l3, resp_to_l3):
367 self.reqToDir = req_to_dir
368 self.respToDir = resp_to_dir
369 self.l3UnblockToDir = l3_unblock_to_dir
370 self.reqToL3 = req_to_l3
371 self.probeToL3 = probe_to_l3
372 self.respToL3 = resp_to_l3
373
374class DirMem(RubyDirectoryMemory, CntrlBase):
375 def create(self, options, ruby_system, system):
376 self.version = self.versionCount()
377
378 phys_mem_size = AddrRange(options.mem_size).size()
379 mem_module_size = phys_mem_size / options.num_dirs
380 dir_size = MemorySize('0B')
381 dir_size.value = mem_module_size
382 self.size = dir_size
383
384class DirCntrl(Directory_Controller, CntrlBase):
385 def create(self, options, ruby_system, system):
386 self.version = self.versionCount()
387
388 self.response_latency = 30
389
390 self.directory = DirMem()
391 self.directory.create(options, ruby_system, system)
392
393 self.L3CacheMemory = L3Cache()
394 self.L3CacheMemory.create(options, ruby_system, system)
395
396 self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
397 self.L3CacheMemory.tagAccessLatency)
398
399 self.number_of_TBEs = options.num_tbes
400
401 self.ruby_system = ruby_system
402
403 if options.recycle_latency:
404 self.recycle_latency = options.recycle_latency
405
406 def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
407 req_to_l3, probe_to_l3, resp_to_l3):
408 self.reqToDir = req_to_dir
409 self.respToDir = resp_to_dir
410 self.l3UnblockToDir = l3_unblock_to_dir
411 self.reqToL3 = req_to_l3
412 self.probeToL3 = probe_to_l3
413 self.respToL3 = resp_to_l3
414
415
416
417def define_options(parser):
418 parser.add_option("--num-subcaches", type="int", default=4)
419 parser.add_option("--l3-data-latency", type="int", default=20)
420 parser.add_option("--l3-tag-latency", type="int", default=15)
421 parser.add_option("--cpu-to-dir-latency", type="int", default=15)
422 parser.add_option("--gpu-to-dir-latency", type="int", default=160)
423 parser.add_option("--no-resource-stalls", action="store_false",
424 default=True)
425 parser.add_option("--num-tbes", type="int", default=256)
426 parser.add_option("--l2-latency", type="int", default=50) # load to use
427 parser.add_option("--num-tccs", type="int", default=1,
428 help="number of TCC directories and banks in the GPU")
429 parser.add_option("--TCP_latency", type="int", default=4,
430 help="TCP latency")
431 parser.add_option("--TCC_latency", type="int", default=16,
432 help="TCC latency")
433 parser.add_option("--tcc-size", type='string', default='256kB',
434 help="agregate tcc size")
435 parser.add_option("--tcp-size", type='string', default='16kB',
436 help="tcp size")
437 parser.add_option("--tcc-dir-factor", type='int', default=4,
438 help="TCCdir size = factor *(TCPs + TCC)")
439
440def create_system(options, full_system, system, dma_devices, ruby_system):
441 if buildEnv['PROTOCOL'] != 'GPU_RfO':
442 panic("This script requires the GPU_RfO protocol to be built.")
443
444 cpu_sequencers = []
445
446 #
447 # The ruby network creation expects the list of nodes in the system to be
448 # consistent with the NetDest list. Therefore the l1 controller nodes
449 # must be listed before the directory nodes and directory nodes before
450 # dma nodes, etc.
451 #
452 cp_cntrl_nodes = []
453 tcp_cntrl_nodes = []
454 sqc_cntrl_nodes = []
455 tcc_cntrl_nodes = []
456 tccdir_cntrl_nodes = []
457 dir_cntrl_nodes = []
458 l3_cntrl_nodes = []
459
460 #
461 # Must create the individual controllers before the network to ensure the
462 # controller constructors are called before the network constructor
463 #
464
465 TCC_bits = int(math.log(options.num_tccs, 2))
466
467 # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
468 # Clusters
469 mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
470 for i in xrange(options.num_dirs):
471
472 dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
473 dir_cntrl.create(options, ruby_system, system)
474 dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
475 #Enough TBEs for all TCP TBEs
476
477 # Connect the Directory controller to the ruby network
478 dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
479 dir_cntrl.requestFromCores.slave = ruby_system.network.master
480
481 dir_cntrl.responseFromCores = MessageBuffer()
482 dir_cntrl.responseFromCores.slave = ruby_system.network.master
483
484 dir_cntrl.unblockFromCores = MessageBuffer()
485 dir_cntrl.unblockFromCores.slave = ruby_system.network.master
486
487 dir_cntrl.probeToCore = MessageBuffer()
488 dir_cntrl.probeToCore.master = ruby_system.network.slave
489
490 dir_cntrl.responseToCore = MessageBuffer()
491 dir_cntrl.responseToCore.master = ruby_system.network.slave
492
493 dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
494 dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
495 dir_cntrl.responseFromMemory = MessageBuffer()
496
497 exec("system.dir_cntrl%d = dir_cntrl" % i)
498 dir_cntrl_nodes.append(dir_cntrl)
499
500 mainCluster.add(dir_cntrl)
501
502 # For an odd number of CPUs, still create the right number of controllers
503 cpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
504 for i in xrange((options.num_cpus + 1) / 2):
505
506 cp_cntrl = CPCntrl()
507 cp_cntrl.create(options, ruby_system, system)
508
509 exec("system.cp_cntrl%d = cp_cntrl" % i)
510 #
511 # Add controllers and sequencers to the appropriate lists
512 #
513 cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
514
515 # Connect the CP controllers and the network
516 cp_cntrl.requestFromCore = MessageBuffer()
517 cp_cntrl.requestFromCore.master = ruby_system.network.slave
518
519 cp_cntrl.responseFromCore = MessageBuffer()
520 cp_cntrl.responseFromCore.master = ruby_system.network.slave
521
522 cp_cntrl.unblockFromCore = MessageBuffer()
523 cp_cntrl.unblockFromCore.master = ruby_system.network.slave
524
525 cp_cntrl.probeToCore = MessageBuffer()
526 cp_cntrl.probeToCore.slave = ruby_system.network.master
527
528 cp_cntrl.responseToCore = MessageBuffer()
529 cp_cntrl.responseToCore.slave = ruby_system.network.master
530
531 cp_cntrl.mandatoryQueue = MessageBuffer()
532 cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
533
534 cpuCluster.add(cp_cntrl)
535
536 gpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
537
538 for i in xrange(options.num_compute_units):
539
540 tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
541 number_of_TBEs = 2560) # max outstanding requests
542 tcp_cntrl.create(options, ruby_system, system)
543
544 exec("system.tcp_cntrl%d = tcp_cntrl" % i)
545 #
546 # Add controllers and sequencers to the appropriate lists
547 #
548 cpu_sequencers.append(tcp_cntrl.coalescer)
549 tcp_cntrl_nodes.append(tcp_cntrl)
550
551 # Connect the TCP controller to the ruby network
552 tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
553 tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
554
555 tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
556 tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
557
558 tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
559 tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
560
561 tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
562 tcp_cntrl.probeToTCP.slave = ruby_system.network.master
563
564 tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
565 tcp_cntrl.responseToTCP.slave = ruby_system.network.master
566
567 tcp_cntrl.mandatoryQueue = MessageBuffer()
568
569 gpuCluster.add(tcp_cntrl)
570
571 for i in xrange(options.num_sqc):
572
573 sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
574 sqc_cntrl.create(options, ruby_system, system)
575
576 exec("system.sqc_cntrl%d = sqc_cntrl" % i)
577 #
578 # Add controllers and sequencers to the appropriate lists
579 #
580 cpu_sequencers.append(sqc_cntrl.sequencer)
581
582 # Connect the SQC controller to the ruby network
583 sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
584 sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
585
586 sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
587 sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
588
589 sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
590 sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
591
592 sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
593 sqc_cntrl.probeToSQC.slave = ruby_system.network.master
594
595 sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
596 sqc_cntrl.responseToSQC.slave = ruby_system.network.master
597
598 sqc_cntrl.mandatoryQueue = MessageBuffer()
599
600 # SQC also in GPU cluster
601 gpuCluster.add(sqc_cntrl)
602
|