GPU_RfO.py revision 11310 
1#
2#  Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3#  All rights reserved.
4#
5#  For use for simulation and test purposes only
6#
7#  Redistribution and use in source and binary forms, with or without
8#  modification, are permitted provided that the following conditions are met:
9#
10#  1. Redistributions of source code must retain the above copyright notice,
11#  this list of conditions and the following disclaimer.
12#
13#  2. Redistributions in binary form must reproduce the above copyright notice,
14#  this list of conditions and the following disclaimer in the documentation
15#  and/or other materials provided with the distribution.
16#
17#  3. Neither the name of the copyright holder nor the names of its contributors
18#  may be used to endorse or promote products derived from this software
19#  without specific prior written permission.
20#
21#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31#  POSSIBILITY OF SUCH DAMAGE.
32#
33#  Author: Lisa Hsu
34#
35
36import math
37import m5
38from m5.objects import *
39from m5.defines import buildEnv
40from Ruby import create_topology
41from Ruby import send_evicts
42
43from Cluster import Cluster
44from Crossbar import Crossbar
45
46class CntrlBase:
47    _seqs = 0
48    @classmethod
49    def seqCount(cls):
50        # Use SeqCount not class since we need global count
51        CntrlBase._seqs += 1
52        return CntrlBase._seqs - 1
53
54    _cntrls = 0
55    @classmethod
56    def cntrlCount(cls):
57        # Use CntlCount not class since we need global count
58        CntrlBase._cntrls += 1
59        return CntrlBase._cntrls - 1
60
61    _version = 0
62    @classmethod
63    def versionCount(cls):
64        cls._version += 1 # Use count for this particular type
65        return cls._version - 1
66
67class TccDirCache(RubyCache):
68    size = "512kB"
69    assoc = 16
70    resourceStalls = False
71    def create(self, options):
72        self.size = MemorySize(options.tcc_size)
73        self.size.value += (options.num_compute_units *
74                            (MemorySize(options.tcp_size).value) *
75                            options.tcc_dir_factor) / long(options.num_tccs)
76        self.start_index_bit = math.log(options.cacheline_size, 2) + \
77                               math.log(options.num_tccs, 2)
78        self.replacement_policy = PseudoLRUReplacementPolicy()
79
80class L1DCache(RubyCache):
81    resourceStalls = False
82    def create(self, options):
83        self.size = MemorySize(options.l1d_size)
84        self.assoc = options.l1d_assoc
85        self.replacement_policy = PseudoLRUReplacementPolicy()
86
87class L1ICache(RubyCache):
88    resourceStalls = False
89    def create(self, options):
90        self.size = MemorySize(options.l1i_size)
91        self.assoc = options.l1i_assoc
92        self.replacement_policy = PseudoLRUReplacementPolicy()
93
94class L2Cache(RubyCache):
95    resourceStalls = False
96    def create(self, options):
97        self.size = MemorySize(options.l2_size)
98        self.assoc = options.l2_assoc
99        self.replacement_policy = PseudoLRUReplacementPolicy()
100
101
102class CPCntrl(CorePair_Controller, CntrlBase):
103
104    def create(self, options, ruby_system, system):
105        self.version = self.versionCount()
106
107        self.L1Icache = L1ICache()
108        self.L1Icache.create(options)
109        self.L1D0cache = L1DCache()
110        self.L1D0cache.create(options)
111        self.L1D1cache = L1DCache()
112        self.L1D1cache.create(options)
113        self.L2cache = L2Cache()
114        self.L2cache.create(options)
115
116        self.sequencer = RubySequencer()
117        self.sequencer.icache_hit_latency = 2
118        self.sequencer.dcache_hit_latency = 2
119        self.sequencer.version = self.seqCount()
120        self.sequencer.icache = self.L1Icache
121        self.sequencer.dcache = self.L1D0cache
122        self.sequencer.ruby_system = ruby_system
123        self.sequencer.coreid = 0
124        self.sequencer.is_cpu_sequencer = True
125
126        self.sequencer1 = RubySequencer()
127        self.sequencer1.version = self.seqCount()
128        self.sequencer1.icache = self.L1Icache
129        self.sequencer1.dcache = self.L1D1cache
130        self.sequencer1.icache_hit_latency = 2
131        self.sequencer1.dcache_hit_latency = 2
132        self.sequencer1.ruby_system = ruby_system
133        self.sequencer1.coreid = 1
134        self.sequencer1.is_cpu_sequencer = True
135
136        self.issue_latency = options.cpu_to_dir_latency
137        self.send_evictions = send_evicts(options)
138
139        self.ruby_system = ruby_system
140
141        if options.recycle_latency:
142            self.recycle_latency = options.recycle_latency
143
144class TCPCache(RubyCache):
145    assoc = 8
146    dataArrayBanks = 16
147    tagArrayBanks = 4
148    dataAccessLatency = 4
149    tagAccessLatency = 1
150    def create(self, options):
151        self.size = MemorySize(options.tcp_size)
152        self.replacement_policy = PseudoLRUReplacementPolicy()
153
154class TCPCntrl(TCP_Controller, CntrlBase):
155
156    def create(self, options, ruby_system, system):
157        self.version = self.versionCount()
158
159        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
160        self.L1cache.resourceStalls = options.no_resource_stalls
161        self.L1cache.create(options)
162
163        self.coalescer = RubyGPUCoalescer()
164        self.coalescer.version = self.seqCount()
165        self.coalescer.icache = self.L1cache
166        self.coalescer.dcache = self.L1cache
167        self.coalescer.ruby_system = ruby_system
168        self.coalescer.support_inst_reqs = False
169        self.coalescer.is_cpu_sequencer = False
170        self.coalescer.max_outstanding_requests = options.simds_per_cu * \
171                                                  options.wfs_per_simd * \
172                                                  options.wf_size
173
174        self.sequencer = RubySequencer()
175        self.sequencer.version = self.seqCount()
176        self.sequencer.icache = self.L1cache
177        self.sequencer.dcache = self.L1cache
178        self.sequencer.ruby_system = ruby_system
179        self.sequencer.is_cpu_sequencer = True
180
181        self.use_seq_not_coal = False
182
183        self.ruby_system = ruby_system
184
185        if options.recycle_latency:
186            self.recycle_latency = options.recycle_latency
187
188    def createCP(self, options, ruby_system, system):
189        self.version = self.versionCount()
190
191        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
192        self.L1cache.resourceStalls = options.no_resource_stalls
193        self.L1cache.create(options)
194
195        self.coalescer = RubyGPUCoalescer()
196        self.coalescer.version = self.seqCount()
197        self.coalescer.icache = self.L1cache
198        self.coalescer.dcache = self.L1cache
199        self.coalescer.ruby_system = ruby_system
200        self.coalescer.support_inst_reqs = False
201        self.coalescer.is_cpu_sequencer = False
202
203        self.sequencer = RubySequencer()
204        self.sequencer.version = self.seqCount()
205        self.sequencer.icache = self.L1cache
206        self.sequencer.dcache = self.L1cache
207        self.sequencer.ruby_system = ruby_system
208        self.sequencer.is_cpu_sequencer = True
209
210        self.use_seq_not_coal = True
211
212        self.ruby_system = ruby_system
213
214        if options.recycle_latency:
215            self.recycle_latency = options.recycle_latency
216
217class SQCCache(RubyCache):
218    size = "32kB"
219    assoc = 8
220    dataArrayBanks = 16
221    tagArrayBanks = 4
222    dataAccessLatency = 4
223    tagAccessLatency = 1
224    def create(self, options):
225        self.replacement_policy = PseudoLRUReplacementPolicy()
226
227class SQCCntrl(SQC_Controller, CntrlBase):
228
229    def create(self, options, ruby_system, system):
230        self.version = self.versionCount()
231
232        self.L1cache = SQCCache()
233        self.L1cache.create(options)
234        self.L1cache.resourceStalls = options.no_resource_stalls
235
236        self.sequencer = RubySequencer()
237
238        self.sequencer.version = self.seqCount()
239        self.sequencer.icache = self.L1cache
240        self.sequencer.dcache = self.L1cache
241        self.sequencer.ruby_system = ruby_system
242        self.sequencer.support_data_reqs = False
243        self.sequencer.is_cpu_sequencer = False
244
245        self.ruby_system = ruby_system
246
247        if options.recycle_latency:
248            self.recycle_latency = options.recycle_latency
249
250    def createCP(self, options, ruby_system, system):
251        self.version = self.versionCount()
252
253        self.L1cache = SQCCache()
254        self.L1cache.create(options)
255        self.L1cache.resourceStalls = options.no_resource_stalls
256
257        self.sequencer = RubySequencer()
258
259        self.sequencer.version = self.seqCount()
260        self.sequencer.icache = self.L1cache
261        self.sequencer.dcache = self.L1cache
262        self.sequencer.ruby_system = ruby_system
263        self.sequencer.support_data_reqs = False
264
265        self.ruby_system = ruby_system
266
267        if options.recycle_latency:
268            self.recycle_latency = options.recycle_latency
269
270
271class TCC(RubyCache):
272    assoc = 16
273    dataAccessLatency = 8
274    tagAccessLatency = 2
275    resourceStalls = True
276    def create(self, options):
277        self.size = MemorySize(options.tcc_size)
278        self.size = self.size / options.num_tccs
279        self.dataArrayBanks = 256 / options.num_tccs #number of data banks
280        self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
281        if ((self.size.value / long(self.assoc)) < 128):
282            self.size.value = long(128 * self.assoc)
283        self.start_index_bit = math.log(options.cacheline_size, 2) + \
284                               math.log(options.num_tccs, 2)
285        self.replacement_policy = PseudoLRUReplacementPolicy()
286
287class TCCCntrl(TCC_Controller, CntrlBase):
288    def create(self, options, ruby_system, system):
289        self.version = self.versionCount()
290        self.L2cache = TCC()
291        self.L2cache.create(options)
292        self.l2_response_latency = options.TCC_latency
293
294        self.number_of_TBEs = 2048
295
296        self.ruby_system = ruby_system
297
298        if options.recycle_latency:
299            self.recycle_latency = options.recycle_latency
300
301    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
302                           tcc_unblock_to_tccdir, req_to_tcc,
303                           probe_to_tcc, resp_to_tcc):
304        self.w_reqToTCCDir = req_to_tccdir
305        self.w_respToTCCDir = resp_to_tccdir
306        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
307        self.w_reqToTCC = req_to_tcc
308        self.w_probeToTCC = probe_to_tcc
309        self.w_respToTCC = resp_to_tcc
310
311class TCCDirCntrl(TCCdir_Controller, CntrlBase):
312    def create(self, options, ruby_system, system):
313        self.version = self.versionCount()
314
315        self.directory = TccDirCache()
316        self.directory.create(options)
317
318        self.number_of_TBEs = 1024
319
320        self.ruby_system = ruby_system
321
322        if options.recycle_latency:
323            self.recycle_latency = options.recycle_latency
324
325    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
326                           tcc_unblock_to_tccdir, req_to_tcc,
327                           probe_to_tcc, resp_to_tcc):
328        self.w_reqToTCCDir = req_to_tccdir
329        self.w_respToTCCDir = resp_to_tccdir
330        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
331        self.w_reqToTCC = req_to_tcc
332        self.w_probeToTCC = probe_to_tcc
333        self.w_respToTCC = resp_to_tcc
334
335class L3Cache(RubyCache):
336    assoc = 8
337    dataArrayBanks = 256
338    tagArrayBanks = 256
339
340    def create(self, options, ruby_system, system):
341        self.size = MemorySize(options.l3_size)
342        self.size.value /= options.num_dirs
343        self.dataArrayBanks /= options.num_dirs
344        self.tagArrayBanks /= options.num_dirs
345        self.dataArrayBanks /= options.num_dirs
346        self.tagArrayBanks /= options.num_dirs
347        self.dataAccessLatency = options.l3_data_latency
348        self.tagAccessLatency = options.l3_tag_latency
349        self.resourceStalls = options.no_resource_stalls
350        self.replacement_policy = PseudoLRUReplacementPolicy()
351
352class L3Cntrl(L3Cache_Controller, CntrlBase):
353    def create(self, options, ruby_system, system):
354        self.version = self.versionCount()
355        self.L3cache = L3Cache()
356        self.L3cache.create(options, ruby_system, system)
357
358        self.l3_response_latency = max(self.L3cache.dataAccessLatency,
359                                       self.L3cache.tagAccessLatency)
360        self.ruby_system = ruby_system
361
362        if options.recycle_latency:
363            self.recycle_latency = options.recycle_latency
364
365    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
366                           req_to_l3, probe_to_l3, resp_to_l3):
367        self.reqToDir = req_to_dir
368        self.respToDir = resp_to_dir
369        self.l3UnblockToDir = l3_unblock_to_dir
370        self.reqToL3 = req_to_l3
371        self.probeToL3 = probe_to_l3
372        self.respToL3 = resp_to_l3
373
374class DirMem(RubyDirectoryMemory, CntrlBase):
375    def create(self, options, ruby_system, system):
376        self.version = self.versionCount()
377
378        phys_mem_size = AddrRange(options.mem_size).size()
379        mem_module_size = phys_mem_size / options.num_dirs
380        dir_size = MemorySize('0B')
381        dir_size.value = mem_module_size
382        self.size = dir_size
383
384class DirCntrl(Directory_Controller, CntrlBase):
385    def create(self, options, ruby_system, system):
386        self.version = self.versionCount()
387
388        self.response_latency = 30
389
390        self.directory = DirMem()
391        self.directory.create(options, ruby_system, system)
392
393        self.L3CacheMemory = L3Cache()
394        self.L3CacheMemory.create(options, ruby_system, system)
395
396        self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
397                                  self.L3CacheMemory.tagAccessLatency)
398
399        self.number_of_TBEs = options.num_tbes
400
401        self.ruby_system = ruby_system
402
403        if options.recycle_latency:
404            self.recycle_latency = options.recycle_latency
405
406    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
407                           req_to_l3, probe_to_l3, resp_to_l3):
408        self.reqToDir = req_to_dir
409        self.respToDir = resp_to_dir
410        self.l3UnblockToDir = l3_unblock_to_dir
411        self.reqToL3 = req_to_l3
412        self.probeToL3 = probe_to_l3
413        self.respToL3 = resp_to_l3
414
415
416
417def define_options(parser):
418    parser.add_option("--num-subcaches", type="int", default=4)
419    parser.add_option("--l3-data-latency", type="int", default=20)
420    parser.add_option("--l3-tag-latency", type="int", default=15)
421    parser.add_option("--cpu-to-dir-latency", type="int", default=15)
422    parser.add_option("--gpu-to-dir-latency", type="int", default=160)
423    parser.add_option("--no-resource-stalls", action="store_false",
424                      default=True)
425    parser.add_option("--num-tbes", type="int", default=256)
426    parser.add_option("--l2-latency", type="int", default=50) # load to use
427    parser.add_option("--num-tccs", type="int", default=1,
428                      help="number of TCC directories and banks in the GPU")
429    parser.add_option("--TCP_latency", type="int", default=4,
430                      help="TCP latency")
431    parser.add_option("--TCC_latency", type="int", default=16,
432                      help="TCC latency")
433    parser.add_option("--tcc-size", type='string', default='256kB',
434                      help="agregate tcc size")
435    parser.add_option("--tcp-size", type='string', default='16kB',
436                      help="tcp size")
437    parser.add_option("--tcc-dir-factor", type='int', default=4,
438                      help="TCCdir size = factor *(TCPs + TCC)")
439
440def create_system(options, full_system, system, dma_devices, ruby_system):
441    if buildEnv['PROTOCOL'] != 'GPU_RfO':
442        panic("This script requires the GPU_RfO protocol to be built.")
443
444    cpu_sequencers = []
445
446    #
447    # The ruby network creation expects the list of nodes in the system to be
448    # consistent with the NetDest list.  Therefore the l1 controller nodes
449    # must be listed before the directory nodes and directory nodes before
450    # dma nodes, etc.
451    #
452    cp_cntrl_nodes = []
453    tcp_cntrl_nodes = []
454    sqc_cntrl_nodes = []
455    tcc_cntrl_nodes = []
456    tccdir_cntrl_nodes = []
457    dir_cntrl_nodes = []
458    l3_cntrl_nodes = []
459
460    #
461    # Must create the individual controllers before the network to ensure the
462    # controller constructors are called before the network constructor
463    #
464
465    TCC_bits = int(math.log(options.num_tccs, 2))
466
467    # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
468    # Clusters
469    mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
470    for i in xrange(options.num_dirs):
471
472        dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
473        dir_cntrl.create(options, ruby_system, system)
474        dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
475        #Enough TBEs for all TCP TBEs
476
477        # Connect the Directory controller to the ruby network
478        dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
479        dir_cntrl.requestFromCores.slave = ruby_system.network.master
480
481        dir_cntrl.responseFromCores = MessageBuffer()
482        dir_cntrl.responseFromCores.slave = ruby_system.network.master
483
484        dir_cntrl.unblockFromCores = MessageBuffer()
485        dir_cntrl.unblockFromCores.slave = ruby_system.network.master
486
487        dir_cntrl.probeToCore = MessageBuffer()
488        dir_cntrl.probeToCore.master = ruby_system.network.slave
489
490        dir_cntrl.responseToCore = MessageBuffer()
491        dir_cntrl.responseToCore.master = ruby_system.network.slave
492
493        dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
494        dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
495        dir_cntrl.responseFromMemory = MessageBuffer()
496
497        exec("system.dir_cntrl%d = dir_cntrl" % i)
498        dir_cntrl_nodes.append(dir_cntrl)
499
500        mainCluster.add(dir_cntrl)
501
502    # For an odd number of CPUs, still create the right number of controllers
503    cpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
504    for i in xrange((options.num_cpus + 1) / 2):
505
506        cp_cntrl = CPCntrl()
507        cp_cntrl.create(options, ruby_system, system)
508
509        exec("system.cp_cntrl%d = cp_cntrl" % i)
510        #
511        # Add controllers and sequencers to the appropriate lists
512        #
513        cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
514
515        # Connect the CP controllers and the network
516        cp_cntrl.requestFromCore = MessageBuffer()
517        cp_cntrl.requestFromCore.master = ruby_system.network.slave
518
519        cp_cntrl.responseFromCore = MessageBuffer()
520        cp_cntrl.responseFromCore.master = ruby_system.network.slave
521
522        cp_cntrl.unblockFromCore = MessageBuffer()
523        cp_cntrl.unblockFromCore.master = ruby_system.network.slave
524
525        cp_cntrl.probeToCore = MessageBuffer()
526        cp_cntrl.probeToCore.slave = ruby_system.network.master
527
528        cp_cntrl.responseToCore = MessageBuffer()
529        cp_cntrl.responseToCore.slave = ruby_system.network.master
530
531        cp_cntrl.mandatoryQueue = MessageBuffer()
532        cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
533
534        cpuCluster.add(cp_cntrl)
535
536    gpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
537
538    for i in xrange(options.num_compute_units):
539
540        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
541                             number_of_TBEs = 2560) # max outstanding requests
542        tcp_cntrl.create(options, ruby_system, system)
543
544        exec("system.tcp_cntrl%d = tcp_cntrl" % i)
545        #
546        # Add controllers and sequencers to the appropriate lists
547        #
548        cpu_sequencers.append(tcp_cntrl.coalescer)
549        tcp_cntrl_nodes.append(tcp_cntrl)
550
551        # Connect the TCP controller to the ruby network
552        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
553        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
554
555        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
556        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
557
558        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
559        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
560
561        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
562        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
563
564        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
565        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
566
567        tcp_cntrl.mandatoryQueue = MessageBuffer()
568
569        gpuCluster.add(tcp_cntrl)
570
571    for i in xrange(options.num_sqc):
572
573        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
574        sqc_cntrl.create(options, ruby_system, system)
575
576        exec("system.sqc_cntrl%d = sqc_cntrl" % i)
577        #
578        # Add controllers and sequencers to the appropriate lists
579        #
580        cpu_sequencers.append(sqc_cntrl.sequencer)
581
582        # Connect the SQC controller to the ruby network
583        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
584        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
585
586        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
587        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
588
589        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
590        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
591
592        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
593        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
594
595        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
596        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
597
598        sqc_cntrl.mandatoryQueue = MessageBuffer()
599
600        # SQC also in GPU cluster
601        gpuCluster.add(sqc_cntrl)
602
603    for i in xrange(options.num_cp):
604
605        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
606                             number_of_TBEs = 2560) # max outstanding requests
607        tcp_cntrl.createCP(options, ruby_system, system)
608
609        exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i))
610        #
611        # Add controllers and sequencers to the appropriate lists
612        #
613        cpu_sequencers.append(tcp_cntrl.sequencer)
614        tcp_cntrl_nodes.append(tcp_cntrl)
615
616        # Connect the TCP controller to the ruby network
617        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
618        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
619
620        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
621        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
622
623        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
624        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
625
626        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
627        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
628
629        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
630        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
631
632        tcp_cntrl.mandatoryQueue = MessageBuffer()
633
634        gpuCluster.add(tcp_cntrl)
635
636        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
637        sqc_cntrl.createCP(options, ruby_system, system)
638
639        exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i))
640        #
641        # Add controllers and sequencers to the appropriate lists
642        #
643        cpu_sequencers.append(sqc_cntrl.sequencer)
644
645        # Connect the SQC controller to the ruby network
646        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
647        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
648
649        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
650        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
651
652        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
653        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
654
655        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
656        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
657
658        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
659        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
660
661        sqc_cntrl.mandatoryQueue = MessageBuffer()
662
663        # SQC also in GPU cluster
664        gpuCluster.add(sqc_cntrl)
665
666    for i in xrange(options.num_tccs):
667
668        tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits,
669                             number_of_TBEs = options.num_compute_units * 2560)
670        #Enough TBEs for all TCP TBEs
671        tcc_cntrl.create(options, ruby_system, system)
672        tcc_cntrl_nodes.append(tcc_cntrl)
673
674        tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits,
675                              number_of_TBEs = options.num_compute_units * 2560)
676        #Enough TBEs for all TCP TBEs
677        tccdir_cntrl.create(options, ruby_system, system)
678        tccdir_cntrl_nodes.append(tccdir_cntrl)
679
680        exec("system.tcc_cntrl%d = tcc_cntrl" % i)
681        exec("system.tccdir_cntrl%d = tccdir_cntrl" % i)
682
683        # connect all of the wire buffers between L3 and dirs up
684        req_to_tccdir = RubyWireBuffer()
685        resp_to_tccdir = RubyWireBuffer()
686        tcc_unblock_to_tccdir = RubyWireBuffer()
687        req_to_tcc = RubyWireBuffer()
688        probe_to_tcc = RubyWireBuffer()
689        resp_to_tcc = RubyWireBuffer()
690
691        tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
692                                     tcc_unblock_to_tccdir, req_to_tcc,
693                                     probe_to_tcc, resp_to_tcc)
694        tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
695                                        tcc_unblock_to_tccdir, req_to_tcc,
696                                        probe_to_tcc, resp_to_tcc)
697
698        # Connect the TCC controller to the ruby network
699        tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True)
700        tcc_cntrl.responseFromTCC.master = ruby_system.network.slave
701
702        tcc_cntrl.responseToTCC = MessageBuffer(ordered = True)
703        tcc_cntrl.responseToTCC.slave = ruby_system.network.master
704
705        # Connect the TCC Dir controller to the ruby network
706        tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True)
707        tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master
708
709        tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True)
710        tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master
711
712        tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True)
713        tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master
714
715        tccdir_cntrl.probeToCore = MessageBuffer(ordered = True)
716        tccdir_cntrl.probeToCore.master = ruby_system.network.slave
717
718        tccdir_cntrl.responseToCore = MessageBuffer(ordered = True)
719        tccdir_cntrl.responseToCore.master = ruby_system.network.slave
720
721        tccdir_cntrl.probeFromNB = MessageBuffer()
722        tccdir_cntrl.probeFromNB.slave = ruby_system.network.master
723
724        tccdir_cntrl.responseFromNB = MessageBuffer()
725        tccdir_cntrl.responseFromNB.slave = ruby_system.network.master
726
727        tccdir_cntrl.requestToNB = MessageBuffer()
728        tccdir_cntrl.requestToNB.master = ruby_system.network.slave
729
730        tccdir_cntrl.responseToNB = MessageBuffer()
731        tccdir_cntrl.responseToNB.master = ruby_system.network.slave
732
733        tccdir_cntrl.unblockToNB = MessageBuffer()
734        tccdir_cntrl.unblockToNB.master = ruby_system.network.slave
735
736        tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True)
737
738        # TCC cntrls added to the GPU cluster
739        gpuCluster.add(tcc_cntrl)
740        gpuCluster.add(tccdir_cntrl)
741
742    # Assuming no DMA devices
743    assert(len(dma_devices) == 0)
744
745    # Add cpu/gpu clusters to main cluster
746    mainCluster.add(cpuCluster)
747    mainCluster.add(gpuCluster)
748
749    ruby_system.network.number_of_virtual_networks = 10
750
751    return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
752