GPU_RfO.py revision 12647 
1# Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
2# All rights reserved.
3#
4# For use for simulation and test purposes only
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9# 1. Redistributions of source code must retain the above copyright notice,
10# this list of conditions and the following disclaimer.
11#
12# 2. Redistributions in binary form must reproduce the above copyright notice,
13# this list of conditions and the following disclaimer in the documentation
14# and/or other materials provided with the distribution.
15#
16# 3. Neither the name of the copyright holder nor the names of its
17# contributors may be used to endorse or promote products derived from this
18# software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
24# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30# POSSIBILITY OF SUCH DAMAGE.
31#
32# Authors: Lisa Hsu
33
34import math
35import m5
36from m5.objects import *
37from m5.defines import buildEnv
38from Ruby import create_topology
39from Ruby import send_evicts
40
41from topologies.Cluster import Cluster
42from topologies.Crossbar import Crossbar
43
44class CntrlBase:
45    _seqs = 0
46    @classmethod
47    def seqCount(cls):
48        # Use SeqCount not class since we need global count
49        CntrlBase._seqs += 1
50        return CntrlBase._seqs - 1
51
52    _cntrls = 0
53    @classmethod
54    def cntrlCount(cls):
55        # Use CntlCount not class since we need global count
56        CntrlBase._cntrls += 1
57        return CntrlBase._cntrls - 1
58
59    _version = 0
60    @classmethod
61    def versionCount(cls):
62        cls._version += 1 # Use count for this particular type
63        return cls._version - 1
64
65class TccDirCache(RubyCache):
66    size = "512kB"
67    assoc = 16
68    resourceStalls = False
69    def create(self, options):
70        self.size = MemorySize(options.tcc_size)
71        self.size.value += (options.num_compute_units *
72                            (MemorySize(options.tcp_size).value) *
73                            options.tcc_dir_factor) / long(options.num_tccs)
74        self.start_index_bit = math.log(options.cacheline_size, 2) + \
75                               math.log(options.num_tccs, 2)
76        self.replacement_policy = PseudoLRUReplacementPolicy()
77
78class L1DCache(RubyCache):
79    resourceStalls = False
80    def create(self, options):
81        self.size = MemorySize(options.l1d_size)
82        self.assoc = options.l1d_assoc
83        self.replacement_policy = PseudoLRUReplacementPolicy()
84
85class L1ICache(RubyCache):
86    resourceStalls = False
87    def create(self, options):
88        self.size = MemorySize(options.l1i_size)
89        self.assoc = options.l1i_assoc
90        self.replacement_policy = PseudoLRUReplacementPolicy()
91
92class L2Cache(RubyCache):
93    resourceStalls = False
94    def create(self, options):
95        self.size = MemorySize(options.l2_size)
96        self.assoc = options.l2_assoc
97        self.replacement_policy = PseudoLRUReplacementPolicy()
98
99
100class CPCntrl(CorePair_Controller, CntrlBase):
101
102    def create(self, options, ruby_system, system):
103        self.version = self.versionCount()
104
105        self.L1Icache = L1ICache()
106        self.L1Icache.create(options)
107        self.L1D0cache = L1DCache()
108        self.L1D0cache.create(options)
109        self.L1D1cache = L1DCache()
110        self.L1D1cache.create(options)
111        self.L2cache = L2Cache()
112        self.L2cache.create(options)
113
114        self.sequencer = RubySequencer()
115        self.sequencer.icache_hit_latency = 2
116        self.sequencer.dcache_hit_latency = 2
117        self.sequencer.version = self.seqCount()
118        self.sequencer.icache = self.L1Icache
119        self.sequencer.dcache = self.L1D0cache
120        self.sequencer.ruby_system = ruby_system
121        self.sequencer.coreid = 0
122        self.sequencer.is_cpu_sequencer = True
123
124        self.sequencer1 = RubySequencer()
125        self.sequencer1.version = self.seqCount()
126        self.sequencer1.icache = self.L1Icache
127        self.sequencer1.dcache = self.L1D1cache
128        self.sequencer1.icache_hit_latency = 2
129        self.sequencer1.dcache_hit_latency = 2
130        self.sequencer1.ruby_system = ruby_system
131        self.sequencer1.coreid = 1
132        self.sequencer1.is_cpu_sequencer = True
133
134        self.issue_latency = options.cpu_to_dir_latency
135        self.send_evictions = send_evicts(options)
136
137        self.ruby_system = ruby_system
138
139        if options.recycle_latency:
140            self.recycle_latency = options.recycle_latency
141
142class TCPCache(RubyCache):
143    assoc = 8
144    dataArrayBanks = 16
145    tagArrayBanks = 4
146    dataAccessLatency = 4
147    tagAccessLatency = 1
148    def create(self, options):
149        self.size = MemorySize(options.tcp_size)
150        self.replacement_policy = PseudoLRUReplacementPolicy()
151
152class TCPCntrl(TCP_Controller, CntrlBase):
153
154    def create(self, options, ruby_system, system):
155        self.version = self.versionCount()
156
157        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
158        self.L1cache.resourceStalls = options.no_resource_stalls
159        self.L1cache.create(options)
160
161        self.coalescer = RubyGPUCoalescer()
162        self.coalescer.version = self.seqCount()
163        self.coalescer.icache = self.L1cache
164        self.coalescer.dcache = self.L1cache
165        self.coalescer.ruby_system = ruby_system
166        self.coalescer.support_inst_reqs = False
167        self.coalescer.is_cpu_sequencer = False
168        self.coalescer.max_outstanding_requests = options.simds_per_cu * \
169                                                  options.wfs_per_simd * \
170                                                  options.wf_size
171
172        self.sequencer = RubySequencer()
173        self.sequencer.version = self.seqCount()
174        self.sequencer.icache = self.L1cache
175        self.sequencer.dcache = self.L1cache
176        self.sequencer.ruby_system = ruby_system
177        self.sequencer.is_cpu_sequencer = True
178
179        self.use_seq_not_coal = False
180
181        self.ruby_system = ruby_system
182
183        if options.recycle_latency:
184            self.recycle_latency = options.recycle_latency
185
186    def createCP(self, options, ruby_system, system):
187        self.version = self.versionCount()
188
189        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
190        self.L1cache.resourceStalls = options.no_resource_stalls
191        self.L1cache.create(options)
192
193        self.coalescer = RubyGPUCoalescer()
194        self.coalescer.version = self.seqCount()
195        self.coalescer.icache = self.L1cache
196        self.coalescer.dcache = self.L1cache
197        self.coalescer.ruby_system = ruby_system
198        self.coalescer.support_inst_reqs = False
199        self.coalescer.is_cpu_sequencer = False
200
201        self.sequencer = RubySequencer()
202        self.sequencer.version = self.seqCount()
203        self.sequencer.icache = self.L1cache
204        self.sequencer.dcache = self.L1cache
205        self.sequencer.ruby_system = ruby_system
206        self.sequencer.is_cpu_sequencer = True
207
208        self.use_seq_not_coal = True
209
210        self.ruby_system = ruby_system
211
212        if options.recycle_latency:
213            self.recycle_latency = options.recycle_latency
214
215class SQCCache(RubyCache):
216    size = "32kB"
217    assoc = 8
218    dataArrayBanks = 16
219    tagArrayBanks = 4
220    dataAccessLatency = 4
221    tagAccessLatency = 1
222    def create(self, options):
223        self.replacement_policy = PseudoLRUReplacementPolicy()
224
225class SQCCntrl(SQC_Controller, CntrlBase):
226
227    def create(self, options, ruby_system, system):
228        self.version = self.versionCount()
229
230        self.L1cache = SQCCache()
231        self.L1cache.create(options)
232        self.L1cache.resourceStalls = options.no_resource_stalls
233
234        self.sequencer = RubySequencer()
235
236        self.sequencer.version = self.seqCount()
237        self.sequencer.icache = self.L1cache
238        self.sequencer.dcache = self.L1cache
239        self.sequencer.ruby_system = ruby_system
240        self.sequencer.support_data_reqs = False
241        self.sequencer.is_cpu_sequencer = False
242
243        self.ruby_system = ruby_system
244
245        if options.recycle_latency:
246            self.recycle_latency = options.recycle_latency
247
248    def createCP(self, options, ruby_system, system):
249        self.version = self.versionCount()
250
251        self.L1cache = SQCCache()
252        self.L1cache.create(options)
253        self.L1cache.resourceStalls = options.no_resource_stalls
254
255        self.sequencer = RubySequencer()
256
257        self.sequencer.version = self.seqCount()
258        self.sequencer.icache = self.L1cache
259        self.sequencer.dcache = self.L1cache
260        self.sequencer.ruby_system = ruby_system
261        self.sequencer.support_data_reqs = False
262
263        self.ruby_system = ruby_system
264
265        if options.recycle_latency:
266            self.recycle_latency = options.recycle_latency
267
268
269class TCC(RubyCache):
270    assoc = 16
271    dataAccessLatency = 8
272    tagAccessLatency = 2
273    resourceStalls = True
274    def create(self, options):
275        self.size = MemorySize(options.tcc_size)
276        self.size = self.size / options.num_tccs
277        self.dataArrayBanks = 256 / options.num_tccs #number of data banks
278        self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
279        if ((self.size.value / long(self.assoc)) < 128):
280            self.size.value = long(128 * self.assoc)
281        self.start_index_bit = math.log(options.cacheline_size, 2) + \
282                               math.log(options.num_tccs, 2)
283        self.replacement_policy = PseudoLRUReplacementPolicy()
284
285class TCCCntrl(TCC_Controller, CntrlBase):
286    def create(self, options, ruby_system, system):
287        self.version = self.versionCount()
288        self.L2cache = TCC()
289        self.L2cache.create(options)
290        self.l2_response_latency = options.TCC_latency
291
292        self.number_of_TBEs = 2048
293
294        self.ruby_system = ruby_system
295
296        if options.recycle_latency:
297            self.recycle_latency = options.recycle_latency
298
299    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
300                           tcc_unblock_to_tccdir, req_to_tcc,
301                           probe_to_tcc, resp_to_tcc):
302        self.w_reqToTCCDir = req_to_tccdir
303        self.w_respToTCCDir = resp_to_tccdir
304        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
305        self.w_reqToTCC = req_to_tcc
306        self.w_probeToTCC = probe_to_tcc
307        self.w_respToTCC = resp_to_tcc
308
309class TCCDirCntrl(TCCdir_Controller, CntrlBase):
310    def create(self, options, ruby_system, system):
311        self.version = self.versionCount()
312
313        self.directory = TccDirCache()
314        self.directory.create(options)
315
316        self.number_of_TBEs = 1024
317
318        self.ruby_system = ruby_system
319
320        if options.recycle_latency:
321            self.recycle_latency = options.recycle_latency
322
323    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
324                           tcc_unblock_to_tccdir, req_to_tcc,
325                           probe_to_tcc, resp_to_tcc):
326        self.w_reqToTCCDir = req_to_tccdir
327        self.w_respToTCCDir = resp_to_tccdir
328        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
329        self.w_reqToTCC = req_to_tcc
330        self.w_probeToTCC = probe_to_tcc
331        self.w_respToTCC = resp_to_tcc
332
333class L3Cache(RubyCache):
334    assoc = 8
335    dataArrayBanks = 256
336    tagArrayBanks = 256
337
338    def create(self, options, ruby_system, system):
339        self.size = MemorySize(options.l3_size)
340        self.size.value /= options.num_dirs
341        self.dataArrayBanks /= options.num_dirs
342        self.tagArrayBanks /= options.num_dirs
343        self.dataArrayBanks /= options.num_dirs
344        self.tagArrayBanks /= options.num_dirs
345        self.dataAccessLatency = options.l3_data_latency
346        self.tagAccessLatency = options.l3_tag_latency
347        self.resourceStalls = options.no_resource_stalls
348        self.replacement_policy = PseudoLRUReplacementPolicy()
349
350class L3Cntrl(L3Cache_Controller, CntrlBase):
351    def create(self, options, ruby_system, system):
352        self.version = self.versionCount()
353        self.L3cache = L3Cache()
354        self.L3cache.create(options, ruby_system, system)
355
356        self.l3_response_latency = max(self.L3cache.dataAccessLatency,
357                                       self.L3cache.tagAccessLatency)
358        self.ruby_system = ruby_system
359
360        if options.recycle_latency:
361            self.recycle_latency = options.recycle_latency
362
363    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
364                           req_to_l3, probe_to_l3, resp_to_l3):
365        self.reqToDir = req_to_dir
366        self.respToDir = resp_to_dir
367        self.l3UnblockToDir = l3_unblock_to_dir
368        self.reqToL3 = req_to_l3
369        self.probeToL3 = probe_to_l3
370        self.respToL3 = resp_to_l3
371
372class DirCntrl(Directory_Controller, CntrlBase):
373    def create(self, options, dir_ranges, ruby_system, system):
374        self.version = self.versionCount()
375
376        self.response_latency = 30
377
378        self.addr_ranges = dir_ranges
379        self.directory = RubyDirectoryMemory()
380
381        self.L3CacheMemory = L3Cache()
382        self.L3CacheMemory.create(options, ruby_system, system)
383
384        self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
385                                  self.L3CacheMemory.tagAccessLatency)
386
387        self.number_of_TBEs = options.num_tbes
388
389        self.ruby_system = ruby_system
390
391        if options.recycle_latency:
392            self.recycle_latency = options.recycle_latency
393
394    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
395                           req_to_l3, probe_to_l3, resp_to_l3):
396        self.reqToDir = req_to_dir
397        self.respToDir = resp_to_dir
398        self.l3UnblockToDir = l3_unblock_to_dir
399        self.reqToL3 = req_to_l3
400        self.probeToL3 = probe_to_l3
401        self.respToL3 = resp_to_l3
402
403
404
405def define_options(parser):
406    parser.add_option("--num-subcaches", type="int", default=4)
407    parser.add_option("--l3-data-latency", type="int", default=20)
408    parser.add_option("--l3-tag-latency", type="int", default=15)
409    parser.add_option("--cpu-to-dir-latency", type="int", default=15)
410    parser.add_option("--gpu-to-dir-latency", type="int", default=160)
411    parser.add_option("--no-resource-stalls", action="store_false",
412                      default=True)
413    parser.add_option("--num-tbes", type="int", default=256)
414    parser.add_option("--l2-latency", type="int", default=50) # load to use
415    parser.add_option("--num-tccs", type="int", default=1,
416                      help="number of TCC directories and banks in the GPU")
417    parser.add_option("--TCP_latency", type="int", default=4,
418                      help="TCP latency")
419    parser.add_option("--TCC_latency", type="int", default=16,
420                      help="TCC latency")
421    parser.add_option("--tcc-size", type='string', default='256kB',
422                      help="agregate tcc size")
423    parser.add_option("--tcp-size", type='string', default='16kB',
424                      help="tcp size")
425    parser.add_option("--tcc-dir-factor", type='int', default=4,
426                      help="TCCdir size = factor *(TCPs + TCC)")
427
428def create_system(options, full_system, system, dma_devices, bootmem,
429                  ruby_system):
430    if buildEnv['PROTOCOL'] != 'GPU_RfO':
431        panic("This script requires the GPU_RfO protocol to be built.")
432
433    cpu_sequencers = []
434
435    #
436    # The ruby network creation expects the list of nodes in the system to be
437    # consistent with the NetDest list.  Therefore the l1 controller nodes
438    # must be listed before the directory nodes and directory nodes before
439    # dma nodes, etc.
440    #
441    cp_cntrl_nodes = []
442    tcp_cntrl_nodes = []
443    sqc_cntrl_nodes = []
444    tcc_cntrl_nodes = []
445    tccdir_cntrl_nodes = []
446    dir_cntrl_nodes = []
447    l3_cntrl_nodes = []
448
449    #
450    # Must create the individual controllers before the network to ensure the
451    # controller constructors are called before the network constructor
452    #
453
454    TCC_bits = int(math.log(options.num_tccs, 2))
455
456    # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
457    # Clusters
458    mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
459
460    if options.numa_high_bit:
461        numa_bit = options.numa_high_bit
462    else:
463        # if the numa_bit is not specified, set the directory bits as the
464        # lowest bits above the block offset bits, and the numa_bit as the
465        # highest of those directory bits
466        dir_bits = int(math.log(options.num_dirs, 2))
467        block_size_bits = int(math.log(options.cacheline_size, 2))
468        numa_bit = block_size_bits + dir_bits - 1
469
470    for i in xrange(options.num_dirs):
471        dir_ranges = []
472        for r in system.mem_ranges:
473            addr_range = m5.objects.AddrRange(r.start, size = r.size(),
474                                              intlvHighBit = numa_bit,
475                                              intlvBits = dir_bits,
476                                              intlvMatch = i)
477            dir_ranges.append(addr_range)
478
479        dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
480        dir_cntrl.create(options, dir_ranges, ruby_system, system)
481        dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
482        #Enough TBEs for all TCP TBEs
483
484        # Connect the Directory controller to the ruby network
485        dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
486        dir_cntrl.requestFromCores.slave = ruby_system.network.master
487
488        dir_cntrl.responseFromCores = MessageBuffer()
489        dir_cntrl.responseFromCores.slave = ruby_system.network.master
490
491        dir_cntrl.unblockFromCores = MessageBuffer()
492        dir_cntrl.unblockFromCores.slave = ruby_system.network.master
493
494        dir_cntrl.probeToCore = MessageBuffer()
495        dir_cntrl.probeToCore.master = ruby_system.network.slave
496
497        dir_cntrl.responseToCore = MessageBuffer()
498        dir_cntrl.responseToCore.master = ruby_system.network.slave
499
500        dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
501        dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
502        dir_cntrl.responseFromMemory = MessageBuffer()
503
504        exec("system.dir_cntrl%d = dir_cntrl" % i)
505        dir_cntrl_nodes.append(dir_cntrl)
506
507        mainCluster.add(dir_cntrl)
508
509    # For an odd number of CPUs, still create the right number of controllers
510    cpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
511    for i in xrange((options.num_cpus + 1) / 2):
512
513        cp_cntrl = CPCntrl()
514        cp_cntrl.create(options, ruby_system, system)
515
516        exec("system.cp_cntrl%d = cp_cntrl" % i)
517        #
518        # Add controllers and sequencers to the appropriate lists
519        #
520        cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
521
522        # Connect the CP controllers and the network
523        cp_cntrl.requestFromCore = MessageBuffer()
524        cp_cntrl.requestFromCore.master = ruby_system.network.slave
525
526        cp_cntrl.responseFromCore = MessageBuffer()
527        cp_cntrl.responseFromCore.master = ruby_system.network.slave
528
529        cp_cntrl.unblockFromCore = MessageBuffer()
530        cp_cntrl.unblockFromCore.master = ruby_system.network.slave
531
532        cp_cntrl.probeToCore = MessageBuffer()
533        cp_cntrl.probeToCore.slave = ruby_system.network.master
534
535        cp_cntrl.responseToCore = MessageBuffer()
536        cp_cntrl.responseToCore.slave = ruby_system.network.master
537
538        cp_cntrl.mandatoryQueue = MessageBuffer()
539        cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
540
541        cpuCluster.add(cp_cntrl)
542
543    gpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
544
545    for i in xrange(options.num_compute_units):
546
547        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
548                             number_of_TBEs = 2560) # max outstanding requests
549        tcp_cntrl.create(options, ruby_system, system)
550
551        exec("system.tcp_cntrl%d = tcp_cntrl" % i)
552        #
553        # Add controllers and sequencers to the appropriate lists
554        #
555        cpu_sequencers.append(tcp_cntrl.coalescer)
556        tcp_cntrl_nodes.append(tcp_cntrl)
557
558        # Connect the TCP controller to the ruby network
559        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
560        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
561
562        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
563        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
564
565        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
566        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
567
568        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
569        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
570
571        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
572        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
573
574        tcp_cntrl.mandatoryQueue = MessageBuffer()
575
576        gpuCluster.add(tcp_cntrl)
577
578    for i in xrange(options.num_sqc):
579
580        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
581        sqc_cntrl.create(options, ruby_system, system)
582
583        exec("system.sqc_cntrl%d = sqc_cntrl" % i)
584        #
585        # Add controllers and sequencers to the appropriate lists
586        #
587        cpu_sequencers.append(sqc_cntrl.sequencer)
588
589        # Connect the SQC controller to the ruby network
590        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
591        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
592
593        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
594        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
595
596        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
597        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
598
599        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
600        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
601
602        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
603        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
604
605        sqc_cntrl.mandatoryQueue = MessageBuffer()
606
607        # SQC also in GPU cluster
608        gpuCluster.add(sqc_cntrl)
609
610    for i in xrange(options.num_cp):
611
612        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
613                             number_of_TBEs = 2560) # max outstanding requests
614        tcp_cntrl.createCP(options, ruby_system, system)
615
616        exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i))
617        #
618        # Add controllers and sequencers to the appropriate lists
619        #
620        cpu_sequencers.append(tcp_cntrl.sequencer)
621        tcp_cntrl_nodes.append(tcp_cntrl)
622
623        # Connect the TCP controller to the ruby network
624        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
625        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
626
627        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
628        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
629
630        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
631        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
632
633        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
634        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
635
636        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
637        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
638
639        tcp_cntrl.mandatoryQueue = MessageBuffer()
640
641        gpuCluster.add(tcp_cntrl)
642
643        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
644        sqc_cntrl.createCP(options, ruby_system, system)
645
646        exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i))
647        #
648        # Add controllers and sequencers to the appropriate lists
649        #
650        cpu_sequencers.append(sqc_cntrl.sequencer)
651
652        # Connect the SQC controller to the ruby network
653        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
654        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
655
656        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
657        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
658
659        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
660        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
661
662        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
663        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
664
665        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
666        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
667
668        sqc_cntrl.mandatoryQueue = MessageBuffer()
669
670        # SQC also in GPU cluster
671        gpuCluster.add(sqc_cntrl)
672
673    for i in xrange(options.num_tccs):
674
675        tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits,
676                             number_of_TBEs = options.num_compute_units * 2560)
677        #Enough TBEs for all TCP TBEs
678        tcc_cntrl.create(options, ruby_system, system)
679        tcc_cntrl_nodes.append(tcc_cntrl)
680
681        tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits,
682                              number_of_TBEs = options.num_compute_units * 2560)
683        #Enough TBEs for all TCP TBEs
684        tccdir_cntrl.create(options, ruby_system, system)
685        tccdir_cntrl_nodes.append(tccdir_cntrl)
686
687        exec("system.tcc_cntrl%d = tcc_cntrl" % i)
688        exec("system.tccdir_cntrl%d = tccdir_cntrl" % i)
689
690        # connect all of the wire buffers between L3 and dirs up
691        req_to_tccdir = RubyWireBuffer()
692        resp_to_tccdir = RubyWireBuffer()
693        tcc_unblock_to_tccdir = RubyWireBuffer()
694        req_to_tcc = RubyWireBuffer()
695        probe_to_tcc = RubyWireBuffer()
696        resp_to_tcc = RubyWireBuffer()
697
698        tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
699                                     tcc_unblock_to_tccdir, req_to_tcc,
700                                     probe_to_tcc, resp_to_tcc)
701        tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
702                                        tcc_unblock_to_tccdir, req_to_tcc,
703                                        probe_to_tcc, resp_to_tcc)
704
705        # Connect the TCC controller to the ruby network
706        tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True)
707        tcc_cntrl.responseFromTCC.master = ruby_system.network.slave
708
709        tcc_cntrl.responseToTCC = MessageBuffer(ordered = True)
710        tcc_cntrl.responseToTCC.slave = ruby_system.network.master
711
712        # Connect the TCC Dir controller to the ruby network
713        tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True)
714        tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master
715
716        tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True)
717        tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master
718
719        tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True)
720        tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master
721
722        tccdir_cntrl.probeToCore = MessageBuffer(ordered = True)
723        tccdir_cntrl.probeToCore.master = ruby_system.network.slave
724
725        tccdir_cntrl.responseToCore = MessageBuffer(ordered = True)
726        tccdir_cntrl.responseToCore.master = ruby_system.network.slave
727
728        tccdir_cntrl.probeFromNB = MessageBuffer()
729        tccdir_cntrl.probeFromNB.slave = ruby_system.network.master
730
731        tccdir_cntrl.responseFromNB = MessageBuffer()
732        tccdir_cntrl.responseFromNB.slave = ruby_system.network.master
733
734        tccdir_cntrl.requestToNB = MessageBuffer()
735        tccdir_cntrl.requestToNB.master = ruby_system.network.slave
736
737        tccdir_cntrl.responseToNB = MessageBuffer()
738        tccdir_cntrl.responseToNB.master = ruby_system.network.slave
739
740        tccdir_cntrl.unblockToNB = MessageBuffer()
741        tccdir_cntrl.unblockToNB.master = ruby_system.network.slave
742
743        tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True)
744
745        # TCC cntrls added to the GPU cluster
746        gpuCluster.add(tcc_cntrl)
747        gpuCluster.add(tccdir_cntrl)
748
749    # Assuming no DMA devices
750    assert(len(dma_devices) == 0)
751
752    # Add cpu/gpu clusters to main cluster
753    mainCluster.add(cpuCluster)
754    mainCluster.add(gpuCluster)
755
756    ruby_system.network.number_of_virtual_networks = 10
757
758    return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
759