GPU_RfO.py revision 13974 
1# Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
2# All rights reserved.
3#
4# For use for simulation and test purposes only
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9# 1. Redistributions of source code must retain the above copyright notice,
10# this list of conditions and the following disclaimer.
11#
12# 2. Redistributions in binary form must reproduce the above copyright notice,
13# this list of conditions and the following disclaimer in the documentation
14# and/or other materials provided with the distribution.
15#
16# 3. Neither the name of the copyright holder nor the names of its
17# contributors may be used to endorse or promote products derived from this
18# software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
24# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30# POSSIBILITY OF SUCH DAMAGE.
31#
32# Authors: Lisa Hsu
33
34import math
35import m5
36from m5.objects import *
37from m5.defines import buildEnv
38from m5.util import addToPath
39from .Ruby import create_topology
40from .Ruby import send_evicts
41
42addToPath('../')
43
44from topologies.Cluster import Cluster
45from topologies.Crossbar import Crossbar
46
47class CntrlBase:
48    _seqs = 0
49    @classmethod
50    def seqCount(cls):
51        # Use SeqCount not class since we need global count
52        CntrlBase._seqs += 1
53        return CntrlBase._seqs - 1
54
55    _cntrls = 0
56    @classmethod
57    def cntrlCount(cls):
58        # Use CntlCount not class since we need global count
59        CntrlBase._cntrls += 1
60        return CntrlBase._cntrls - 1
61
62    _version = 0
63    @classmethod
64    def versionCount(cls):
65        cls._version += 1 # Use count for this particular type
66        return cls._version - 1
67
68class TccDirCache(RubyCache):
69    size = "512kB"
70    assoc = 16
71    resourceStalls = False
72    def create(self, options):
73        self.size = MemorySize(options.tcc_size)
74        self.size.value += (options.num_compute_units *
75                            (MemorySize(options.tcp_size).value) *
76                            options.tcc_dir_factor) / long(options.num_tccs)
77        self.start_index_bit = math.log(options.cacheline_size, 2) + \
78                               math.log(options.num_tccs, 2)
79        self.replacement_policy = PseudoLRUReplacementPolicy()
80
81class L1DCache(RubyCache):
82    resourceStalls = False
83    def create(self, options):
84        self.size = MemorySize(options.l1d_size)
85        self.assoc = options.l1d_assoc
86        self.replacement_policy = PseudoLRUReplacementPolicy()
87
88class L1ICache(RubyCache):
89    resourceStalls = False
90    def create(self, options):
91        self.size = MemorySize(options.l1i_size)
92        self.assoc = options.l1i_assoc
93        self.replacement_policy = PseudoLRUReplacementPolicy()
94
95class L2Cache(RubyCache):
96    resourceStalls = False
97    def create(self, options):
98        self.size = MemorySize(options.l2_size)
99        self.assoc = options.l2_assoc
100        self.replacement_policy = PseudoLRUReplacementPolicy()
101
102
103class CPCntrl(CorePair_Controller, CntrlBase):
104
105    def create(self, options, ruby_system, system):
106        self.version = self.versionCount()
107
108        self.L1Icache = L1ICache()
109        self.L1Icache.create(options)
110        self.L1D0cache = L1DCache()
111        self.L1D0cache.create(options)
112        self.L1D1cache = L1DCache()
113        self.L1D1cache.create(options)
114        self.L2cache = L2Cache()
115        self.L2cache.create(options)
116
117        self.sequencer = RubySequencer()
118        self.sequencer.version = self.seqCount()
119        self.sequencer.icache = self.L1Icache
120        self.sequencer.dcache = self.L1D0cache
121        self.sequencer.ruby_system = ruby_system
122        self.sequencer.coreid = 0
123        self.sequencer.is_cpu_sequencer = True
124
125        self.sequencer1 = RubySequencer()
126        self.sequencer1.version = self.seqCount()
127        self.sequencer1.icache = self.L1Icache
128        self.sequencer1.dcache = self.L1D1cache
129        self.sequencer1.ruby_system = ruby_system
130        self.sequencer1.coreid = 1
131        self.sequencer1.is_cpu_sequencer = True
132
133        # Defines icache/dcache hit latency
134        self.mandatory_queue_latency = 2
135
136        self.issue_latency = options.cpu_to_dir_latency
137        self.send_evictions = send_evicts(options)
138
139        self.ruby_system = ruby_system
140
141        if options.recycle_latency:
142            self.recycle_latency = options.recycle_latency
143
144class TCPCache(RubyCache):
145    assoc = 8
146    dataArrayBanks = 16
147    tagArrayBanks = 4
148    dataAccessLatency = 4
149    tagAccessLatency = 1
150    def create(self, options):
151        self.size = MemorySize(options.tcp_size)
152        self.replacement_policy = PseudoLRUReplacementPolicy()
153
154class TCPCntrl(TCP_Controller, CntrlBase):
155
156    def create(self, options, ruby_system, system):
157        self.version = self.versionCount()
158
159        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
160        self.L1cache.resourceStalls = options.no_resource_stalls
161        self.L1cache.create(options)
162
163        self.coalescer = RubyGPUCoalescer()
164        self.coalescer.version = self.seqCount()
165        self.coalescer.icache = self.L1cache
166        self.coalescer.dcache = self.L1cache
167        self.coalescer.ruby_system = ruby_system
168        self.coalescer.support_inst_reqs = False
169        self.coalescer.is_cpu_sequencer = False
170        self.coalescer.max_outstanding_requests = options.simds_per_cu * \
171                                                  options.wfs_per_simd * \
172                                                  options.wf_size
173
174        self.sequencer = RubySequencer()
175        self.sequencer.version = self.seqCount()
176        self.sequencer.icache = self.L1cache
177        self.sequencer.dcache = self.L1cache
178        self.sequencer.ruby_system = ruby_system
179        self.sequencer.is_cpu_sequencer = True
180
181        self.use_seq_not_coal = False
182
183        self.ruby_system = ruby_system
184
185        if options.recycle_latency:
186            self.recycle_latency = options.recycle_latency
187
188    def createCP(self, options, ruby_system, system):
189        self.version = self.versionCount()
190
191        self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
192        self.L1cache.resourceStalls = options.no_resource_stalls
193        self.L1cache.create(options)
194
195        self.coalescer = RubyGPUCoalescer()
196        self.coalescer.version = self.seqCount()
197        self.coalescer.icache = self.L1cache
198        self.coalescer.dcache = self.L1cache
199        self.coalescer.ruby_system = ruby_system
200        self.coalescer.support_inst_reqs = False
201        self.coalescer.is_cpu_sequencer = False
202
203        self.sequencer = RubySequencer()
204        self.sequencer.version = self.seqCount()
205        self.sequencer.icache = self.L1cache
206        self.sequencer.dcache = self.L1cache
207        self.sequencer.ruby_system = ruby_system
208        self.sequencer.is_cpu_sequencer = True
209
210        self.use_seq_not_coal = True
211
212        self.ruby_system = ruby_system
213
214        if options.recycle_latency:
215            self.recycle_latency = options.recycle_latency
216
217class SQCCache(RubyCache):
218    size = "32kB"
219    assoc = 8
220    dataArrayBanks = 16
221    tagArrayBanks = 4
222    dataAccessLatency = 4
223    tagAccessLatency = 1
224    def create(self, options):
225        self.replacement_policy = PseudoLRUReplacementPolicy()
226
227class SQCCntrl(SQC_Controller, CntrlBase):
228
229    def create(self, options, ruby_system, system):
230        self.version = self.versionCount()
231
232        self.L1cache = SQCCache()
233        self.L1cache.create(options)
234        self.L1cache.resourceStalls = options.no_resource_stalls
235
236        self.sequencer = RubySequencer()
237
238        self.sequencer.version = self.seqCount()
239        self.sequencer.icache = self.L1cache
240        self.sequencer.dcache = self.L1cache
241        self.sequencer.ruby_system = ruby_system
242        self.sequencer.support_data_reqs = False
243        self.sequencer.is_cpu_sequencer = False
244
245        self.ruby_system = ruby_system
246
247        if options.recycle_latency:
248            self.recycle_latency = options.recycle_latency
249
250    def createCP(self, options, ruby_system, system):
251        self.version = self.versionCount()
252
253        self.L1cache = SQCCache()
254        self.L1cache.create(options)
255        self.L1cache.resourceStalls = options.no_resource_stalls
256
257        self.sequencer = RubySequencer()
258
259        self.sequencer.version = self.seqCount()
260        self.sequencer.icache = self.L1cache
261        self.sequencer.dcache = self.L1cache
262        self.sequencer.ruby_system = ruby_system
263        self.sequencer.support_data_reqs = False
264
265        self.ruby_system = ruby_system
266
267        if options.recycle_latency:
268            self.recycle_latency = options.recycle_latency
269
270
271class TCC(RubyCache):
272    assoc = 16
273    dataAccessLatency = 8
274    tagAccessLatency = 2
275    resourceStalls = True
276    def create(self, options):
277        self.size = MemorySize(options.tcc_size)
278        self.size = self.size / options.num_tccs
279        self.dataArrayBanks = 256 / options.num_tccs #number of data banks
280        self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
281        if ((self.size.value / long(self.assoc)) < 128):
282            self.size.value = long(128 * self.assoc)
283        self.start_index_bit = math.log(options.cacheline_size, 2) + \
284                               math.log(options.num_tccs, 2)
285        self.replacement_policy = PseudoLRUReplacementPolicy()
286
287class TCCCntrl(TCC_Controller, CntrlBase):
288    def create(self, options, ruby_system, system):
289        self.version = self.versionCount()
290        self.L2cache = TCC()
291        self.L2cache.create(options)
292        self.l2_response_latency = options.TCC_latency
293
294        self.number_of_TBEs = 2048
295
296        self.ruby_system = ruby_system
297
298        if options.recycle_latency:
299            self.recycle_latency = options.recycle_latency
300
301    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
302                           tcc_unblock_to_tccdir, req_to_tcc,
303                           probe_to_tcc, resp_to_tcc):
304        self.w_reqToTCCDir = req_to_tccdir
305        self.w_respToTCCDir = resp_to_tccdir
306        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
307        self.w_reqToTCC = req_to_tcc
308        self.w_probeToTCC = probe_to_tcc
309        self.w_respToTCC = resp_to_tcc
310
311class TCCDirCntrl(TCCdir_Controller, CntrlBase):
312    def create(self, options, ruby_system, system):
313        self.version = self.versionCount()
314
315        self.directory = TccDirCache()
316        self.directory.create(options)
317
318        self.number_of_TBEs = 1024
319
320        self.ruby_system = ruby_system
321
322        if options.recycle_latency:
323            self.recycle_latency = options.recycle_latency
324
325    def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
326                           tcc_unblock_to_tccdir, req_to_tcc,
327                           probe_to_tcc, resp_to_tcc):
328        self.w_reqToTCCDir = req_to_tccdir
329        self.w_respToTCCDir = resp_to_tccdir
330        self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
331        self.w_reqToTCC = req_to_tcc
332        self.w_probeToTCC = probe_to_tcc
333        self.w_respToTCC = resp_to_tcc
334
335class L3Cache(RubyCache):
336    assoc = 8
337    dataArrayBanks = 256
338    tagArrayBanks = 256
339
340    def create(self, options, ruby_system, system):
341        self.size = MemorySize(options.l3_size)
342        self.size.value /= options.num_dirs
343        self.dataArrayBanks /= options.num_dirs
344        self.tagArrayBanks /= options.num_dirs
345        self.dataArrayBanks /= options.num_dirs
346        self.tagArrayBanks /= options.num_dirs
347        self.dataAccessLatency = options.l3_data_latency
348        self.tagAccessLatency = options.l3_tag_latency
349        self.resourceStalls = options.no_resource_stalls
350        self.replacement_policy = PseudoLRUReplacementPolicy()
351
352class L3Cntrl(L3Cache_Controller, CntrlBase):
353    def create(self, options, ruby_system, system):
354        self.version = self.versionCount()
355        self.L3cache = L3Cache()
356        self.L3cache.create(options, ruby_system, system)
357
358        self.l3_response_latency = max(self.L3cache.dataAccessLatency,
359                                       self.L3cache.tagAccessLatency)
360        self.ruby_system = ruby_system
361
362        if options.recycle_latency:
363            self.recycle_latency = options.recycle_latency
364
365    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
366                           req_to_l3, probe_to_l3, resp_to_l3):
367        self.reqToDir = req_to_dir
368        self.respToDir = resp_to_dir
369        self.l3UnblockToDir = l3_unblock_to_dir
370        self.reqToL3 = req_to_l3
371        self.probeToL3 = probe_to_l3
372        self.respToL3 = resp_to_l3
373
374class DirCntrl(Directory_Controller, CntrlBase):
375    def create(self, options, dir_ranges, ruby_system, system):
376        self.version = self.versionCount()
377
378        self.response_latency = 30
379
380        self.addr_ranges = dir_ranges
381        self.directory = RubyDirectoryMemory()
382
383        self.L3CacheMemory = L3Cache()
384        self.L3CacheMemory.create(options, ruby_system, system)
385
386        self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
387                                  self.L3CacheMemory.tagAccessLatency)
388
389        self.number_of_TBEs = options.num_tbes
390
391        self.ruby_system = ruby_system
392
393        if options.recycle_latency:
394            self.recycle_latency = options.recycle_latency
395
396    def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
397                           req_to_l3, probe_to_l3, resp_to_l3):
398        self.reqToDir = req_to_dir
399        self.respToDir = resp_to_dir
400        self.l3UnblockToDir = l3_unblock_to_dir
401        self.reqToL3 = req_to_l3
402        self.probeToL3 = probe_to_l3
403        self.respToL3 = resp_to_l3
404
405
406
407def define_options(parser):
408    parser.add_option("--num-subcaches", type="int", default=4)
409    parser.add_option("--l3-data-latency", type="int", default=20)
410    parser.add_option("--l3-tag-latency", type="int", default=15)
411    parser.add_option("--cpu-to-dir-latency", type="int", default=15)
412    parser.add_option("--gpu-to-dir-latency", type="int", default=160)
413    parser.add_option("--no-resource-stalls", action="store_false",
414                      default=True)
415    parser.add_option("--num-tbes", type="int", default=256)
416    parser.add_option("--l2-latency", type="int", default=50) # load to use
417    parser.add_option("--num-tccs", type="int", default=1,
418                      help="number of TCC directories and banks in the GPU")
419    parser.add_option("--TCP_latency", type="int", default=4,
420                      help="TCP latency")
421    parser.add_option("--TCC_latency", type="int", default=16,
422                      help="TCC latency")
423    parser.add_option("--tcc-size", type='string', default='256kB',
424                      help="agregate tcc size")
425    parser.add_option("--tcp-size", type='string', default='16kB',
426                      help="tcp size")
427    parser.add_option("--tcc-dir-factor", type='int', default=4,
428                      help="TCCdir size = factor *(TCPs + TCC)")
429
430def create_system(options, full_system, system, dma_devices, bootmem,
431                  ruby_system):
432    if buildEnv['PROTOCOL'] != 'GPU_RfO':
433        panic("This script requires the GPU_RfO protocol to be built.")
434
435    cpu_sequencers = []
436
437    #
438    # The ruby network creation expects the list of nodes in the system to be
439    # consistent with the NetDest list.  Therefore the l1 controller nodes
440    # must be listed before the directory nodes and directory nodes before
441    # dma nodes, etc.
442    #
443    cp_cntrl_nodes = []
444    tcp_cntrl_nodes = []
445    sqc_cntrl_nodes = []
446    tcc_cntrl_nodes = []
447    tccdir_cntrl_nodes = []
448    dir_cntrl_nodes = []
449    l3_cntrl_nodes = []
450
451    #
452    # Must create the individual controllers before the network to ensure the
453    # controller constructors are called before the network constructor
454    #
455
456    TCC_bits = int(math.log(options.num_tccs, 2))
457
458    # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
459    # Clusters
460    mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
461
462    if options.numa_high_bit:
463        numa_bit = options.numa_high_bit
464    else:
465        # if the numa_bit is not specified, set the directory bits as the
466        # lowest bits above the block offset bits, and the numa_bit as the
467        # highest of those directory bits
468        dir_bits = int(math.log(options.num_dirs, 2))
469        block_size_bits = int(math.log(options.cacheline_size, 2))
470        numa_bit = block_size_bits + dir_bits - 1
471
472    for i in range(options.num_dirs):
473        dir_ranges = []
474        for r in system.mem_ranges:
475            addr_range = m5.objects.AddrRange(r.start, size = r.size(),
476                                              intlvHighBit = numa_bit,
477                                              intlvBits = dir_bits,
478                                              intlvMatch = i)
479            dir_ranges.append(addr_range)
480
481        dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
482        dir_cntrl.create(options, dir_ranges, ruby_system, system)
483        dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
484        #Enough TBEs for all TCP TBEs
485
486        # Connect the Directory controller to the ruby network
487        dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
488        dir_cntrl.requestFromCores.slave = ruby_system.network.master
489
490        dir_cntrl.responseFromCores = MessageBuffer()
491        dir_cntrl.responseFromCores.slave = ruby_system.network.master
492
493        dir_cntrl.unblockFromCores = MessageBuffer()
494        dir_cntrl.unblockFromCores.slave = ruby_system.network.master
495
496        dir_cntrl.probeToCore = MessageBuffer()
497        dir_cntrl.probeToCore.master = ruby_system.network.slave
498
499        dir_cntrl.responseToCore = MessageBuffer()
500        dir_cntrl.responseToCore.master = ruby_system.network.slave
501
502        dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
503        dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
504        dir_cntrl.responseFromMemory = MessageBuffer()
505
506        exec("system.dir_cntrl%d = dir_cntrl" % i)
507        dir_cntrl_nodes.append(dir_cntrl)
508
509        mainCluster.add(dir_cntrl)
510
511    # For an odd number of CPUs, still create the right number of controllers
512    cpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
513    for i in range((options.num_cpus + 1) // 2):
514
515        cp_cntrl = CPCntrl()
516        cp_cntrl.create(options, ruby_system, system)
517
518        exec("system.cp_cntrl%d = cp_cntrl" % i)
519        #
520        # Add controllers and sequencers to the appropriate lists
521        #
522        cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
523
524        # Connect the CP controllers and the network
525        cp_cntrl.requestFromCore = MessageBuffer()
526        cp_cntrl.requestFromCore.master = ruby_system.network.slave
527
528        cp_cntrl.responseFromCore = MessageBuffer()
529        cp_cntrl.responseFromCore.master = ruby_system.network.slave
530
531        cp_cntrl.unblockFromCore = MessageBuffer()
532        cp_cntrl.unblockFromCore.master = ruby_system.network.slave
533
534        cp_cntrl.probeToCore = MessageBuffer()
535        cp_cntrl.probeToCore.slave = ruby_system.network.master
536
537        cp_cntrl.responseToCore = MessageBuffer()
538        cp_cntrl.responseToCore.slave = ruby_system.network.master
539
540        cp_cntrl.mandatoryQueue = MessageBuffer()
541        cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
542
543        cpuCluster.add(cp_cntrl)
544
545    gpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
546
547    for i in range(options.num_compute_units):
548
549        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
550                             number_of_TBEs = 2560) # max outstanding requests
551        tcp_cntrl.create(options, ruby_system, system)
552
553        exec("system.tcp_cntrl%d = tcp_cntrl" % i)
554        #
555        # Add controllers and sequencers to the appropriate lists
556        #
557        cpu_sequencers.append(tcp_cntrl.coalescer)
558        tcp_cntrl_nodes.append(tcp_cntrl)
559
560        # Connect the TCP controller to the ruby network
561        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
562        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
563
564        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
565        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
566
567        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
568        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
569
570        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
571        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
572
573        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
574        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
575
576        tcp_cntrl.mandatoryQueue = MessageBuffer()
577
578        gpuCluster.add(tcp_cntrl)
579
580    for i in range(options.num_sqc):
581
582        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
583        sqc_cntrl.create(options, ruby_system, system)
584
585        exec("system.sqc_cntrl%d = sqc_cntrl" % i)
586        #
587        # Add controllers and sequencers to the appropriate lists
588        #
589        cpu_sequencers.append(sqc_cntrl.sequencer)
590
591        # Connect the SQC controller to the ruby network
592        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
593        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
594
595        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
596        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
597
598        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
599        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
600
601        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
602        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
603
604        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
605        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
606
607        sqc_cntrl.mandatoryQueue = MessageBuffer()
608
609        # SQC also in GPU cluster
610        gpuCluster.add(sqc_cntrl)
611
612    for i in range(options.num_cp):
613
614        tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
615                             number_of_TBEs = 2560) # max outstanding requests
616        tcp_cntrl.createCP(options, ruby_system, system)
617
618        exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i))
619        #
620        # Add controllers and sequencers to the appropriate lists
621        #
622        cpu_sequencers.append(tcp_cntrl.sequencer)
623        tcp_cntrl_nodes.append(tcp_cntrl)
624
625        # Connect the TCP controller to the ruby network
626        tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
627        tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
628
629        tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
630        tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
631
632        tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
633        tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
634
635        tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
636        tcp_cntrl.probeToTCP.slave = ruby_system.network.master
637
638        tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
639        tcp_cntrl.responseToTCP.slave = ruby_system.network.master
640
641        tcp_cntrl.mandatoryQueue = MessageBuffer()
642
643        gpuCluster.add(tcp_cntrl)
644
645        sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
646        sqc_cntrl.createCP(options, ruby_system, system)
647
648        exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i))
649        #
650        # Add controllers and sequencers to the appropriate lists
651        #
652        cpu_sequencers.append(sqc_cntrl.sequencer)
653
654        # Connect the SQC controller to the ruby network
655        sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
656        sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
657
658        sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
659        sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
660
661        sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
662        sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
663
664        sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
665        sqc_cntrl.probeToSQC.slave = ruby_system.network.master
666
667        sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
668        sqc_cntrl.responseToSQC.slave = ruby_system.network.master
669
670        sqc_cntrl.mandatoryQueue = MessageBuffer()
671
672        # SQC also in GPU cluster
673        gpuCluster.add(sqc_cntrl)
674
675    for i in range(options.num_tccs):
676
677        tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits,
678                             number_of_TBEs = options.num_compute_units * 2560)
679        #Enough TBEs for all TCP TBEs
680        tcc_cntrl.create(options, ruby_system, system)
681        tcc_cntrl_nodes.append(tcc_cntrl)
682
683        tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits,
684                              number_of_TBEs = options.num_compute_units * 2560)
685        #Enough TBEs for all TCP TBEs
686        tccdir_cntrl.create(options, ruby_system, system)
687        tccdir_cntrl_nodes.append(tccdir_cntrl)
688
689        exec("system.tcc_cntrl%d = tcc_cntrl" % i)
690        exec("system.tccdir_cntrl%d = tccdir_cntrl" % i)
691
692        # connect all of the wire buffers between L3 and dirs up
693        req_to_tccdir = RubyWireBuffer()
694        resp_to_tccdir = RubyWireBuffer()
695        tcc_unblock_to_tccdir = RubyWireBuffer()
696        req_to_tcc = RubyWireBuffer()
697        probe_to_tcc = RubyWireBuffer()
698        resp_to_tcc = RubyWireBuffer()
699
700        tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
701                                     tcc_unblock_to_tccdir, req_to_tcc,
702                                     probe_to_tcc, resp_to_tcc)
703        tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
704                                        tcc_unblock_to_tccdir, req_to_tcc,
705                                        probe_to_tcc, resp_to_tcc)
706
707        # Connect the TCC controller to the ruby network
708        tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True)
709        tcc_cntrl.responseFromTCC.master = ruby_system.network.slave
710
711        tcc_cntrl.responseToTCC = MessageBuffer(ordered = True)
712        tcc_cntrl.responseToTCC.slave = ruby_system.network.master
713
714        # Connect the TCC Dir controller to the ruby network
715        tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True)
716        tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master
717
718        tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True)
719        tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master
720
721        tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True)
722        tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master
723
724        tccdir_cntrl.probeToCore = MessageBuffer(ordered = True)
725        tccdir_cntrl.probeToCore.master = ruby_system.network.slave
726
727        tccdir_cntrl.responseToCore = MessageBuffer(ordered = True)
728        tccdir_cntrl.responseToCore.master = ruby_system.network.slave
729
730        tccdir_cntrl.probeFromNB = MessageBuffer()
731        tccdir_cntrl.probeFromNB.slave = ruby_system.network.master
732
733        tccdir_cntrl.responseFromNB = MessageBuffer()
734        tccdir_cntrl.responseFromNB.slave = ruby_system.network.master
735
736        tccdir_cntrl.requestToNB = MessageBuffer()
737        tccdir_cntrl.requestToNB.master = ruby_system.network.slave
738
739        tccdir_cntrl.responseToNB = MessageBuffer()
740        tccdir_cntrl.responseToNB.master = ruby_system.network.slave
741
742        tccdir_cntrl.unblockToNB = MessageBuffer()
743        tccdir_cntrl.unblockToNB.master = ruby_system.network.slave
744
745        tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True)
746
747        # TCC cntrls added to the GPU cluster
748        gpuCluster.add(tcc_cntrl)
749        gpuCluster.add(tccdir_cntrl)
750
751    # Assuming no DMA devices
752    assert(len(dma_devices) == 0)
753
754    # Add cpu/gpu clusters to main cluster
755    mainCluster.add(cpuCluster)
756    mainCluster.add(gpuCluster)
757
758    ruby_system.network.number_of_virtual_networks = 10
759
760    return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
761