SConscript revision 12315
1# -*- mode:python -*-
2
3# Copyright (c) 2004-2005 The Regents of The University of Michigan
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# Authors: Nathan Binkert
30
31import array
32import bisect
33import imp
34import marshal
35import os
36import re
37import subprocess
38import sys
39import zlib
40
41from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
42
43import SCons
44
45from gem5_scons import Transform
46
47# This file defines how to build a particular configuration of gem5
48# based on variable settings in the 'env' build environment.
49
50Import('*')
51
52# Children need to see the environment
53Export('env')
54
55build_env = [(opt, env[opt]) for opt in export_vars]
56
57from m5.util import code_formatter, compareVersions
58
59########################################################################
60# Code for adding source files of various types
61#
62# When specifying a source file of some type, a set of tags can be
63# specified for that file.
64
65class SourceList(list):
66    def with_tags_that(self, predicate):
67        '''Return a list of sources with tags that satisfy a predicate.'''
68        def match(source):
69            return predicate(source.tags)
70        return SourceList(filter(match, self))
71
72    def with_any_tags(self, *tags):
73        '''Return a list of sources with any of the supplied tags.'''
74        return self.with_tags_that(lambda stags: len(tags & stags) > 0)
75
76    def with_all_tags(self, *tags):
77        '''Return a list of sources with all of the supplied tags.'''
78        return self.with_tags_that(lambda stags: tags <= stags)
79
80    def with_tag(self, tag):
81        '''Return a list of sources with the supplied tag.'''
82        return self.with_tags_that(lambda stags: tag in stags)
83
84    def without_tags(self, *tags):
85        '''Return a list of sources without any of the supplied tags.'''
86        return self.with_tags_that(lambda stags: len(tags & stags) == 0)
87
88    def without_tag(self, tag):
89        '''Return a list of sources with the supplied tag.'''
90        return self.with_tags_that(lambda stags: tag not in stags)
91
92class SourceMeta(type):
93    '''Meta class for source files that keeps track of all files of a
94    particular type.'''
95    def __init__(cls, name, bases, dict):
96        super(SourceMeta, cls).__init__(name, bases, dict)
97        cls.all = SourceList()
98
99class SourceFile(object):
100    '''Base object that encapsulates the notion of a source file.
101    This includes, the source node, target node, various manipulations
102    of those.  A source file also specifies a set of tags which
103    describing arbitrary properties of the source file.'''
104    __metaclass__ = SourceMeta
105    def __init__(self, source, tags=None, add_tags=None):
106        if tags is None:
107            tags='gem5 lib'
108        if isinstance(tags, basestring):
109            tags = set([tags])
110        if isinstance(add_tags, basestring):
111            add_tags = set([add_tags])
112        if add_tags:
113            tags = tags | add_tags
114        self.tags = set(tags)
115
116        tnode = source
117        if not isinstance(source, SCons.Node.FS.File):
118            tnode = File(source)
119
120        self.tnode = tnode
121        self.snode = tnode.srcnode()
122
123        for base in type(self).__mro__:
124            if issubclass(base, SourceFile):
125                base.all.append(self)
126
127    @property
128    def filename(self):
129        return str(self.tnode)
130
131    @property
132    def dirname(self):
133        return dirname(self.filename)
134
135    @property
136    def basename(self):
137        return basename(self.filename)
138
139    @property
140    def extname(self):
141        index = self.basename.rfind('.')
142        if index <= 0:
143            # dot files aren't extensions
144            return self.basename, None
145
146        return self.basename[:index], self.basename[index+1:]
147
148    def __lt__(self, other): return self.filename < other.filename
149    def __le__(self, other): return self.filename <= other.filename
150    def __gt__(self, other): return self.filename > other.filename
151    def __ge__(self, other): return self.filename >= other.filename
152    def __eq__(self, other): return self.filename == other.filename
153    def __ne__(self, other): return self.filename != other.filename
154
155class Source(SourceFile):
156    ungrouped_tag = 'No link group'
157    source_groups = set()
158
159    _current_group_tag = ungrouped_tag
160
161    @staticmethod
162    def link_group_tag(group):
163        return 'link group: %s' % group
164
165    @classmethod
166    def set_group(cls, group):
167        new_tag = Source.link_group_tag(group)
168        Source._current_group_tag = new_tag
169        Source.source_groups.add(group)
170
171    def _add_link_group_tag(self):
172        self.tags.add(Source._current_group_tag)
173
174    '''Add a c/c++ source file to the build'''
175    def __init__(self, source, tags=None, add_tags=None):
176        '''specify the source file, and any tags'''
177        super(Source, self).__init__(source, tags, add_tags)
178        self._add_link_group_tag()
179
180class PySource(SourceFile):
181    '''Add a python source file to the named package'''
182    invalid_sym_char = re.compile('[^A-z0-9_]')
183    modules = {}
184    tnodes = {}
185    symnames = {}
186
187    def __init__(self, package, source, tags=None, add_tags=None):
188        '''specify the python package, the source file, and any tags'''
189        super(PySource, self).__init__(source, tags, add_tags)
190
191        modname,ext = self.extname
192        assert ext == 'py'
193
194        if package:
195            path = package.split('.')
196        else:
197            path = []
198
199        modpath = path[:]
200        if modname != '__init__':
201            modpath += [ modname ]
202        modpath = '.'.join(modpath)
203
204        arcpath = path + [ self.basename ]
205        abspath = self.snode.abspath
206        if not exists(abspath):
207            abspath = self.tnode.abspath
208
209        self.package = package
210        self.modname = modname
211        self.modpath = modpath
212        self.arcname = joinpath(*arcpath)
213        self.abspath = abspath
214        self.compiled = File(self.filename + 'c')
215        self.cpp = File(self.filename + '.cc')
216        self.symname = PySource.invalid_sym_char.sub('_', modpath)
217
218        PySource.modules[modpath] = self
219        PySource.tnodes[self.tnode] = self
220        PySource.symnames[self.symname] = self
221
222class SimObject(PySource):
223    '''Add a SimObject python file as a python source object and add
224    it to a list of sim object modules'''
225
226    fixed = False
227    modnames = []
228
229    def __init__(self, source, tags=None, add_tags=None):
230        '''Specify the source file and any tags (automatically in
231        the m5.objects package)'''
232        super(SimObject, self).__init__('m5.objects', source, tags, add_tags)
233        if self.fixed:
234            raise AttributeError, "Too late to call SimObject now."
235
236        bisect.insort_right(SimObject.modnames, self.modname)
237
238class ProtoBuf(SourceFile):
239    '''Add a Protocol Buffer to build'''
240
241    def __init__(self, source, tags=None, add_tags=None):
242        '''Specify the source file, and any tags'''
243        super(ProtoBuf, self).__init__(source, tags, add_tags)
244
245        # Get the file name and the extension
246        modname,ext = self.extname
247        assert ext == 'proto'
248
249        # Currently, we stick to generating the C++ headers, so we
250        # only need to track the source and header.
251        self.cc_file = File(modname + '.pb.cc')
252        self.hh_file = File(modname + '.pb.h')
253
254class UnitTest(object):
255    '''Create a UnitTest'''
256
257    all = []
258    def __init__(self, target, *sources, **kwargs):
259        '''Specify the target name and any sources.  Sources that are
260        not SourceFiles are evalued with Source().  All files are
261        tagged with the name of the UnitTest target.'''
262
263        srcs = SourceList()
264        for src in sources:
265            if not isinstance(src, SourceFile):
266                src = Source(src, tags=str(target))
267            srcs.append(src)
268
269        self.sources = srcs
270        self.target = target
271        self.main = kwargs.get('main', False)
272        self.all.append(self)
273
274class GTest(UnitTest):
275    '''Create a unit test based on the google test framework.'''
276
277    all = []
278    def __init__(self, *args, **kwargs):
279        super(GTest, self).__init__(*args, **kwargs)
280        self.dir = Dir('.')
281
282# Children should have access
283Export('Source')
284Export('PySource')
285Export('SimObject')
286Export('ProtoBuf')
287Export('UnitTest')
288Export('GTest')
289
290########################################################################
291#
292# Debug Flags
293#
294debug_flags = {}
295def DebugFlag(name, desc=None):
296    if name in debug_flags:
297        raise AttributeError, "Flag %s already specified" % name
298    debug_flags[name] = (name, (), desc)
299
300def CompoundFlag(name, flags, desc=None):
301    if name in debug_flags:
302        raise AttributeError, "Flag %s already specified" % name
303
304    compound = tuple(flags)
305    debug_flags[name] = (name, compound, desc)
306
307Export('DebugFlag')
308Export('CompoundFlag')
309
310########################################################################
311#
312# Set some compiler variables
313#
314
315# Include file paths are rooted in this directory.  SCons will
316# automatically expand '.' to refer to both the source directory and
317# the corresponding build directory to pick up generated include
318# files.
319env.Append(CPPPATH=Dir('.'))
320
321for extra_dir in extras_dir_list:
322    env.Append(CPPPATH=Dir(extra_dir))
323
324# Workaround for bug in SCons version > 0.97d20071212
325# Scons bug id: 2006 gem5 Bug id: 308
326for root, dirs, files in os.walk(base_dir, topdown=True):
327    Dir(root[len(base_dir) + 1:])
328
329########################################################################
330#
331# Walk the tree and execute all SConscripts in subdirectories
332#
333
334here = Dir('.').srcnode().abspath
335for root, dirs, files in os.walk(base_dir, topdown=True):
336    if root == here:
337        # we don't want to recurse back into this SConscript
338        continue
339
340    if 'SConscript' in files:
341        build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
342        Source.set_group(build_dir)
343        SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
344
345for extra_dir in extras_dir_list:
346    prefix_len = len(dirname(extra_dir)) + 1
347
348    # Also add the corresponding build directory to pick up generated
349    # include files.
350    env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
351
352    for root, dirs, files in os.walk(extra_dir, topdown=True):
353        # if build lives in the extras directory, don't walk down it
354        if 'build' in dirs:
355            dirs.remove('build')
356
357        if 'SConscript' in files:
358            build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
359            SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
360
361for opt in export_vars:
362    env.ConfigFile(opt)
363
364def makeTheISA(source, target, env):
365    isas = [ src.get_contents() for src in source ]
366    target_isa = env['TARGET_ISA']
367    def define(isa):
368        return isa.upper() + '_ISA'
369
370    def namespace(isa):
371        return isa[0].upper() + isa[1:].lower() + 'ISA'
372
373
374    code = code_formatter()
375    code('''\
376#ifndef __CONFIG_THE_ISA_HH__
377#define __CONFIG_THE_ISA_HH__
378
379''')
380
381    # create defines for the preprocessing and compile-time determination
382    for i,isa in enumerate(isas):
383        code('#define $0 $1', define(isa), i + 1)
384    code()
385
386    # create an enum for any run-time determination of the ISA, we
387    # reuse the same name as the namespaces
388    code('enum class Arch {')
389    for i,isa in enumerate(isas):
390        if i + 1 == len(isas):
391            code('  $0 = $1', namespace(isa), define(isa))
392        else:
393            code('  $0 = $1,', namespace(isa), define(isa))
394    code('};')
395
396    code('''
397
398#define THE_ISA ${{define(target_isa)}}
399#define TheISA ${{namespace(target_isa)}}
400#define THE_ISA_STR "${{target_isa}}"
401
402#endif // __CONFIG_THE_ISA_HH__''')
403
404    code.write(str(target[0]))
405
406env.Command('config/the_isa.hh', map(Value, all_isa_list),
407            MakeAction(makeTheISA, Transform("CFG ISA", 0)))
408
409def makeTheGPUISA(source, target, env):
410    isas = [ src.get_contents() for src in source ]
411    target_gpu_isa = env['TARGET_GPU_ISA']
412    def define(isa):
413        return isa.upper() + '_ISA'
414
415    def namespace(isa):
416        return isa[0].upper() + isa[1:].lower() + 'ISA'
417
418
419    code = code_formatter()
420    code('''\
421#ifndef __CONFIG_THE_GPU_ISA_HH__
422#define __CONFIG_THE_GPU_ISA_HH__
423
424''')
425
426    # create defines for the preprocessing and compile-time determination
427    for i,isa in enumerate(isas):
428        code('#define $0 $1', define(isa), i + 1)
429    code()
430
431    # create an enum for any run-time determination of the ISA, we
432    # reuse the same name as the namespaces
433    code('enum class GPUArch {')
434    for i,isa in enumerate(isas):
435        if i + 1 == len(isas):
436            code('  $0 = $1', namespace(isa), define(isa))
437        else:
438            code('  $0 = $1,', namespace(isa), define(isa))
439    code('};')
440
441    code('''
442
443#define THE_GPU_ISA ${{define(target_gpu_isa)}}
444#define TheGpuISA ${{namespace(target_gpu_isa)}}
445#define THE_GPU_ISA_STR "${{target_gpu_isa}}"
446
447#endif // __CONFIG_THE_GPU_ISA_HH__''')
448
449    code.write(str(target[0]))
450
451env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
452            MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
453
454########################################################################
455#
456# Prevent any SimObjects from being added after this point, they
457# should all have been added in the SConscripts above
458#
459SimObject.fixed = True
460
461class DictImporter(object):
462    '''This importer takes a dictionary of arbitrary module names that
463    map to arbitrary filenames.'''
464    def __init__(self, modules):
465        self.modules = modules
466        self.installed = set()
467
468    def __del__(self):
469        self.unload()
470
471    def unload(self):
472        import sys
473        for module in self.installed:
474            del sys.modules[module]
475        self.installed = set()
476
477    def find_module(self, fullname, path):
478        if fullname == 'm5.defines':
479            return self
480
481        if fullname == 'm5.objects':
482            return self
483
484        if fullname.startswith('_m5'):
485            return None
486
487        source = self.modules.get(fullname, None)
488        if source is not None and fullname.startswith('m5.objects'):
489            return self
490
491        return None
492
493    def load_module(self, fullname):
494        mod = imp.new_module(fullname)
495        sys.modules[fullname] = mod
496        self.installed.add(fullname)
497
498        mod.__loader__ = self
499        if fullname == 'm5.objects':
500            mod.__path__ = fullname.split('.')
501            return mod
502
503        if fullname == 'm5.defines':
504            mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
505            return mod
506
507        source = self.modules[fullname]
508        if source.modname == '__init__':
509            mod.__path__ = source.modpath
510        mod.__file__ = source.abspath
511
512        exec file(source.abspath, 'r') in mod.__dict__
513
514        return mod
515
516import m5.SimObject
517import m5.params
518from m5.util import code_formatter
519
520m5.SimObject.clear()
521m5.params.clear()
522
523# install the python importer so we can grab stuff from the source
524# tree itself.  We can't have SimObjects added after this point or
525# else we won't know about them for the rest of the stuff.
526importer = DictImporter(PySource.modules)
527sys.meta_path[0:0] = [ importer ]
528
529# import all sim objects so we can populate the all_objects list
530# make sure that we're working with a list, then let's sort it
531for modname in SimObject.modnames:
532    exec('from m5.objects import %s' % modname)
533
534# we need to unload all of the currently imported modules so that they
535# will be re-imported the next time the sconscript is run
536importer.unload()
537sys.meta_path.remove(importer)
538
539sim_objects = m5.SimObject.allClasses
540all_enums = m5.params.allEnums
541
542for name,obj in sorted(sim_objects.iteritems()):
543    for param in obj._params.local.values():
544        # load the ptype attribute now because it depends on the
545        # current version of SimObject.allClasses, but when scons
546        # actually uses the value, all versions of
547        # SimObject.allClasses will have been loaded
548        param.ptype
549
550########################################################################
551#
552# calculate extra dependencies
553#
554module_depends = ["m5", "m5.SimObject", "m5.params"]
555depends = [ PySource.modules[dep].snode for dep in module_depends ]
556depends.sort(key = lambda x: x.name)
557
558########################################################################
559#
560# Commands for the basic automatically generated python files
561#
562
563# Generate Python file containing a dict specifying the current
564# buildEnv flags.
565def makeDefinesPyFile(target, source, env):
566    build_env = source[0].get_contents()
567
568    code = code_formatter()
569    code("""
570import _m5.core
571import m5.util
572
573buildEnv = m5.util.SmartDict($build_env)
574
575compileDate = _m5.core.compileDate
576_globals = globals()
577for key,val in _m5.core.__dict__.iteritems():
578    if key.startswith('flag_'):
579        flag = key[5:]
580        _globals[flag] = val
581del _globals
582""")
583    code.write(target[0].abspath)
584
585defines_info = Value(build_env)
586# Generate a file with all of the compile options in it
587env.Command('python/m5/defines.py', defines_info,
588            MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
589PySource('m5', 'python/m5/defines.py')
590
591# Generate python file containing info about the M5 source code
592def makeInfoPyFile(target, source, env):
593    code = code_formatter()
594    for src in source:
595        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
596        code('$src = ${{repr(data)}}')
597    code.write(str(target[0]))
598
599# Generate a file that wraps the basic top level files
600env.Command('python/m5/info.py',
601            [ '#/COPYING', '#/LICENSE', '#/README', ],
602            MakeAction(makeInfoPyFile, Transform("INFO")))
603PySource('m5', 'python/m5/info.py')
604
605########################################################################
606#
607# Create all of the SimObject param headers and enum headers
608#
609
610def createSimObjectParamStruct(target, source, env):
611    assert len(target) == 1 and len(source) == 1
612
613    name = source[0].get_text_contents()
614    obj = sim_objects[name]
615
616    code = code_formatter()
617    obj.cxx_param_decl(code)
618    code.write(target[0].abspath)
619
620def createSimObjectCxxConfig(is_header):
621    def body(target, source, env):
622        assert len(target) == 1 and len(source) == 1
623
624        name = str(source[0].get_contents())
625        obj = sim_objects[name]
626
627        code = code_formatter()
628        obj.cxx_config_param_file(code, is_header)
629        code.write(target[0].abspath)
630    return body
631
632def createEnumStrings(target, source, env):
633    assert len(target) == 1 and len(source) == 2
634
635    name = source[0].get_text_contents()
636    use_python = source[1].read()
637    obj = all_enums[name]
638
639    code = code_formatter()
640    obj.cxx_def(code)
641    if use_python:
642        obj.pybind_def(code)
643    code.write(target[0].abspath)
644
645def createEnumDecls(target, source, env):
646    assert len(target) == 1 and len(source) == 1
647
648    name = source[0].get_text_contents()
649    obj = all_enums[name]
650
651    code = code_formatter()
652    obj.cxx_decl(code)
653    code.write(target[0].abspath)
654
655def createSimObjectPyBindWrapper(target, source, env):
656    name = source[0].get_text_contents()
657    obj = sim_objects[name]
658
659    code = code_formatter()
660    obj.pybind_decl(code)
661    code.write(target[0].abspath)
662
663# Generate all of the SimObject param C++ struct header files
664params_hh_files = []
665for name,simobj in sorted(sim_objects.iteritems()):
666    py_source = PySource.modules[simobj.__module__]
667    extra_deps = [ py_source.tnode ]
668
669    hh_file = File('params/%s.hh' % name)
670    params_hh_files.append(hh_file)
671    env.Command(hh_file, Value(name),
672                MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
673    env.Depends(hh_file, depends + extra_deps)
674
675# C++ parameter description files
676if GetOption('with_cxx_config'):
677    for name,simobj in sorted(sim_objects.iteritems()):
678        py_source = PySource.modules[simobj.__module__]
679        extra_deps = [ py_source.tnode ]
680
681        cxx_config_hh_file = File('cxx_config/%s.hh' % name)
682        cxx_config_cc_file = File('cxx_config/%s.cc' % name)
683        env.Command(cxx_config_hh_file, Value(name),
684                    MakeAction(createSimObjectCxxConfig(True),
685                    Transform("CXXCPRHH")))
686        env.Command(cxx_config_cc_file, Value(name),
687                    MakeAction(createSimObjectCxxConfig(False),
688                    Transform("CXXCPRCC")))
689        env.Depends(cxx_config_hh_file, depends + extra_deps +
690                    [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
691        env.Depends(cxx_config_cc_file, depends + extra_deps +
692                    [cxx_config_hh_file])
693        Source(cxx_config_cc_file)
694
695    cxx_config_init_cc_file = File('cxx_config/init.cc')
696
697    def createCxxConfigInitCC(target, source, env):
698        assert len(target) == 1 and len(source) == 1
699
700        code = code_formatter()
701
702        for name,simobj in sorted(sim_objects.iteritems()):
703            if not hasattr(simobj, 'abstract') or not simobj.abstract:
704                code('#include "cxx_config/${name}.hh"')
705        code()
706        code('void cxxConfigInit()')
707        code('{')
708        code.indent()
709        for name,simobj in sorted(sim_objects.iteritems()):
710            not_abstract = not hasattr(simobj, 'abstract') or \
711                not simobj.abstract
712            if not_abstract and 'type' in simobj.__dict__:
713                code('cxx_config_directory["${name}"] = '
714                     '${name}CxxConfigParams::makeDirectoryEntry();')
715        code.dedent()
716        code('}')
717        code.write(target[0].abspath)
718
719    py_source = PySource.modules[simobj.__module__]
720    extra_deps = [ py_source.tnode ]
721    env.Command(cxx_config_init_cc_file, Value(name),
722        MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
723    cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
724        for name,simobj in sorted(sim_objects.iteritems())
725        if not hasattr(simobj, 'abstract') or not simobj.abstract]
726    Depends(cxx_config_init_cc_file, cxx_param_hh_files +
727            [File('sim/cxx_config.hh')])
728    Source(cxx_config_init_cc_file)
729
730# Generate all enum header files
731for name,enum in sorted(all_enums.iteritems()):
732    py_source = PySource.modules[enum.__module__]
733    extra_deps = [ py_source.tnode ]
734
735    cc_file = File('enums/%s.cc' % name)
736    env.Command(cc_file, [Value(name), Value(env['USE_PYTHON'])],
737                MakeAction(createEnumStrings, Transform("ENUM STR")))
738    env.Depends(cc_file, depends + extra_deps)
739    Source(cc_file)
740
741    hh_file = File('enums/%s.hh' % name)
742    env.Command(hh_file, Value(name),
743                MakeAction(createEnumDecls, Transform("ENUMDECL")))
744    env.Depends(hh_file, depends + extra_deps)
745
746# Generate SimObject Python bindings wrapper files
747if env['USE_PYTHON']:
748    for name,simobj in sorted(sim_objects.iteritems()):
749        py_source = PySource.modules[simobj.__module__]
750        extra_deps = [ py_source.tnode ]
751        cc_file = File('python/_m5/param_%s.cc' % name)
752        env.Command(cc_file, Value(name),
753                    MakeAction(createSimObjectPyBindWrapper,
754                               Transform("SO PyBind")))
755        env.Depends(cc_file, depends + extra_deps)
756        Source(cc_file)
757
758# Build all protocol buffers if we have got protoc and protobuf available
759if env['HAVE_PROTOBUF']:
760    for proto in ProtoBuf.all:
761        # Use both the source and header as the target, and the .proto
762        # file as the source. When executing the protoc compiler, also
763        # specify the proto_path to avoid having the generated files
764        # include the path.
765        env.Command([proto.cc_file, proto.hh_file], proto.tnode,
766                    MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
767                               '--proto_path ${SOURCE.dir} $SOURCE',
768                               Transform("PROTOC")))
769
770        # Add the C++ source file
771        Source(proto.cc_file, tags=proto.tags)
772elif ProtoBuf.all:
773    print 'Got protobuf to build, but lacks support!'
774    Exit(1)
775
776#
777# Handle debug flags
778#
779def makeDebugFlagCC(target, source, env):
780    assert(len(target) == 1 and len(source) == 1)
781
782    code = code_formatter()
783
784    # delay definition of CompoundFlags until after all the definition
785    # of all constituent SimpleFlags
786    comp_code = code_formatter()
787
788    # file header
789    code('''
790/*
791 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
792 */
793
794#include "base/debug.hh"
795
796namespace Debug {
797
798''')
799
800    for name, flag in sorted(source[0].read().iteritems()):
801        n, compound, desc = flag
802        assert n == name
803
804        if not compound:
805            code('SimpleFlag $name("$name", "$desc");')
806        else:
807            comp_code('CompoundFlag $name("$name", "$desc",')
808            comp_code.indent()
809            last = len(compound) - 1
810            for i,flag in enumerate(compound):
811                if i != last:
812                    comp_code('&$flag,')
813                else:
814                    comp_code('&$flag);')
815            comp_code.dedent()
816
817    code.append(comp_code)
818    code()
819    code('} // namespace Debug')
820
821    code.write(str(target[0]))
822
823def makeDebugFlagHH(target, source, env):
824    assert(len(target) == 1 and len(source) == 1)
825
826    val = eval(source[0].get_contents())
827    name, compound, desc = val
828
829    code = code_formatter()
830
831    # file header boilerplate
832    code('''\
833/*
834 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
835 */
836
837#ifndef __DEBUG_${name}_HH__
838#define __DEBUG_${name}_HH__
839
840namespace Debug {
841''')
842
843    if compound:
844        code('class CompoundFlag;')
845    code('class SimpleFlag;')
846
847    if compound:
848        code('extern CompoundFlag $name;')
849        for flag in compound:
850            code('extern SimpleFlag $flag;')
851    else:
852        code('extern SimpleFlag $name;')
853
854    code('''
855}
856
857#endif // __DEBUG_${name}_HH__
858''')
859
860    code.write(str(target[0]))
861
862for name,flag in sorted(debug_flags.iteritems()):
863    n, compound, desc = flag
864    assert n == name
865
866    hh_file = 'debug/%s.hh' % name
867    env.Command(hh_file, Value(flag),
868                MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
869
870env.Command('debug/flags.cc', Value(debug_flags),
871            MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
872Source('debug/flags.cc')
873
874# version tags
875tags = \
876env.Command('sim/tags.cc', None,
877            MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
878                       Transform("VER TAGS")))
879env.AlwaysBuild(tags)
880
881# Embed python files.  All .py files that have been indicated by a
882# PySource() call in a SConscript need to be embedded into the M5
883# library.  To do that, we compile the file to byte code, marshal the
884# byte code, compress it, and then generate a c++ file that
885# inserts the result into an array.
886def embedPyFile(target, source, env):
887    def c_str(string):
888        if string is None:
889            return "0"
890        return '"%s"' % string
891
892    '''Action function to compile a .py into a code object, marshal
893    it, compress it, and stick it into an asm file so the code appears
894    as just bytes with a label in the data section'''
895
896    src = file(str(source[0]), 'r').read()
897
898    pysource = PySource.tnodes[source[0]]
899    compiled = compile(src, pysource.abspath, 'exec')
900    marshalled = marshal.dumps(compiled)
901    compressed = zlib.compress(marshalled)
902    data = compressed
903    sym = pysource.symname
904
905    code = code_formatter()
906    code('''\
907#include "sim/init.hh"
908
909namespace {
910
911const uint8_t data_${sym}[] = {
912''')
913    code.indent()
914    step = 16
915    for i in xrange(0, len(data), step):
916        x = array.array('B', data[i:i+step])
917        code(''.join('%d,' % d for d in x))
918    code.dedent()
919
920    code('''};
921
922EmbeddedPython embedded_${sym}(
923    ${{c_str(pysource.arcname)}},
924    ${{c_str(pysource.abspath)}},
925    ${{c_str(pysource.modpath)}},
926    data_${sym},
927    ${{len(data)}},
928    ${{len(marshalled)}});
929
930} // anonymous namespace
931''')
932    code.write(str(target[0]))
933
934for source in PySource.all:
935    env.Command(source.cpp, source.tnode,
936                MakeAction(embedPyFile, Transform("EMBED PY")))
937    Source(source.cpp, tags=source.tags, add_tags='python')
938
939########################################################################
940#
941# Define binaries.  Each different build type (debug, opt, etc.) gets
942# a slightly different build environment.
943#
944
945# List of constructed environments to pass back to SConstruct
946date_source = Source('base/date.cc', tags=[])
947
948# Function to create a new build environment as clone of current
949# environment 'env' with modified object suffix and optional stripped
950# binary.  Additional keyword arguments are appended to corresponding
951# build environment vars.
952def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
953    # SCons doesn't know to append a library suffix when there is a '.' in the
954    # name.  Use '_' instead.
955    libname = 'gem5_' + label
956    exename = 'gem5.' + label
957    secondary_exename = 'm5.' + label
958
959    new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
960    new_env.Label = label
961    new_env.Append(**kwargs)
962
963    make_static = lambda source: new_env.StaticObject(source.tnode)
964    make_shared = lambda source: new_env.SharedObject(source.tnode)
965
966    lib_sources = Source.all.with_tag('gem5 lib')
967
968    # Without Python, leave out all Python content from the library
969    # builds.  The option doesn't affect gem5 built as a program
970    if GetOption('without_python'):
971        lib_sources = lib_sources.without_tag('python')
972
973    static_objs = []
974    shared_objs = []
975
976    for s in lib_sources.with_tag(Source.ungrouped_tag):
977        static_objs.append(make_static(s))
978        shared_objs.append(make_shared(s))
979
980    for group in Source.source_groups:
981        srcs = lib_sources.with_tag(Source.link_group_tag(group))
982        if not srcs:
983            continue
984
985        group_static = [ make_static(s) for s in srcs ]
986        group_shared = [ make_shared(s) for s in srcs ]
987
988        # If partial linking is disabled, add these sources to the build
989        # directly, and short circuit this loop.
990        if disable_partial:
991            static_objs.extend(group_static)
992            shared_objs.extend(group_shared)
993            continue
994
995        # Set up the static partially linked objects.
996        file_name = new_env.subst("${OBJPREFIX}lib${OBJSUFFIX}.partial")
997        target = File(joinpath(group, file_name))
998        partial = env.PartialStatic(target=target, source=group_static)
999        static_objs.extend(partial)
1000
1001        # Set up the shared partially linked objects.
1002        file_name = new_env.subst("${SHOBJPREFIX}lib${SHOBJSUFFIX}.partial")
1003        target = File(joinpath(group, file_name))
1004        partial = env.PartialShared(target=target, source=group_shared)
1005        shared_objs.extend(partial)
1006
1007    static_date = make_static(date_source)
1008    new_env.Depends(static_date, static_objs)
1009    static_objs.extend(static_date)
1010
1011    shared_date = make_shared(date_source)
1012    new_env.Depends(shared_date, shared_objs)
1013    shared_objs.extend(shared_date)
1014
1015    # First make a library of everything but main() so other programs can
1016    # link against m5.
1017    static_lib = new_env.StaticLibrary(libname, static_objs)
1018    shared_lib = new_env.SharedLibrary(libname, shared_objs)
1019
1020    # Now link a stub with main() and the static library.
1021    main_objs = [ make_static(s) for s in Source.all.with_tag('main') ]
1022
1023    for test in UnitTest.all:
1024        test_sources = Source.all.with_tag(str(test.target))
1025        test_objs = [ make_static(s) for s in test_sources ]
1026        if test.main:
1027            test_objs += main_objs
1028        path = 'unittest/%s.%s' % (test.target, label)
1029        new_env.Program(path, test_objs + static_objs)
1030
1031    gtest_env = new_env.Clone()
1032    gtest_env.Append(LIBS=gtest_env['GTEST_LIBS'])
1033    gtest_env.Append(CPPFLAGS=gtest_env['GTEST_CPPFLAGS'])
1034    for test in GTest.all:
1035        test_sources = Source.all.with_tag(str(test.target))
1036        test_objs = [ gtest_env.StaticObject(s.tnode) for s in test_sources ]
1037        gtest_env.Program(test.dir.File('%s.%s' % (test.target, label)),
1038                          test_objs)
1039
1040    progname = exename
1041    if strip:
1042        progname += '.unstripped'
1043
1044    targets = new_env.Program(progname, main_objs + static_objs)
1045
1046    if strip:
1047        if sys.platform == 'sunos5':
1048            cmd = 'cp $SOURCE $TARGET; strip $TARGET'
1049        else:
1050            cmd = 'strip $SOURCE -o $TARGET'
1051        targets = new_env.Command(exename, progname,
1052                    MakeAction(cmd, Transform("STRIP")))
1053
1054    new_env.Command(secondary_exename, exename,
1055            MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1056
1057    new_env.M5Binary = targets[0]
1058
1059    # Set up regression tests.
1060    SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1061               variant_dir=Dir('tests').Dir(new_env.Label),
1062               exports={ 'env' : new_env }, duplicate=False)
1063
1064# Start out with the compiler flags common to all compilers,
1065# i.e. they all use -g for opt and -g -pg for prof
1066ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1067           'perf' : ['-g']}
1068
1069# Start out with the linker flags common to all linkers, i.e. -pg for
1070# prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1071# no-as-needed and as-needed as the binutils linker is too clever and
1072# simply doesn't link to the library otherwise.
1073ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1074           'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1075
1076# For Link Time Optimization, the optimisation flags used to compile
1077# individual files are decoupled from those used at link time
1078# (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1079# to also update the linker flags based on the target.
1080if env['GCC']:
1081    if sys.platform == 'sunos5':
1082        ccflags['debug'] += ['-gstabs+']
1083    else:
1084        ccflags['debug'] += ['-ggdb3']
1085    ldflags['debug'] += ['-O0']
1086    # opt, fast, prof and perf all share the same cc flags, also add
1087    # the optimization to the ldflags as LTO defers the optimization
1088    # to link time
1089    for target in ['opt', 'fast', 'prof', 'perf']:
1090        ccflags[target] += ['-O3']
1091        ldflags[target] += ['-O3']
1092
1093    ccflags['fast'] += env['LTO_CCFLAGS']
1094    ldflags['fast'] += env['LTO_LDFLAGS']
1095elif env['CLANG']:
1096    ccflags['debug'] += ['-g', '-O0']
1097    # opt, fast, prof and perf all share the same cc flags
1098    for target in ['opt', 'fast', 'prof', 'perf']:
1099        ccflags[target] += ['-O3']
1100else:
1101    print 'Unknown compiler, please fix compiler options'
1102    Exit(1)
1103
1104
1105# To speed things up, we only instantiate the build environments we
1106# need.  We try to identify the needed environment for each target; if
1107# we can't, we fall back on instantiating all the environments just to
1108# be safe.
1109target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1110obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1111              'gpo' : 'perf'}
1112
1113def identifyTarget(t):
1114    ext = t.split('.')[-1]
1115    if ext in target_types:
1116        return ext
1117    if obj2target.has_key(ext):
1118        return obj2target[ext]
1119    match = re.search(r'/tests/([^/]+)/', t)
1120    if match and match.group(1) in target_types:
1121        return match.group(1)
1122    return 'all'
1123
1124needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1125if 'all' in needed_envs:
1126    needed_envs += target_types
1127
1128# Debug binary
1129if 'debug' in needed_envs:
1130    makeEnv(env, 'debug', '.do',
1131            CCFLAGS = Split(ccflags['debug']),
1132            CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1133            LINKFLAGS = Split(ldflags['debug']))
1134
1135# Optimized binary
1136if 'opt' in needed_envs:
1137    makeEnv(env, 'opt', '.o',
1138            CCFLAGS = Split(ccflags['opt']),
1139            CPPDEFINES = ['TRACING_ON=1'],
1140            LINKFLAGS = Split(ldflags['opt']))
1141
1142# "Fast" binary
1143if 'fast' in needed_envs:
1144    disable_partial = \
1145            env.get('BROKEN_INCREMENTAL_LTO', False) and \
1146            GetOption('force_lto')
1147    makeEnv(env, 'fast', '.fo', strip = True,
1148            CCFLAGS = Split(ccflags['fast']),
1149            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1150            LINKFLAGS = Split(ldflags['fast']),
1151            disable_partial=disable_partial)
1152
1153# Profiled binary using gprof
1154if 'prof' in needed_envs:
1155    makeEnv(env, 'prof', '.po',
1156            CCFLAGS = Split(ccflags['prof']),
1157            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1158            LINKFLAGS = Split(ldflags['prof']))
1159
1160# Profiled binary using google-pprof
1161if 'perf' in needed_envs:
1162    makeEnv(env, 'perf', '.gpo',
1163            CCFLAGS = Split(ccflags['perf']),
1164            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1165            LINKFLAGS = Split(ldflags['perf']))
1166