SConscript revision 11974:006d830b4a4e
1# -*- mode:python -*-
2
3# Copyright (c) 2004-2005 The Regents of The University of Michigan
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# Authors: Nathan Binkert
30
31import array
32import bisect
33import imp
34import marshal
35import os
36import re
37import subprocess
38import sys
39import zlib
40
41from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
42
43import SCons
44
45# This file defines how to build a particular configuration of gem5
46# based on variable settings in the 'env' build environment.
47
48Import('*')
49
50# Children need to see the environment
51Export('env')
52
53build_env = [(opt, env[opt]) for opt in export_vars]
54
55from m5.util import code_formatter, compareVersions
56
57########################################################################
58# Code for adding source files of various types
59#
60# When specifying a source file of some type, a set of guards can be
61# specified for that file.  When get() is used to find the files, if
62# get specifies a set of filters, only files that match those filters
63# will be accepted (unspecified filters on files are assumed to be
64# false).  Current filters are:
65#     main -- specifies the gem5 main() function
66#     skip_lib -- do not put this file into the gem5 library
67#     skip_no_python -- do not put this file into a no_python library
68#       as it embeds compiled Python
69#     <unittest> -- unit tests use filters based on the unit test name
70#
71# A parent can now be specified for a source file and default filter
72# values will be retrieved recursively from parents (children override
73# parents).
74#
75class SourceMeta(type):
76    '''Meta class for source files that keeps track of all files of a
77    particular type and has a get function for finding all functions
78    of a certain type that match a set of guards'''
79    def __init__(cls, name, bases, dict):
80        super(SourceMeta, cls).__init__(name, bases, dict)
81        cls.all = []
82
83    def get(cls, **guards):
84        '''Find all files that match the specified guards.  If a source
85        file does not specify a flag, the default is False'''
86        for src in cls.all:
87            for flag,value in guards.iteritems():
88                # if the flag is found and has a different value, skip
89                # this file
90                if src.all_guards.get(flag, False) != value:
91                    break
92            else:
93                yield src
94
95class SourceFile(object):
96    '''Base object that encapsulates the notion of a source file.
97    This includes, the source node, target node, various manipulations
98    of those.  A source file also specifies a set of guards which
99    describing which builds the source file applies to.  A parent can
100    also be specified to get default guards from'''
101    __metaclass__ = SourceMeta
102    def __init__(self, source, parent=None, **guards):
103        self.guards = guards
104        self.parent = parent
105
106        tnode = source
107        if not isinstance(source, SCons.Node.FS.File):
108            tnode = File(source)
109
110        self.tnode = tnode
111        self.snode = tnode.srcnode()
112
113        for base in type(self).__mro__:
114            if issubclass(base, SourceFile):
115                base.all.append(self)
116
117    @property
118    def filename(self):
119        return str(self.tnode)
120
121    @property
122    def dirname(self):
123        return dirname(self.filename)
124
125    @property
126    def basename(self):
127        return basename(self.filename)
128
129    @property
130    def extname(self):
131        index = self.basename.rfind('.')
132        if index <= 0:
133            # dot files aren't extensions
134            return self.basename, None
135
136        return self.basename[:index], self.basename[index+1:]
137
138    @property
139    def all_guards(self):
140        '''find all guards for this object getting default values
141        recursively from its parents'''
142        guards = {}
143        if self.parent:
144            guards.update(self.parent.guards)
145        guards.update(self.guards)
146        return guards
147
148    def __lt__(self, other): return self.filename < other.filename
149    def __le__(self, other): return self.filename <= other.filename
150    def __gt__(self, other): return self.filename > other.filename
151    def __ge__(self, other): return self.filename >= other.filename
152    def __eq__(self, other): return self.filename == other.filename
153    def __ne__(self, other): return self.filename != other.filename
154
155    @staticmethod
156    def done():
157        def disabled(cls, name, *ignored):
158            raise RuntimeError("Additional SourceFile '%s'" % name,\
159                  "declared, but targets deps are already fixed.")
160        SourceFile.__init__ = disabled
161
162
163class Source(SourceFile):
164    '''Add a c/c++ source file to the build'''
165    def __init__(self, source, Werror=True, swig=False, **guards):
166        '''specify the source file, and any guards'''
167        super(Source, self).__init__(source, **guards)
168
169        self.Werror = Werror
170        self.swig = swig
171
172class PySource(SourceFile):
173    '''Add a python source file to the named package'''
174    invalid_sym_char = re.compile('[^A-z0-9_]')
175    modules = {}
176    tnodes = {}
177    symnames = {}
178
179    def __init__(self, package, source, **guards):
180        '''specify the python package, the source file, and any guards'''
181        super(PySource, self).__init__(source, **guards)
182
183        modname,ext = self.extname
184        assert ext == 'py'
185
186        if package:
187            path = package.split('.')
188        else:
189            path = []
190
191        modpath = path[:]
192        if modname != '__init__':
193            modpath += [ modname ]
194        modpath = '.'.join(modpath)
195
196        arcpath = path + [ self.basename ]
197        abspath = self.snode.abspath
198        if not exists(abspath):
199            abspath = self.tnode.abspath
200
201        self.package = package
202        self.modname = modname
203        self.modpath = modpath
204        self.arcname = joinpath(*arcpath)
205        self.abspath = abspath
206        self.compiled = File(self.filename + 'c')
207        self.cpp = File(self.filename + '.cc')
208        self.symname = PySource.invalid_sym_char.sub('_', modpath)
209
210        PySource.modules[modpath] = self
211        PySource.tnodes[self.tnode] = self
212        PySource.symnames[self.symname] = self
213
214class SimObject(PySource):
215    '''Add a SimObject python file as a python source object and add
216    it to a list of sim object modules'''
217
218    fixed = False
219    modnames = []
220
221    def __init__(self, source, **guards):
222        '''Specify the source file and any guards (automatically in
223        the m5.objects package)'''
224        super(SimObject, self).__init__('m5.objects', source, **guards)
225        if self.fixed:
226            raise AttributeError, "Too late to call SimObject now."
227
228        bisect.insort_right(SimObject.modnames, self.modname)
229
230class SwigSource(SourceFile):
231    '''Add a swig file to build'''
232
233    def __init__(self, package, source, **guards):
234        '''Specify the python package, the source file, and any guards'''
235        super(SwigSource, self).__init__(source, skip_no_python=True, **guards)
236
237        modname,ext = self.extname
238        assert ext == 'i'
239
240        self.package = package
241        self.module = modname
242        cc_file = joinpath(self.dirname, modname + '_wrap.cc')
243        py_file = joinpath(self.dirname, modname + '.py')
244
245        self.cc_source = Source(cc_file, swig=True, parent=self, **guards)
246        self.py_source = PySource(package, py_file, parent=self, **guards)
247
248class ProtoBuf(SourceFile):
249    '''Add a Protocol Buffer to build'''
250
251    def __init__(self, source, **guards):
252        '''Specify the source file, and any guards'''
253        super(ProtoBuf, self).__init__(source, **guards)
254
255        # Get the file name and the extension
256        modname,ext = self.extname
257        assert ext == 'proto'
258
259        # Currently, we stick to generating the C++ headers, so we
260        # only need to track the source and header.
261        self.cc_file = File(modname + '.pb.cc')
262        self.hh_file = File(modname + '.pb.h')
263
264class UnitTest(object):
265    '''Create a UnitTest'''
266
267    all = []
268    def __init__(self, target, *sources, **kwargs):
269        '''Specify the target name and any sources.  Sources that are
270        not SourceFiles are evalued with Source().  All files are
271        guarded with a guard of the same name as the UnitTest
272        target.'''
273
274        srcs = []
275        for src in sources:
276            if not isinstance(src, SourceFile):
277                src = Source(src, skip_lib=True)
278            src.guards[target] = True
279            srcs.append(src)
280
281        self.sources = srcs
282        self.target = target
283        self.main = kwargs.get('main', False)
284        UnitTest.all.append(self)
285
286# Children should have access
287Export('Source')
288Export('PySource')
289Export('SimObject')
290Export('SwigSource')
291Export('ProtoBuf')
292Export('UnitTest')
293
294########################################################################
295#
296# Debug Flags
297#
298debug_flags = {}
299def DebugFlag(name, desc=None):
300    if name in debug_flags:
301        raise AttributeError, "Flag %s already specified" % name
302    debug_flags[name] = (name, (), desc)
303
304def CompoundFlag(name, flags, desc=None):
305    if name in debug_flags:
306        raise AttributeError, "Flag %s already specified" % name
307
308    compound = tuple(flags)
309    debug_flags[name] = (name, compound, desc)
310
311Export('DebugFlag')
312Export('CompoundFlag')
313
314########################################################################
315#
316# Set some compiler variables
317#
318
319# Include file paths are rooted in this directory.  SCons will
320# automatically expand '.' to refer to both the source directory and
321# the corresponding build directory to pick up generated include
322# files.
323env.Append(CPPPATH=Dir('.'))
324
325for extra_dir in extras_dir_list:
326    env.Append(CPPPATH=Dir(extra_dir))
327
328# Workaround for bug in SCons version > 0.97d20071212
329# Scons bug id: 2006 gem5 Bug id: 308
330for root, dirs, files in os.walk(base_dir, topdown=True):
331    Dir(root[len(base_dir) + 1:])
332
333########################################################################
334#
335# Walk the tree and execute all SConscripts in subdirectories
336#
337
338here = Dir('.').srcnode().abspath
339for root, dirs, files in os.walk(base_dir, topdown=True):
340    if root == here:
341        # we don't want to recurse back into this SConscript
342        continue
343
344    if 'SConscript' in files:
345        build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
346        SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
347
348for extra_dir in extras_dir_list:
349    prefix_len = len(dirname(extra_dir)) + 1
350
351    # Also add the corresponding build directory to pick up generated
352    # include files.
353    env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
354
355    for root, dirs, files in os.walk(extra_dir, topdown=True):
356        # if build lives in the extras directory, don't walk down it
357        if 'build' in dirs:
358            dirs.remove('build')
359
360        if 'SConscript' in files:
361            build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
362            SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
363
364for opt in export_vars:
365    env.ConfigFile(opt)
366
367def makeTheISA(source, target, env):
368    isas = [ src.get_contents() for src in source ]
369    target_isa = env['TARGET_ISA']
370    def define(isa):
371        return isa.upper() + '_ISA'
372
373    def namespace(isa):
374        return isa[0].upper() + isa[1:].lower() + 'ISA'
375
376
377    code = code_formatter()
378    code('''\
379#ifndef __CONFIG_THE_ISA_HH__
380#define __CONFIG_THE_ISA_HH__
381
382''')
383
384    # create defines for the preprocessing and compile-time determination
385    for i,isa in enumerate(isas):
386        code('#define $0 $1', define(isa), i + 1)
387    code()
388
389    # create an enum for any run-time determination of the ISA, we
390    # reuse the same name as the namespaces
391    code('enum class Arch {')
392    for i,isa in enumerate(isas):
393        if i + 1 == len(isas):
394            code('  $0 = $1', namespace(isa), define(isa))
395        else:
396            code('  $0 = $1,', namespace(isa), define(isa))
397    code('};')
398
399    code('''
400
401#define THE_ISA ${{define(target_isa)}}
402#define TheISA ${{namespace(target_isa)}}
403#define THE_ISA_STR "${{target_isa}}"
404
405#endif // __CONFIG_THE_ISA_HH__''')
406
407    code.write(str(target[0]))
408
409env.Command('config/the_isa.hh', map(Value, all_isa_list),
410            MakeAction(makeTheISA, Transform("CFG ISA", 0)))
411
412def makeTheGPUISA(source, target, env):
413    isas = [ src.get_contents() for src in source ]
414    target_gpu_isa = env['TARGET_GPU_ISA']
415    def define(isa):
416        return isa.upper() + '_ISA'
417
418    def namespace(isa):
419        return isa[0].upper() + isa[1:].lower() + 'ISA'
420
421
422    code = code_formatter()
423    code('''\
424#ifndef __CONFIG_THE_GPU_ISA_HH__
425#define __CONFIG_THE_GPU_ISA_HH__
426
427''')
428
429    # create defines for the preprocessing and compile-time determination
430    for i,isa in enumerate(isas):
431        code('#define $0 $1', define(isa), i + 1)
432    code()
433
434    # create an enum for any run-time determination of the ISA, we
435    # reuse the same name as the namespaces
436    code('enum class GPUArch {')
437    for i,isa in enumerate(isas):
438        if i + 1 == len(isas):
439            code('  $0 = $1', namespace(isa), define(isa))
440        else:
441            code('  $0 = $1,', namespace(isa), define(isa))
442    code('};')
443
444    code('''
445
446#define THE_GPU_ISA ${{define(target_gpu_isa)}}
447#define TheGpuISA ${{namespace(target_gpu_isa)}}
448#define THE_GPU_ISA_STR "${{target_gpu_isa}}"
449
450#endif // __CONFIG_THE_GPU_ISA_HH__''')
451
452    code.write(str(target[0]))
453
454env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
455            MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
456
457########################################################################
458#
459# Prevent any SimObjects from being added after this point, they
460# should all have been added in the SConscripts above
461#
462SimObject.fixed = True
463
464class DictImporter(object):
465    '''This importer takes a dictionary of arbitrary module names that
466    map to arbitrary filenames.'''
467    def __init__(self, modules):
468        self.modules = modules
469        self.installed = set()
470
471    def __del__(self):
472        self.unload()
473
474    def unload(self):
475        import sys
476        for module in self.installed:
477            del sys.modules[module]
478        self.installed = set()
479
480    def find_module(self, fullname, path):
481        if fullname == 'm5.defines':
482            return self
483
484        if fullname == 'm5.objects':
485            return self
486
487        if fullname.startswith('_m5'):
488            return None
489
490        source = self.modules.get(fullname, None)
491        if source is not None and fullname.startswith('m5.objects'):
492            return self
493
494        return None
495
496    def load_module(self, fullname):
497        mod = imp.new_module(fullname)
498        sys.modules[fullname] = mod
499        self.installed.add(fullname)
500
501        mod.__loader__ = self
502        if fullname == 'm5.objects':
503            mod.__path__ = fullname.split('.')
504            return mod
505
506        if fullname == 'm5.defines':
507            mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
508            return mod
509
510        source = self.modules[fullname]
511        if source.modname == '__init__':
512            mod.__path__ = source.modpath
513        mod.__file__ = source.abspath
514
515        exec file(source.abspath, 'r') in mod.__dict__
516
517        return mod
518
519import m5.SimObject
520import m5.params
521from m5.util import code_formatter
522
523m5.SimObject.clear()
524m5.params.clear()
525
526# install the python importer so we can grab stuff from the source
527# tree itself.  We can't have SimObjects added after this point or
528# else we won't know about them for the rest of the stuff.
529importer = DictImporter(PySource.modules)
530sys.meta_path[0:0] = [ importer ]
531
532# import all sim objects so we can populate the all_objects list
533# make sure that we're working with a list, then let's sort it
534for modname in SimObject.modnames:
535    exec('from m5.objects import %s' % modname)
536
537# we need to unload all of the currently imported modules so that they
538# will be re-imported the next time the sconscript is run
539importer.unload()
540sys.meta_path.remove(importer)
541
542sim_objects = m5.SimObject.allClasses
543all_enums = m5.params.allEnums
544
545if m5.SimObject.noCxxHeader:
546    print >> sys.stderr, \
547        "warning: At least one SimObject lacks a header specification. " \
548        "This can cause unexpected results in the generated SWIG " \
549        "wrappers."
550
551# Find param types that need to be explicitly wrapped with swig.
552# These will be recognized because the ParamDesc will have a
553# swig_decl() method.  Most param types are based on types that don't
554# need this, either because they're based on native types (like Int)
555# or because they're SimObjects (which get swigged independently).
556# For now the only things handled here are VectorParam types.
557params_to_swig = {}
558for name,obj in sorted(sim_objects.iteritems()):
559    for param in obj._params.local.values():
560        # load the ptype attribute now because it depends on the
561        # current version of SimObject.allClasses, but when scons
562        # actually uses the value, all versions of
563        # SimObject.allClasses will have been loaded
564        param.ptype
565
566        if not hasattr(param, 'swig_decl'):
567            continue
568        pname = param.ptype_str
569        if pname not in params_to_swig:
570            params_to_swig[pname] = param
571
572########################################################################
573#
574# calculate extra dependencies
575#
576module_depends = ["m5", "m5.SimObject", "m5.params"]
577depends = [ PySource.modules[dep].snode for dep in module_depends ]
578depends.sort(key = lambda x: x.name)
579
580########################################################################
581#
582# Commands for the basic automatically generated python files
583#
584
585# Generate Python file containing a dict specifying the current
586# buildEnv flags.
587def makeDefinesPyFile(target, source, env):
588    build_env = source[0].get_contents()
589
590    code = code_formatter()
591    code("""
592import _m5.core
593import m5.util
594
595buildEnv = m5.util.SmartDict($build_env)
596
597compileDate = _m5.core.compileDate
598_globals = globals()
599for key,val in _m5.core.__dict__.iteritems():
600    if key.startswith('flag_'):
601        flag = key[5:]
602        _globals[flag] = val
603del _globals
604""")
605    code.write(target[0].abspath)
606
607defines_info = Value(build_env)
608# Generate a file with all of the compile options in it
609env.Command('python/m5/defines.py', defines_info,
610            MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
611PySource('m5', 'python/m5/defines.py')
612
613# Generate python file containing info about the M5 source code
614def makeInfoPyFile(target, source, env):
615    code = code_formatter()
616    for src in source:
617        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
618        code('$src = ${{repr(data)}}')
619    code.write(str(target[0]))
620
621# Generate a file that wraps the basic top level files
622env.Command('python/m5/info.py',
623            [ '#/COPYING', '#/LICENSE', '#/README', ],
624            MakeAction(makeInfoPyFile, Transform("INFO")))
625PySource('m5', 'python/m5/info.py')
626
627########################################################################
628#
629# Create all of the SimObject param headers and enum headers
630#
631
632def createSimObjectParamStruct(target, source, env):
633    assert len(target) == 1 and len(source) == 1
634
635    name = str(source[0].get_contents())
636    obj = sim_objects[name]
637
638    code = code_formatter()
639    obj.cxx_param_decl(code)
640    code.write(target[0].abspath)
641
642def createSimObjectCxxConfig(is_header):
643    def body(target, source, env):
644        assert len(target) == 1 and len(source) == 1
645
646        name = str(source[0].get_contents())
647        obj = sim_objects[name]
648
649        code = code_formatter()
650        obj.cxx_config_param_file(code, is_header)
651        code.write(target[0].abspath)
652    return body
653
654def createParamSwigWrapper(target, source, env):
655    assert len(target) == 1 and len(source) == 1
656
657    name = str(source[0].get_contents())
658    param = params_to_swig[name]
659
660    code = code_formatter()
661    param.swig_decl(code)
662    code.write(target[0].abspath)
663
664def createEnumStrings(target, source, env):
665    assert len(target) == 1 and len(source) == 1
666
667    name = str(source[0].get_contents())
668    obj = all_enums[name]
669
670    code = code_formatter()
671    obj.cxx_def(code)
672    code.write(target[0].abspath)
673
674def createEnumDecls(target, source, env):
675    assert len(target) == 1 and len(source) == 1
676
677    name = str(source[0].get_contents())
678    obj = all_enums[name]
679
680    code = code_formatter()
681    obj.cxx_decl(code)
682    code.write(target[0].abspath)
683
684def createEnumSwigWrapper(target, source, env):
685    assert len(target) == 1 and len(source) == 1
686
687    name = str(source[0].get_contents())
688    obj = all_enums[name]
689
690    code = code_formatter()
691    obj.swig_decl(code)
692    code.write(target[0].abspath)
693
694def createSimObjectSwigWrapper(target, source, env):
695    name = source[0].get_contents()
696    obj = sim_objects[name]
697
698    code = code_formatter()
699    obj.swig_decl(code)
700    code.write(target[0].abspath)
701
702# dummy target for generated code
703# we start out with all the Source files so they get copied to build/*/ also.
704SWIG = env.Dummy('swig', [s.tnode for s in Source.get()])
705
706# Generate all of the SimObject param C++ struct header files
707params_hh_files = []
708for name,simobj in sorted(sim_objects.iteritems()):
709    py_source = PySource.modules[simobj.__module__]
710    extra_deps = [ py_source.tnode ]
711
712    hh_file = File('params/%s.hh' % name)
713    params_hh_files.append(hh_file)
714    env.Command(hh_file, Value(name),
715                MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
716    env.Depends(hh_file, depends + extra_deps)
717    env.Depends(SWIG, hh_file)
718
719# C++ parameter description files
720if GetOption('with_cxx_config'):
721    for name,simobj in sorted(sim_objects.iteritems()):
722        py_source = PySource.modules[simobj.__module__]
723        extra_deps = [ py_source.tnode ]
724
725        cxx_config_hh_file = File('cxx_config/%s.hh' % name)
726        cxx_config_cc_file = File('cxx_config/%s.cc' % name)
727        env.Command(cxx_config_hh_file, Value(name),
728                    MakeAction(createSimObjectCxxConfig(True),
729                    Transform("CXXCPRHH")))
730        env.Command(cxx_config_cc_file, Value(name),
731                    MakeAction(createSimObjectCxxConfig(False),
732                    Transform("CXXCPRCC")))
733        env.Depends(cxx_config_hh_file, depends + extra_deps +
734                    [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
735        env.Depends(cxx_config_cc_file, depends + extra_deps +
736                    [cxx_config_hh_file])
737        Source(cxx_config_cc_file)
738
739    cxx_config_init_cc_file = File('cxx_config/init.cc')
740
741    def createCxxConfigInitCC(target, source, env):
742        assert len(target) == 1 and len(source) == 1
743
744        code = code_formatter()
745
746        for name,simobj in sorted(sim_objects.iteritems()):
747            if not hasattr(simobj, 'abstract') or not simobj.abstract:
748                code('#include "cxx_config/${name}.hh"')
749        code()
750        code('void cxxConfigInit()')
751        code('{')
752        code.indent()
753        for name,simobj in sorted(sim_objects.iteritems()):
754            not_abstract = not hasattr(simobj, 'abstract') or \
755                not simobj.abstract
756            if not_abstract and 'type' in simobj.__dict__:
757                code('cxx_config_directory["${name}"] = '
758                     '${name}CxxConfigParams::makeDirectoryEntry();')
759        code.dedent()
760        code('}')
761        code.write(target[0].abspath)
762
763    py_source = PySource.modules[simobj.__module__]
764    extra_deps = [ py_source.tnode ]
765    env.Command(cxx_config_init_cc_file, Value(name),
766        MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
767    cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
768        for name,simobj in sorted(sim_objects.iteritems())
769        if not hasattr(simobj, 'abstract') or not simobj.abstract]
770    Depends(cxx_config_init_cc_file, cxx_param_hh_files +
771            [File('sim/cxx_config.hh')])
772    Source(cxx_config_init_cc_file)
773
774# Generate any needed param SWIG wrapper files
775params_i_files = []
776for name,param in sorted(params_to_swig.iteritems()):
777    i_file = File('python/_m5/%s.i' % (param.swig_module_name()))
778    params_i_files.append(i_file)
779    env.Command(i_file, Value(name),
780                MakeAction(createParamSwigWrapper, Transform("SW PARAM")))
781    env.Depends(i_file, depends)
782    env.Depends(SWIG, i_file)
783    SwigSource('_m5', i_file)
784
785# Generate all enum header files
786for name,enum in sorted(all_enums.iteritems()):
787    py_source = PySource.modules[enum.__module__]
788    extra_deps = [ py_source.tnode ]
789
790    cc_file = File('enums/%s.cc' % name)
791    env.Command(cc_file, Value(name),
792                MakeAction(createEnumStrings, Transform("ENUM STR")))
793    env.Depends(cc_file, depends + extra_deps)
794    env.Depends(SWIG, cc_file)
795    Source(cc_file)
796
797    hh_file = File('enums/%s.hh' % name)
798    env.Command(hh_file, Value(name),
799                MakeAction(createEnumDecls, Transform("ENUMDECL")))
800    env.Depends(hh_file, depends + extra_deps)
801    env.Depends(SWIG, hh_file)
802
803    i_file = File('python/_m5/enum_%s.i' % name)
804    env.Command(i_file, Value(name),
805                MakeAction(createEnumSwigWrapper, Transform("ENUMSWIG")))
806    env.Depends(i_file, depends + extra_deps)
807    env.Depends(SWIG, i_file)
808    SwigSource('_m5', i_file)
809
810# Generate SimObject SWIG wrapper files
811for name,simobj in sorted(sim_objects.iteritems()):
812    py_source = PySource.modules[simobj.__module__]
813    extra_deps = [ py_source.tnode ]
814    i_file = File('python/_m5/param_%s.i' % name)
815    env.Command(i_file, Value(name),
816                MakeAction(createSimObjectSwigWrapper, Transform("SO SWIG")))
817    env.Depends(i_file, depends + extra_deps)
818    SwigSource('_m5', i_file)
819
820# Generate the main swig init file
821def makeEmbeddedSwigInit(package):
822    def body(target, source, env):
823        assert len(target) == 1 and len(source) == 1
824
825        code = code_formatter()
826        module = source[0].get_contents()
827        # Provide the full context so that the swig-generated call to
828        # Py_InitModule ends up placing the embedded module in the
829        # right package.
830        context = str(package) + "._" + str(module)
831        code('''\
832        #include "sim/init.hh"
833
834        extern "C" {
835            void init_${module}();
836        }
837
838        EmbeddedSwig embed_swig_${module}(init_${module}, "${context}");
839        ''')
840        code.write(str(target[0]))
841    return body
842
843# Build all swig modules
844for swig in SwigSource.all:
845    env.Command([swig.cc_source.tnode, swig.py_source.tnode], swig.tnode,
846                MakeAction('$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
847                '-o ${TARGETS[0]} $SOURCES', Transform("SWIG")))
848    cc_file = str(swig.tnode)
849    init_file = '%s/%s_init.cc' % (dirname(cc_file), basename(cc_file))
850    env.Command(init_file, Value(swig.module),
851                MakeAction(makeEmbeddedSwigInit(swig.package),
852                           Transform("EMBED SW")))
853    env.Depends(SWIG, init_file)
854    Source(init_file, **swig.guards)
855
856# Build all protocol buffers if we have got protoc and protobuf available
857if env['HAVE_PROTOBUF']:
858    for proto in ProtoBuf.all:
859        # Use both the source and header as the target, and the .proto
860        # file as the source. When executing the protoc compiler, also
861        # specify the proto_path to avoid having the generated files
862        # include the path.
863        env.Command([proto.cc_file, proto.hh_file], proto.tnode,
864                    MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
865                               '--proto_path ${SOURCE.dir} $SOURCE',
866                               Transform("PROTOC")))
867
868        env.Depends(SWIG, [proto.cc_file, proto.hh_file])
869        # Add the C++ source file
870        Source(proto.cc_file, **proto.guards)
871elif ProtoBuf.all:
872    print 'Got protobuf to build, but lacks support!'
873    Exit(1)
874
875#
876# Handle debug flags
877#
878def makeDebugFlagCC(target, source, env):
879    assert(len(target) == 1 and len(source) == 1)
880
881    code = code_formatter()
882
883    # delay definition of CompoundFlags until after all the definition
884    # of all constituent SimpleFlags
885    comp_code = code_formatter()
886
887    # file header
888    code('''
889/*
890 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
891 */
892
893#include "base/debug.hh"
894
895namespace Debug {
896
897''')
898
899    for name, flag in sorted(source[0].read().iteritems()):
900        n, compound, desc = flag
901        assert n == name
902
903        if not compound:
904            code('SimpleFlag $name("$name", "$desc");')
905        else:
906            comp_code('CompoundFlag $name("$name", "$desc",')
907            comp_code.indent()
908            last = len(compound) - 1
909            for i,flag in enumerate(compound):
910                if i != last:
911                    comp_code('&$flag,')
912                else:
913                    comp_code('&$flag);')
914            comp_code.dedent()
915
916    code.append(comp_code)
917    code()
918    code('} // namespace Debug')
919
920    code.write(str(target[0]))
921
922def makeDebugFlagHH(target, source, env):
923    assert(len(target) == 1 and len(source) == 1)
924
925    val = eval(source[0].get_contents())
926    name, compound, desc = val
927
928    code = code_formatter()
929
930    # file header boilerplate
931    code('''\
932/*
933 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
934 */
935
936#ifndef __DEBUG_${name}_HH__
937#define __DEBUG_${name}_HH__
938
939namespace Debug {
940''')
941
942    if compound:
943        code('class CompoundFlag;')
944    code('class SimpleFlag;')
945
946    if compound:
947        code('extern CompoundFlag $name;')
948        for flag in compound:
949            code('extern SimpleFlag $flag;')
950    else:
951        code('extern SimpleFlag $name;')
952
953    code('''
954}
955
956#endif // __DEBUG_${name}_HH__
957''')
958
959    code.write(str(target[0]))
960
961for name,flag in sorted(debug_flags.iteritems()):
962    n, compound, desc = flag
963    assert n == name
964
965    hh_file = 'debug/%s.hh' % name
966    env.Command(hh_file, Value(flag),
967                MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
968    env.Depends(SWIG, hh_file)
969
970env.Command('debug/flags.cc', Value(debug_flags),
971            MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
972env.Depends(SWIG, 'debug/flags.cc')
973Source('debug/flags.cc')
974
975# version tags
976tags = \
977env.Command('sim/tags.cc', None,
978            MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
979                       Transform("VER TAGS")))
980env.AlwaysBuild(tags)
981
982# Embed python files.  All .py files that have been indicated by a
983# PySource() call in a SConscript need to be embedded into the M5
984# library.  To do that, we compile the file to byte code, marshal the
985# byte code, compress it, and then generate a c++ file that
986# inserts the result into an array.
987def embedPyFile(target, source, env):
988    def c_str(string):
989        if string is None:
990            return "0"
991        return '"%s"' % string
992
993    '''Action function to compile a .py into a code object, marshal
994    it, compress it, and stick it into an asm file so the code appears
995    as just bytes with a label in the data section'''
996
997    src = file(str(source[0]), 'r').read()
998
999    pysource = PySource.tnodes[source[0]]
1000    compiled = compile(src, pysource.abspath, 'exec')
1001    marshalled = marshal.dumps(compiled)
1002    compressed = zlib.compress(marshalled)
1003    data = compressed
1004    sym = pysource.symname
1005
1006    code = code_formatter()
1007    code('''\
1008#include "sim/init.hh"
1009
1010namespace {
1011
1012const uint8_t data_${sym}[] = {
1013''')
1014    code.indent()
1015    step = 16
1016    for i in xrange(0, len(data), step):
1017        x = array.array('B', data[i:i+step])
1018        code(''.join('%d,' % d for d in x))
1019    code.dedent()
1020
1021    code('''};
1022
1023EmbeddedPython embedded_${sym}(
1024    ${{c_str(pysource.arcname)}},
1025    ${{c_str(pysource.abspath)}},
1026    ${{c_str(pysource.modpath)}},
1027    data_${sym},
1028    ${{len(data)}},
1029    ${{len(marshalled)}});
1030
1031} // anonymous namespace
1032''')
1033    code.write(str(target[0]))
1034
1035for source in PySource.all:
1036    env.Command(source.cpp, source.tnode,
1037                MakeAction(embedPyFile, Transform("EMBED PY")))
1038    env.Depends(SWIG, source.cpp)
1039    Source(source.cpp, skip_no_python=True)
1040
1041########################################################################
1042#
1043# Define binaries.  Each different build type (debug, opt, etc.) gets
1044# a slightly different build environment.
1045#
1046
1047# List of constructed environments to pass back to SConstruct
1048date_source = Source('base/date.cc', skip_lib=True)
1049
1050# Capture this directory for the closure makeEnv, otherwise when it is
1051# called, it won't know what directory it should use.
1052variant_dir = Dir('.').path
1053def variant(*path):
1054    return os.path.join(variant_dir, *path)
1055def variantd(*path):
1056    return variant(*path)+'/'
1057
1058# Function to create a new build environment as clone of current
1059# environment 'env' with modified object suffix and optional stripped
1060# binary.  Additional keyword arguments are appended to corresponding
1061# build environment vars.
1062def makeEnv(env, label, objsfx, strip = False, **kwargs):
1063    # SCons doesn't know to append a library suffix when there is a '.' in the
1064    # name.  Use '_' instead.
1065    libname = variant('gem5_' + label)
1066    exename = variant('gem5.' + label)
1067    secondary_exename = variant('m5.' + label)
1068
1069    new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
1070    new_env.Label = label
1071    new_env.Append(**kwargs)
1072
1073    swig_env = new_env.Clone()
1074
1075    # Both gcc and clang have issues with unused labels and values in
1076    # the SWIG generated code
1077    swig_env.Append(CCFLAGS=['-Wno-unused-label', '-Wno-unused-value'])
1078
1079    if env['GCC']:
1080        # Depending on the SWIG version, we also need to supress
1081        # warnings about uninitialized variables and missing field
1082        # initializers.
1083        swig_env.Append(CCFLAGS=['-Wno-uninitialized',
1084                                 '-Wno-missing-field-initializers',
1085                                 '-Wno-unused-but-set-variable',
1086                                 '-Wno-maybe-uninitialized',
1087                                 '-Wno-type-limits'])
1088
1089
1090        # The address sanitizer is available for gcc >= 4.8
1091        if GetOption('with_asan'):
1092            if GetOption('with_ubsan') and \
1093                    compareVersions(env['GCC_VERSION'], '4.9') >= 0:
1094                new_env.Append(CCFLAGS=['-fsanitize=address,undefined',
1095                                        '-fno-omit-frame-pointer'])
1096                new_env.Append(LINKFLAGS='-fsanitize=address,undefined')
1097            else:
1098                new_env.Append(CCFLAGS=['-fsanitize=address',
1099                                        '-fno-omit-frame-pointer'])
1100                new_env.Append(LINKFLAGS='-fsanitize=address')
1101        # Only gcc >= 4.9 supports UBSan, so check both the version
1102        # and the command-line option before adding the compiler and
1103        # linker flags.
1104        elif GetOption('with_ubsan') and \
1105                compareVersions(env['GCC_VERSION'], '4.9') >= 0:
1106            new_env.Append(CCFLAGS='-fsanitize=undefined')
1107            new_env.Append(LINKFLAGS='-fsanitize=undefined')
1108
1109
1110    if env['CLANG']:
1111        swig_env.Append(CCFLAGS=['-Wno-sometimes-uninitialized',
1112                                 '-Wno-deprecated-register',
1113                                 '-Wno-tautological-compare'])
1114
1115        # We require clang >= 3.1, so there is no need to check any
1116        # versions here.
1117        if GetOption('with_ubsan'):
1118            if GetOption('with_asan'):
1119                new_env.Append(CCFLAGS=['-fsanitize=address,undefined',
1120                                        '-fno-omit-frame-pointer'])
1121                new_env.Append(LINKFLAGS='-fsanitize=address,undefined')
1122            else:
1123                new_env.Append(CCFLAGS='-fsanitize=undefined')
1124                new_env.Append(LINKFLAGS='-fsanitize=undefined')
1125
1126        elif GetOption('with_asan'):
1127            new_env.Append(CCFLAGS=['-fsanitize=address',
1128                                    '-fno-omit-frame-pointer'])
1129            new_env.Append(LINKFLAGS='-fsanitize=address')
1130
1131    werror_env = new_env.Clone()
1132    # Treat warnings as errors but white list some warnings that we
1133    # want to allow (e.g., deprecation warnings).
1134    werror_env.Append(CCFLAGS=['-Werror',
1135                               '-Wno-error=deprecated-declarations',
1136                               '-Wno-error=deprecated',
1137                               ])
1138
1139    def make_obj(source, static, extra_deps = None):
1140        '''This function adds the specified source to the correct
1141        build environment, and returns the corresponding SCons Object
1142        nodes'''
1143
1144        if source.swig:
1145            env = swig_env
1146        elif source.Werror:
1147            env = werror_env
1148        else:
1149            env = new_env
1150
1151        if static:
1152            obj = env.StaticObject(source.tnode)
1153        else:
1154            obj = env.SharedObject(source.tnode)
1155
1156        if extra_deps:
1157            env.Depends(obj, extra_deps)
1158
1159        return obj
1160
1161    lib_guards = {'main': False, 'skip_lib': False}
1162
1163    # Without Python, leave out all SWIG and Python content from the
1164    # library builds.  The option doesn't affect gem5 built as a program
1165    if GetOption('without_python'):
1166        lib_guards['skip_no_python'] = False
1167
1168    static_objs = [ make_obj(s, True) for s in Source.get(**lib_guards) ]
1169    shared_objs = [ make_obj(s, False) for s in Source.get(**lib_guards) ]
1170
1171    static_date = make_obj(date_source, static=True, extra_deps=static_objs)
1172    static_objs.append(static_date)
1173
1174    shared_date = make_obj(date_source, static=False, extra_deps=shared_objs)
1175    shared_objs.append(shared_date)
1176
1177    # First make a library of everything but main() so other programs can
1178    # link against m5.
1179    static_lib = new_env.StaticLibrary(libname, static_objs)
1180    shared_lib = new_env.SharedLibrary(libname, shared_objs)
1181
1182    # Now link a stub with main() and the static library.
1183    main_objs = [ make_obj(s, True) for s in Source.get(main=True) ]
1184
1185    for test in UnitTest.all:
1186        flags = { test.target : True }
1187        test_sources = Source.get(**flags)
1188        test_objs = [ make_obj(s, static=True) for s in test_sources ]
1189        if test.main:
1190            test_objs += main_objs
1191        path = variant('unittest/%s.%s' % (test.target, label))
1192        new_env.Program(path, test_objs + static_objs)
1193
1194    progname = exename
1195    if strip:
1196        progname += '.unstripped'
1197
1198    # When linking the gem5 binary, the command line can be too big for the
1199    # shell to handle. Use "subprocess" to spawn processes without passing
1200    # through the shell to avoid this problem. That means we also can't use
1201    # shell syntax in any of the commands this will run, but that isn't
1202    # currently an issue.
1203    def spawn_with_subprocess(sh, escape, cmd, args, env):
1204        return subprocess.call(args, env=env)
1205
1206    # Since we're not running through a shell, no escaping is necessary either.
1207    targets = new_env.Program(progname, main_objs + static_objs,
1208                              SPAWN=spawn_with_subprocess,
1209                              ESCAPE=lambda x: x)
1210
1211    if strip:
1212        if sys.platform == 'sunos5':
1213            cmd = 'cp $SOURCE $TARGET; strip $TARGET'
1214        else:
1215            cmd = 'strip $SOURCE -o $TARGET'
1216        targets = new_env.Command(exename, progname,
1217                    MakeAction(cmd, Transform("STRIP")))
1218
1219    new_env.Command(secondary_exename, exename,
1220            MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1221
1222    new_env.M5Binary = targets[0]
1223    return new_env
1224
1225# Start out with the compiler flags common to all compilers,
1226# i.e. they all use -g for opt and -g -pg for prof
1227ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1228           'perf' : ['-g']}
1229
1230# Start out with the linker flags common to all linkers, i.e. -pg for
1231# prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1232# no-as-needed and as-needed as the binutils linker is too clever and
1233# simply doesn't link to the library otherwise.
1234ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1235           'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1236
1237# For Link Time Optimization, the optimisation flags used to compile
1238# individual files are decoupled from those used at link time
1239# (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1240# to also update the linker flags based on the target.
1241if env['GCC']:
1242    if sys.platform == 'sunos5':
1243        ccflags['debug'] += ['-gstabs+']
1244    else:
1245        ccflags['debug'] += ['-ggdb3']
1246    ldflags['debug'] += ['-O0']
1247    # opt, fast, prof and perf all share the same cc flags, also add
1248    # the optimization to the ldflags as LTO defers the optimization
1249    # to link time
1250    for target in ['opt', 'fast', 'prof', 'perf']:
1251        ccflags[target] += ['-O3']
1252        ldflags[target] += ['-O3']
1253
1254    ccflags['fast'] += env['LTO_CCFLAGS']
1255    ldflags['fast'] += env['LTO_LDFLAGS']
1256elif env['CLANG']:
1257    ccflags['debug'] += ['-g', '-O0']
1258    # opt, fast, prof and perf all share the same cc flags
1259    for target in ['opt', 'fast', 'prof', 'perf']:
1260        ccflags[target] += ['-O3']
1261else:
1262    print 'Unknown compiler, please fix compiler options'
1263    Exit(1)
1264
1265
1266# To speed things up, we only instantiate the build environments we
1267# need.  We try to identify the needed environment for each target; if
1268# we can't, we fall back on instantiating all the environments just to
1269# be safe.
1270target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1271obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1272              'gpo' : 'perf'}
1273
1274def identifyTarget(t):
1275    ext = t.split('.')[-1]
1276    if ext in target_types:
1277        return ext
1278    if obj2target.has_key(ext):
1279        return obj2target[ext]
1280    match = re.search(r'/tests/([^/]+)/', t)
1281    if match and match.group(1) in target_types:
1282        return match.group(1)
1283    return 'all'
1284
1285needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1286if 'all' in needed_envs:
1287    needed_envs += target_types
1288
1289def makeEnvirons(target, source, env):
1290    # cause any later Source() calls to be fatal, as a diagnostic.
1291    Source.done()
1292
1293    envList = []
1294
1295    # Debug binary
1296    if 'debug' in needed_envs:
1297        envList.append(
1298            makeEnv(env, 'debug', '.do',
1299                    CCFLAGS = Split(ccflags['debug']),
1300                    CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1301                    LINKFLAGS = Split(ldflags['debug'])))
1302
1303    # Optimized binary
1304    if 'opt' in needed_envs:
1305        envList.append(
1306            makeEnv(env, 'opt', '.o',
1307                    CCFLAGS = Split(ccflags['opt']),
1308                    CPPDEFINES = ['TRACING_ON=1'],
1309                    LINKFLAGS = Split(ldflags['opt'])))
1310
1311    # "Fast" binary
1312    if 'fast' in needed_envs:
1313        envList.append(
1314            makeEnv(env, 'fast', '.fo', strip = True,
1315                    CCFLAGS = Split(ccflags['fast']),
1316                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1317                    LINKFLAGS = Split(ldflags['fast'])))
1318
1319    # Profiled binary using gprof
1320    if 'prof' in needed_envs:
1321        envList.append(
1322            makeEnv(env, 'prof', '.po',
1323                    CCFLAGS = Split(ccflags['prof']),
1324                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1325                    LINKFLAGS = Split(ldflags['prof'])))
1326
1327    # Profiled binary using google-pprof
1328    if 'perf' in needed_envs:
1329        envList.append(
1330            makeEnv(env, 'perf', '.gpo',
1331                    CCFLAGS = Split(ccflags['perf']),
1332                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1333                    LINKFLAGS = Split(ldflags['perf'])))
1334
1335    # Set up the regression tests for each build.
1336    for e in envList:
1337        SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1338                   variant_dir = variantd('tests', e.Label),
1339                   exports = { 'env' : e }, duplicate = False)
1340
1341# The MakeEnvirons Builder defers the full dependency collection until
1342# after processing the ISA definition (due to dynamically generated
1343# source files).  Add this dependency to all targets so they will wait
1344# until the environments are completely set up.  Otherwise, a second
1345# process (e.g. -j2 or higher) will try to compile the requested target,
1346# not know how, and fail.
1347env.Append(BUILDERS = {'MakeEnvirons' :
1348                        Builder(action=MakeAction(makeEnvirons,
1349                                                  Transform("ENVIRONS", 1)))})
1350
1351isa_target = env['PHONY_BASE'] + '-deps'
1352environs   = env['PHONY_BASE'] + '-environs'
1353env.Depends('#all-deps',     isa_target)
1354env.Depends('#all-environs', environs)
1355env.ScanISA(isa_target, File('arch/%s/generated/inc.d' % env['TARGET_ISA']))
1356envSetup = env.MakeEnvirons(environs, isa_target)
1357
1358# make sure no -deps targets occur before all ISAs are complete
1359env.Depends(isa_target, '#all-isas')
1360# likewise for -environs targets and all the -deps targets
1361env.Depends(environs, '#all-deps')
1362