SConscript revision 11718
1# -*- mode:python -*-
2
3# Copyright (c) 2004-2005 The Regents of The University of Michigan
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# Authors: Nathan Binkert
30
31import array
32import bisect
33import imp
34import marshal
35import os
36import re
37import sys
38import zlib
39
40from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
41
42import SCons
43
44# This file defines how to build a particular configuration of gem5
45# based on variable settings in the 'env' build environment.
46
47Import('*')
48
49# Children need to see the environment
50Export('env')
51
52build_env = [(opt, env[opt]) for opt in export_vars]
53
54from m5.util import code_formatter, compareVersions
55
56########################################################################
57# Code for adding source files of various types
58#
59# When specifying a source file of some type, a set of guards can be
60# specified for that file.  When get() is used to find the files, if
61# get specifies a set of filters, only files that match those filters
62# will be accepted (unspecified filters on files are assumed to be
63# false).  Current filters are:
64#     main -- specifies the gem5 main() function
65#     skip_lib -- do not put this file into the gem5 library
66#     skip_no_python -- do not put this file into a no_python library
67#       as it embeds compiled Python
68#     <unittest> -- unit tests use filters based on the unit test name
69#
70# A parent can now be specified for a source file and default filter
71# values will be retrieved recursively from parents (children override
72# parents).
73#
74class SourceMeta(type):
75    '''Meta class for source files that keeps track of all files of a
76    particular type and has a get function for finding all functions
77    of a certain type that match a set of guards'''
78    def __init__(cls, name, bases, dict):
79        super(SourceMeta, cls).__init__(name, bases, dict)
80        cls.all = []
81
82    def get(cls, **guards):
83        '''Find all files that match the specified guards.  If a source
84        file does not specify a flag, the default is False'''
85        for src in cls.all:
86            for flag,value in guards.iteritems():
87                # if the flag is found and has a different value, skip
88                # this file
89                if src.all_guards.get(flag, False) != value:
90                    break
91            else:
92                yield src
93
94class SourceFile(object):
95    '''Base object that encapsulates the notion of a source file.
96    This includes, the source node, target node, various manipulations
97    of those.  A source file also specifies a set of guards which
98    describing which builds the source file applies to.  A parent can
99    also be specified to get default guards from'''
100    __metaclass__ = SourceMeta
101    def __init__(self, source, parent=None, **guards):
102        self.guards = guards
103        self.parent = parent
104
105        tnode = source
106        if not isinstance(source, SCons.Node.FS.File):
107            tnode = File(source)
108
109        self.tnode = tnode
110        self.snode = tnode.srcnode()
111
112        for base in type(self).__mro__:
113            if issubclass(base, SourceFile):
114                base.all.append(self)
115
116    @property
117    def filename(self):
118        return str(self.tnode)
119
120    @property
121    def dirname(self):
122        return dirname(self.filename)
123
124    @property
125    def basename(self):
126        return basename(self.filename)
127
128    @property
129    def extname(self):
130        index = self.basename.rfind('.')
131        if index <= 0:
132            # dot files aren't extensions
133            return self.basename, None
134
135        return self.basename[:index], self.basename[index+1:]
136
137    @property
138    def all_guards(self):
139        '''find all guards for this object getting default values
140        recursively from its parents'''
141        guards = {}
142        if self.parent:
143            guards.update(self.parent.guards)
144        guards.update(self.guards)
145        return guards
146
147    def __lt__(self, other): return self.filename < other.filename
148    def __le__(self, other): return self.filename <= other.filename
149    def __gt__(self, other): return self.filename > other.filename
150    def __ge__(self, other): return self.filename >= other.filename
151    def __eq__(self, other): return self.filename == other.filename
152    def __ne__(self, other): return self.filename != other.filename
153
154    @staticmethod
155    def done():
156        def disabled(cls, name, *ignored):
157            raise RuntimeError("Additional SourceFile '%s'" % name,\
158                  "declared, but targets deps are already fixed.")
159        SourceFile.__init__ = disabled
160
161
162class Source(SourceFile):
163    '''Add a c/c++ source file to the build'''
164    def __init__(self, source, Werror=True, swig=False, **guards):
165        '''specify the source file, and any guards'''
166        super(Source, self).__init__(source, **guards)
167
168        self.Werror = Werror
169        self.swig = swig
170
171class PySource(SourceFile):
172    '''Add a python source file to the named package'''
173    invalid_sym_char = re.compile('[^A-z0-9_]')
174    modules = {}
175    tnodes = {}
176    symnames = {}
177
178    def __init__(self, package, source, **guards):
179        '''specify the python package, the source file, and any guards'''
180        super(PySource, self).__init__(source, **guards)
181
182        modname,ext = self.extname
183        assert ext == 'py'
184
185        if package:
186            path = package.split('.')
187        else:
188            path = []
189
190        modpath = path[:]
191        if modname != '__init__':
192            modpath += [ modname ]
193        modpath = '.'.join(modpath)
194
195        arcpath = path + [ self.basename ]
196        abspath = self.snode.abspath
197        if not exists(abspath):
198            abspath = self.tnode.abspath
199
200        self.package = package
201        self.modname = modname
202        self.modpath = modpath
203        self.arcname = joinpath(*arcpath)
204        self.abspath = abspath
205        self.compiled = File(self.filename + 'c')
206        self.cpp = File(self.filename + '.cc')
207        self.symname = PySource.invalid_sym_char.sub('_', modpath)
208
209        PySource.modules[modpath] = self
210        PySource.tnodes[self.tnode] = self
211        PySource.symnames[self.symname] = self
212
213class SimObject(PySource):
214    '''Add a SimObject python file as a python source object and add
215    it to a list of sim object modules'''
216
217    fixed = False
218    modnames = []
219
220    def __init__(self, source, **guards):
221        '''Specify the source file and any guards (automatically in
222        the m5.objects package)'''
223        super(SimObject, self).__init__('m5.objects', source, **guards)
224        if self.fixed:
225            raise AttributeError, "Too late to call SimObject now."
226
227        bisect.insort_right(SimObject.modnames, self.modname)
228
229class SwigSource(SourceFile):
230    '''Add a swig file to build'''
231
232    def __init__(self, package, source, **guards):
233        '''Specify the python package, the source file, and any guards'''
234        super(SwigSource, self).__init__(source, skip_no_python=True, **guards)
235
236        modname,ext = self.extname
237        assert ext == 'i'
238
239        self.package = package
240        self.module = modname
241        cc_file = joinpath(self.dirname, modname + '_wrap.cc')
242        py_file = joinpath(self.dirname, modname + '.py')
243
244        self.cc_source = Source(cc_file, swig=True, parent=self, **guards)
245        self.py_source = PySource(package, py_file, parent=self, **guards)
246
247class ProtoBuf(SourceFile):
248    '''Add a Protocol Buffer to build'''
249
250    def __init__(self, source, **guards):
251        '''Specify the source file, and any guards'''
252        super(ProtoBuf, self).__init__(source, **guards)
253
254        # Get the file name and the extension
255        modname,ext = self.extname
256        assert ext == 'proto'
257
258        # Currently, we stick to generating the C++ headers, so we
259        # only need to track the source and header.
260        self.cc_file = File(modname + '.pb.cc')
261        self.hh_file = File(modname + '.pb.h')
262
263class UnitTest(object):
264    '''Create a UnitTest'''
265
266    all = []
267    def __init__(self, target, *sources, **kwargs):
268        '''Specify the target name and any sources.  Sources that are
269        not SourceFiles are evalued with Source().  All files are
270        guarded with a guard of the same name as the UnitTest
271        target.'''
272
273        srcs = []
274        for src in sources:
275            if not isinstance(src, SourceFile):
276                src = Source(src, skip_lib=True)
277            src.guards[target] = True
278            srcs.append(src)
279
280        self.sources = srcs
281        self.target = target
282        self.main = kwargs.get('main', False)
283        UnitTest.all.append(self)
284
285# Children should have access
286Export('Source')
287Export('PySource')
288Export('SimObject')
289Export('SwigSource')
290Export('ProtoBuf')
291Export('UnitTest')
292
293########################################################################
294#
295# Debug Flags
296#
297debug_flags = {}
298def DebugFlag(name, desc=None):
299    if name in debug_flags:
300        raise AttributeError, "Flag %s already specified" % name
301    debug_flags[name] = (name, (), desc)
302
303def CompoundFlag(name, flags, desc=None):
304    if name in debug_flags:
305        raise AttributeError, "Flag %s already specified" % name
306
307    compound = tuple(flags)
308    debug_flags[name] = (name, compound, desc)
309
310Export('DebugFlag')
311Export('CompoundFlag')
312
313########################################################################
314#
315# Set some compiler variables
316#
317
318# Include file paths are rooted in this directory.  SCons will
319# automatically expand '.' to refer to both the source directory and
320# the corresponding build directory to pick up generated include
321# files.
322env.Append(CPPPATH=Dir('.'))
323
324for extra_dir in extras_dir_list:
325    env.Append(CPPPATH=Dir(extra_dir))
326
327# Workaround for bug in SCons version > 0.97d20071212
328# Scons bug id: 2006 gem5 Bug id: 308
329for root, dirs, files in os.walk(base_dir, topdown=True):
330    Dir(root[len(base_dir) + 1:])
331
332########################################################################
333#
334# Walk the tree and execute all SConscripts in subdirectories
335#
336
337here = Dir('.').srcnode().abspath
338for root, dirs, files in os.walk(base_dir, topdown=True):
339    if root == here:
340        # we don't want to recurse back into this SConscript
341        continue
342
343    if 'SConscript' in files:
344        build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
345        SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
346
347for extra_dir in extras_dir_list:
348    prefix_len = len(dirname(extra_dir)) + 1
349
350    # Also add the corresponding build directory to pick up generated
351    # include files.
352    env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
353
354    for root, dirs, files in os.walk(extra_dir, topdown=True):
355        # if build lives in the extras directory, don't walk down it
356        if 'build' in dirs:
357            dirs.remove('build')
358
359        if 'SConscript' in files:
360            build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
361            SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
362
363for opt in export_vars:
364    env.ConfigFile(opt)
365
366def makeTheISA(source, target, env):
367    isas = [ src.get_contents() for src in source ]
368    target_isa = env['TARGET_ISA']
369    def define(isa):
370        return isa.upper() + '_ISA'
371
372    def namespace(isa):
373        return isa[0].upper() + isa[1:].lower() + 'ISA'
374
375
376    code = code_formatter()
377    code('''\
378#ifndef __CONFIG_THE_ISA_HH__
379#define __CONFIG_THE_ISA_HH__
380
381''')
382
383    # create defines for the preprocessing and compile-time determination
384    for i,isa in enumerate(isas):
385        code('#define $0 $1', define(isa), i + 1)
386    code()
387
388    # create an enum for any run-time determination of the ISA, we
389    # reuse the same name as the namespaces
390    code('enum class Arch {')
391    for i,isa in enumerate(isas):
392        if i + 1 == len(isas):
393            code('  $0 = $1', namespace(isa), define(isa))
394        else:
395            code('  $0 = $1,', namespace(isa), define(isa))
396    code('};')
397
398    code('''
399
400#define THE_ISA ${{define(target_isa)}}
401#define TheISA ${{namespace(target_isa)}}
402#define THE_ISA_STR "${{target_isa}}"
403
404#endif // __CONFIG_THE_ISA_HH__''')
405
406    code.write(str(target[0]))
407
408env.Command('config/the_isa.hh', map(Value, all_isa_list),
409            MakeAction(makeTheISA, Transform("CFG ISA", 0)))
410
411def makeTheGPUISA(source, target, env):
412    isas = [ src.get_contents() for src in source ]
413    target_gpu_isa = env['TARGET_GPU_ISA']
414    def define(isa):
415        return isa.upper() + '_ISA'
416
417    def namespace(isa):
418        return isa[0].upper() + isa[1:].lower() + 'ISA'
419
420
421    code = code_formatter()
422    code('''\
423#ifndef __CONFIG_THE_GPU_ISA_HH__
424#define __CONFIG_THE_GPU_ISA_HH__
425
426''')
427
428    # create defines for the preprocessing and compile-time determination
429    for i,isa in enumerate(isas):
430        code('#define $0 $1', define(isa), i + 1)
431    code()
432
433    # create an enum for any run-time determination of the ISA, we
434    # reuse the same name as the namespaces
435    code('enum class GPUArch {')
436    for i,isa in enumerate(isas):
437        if i + 1 == len(isas):
438            code('  $0 = $1', namespace(isa), define(isa))
439        else:
440            code('  $0 = $1,', namespace(isa), define(isa))
441    code('};')
442
443    code('''
444
445#define THE_GPU_ISA ${{define(target_gpu_isa)}}
446#define TheGpuISA ${{namespace(target_gpu_isa)}}
447#define THE_GPU_ISA_STR "${{target_gpu_isa}}"
448
449#endif // __CONFIG_THE_GPU_ISA_HH__''')
450
451    code.write(str(target[0]))
452
453env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
454            MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
455
456########################################################################
457#
458# Prevent any SimObjects from being added after this point, they
459# should all have been added in the SConscripts above
460#
461SimObject.fixed = True
462
463class DictImporter(object):
464    '''This importer takes a dictionary of arbitrary module names that
465    map to arbitrary filenames.'''
466    def __init__(self, modules):
467        self.modules = modules
468        self.installed = set()
469
470    def __del__(self):
471        self.unload()
472
473    def unload(self):
474        import sys
475        for module in self.installed:
476            del sys.modules[module]
477        self.installed = set()
478
479    def find_module(self, fullname, path):
480        if fullname == 'm5.defines':
481            return self
482
483        if fullname == 'm5.objects':
484            return self
485
486        if fullname.startswith('m5.internal'):
487            return None
488
489        source = self.modules.get(fullname, None)
490        if source is not None and fullname.startswith('m5.objects'):
491            return self
492
493        return None
494
495    def load_module(self, fullname):
496        mod = imp.new_module(fullname)
497        sys.modules[fullname] = mod
498        self.installed.add(fullname)
499
500        mod.__loader__ = self
501        if fullname == 'm5.objects':
502            mod.__path__ = fullname.split('.')
503            return mod
504
505        if fullname == 'm5.defines':
506            mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
507            return mod
508
509        source = self.modules[fullname]
510        if source.modname == '__init__':
511            mod.__path__ = source.modpath
512        mod.__file__ = source.abspath
513
514        exec file(source.abspath, 'r') in mod.__dict__
515
516        return mod
517
518import m5.SimObject
519import m5.params
520from m5.util import code_formatter
521
522m5.SimObject.clear()
523m5.params.clear()
524
525# install the python importer so we can grab stuff from the source
526# tree itself.  We can't have SimObjects added after this point or
527# else we won't know about them for the rest of the stuff.
528importer = DictImporter(PySource.modules)
529sys.meta_path[0:0] = [ importer ]
530
531# import all sim objects so we can populate the all_objects list
532# make sure that we're working with a list, then let's sort it
533for modname in SimObject.modnames:
534    exec('from m5.objects import %s' % modname)
535
536# we need to unload all of the currently imported modules so that they
537# will be re-imported the next time the sconscript is run
538importer.unload()
539sys.meta_path.remove(importer)
540
541sim_objects = m5.SimObject.allClasses
542all_enums = m5.params.allEnums
543
544if m5.SimObject.noCxxHeader:
545    print >> sys.stderr, \
546        "warning: At least one SimObject lacks a header specification. " \
547        "This can cause unexpected results in the generated SWIG " \
548        "wrappers."
549
550# Find param types that need to be explicitly wrapped with swig.
551# These will be recognized because the ParamDesc will have a
552# swig_decl() method.  Most param types are based on types that don't
553# need this, either because they're based on native types (like Int)
554# or because they're SimObjects (which get swigged independently).
555# For now the only things handled here are VectorParam types.
556params_to_swig = {}
557for name,obj in sorted(sim_objects.iteritems()):
558    for param in obj._params.local.values():
559        # load the ptype attribute now because it depends on the
560        # current version of SimObject.allClasses, but when scons
561        # actually uses the value, all versions of
562        # SimObject.allClasses will have been loaded
563        param.ptype
564
565        if not hasattr(param, 'swig_decl'):
566            continue
567        pname = param.ptype_str
568        if pname not in params_to_swig:
569            params_to_swig[pname] = param
570
571########################################################################
572#
573# calculate extra dependencies
574#
575module_depends = ["m5", "m5.SimObject", "m5.params"]
576depends = [ PySource.modules[dep].snode for dep in module_depends ]
577depends.sort(key = lambda x: x.name)
578
579########################################################################
580#
581# Commands for the basic automatically generated python files
582#
583
584# Generate Python file containing a dict specifying the current
585# buildEnv flags.
586def makeDefinesPyFile(target, source, env):
587    build_env = source[0].get_contents()
588
589    code = code_formatter()
590    code("""
591import m5.internal
592import m5.util
593
594buildEnv = m5.util.SmartDict($build_env)
595
596compileDate = m5.internal.core.compileDate
597_globals = globals()
598for key,val in m5.internal.core.__dict__.iteritems():
599    if key.startswith('flag_'):
600        flag = key[5:]
601        _globals[flag] = val
602del _globals
603""")
604    code.write(target[0].abspath)
605
606defines_info = Value(build_env)
607# Generate a file with all of the compile options in it
608env.Command('python/m5/defines.py', defines_info,
609            MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
610PySource('m5', 'python/m5/defines.py')
611
612# Generate python file containing info about the M5 source code
613def makeInfoPyFile(target, source, env):
614    code = code_formatter()
615    for src in source:
616        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
617        code('$src = ${{repr(data)}}')
618    code.write(str(target[0]))
619
620# Generate a file that wraps the basic top level files
621env.Command('python/m5/info.py',
622            [ '#/COPYING', '#/LICENSE', '#/README', ],
623            MakeAction(makeInfoPyFile, Transform("INFO")))
624PySource('m5', 'python/m5/info.py')
625
626########################################################################
627#
628# Create all of the SimObject param headers and enum headers
629#
630
631def createSimObjectParamStruct(target, source, env):
632    assert len(target) == 1 and len(source) == 1
633
634    name = str(source[0].get_contents())
635    obj = sim_objects[name]
636
637    code = code_formatter()
638    obj.cxx_param_decl(code)
639    code.write(target[0].abspath)
640
641def createSimObjectCxxConfig(is_header):
642    def body(target, source, env):
643        assert len(target) == 1 and len(source) == 1
644
645        name = str(source[0].get_contents())
646        obj = sim_objects[name]
647
648        code = code_formatter()
649        obj.cxx_config_param_file(code, is_header)
650        code.write(target[0].abspath)
651    return body
652
653def createParamSwigWrapper(target, source, env):
654    assert len(target) == 1 and len(source) == 1
655
656    name = str(source[0].get_contents())
657    param = params_to_swig[name]
658
659    code = code_formatter()
660    param.swig_decl(code)
661    code.write(target[0].abspath)
662
663def createEnumStrings(target, source, env):
664    assert len(target) == 1 and len(source) == 1
665
666    name = str(source[0].get_contents())
667    obj = all_enums[name]
668
669    code = code_formatter()
670    obj.cxx_def(code)
671    code.write(target[0].abspath)
672
673def createEnumDecls(target, source, env):
674    assert len(target) == 1 and len(source) == 1
675
676    name = str(source[0].get_contents())
677    obj = all_enums[name]
678
679    code = code_formatter()
680    obj.cxx_decl(code)
681    code.write(target[0].abspath)
682
683def createEnumSwigWrapper(target, source, env):
684    assert len(target) == 1 and len(source) == 1
685
686    name = str(source[0].get_contents())
687    obj = all_enums[name]
688
689    code = code_formatter()
690    obj.swig_decl(code)
691    code.write(target[0].abspath)
692
693def createSimObjectSwigWrapper(target, source, env):
694    name = source[0].get_contents()
695    obj = sim_objects[name]
696
697    code = code_formatter()
698    obj.swig_decl(code)
699    code.write(target[0].abspath)
700
701# dummy target for generated code
702# we start out with all the Source files so they get copied to build/*/ also.
703SWIG = env.Dummy('swig', [s.tnode for s in Source.get()])
704
705# Generate all of the SimObject param C++ struct header files
706params_hh_files = []
707for name,simobj in sorted(sim_objects.iteritems()):
708    py_source = PySource.modules[simobj.__module__]
709    extra_deps = [ py_source.tnode ]
710
711    hh_file = File('params/%s.hh' % name)
712    params_hh_files.append(hh_file)
713    env.Command(hh_file, Value(name),
714                MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
715    env.Depends(hh_file, depends + extra_deps)
716    env.Depends(SWIG, hh_file)
717
718# C++ parameter description files
719if GetOption('with_cxx_config'):
720    for name,simobj in sorted(sim_objects.iteritems()):
721        py_source = PySource.modules[simobj.__module__]
722        extra_deps = [ py_source.tnode ]
723
724        cxx_config_hh_file = File('cxx_config/%s.hh' % name)
725        cxx_config_cc_file = File('cxx_config/%s.cc' % name)
726        env.Command(cxx_config_hh_file, Value(name),
727                    MakeAction(createSimObjectCxxConfig(True),
728                    Transform("CXXCPRHH")))
729        env.Command(cxx_config_cc_file, Value(name),
730                    MakeAction(createSimObjectCxxConfig(False),
731                    Transform("CXXCPRCC")))
732        env.Depends(cxx_config_hh_file, depends + extra_deps +
733                    [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
734        env.Depends(cxx_config_cc_file, depends + extra_deps +
735                    [cxx_config_hh_file])
736        Source(cxx_config_cc_file)
737
738    cxx_config_init_cc_file = File('cxx_config/init.cc')
739
740    def createCxxConfigInitCC(target, source, env):
741        assert len(target) == 1 and len(source) == 1
742
743        code = code_formatter()
744
745        for name,simobj in sorted(sim_objects.iteritems()):
746            if not hasattr(simobj, 'abstract') or not simobj.abstract:
747                code('#include "cxx_config/${name}.hh"')
748        code()
749        code('void cxxConfigInit()')
750        code('{')
751        code.indent()
752        for name,simobj in sorted(sim_objects.iteritems()):
753            not_abstract = not hasattr(simobj, 'abstract') or \
754                not simobj.abstract
755            if not_abstract and 'type' in simobj.__dict__:
756                code('cxx_config_directory["${name}"] = '
757                     '${name}CxxConfigParams::makeDirectoryEntry();')
758        code.dedent()
759        code('}')
760        code.write(target[0].abspath)
761
762    py_source = PySource.modules[simobj.__module__]
763    extra_deps = [ py_source.tnode ]
764    env.Command(cxx_config_init_cc_file, Value(name),
765        MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
766    cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
767        for name,simobj in sorted(sim_objects.iteritems())
768        if not hasattr(simobj, 'abstract') or not simobj.abstract]
769    Depends(cxx_config_init_cc_file, cxx_param_hh_files +
770            [File('sim/cxx_config.hh')])
771    Source(cxx_config_init_cc_file)
772
773# Generate any needed param SWIG wrapper files
774params_i_files = []
775for name,param in sorted(params_to_swig.iteritems()):
776    i_file = File('python/m5/internal/%s.i' % (param.swig_module_name()))
777    params_i_files.append(i_file)
778    env.Command(i_file, Value(name),
779                MakeAction(createParamSwigWrapper, Transform("SW PARAM")))
780    env.Depends(i_file, depends)
781    env.Depends(SWIG, i_file)
782    SwigSource('m5.internal', i_file)
783
784# Generate all enum header files
785for name,enum in sorted(all_enums.iteritems()):
786    py_source = PySource.modules[enum.__module__]
787    extra_deps = [ py_source.tnode ]
788
789    cc_file = File('enums/%s.cc' % name)
790    env.Command(cc_file, Value(name),
791                MakeAction(createEnumStrings, Transform("ENUM STR")))
792    env.Depends(cc_file, depends + extra_deps)
793    env.Depends(SWIG, cc_file)
794    Source(cc_file)
795
796    hh_file = File('enums/%s.hh' % name)
797    env.Command(hh_file, Value(name),
798                MakeAction(createEnumDecls, Transform("ENUMDECL")))
799    env.Depends(hh_file, depends + extra_deps)
800    env.Depends(SWIG, hh_file)
801
802    i_file = File('python/m5/internal/enum_%s.i' % name)
803    env.Command(i_file, Value(name),
804                MakeAction(createEnumSwigWrapper, Transform("ENUMSWIG")))
805    env.Depends(i_file, depends + extra_deps)
806    env.Depends(SWIG, i_file)
807    SwigSource('m5.internal', i_file)
808
809# Generate SimObject SWIG wrapper files
810for name,simobj in sorted(sim_objects.iteritems()):
811    py_source = PySource.modules[simobj.__module__]
812    extra_deps = [ py_source.tnode ]
813    i_file = File('python/m5/internal/param_%s.i' % name)
814    env.Command(i_file, Value(name),
815                MakeAction(createSimObjectSwigWrapper, Transform("SO SWIG")))
816    env.Depends(i_file, depends + extra_deps)
817    SwigSource('m5.internal', i_file)
818
819# Generate the main swig init file
820def makeEmbeddedSwigInit(package):
821    def body(target, source, env):
822        assert len(target) == 1 and len(source) == 1
823
824        code = code_formatter()
825        module = source[0].get_contents()
826        # Provide the full context so that the swig-generated call to
827        # Py_InitModule ends up placing the embedded module in the
828        # right package.
829        context = str(package) + "._" + str(module)
830        code('''\
831        #include "sim/init.hh"
832
833        extern "C" {
834            void init_${module}();
835        }
836
837        EmbeddedSwig embed_swig_${module}(init_${module}, "${context}");
838        ''')
839        code.write(str(target[0]))
840    return body
841
842# Build all swig modules
843for swig in SwigSource.all:
844    env.Command([swig.cc_source.tnode, swig.py_source.tnode], swig.tnode,
845                MakeAction('$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
846                '-o ${TARGETS[0]} $SOURCES', Transform("SWIG")))
847    cc_file = str(swig.tnode)
848    init_file = '%s/%s_init.cc' % (dirname(cc_file), basename(cc_file))
849    env.Command(init_file, Value(swig.module),
850                MakeAction(makeEmbeddedSwigInit(swig.package),
851                           Transform("EMBED SW")))
852    env.Depends(SWIG, init_file)
853    Source(init_file, **swig.guards)
854
855# Build all protocol buffers if we have got protoc and protobuf available
856if env['HAVE_PROTOBUF']:
857    for proto in ProtoBuf.all:
858        # Use both the source and header as the target, and the .proto
859        # file as the source. When executing the protoc compiler, also
860        # specify the proto_path to avoid having the generated files
861        # include the path.
862        env.Command([proto.cc_file, proto.hh_file], proto.tnode,
863                    MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
864                               '--proto_path ${SOURCE.dir} $SOURCE',
865                               Transform("PROTOC")))
866
867        env.Depends(SWIG, [proto.cc_file, proto.hh_file])
868        # Add the C++ source file
869        Source(proto.cc_file, **proto.guards)
870elif ProtoBuf.all:
871    print 'Got protobuf to build, but lacks support!'
872    Exit(1)
873
874#
875# Handle debug flags
876#
877def makeDebugFlagCC(target, source, env):
878    assert(len(target) == 1 and len(source) == 1)
879
880    code = code_formatter()
881
882    # delay definition of CompoundFlags until after all the definition
883    # of all constituent SimpleFlags
884    comp_code = code_formatter()
885
886    # file header
887    code('''
888/*
889 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
890 */
891
892#include "base/debug.hh"
893
894namespace Debug {
895
896''')
897
898    for name, flag in sorted(source[0].read().iteritems()):
899        n, compound, desc = flag
900        assert n == name
901
902        if not compound:
903            code('SimpleFlag $name("$name", "$desc");')
904        else:
905            comp_code('CompoundFlag $name("$name", "$desc",')
906            comp_code.indent()
907            last = len(compound) - 1
908            for i,flag in enumerate(compound):
909                if i != last:
910                    comp_code('&$flag,')
911                else:
912                    comp_code('&$flag);')
913            comp_code.dedent()
914
915    code.append(comp_code)
916    code()
917    code('} // namespace Debug')
918
919    code.write(str(target[0]))
920
921def makeDebugFlagHH(target, source, env):
922    assert(len(target) == 1 and len(source) == 1)
923
924    val = eval(source[0].get_contents())
925    name, compound, desc = val
926
927    code = code_formatter()
928
929    # file header boilerplate
930    code('''\
931/*
932 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
933 */
934
935#ifndef __DEBUG_${name}_HH__
936#define __DEBUG_${name}_HH__
937
938namespace Debug {
939''')
940
941    if compound:
942        code('class CompoundFlag;')
943    code('class SimpleFlag;')
944
945    if compound:
946        code('extern CompoundFlag $name;')
947        for flag in compound:
948            code('extern SimpleFlag $flag;')
949    else:
950        code('extern SimpleFlag $name;')
951
952    code('''
953}
954
955#endif // __DEBUG_${name}_HH__
956''')
957
958    code.write(str(target[0]))
959
960for name,flag in sorted(debug_flags.iteritems()):
961    n, compound, desc = flag
962    assert n == name
963
964    hh_file = 'debug/%s.hh' % name
965    env.Command(hh_file, Value(flag),
966                MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
967    env.Depends(SWIG, hh_file)
968
969env.Command('debug/flags.cc', Value(debug_flags),
970            MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
971env.Depends(SWIG, 'debug/flags.cc')
972Source('debug/flags.cc')
973
974# version tags
975tags = \
976env.Command('sim/tags.cc', None,
977            MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
978                       Transform("VER TAGS")))
979env.AlwaysBuild(tags)
980
981# Embed python files.  All .py files that have been indicated by a
982# PySource() call in a SConscript need to be embedded into the M5
983# library.  To do that, we compile the file to byte code, marshal the
984# byte code, compress it, and then generate a c++ file that
985# inserts the result into an array.
986def embedPyFile(target, source, env):
987    def c_str(string):
988        if string is None:
989            return "0"
990        return '"%s"' % string
991
992    '''Action function to compile a .py into a code object, marshal
993    it, compress it, and stick it into an asm file so the code appears
994    as just bytes with a label in the data section'''
995
996    src = file(str(source[0]), 'r').read()
997
998    pysource = PySource.tnodes[source[0]]
999    compiled = compile(src, pysource.abspath, 'exec')
1000    marshalled = marshal.dumps(compiled)
1001    compressed = zlib.compress(marshalled)
1002    data = compressed
1003    sym = pysource.symname
1004
1005    code = code_formatter()
1006    code('''\
1007#include "sim/init.hh"
1008
1009namespace {
1010
1011const uint8_t data_${sym}[] = {
1012''')
1013    code.indent()
1014    step = 16
1015    for i in xrange(0, len(data), step):
1016        x = array.array('B', data[i:i+step])
1017        code(''.join('%d,' % d for d in x))
1018    code.dedent()
1019
1020    code('''};
1021
1022EmbeddedPython embedded_${sym}(
1023    ${{c_str(pysource.arcname)}},
1024    ${{c_str(pysource.abspath)}},
1025    ${{c_str(pysource.modpath)}},
1026    data_${sym},
1027    ${{len(data)}},
1028    ${{len(marshalled)}});
1029
1030} // anonymous namespace
1031''')
1032    code.write(str(target[0]))
1033
1034for source in PySource.all:
1035    env.Command(source.cpp, source.tnode,
1036                MakeAction(embedPyFile, Transform("EMBED PY")))
1037    env.Depends(SWIG, source.cpp)
1038    Source(source.cpp, skip_no_python=True)
1039
1040########################################################################
1041#
1042# Define binaries.  Each different build type (debug, opt, etc.) gets
1043# a slightly different build environment.
1044#
1045
1046# List of constructed environments to pass back to SConstruct
1047date_source = Source('base/date.cc', skip_lib=True)
1048
1049# Capture this directory for the closure makeEnv, otherwise when it is
1050# called, it won't know what directory it should use.
1051variant_dir = Dir('.').path
1052def variant(*path):
1053    return os.path.join(variant_dir, *path)
1054def variantd(*path):
1055    return variant(*path)+'/'
1056
1057# Function to create a new build environment as clone of current
1058# environment 'env' with modified object suffix and optional stripped
1059# binary.  Additional keyword arguments are appended to corresponding
1060# build environment vars.
1061def makeEnv(env, label, objsfx, strip = False, **kwargs):
1062    # SCons doesn't know to append a library suffix when there is a '.' in the
1063    # name.  Use '_' instead.
1064    libname = variant('gem5_' + label)
1065    exename = variant('gem5.' + label)
1066    secondary_exename = variant('m5.' + label)
1067
1068    new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
1069    new_env.Label = label
1070    new_env.Append(**kwargs)
1071
1072    swig_env = new_env.Clone()
1073
1074    # Both gcc and clang have issues with unused labels and values in
1075    # the SWIG generated code
1076    swig_env.Append(CCFLAGS=['-Wno-unused-label', '-Wno-unused-value'])
1077
1078    if env['GCC']:
1079        # Depending on the SWIG version, we also need to supress
1080        # warnings about uninitialized variables and missing field
1081        # initializers.
1082        swig_env.Append(CCFLAGS=['-Wno-uninitialized',
1083                                 '-Wno-missing-field-initializers',
1084                                 '-Wno-unused-but-set-variable',
1085                                 '-Wno-maybe-uninitialized',
1086                                 '-Wno-type-limits'])
1087
1088
1089        # The address sanitizer is available for gcc >= 4.8
1090        if GetOption('with_asan'):
1091            if GetOption('with_ubsan') and \
1092                    compareVersions(env['GCC_VERSION'], '4.9') >= 0:
1093                new_env.Append(CCFLAGS=['-fsanitize=address,undefined',
1094                                        '-fno-omit-frame-pointer'])
1095                new_env.Append(LINKFLAGS='-fsanitize=address,undefined')
1096            else:
1097                new_env.Append(CCFLAGS=['-fsanitize=address',
1098                                        '-fno-omit-frame-pointer'])
1099                new_env.Append(LINKFLAGS='-fsanitize=address')
1100        # Only gcc >= 4.9 supports UBSan, so check both the version
1101        # and the command-line option before adding the compiler and
1102        # linker flags.
1103        elif GetOption('with_ubsan') and \
1104                compareVersions(env['GCC_VERSION'], '4.9') >= 0:
1105            new_env.Append(CCFLAGS='-fsanitize=undefined')
1106            new_env.Append(LINKFLAGS='-fsanitize=undefined')
1107
1108
1109    if env['CLANG']:
1110        swig_env.Append(CCFLAGS=['-Wno-sometimes-uninitialized',
1111                                 '-Wno-deprecated-register',
1112                                 '-Wno-tautological-compare'])
1113
1114        # We require clang >= 3.1, so there is no need to check any
1115        # versions here.
1116        if GetOption('with_ubsan'):
1117            if GetOption('with_asan'):
1118                new_env.Append(CCFLAGS=['-fsanitize=address,undefined',
1119                                        '-fno-omit-frame-pointer'])
1120                new_env.Append(LINKFLAGS='-fsanitize=address,undefined')
1121            else:
1122                new_env.Append(CCFLAGS='-fsanitize=undefined')
1123                new_env.Append(LINKFLAGS='-fsanitize=undefined')
1124
1125        elif GetOption('with_asan'):
1126            new_env.Append(CCFLAGS=['-fsanitize=address',
1127                                    '-fno-omit-frame-pointer'])
1128            new_env.Append(LINKFLAGS='-fsanitize=address')
1129
1130    werror_env = new_env.Clone()
1131    # Treat warnings as errors but white list some warnings that we
1132    # want to allow (e.g., deprecation warnings).
1133    werror_env.Append(CCFLAGS=['-Werror',
1134                               '-Wno-error=deprecated-declarations',
1135                               '-Wno-error=deprecated',
1136                               ])
1137
1138    def make_obj(source, static, extra_deps = None):
1139        '''This function adds the specified source to the correct
1140        build environment, and returns the corresponding SCons Object
1141        nodes'''
1142
1143        if source.swig:
1144            env = swig_env
1145        elif source.Werror:
1146            env = werror_env
1147        else:
1148            env = new_env
1149
1150        if static:
1151            obj = env.StaticObject(source.tnode)
1152        else:
1153            obj = env.SharedObject(source.tnode)
1154
1155        if extra_deps:
1156            env.Depends(obj, extra_deps)
1157
1158        return obj
1159
1160    lib_guards = {'main': False, 'skip_lib': False}
1161
1162    # Without Python, leave out all SWIG and Python content from the
1163    # library builds.  The option doesn't affect gem5 built as a program
1164    if GetOption('without_python'):
1165        lib_guards['skip_no_python'] = False
1166
1167    static_objs = [ make_obj(s, True) for s in Source.get(**lib_guards) ]
1168    shared_objs = [ make_obj(s, False) for s in Source.get(**lib_guards) ]
1169
1170    static_date = make_obj(date_source, static=True, extra_deps=static_objs)
1171    static_objs.append(static_date)
1172
1173    shared_date = make_obj(date_source, static=False, extra_deps=shared_objs)
1174    shared_objs.append(shared_date)
1175
1176    # First make a library of everything but main() so other programs can
1177    # link against m5.
1178    static_lib = new_env.StaticLibrary(libname, static_objs)
1179    shared_lib = new_env.SharedLibrary(libname, shared_objs)
1180
1181    # Now link a stub with main() and the static library.
1182    main_objs = [ make_obj(s, True) for s in Source.get(main=True) ]
1183
1184    for test in UnitTest.all:
1185        flags = { test.target : True }
1186        test_sources = Source.get(**flags)
1187        test_objs = [ make_obj(s, static=True) for s in test_sources ]
1188        if test.main:
1189            test_objs += main_objs
1190        path = variant('unittest/%s.%s' % (test.target, label))
1191        new_env.Program(path, test_objs + static_objs)
1192
1193    progname = exename
1194    if strip:
1195        progname += '.unstripped'
1196
1197    targets = new_env.Program(progname, main_objs + static_objs)
1198
1199    if strip:
1200        if sys.platform == 'sunos5':
1201            cmd = 'cp $SOURCE $TARGET; strip $TARGET'
1202        else:
1203            cmd = 'strip $SOURCE -o $TARGET'
1204        targets = new_env.Command(exename, progname,
1205                    MakeAction(cmd, Transform("STRIP")))
1206
1207    new_env.Command(secondary_exename, exename,
1208            MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1209
1210    new_env.M5Binary = targets[0]
1211    return new_env
1212
1213# Start out with the compiler flags common to all compilers,
1214# i.e. they all use -g for opt and -g -pg for prof
1215ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1216           'perf' : ['-g']}
1217
1218# Start out with the linker flags common to all linkers, i.e. -pg for
1219# prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1220# no-as-needed and as-needed as the binutils linker is too clever and
1221# simply doesn't link to the library otherwise.
1222ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1223           'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1224
1225# For Link Time Optimization, the optimisation flags used to compile
1226# individual files are decoupled from those used at link time
1227# (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1228# to also update the linker flags based on the target.
1229if env['GCC']:
1230    if sys.platform == 'sunos5':
1231        ccflags['debug'] += ['-gstabs+']
1232    else:
1233        ccflags['debug'] += ['-ggdb3']
1234    ldflags['debug'] += ['-O0']
1235    # opt, fast, prof and perf all share the same cc flags, also add
1236    # the optimization to the ldflags as LTO defers the optimization
1237    # to link time
1238    for target in ['opt', 'fast', 'prof', 'perf']:
1239        ccflags[target] += ['-O3']
1240        ldflags[target] += ['-O3']
1241
1242    ccflags['fast'] += env['LTO_CCFLAGS']
1243    ldflags['fast'] += env['LTO_LDFLAGS']
1244elif env['CLANG']:
1245    ccflags['debug'] += ['-g', '-O0']
1246    # opt, fast, prof and perf all share the same cc flags
1247    for target in ['opt', 'fast', 'prof', 'perf']:
1248        ccflags[target] += ['-O3']
1249else:
1250    print 'Unknown compiler, please fix compiler options'
1251    Exit(1)
1252
1253
1254# To speed things up, we only instantiate the build environments we
1255# need.  We try to identify the needed environment for each target; if
1256# we can't, we fall back on instantiating all the environments just to
1257# be safe.
1258target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1259obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1260              'gpo' : 'perf'}
1261
1262def identifyTarget(t):
1263    ext = t.split('.')[-1]
1264    if ext in target_types:
1265        return ext
1266    if obj2target.has_key(ext):
1267        return obj2target[ext]
1268    match = re.search(r'/tests/([^/]+)/', t)
1269    if match and match.group(1) in target_types:
1270        return match.group(1)
1271    return 'all'
1272
1273needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1274if 'all' in needed_envs:
1275    needed_envs += target_types
1276
1277def makeEnvirons(target, source, env):
1278    # cause any later Source() calls to be fatal, as a diagnostic.
1279    Source.done()
1280
1281    envList = []
1282
1283    # Debug binary
1284    if 'debug' in needed_envs:
1285        envList.append(
1286            makeEnv(env, 'debug', '.do',
1287                    CCFLAGS = Split(ccflags['debug']),
1288                    CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1289                    LINKFLAGS = Split(ldflags['debug'])))
1290
1291    # Optimized binary
1292    if 'opt' in needed_envs:
1293        envList.append(
1294            makeEnv(env, 'opt', '.o',
1295                    CCFLAGS = Split(ccflags['opt']),
1296                    CPPDEFINES = ['TRACING_ON=1'],
1297                    LINKFLAGS = Split(ldflags['opt'])))
1298
1299    # "Fast" binary
1300    if 'fast' in needed_envs:
1301        envList.append(
1302            makeEnv(env, 'fast', '.fo', strip = True,
1303                    CCFLAGS = Split(ccflags['fast']),
1304                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1305                    LINKFLAGS = Split(ldflags['fast'])))
1306
1307    # Profiled binary using gprof
1308    if 'prof' in needed_envs:
1309        envList.append(
1310            makeEnv(env, 'prof', '.po',
1311                    CCFLAGS = Split(ccflags['prof']),
1312                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1313                    LINKFLAGS = Split(ldflags['prof'])))
1314
1315    # Profiled binary using google-pprof
1316    if 'perf' in needed_envs:
1317        envList.append(
1318            makeEnv(env, 'perf', '.gpo',
1319                    CCFLAGS = Split(ccflags['perf']),
1320                    CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1321                    LINKFLAGS = Split(ldflags['perf'])))
1322
1323    # Set up the regression tests for each build.
1324    for e in envList:
1325        SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1326                   variant_dir = variantd('tests', e.Label),
1327                   exports = { 'env' : e }, duplicate = False)
1328
1329# The MakeEnvirons Builder defers the full dependency collection until
1330# after processing the ISA definition (due to dynamically generated
1331# source files).  Add this dependency to all targets so they will wait
1332# until the environments are completely set up.  Otherwise, a second
1333# process (e.g. -j2 or higher) will try to compile the requested target,
1334# not know how, and fail.
1335env.Append(BUILDERS = {'MakeEnvirons' :
1336                        Builder(action=MakeAction(makeEnvirons,
1337                                                  Transform("ENVIRONS", 1)))})
1338
1339isa_target = env['PHONY_BASE'] + '-deps'
1340environs   = env['PHONY_BASE'] + '-environs'
1341env.Depends('#all-deps',     isa_target)
1342env.Depends('#all-environs', environs)
1343env.ScanISA(isa_target, File('arch/%s/generated/inc.d' % env['TARGET_ISA']))
1344envSetup = env.MakeEnvirons(environs, isa_target)
1345
1346# make sure no -deps targets occur before all ISAs are complete
1347env.Depends(isa_target, '#all-isas')
1348# likewise for -environs targets and all the -deps targets
1349env.Depends(environs, '#all-deps')
1350