SConscript revision 12307
1# -*- mode:python -*-
2
3# Copyright (c) 2004-2005 The Regents of The University of Michigan
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# Authors: Nathan Binkert
30
31import array
32import bisect
33import imp
34import marshal
35import os
36import re
37import subprocess
38import sys
39import zlib
40
41from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
42
43import SCons
44
45from gem5_scons import Transform
46
47# This file defines how to build a particular configuration of gem5
48# based on variable settings in the 'env' build environment.
49
50Import('*')
51
52# Children need to see the environment
53Export('env')
54
55build_env = [(opt, env[opt]) for opt in export_vars]
56
57from m5.util import code_formatter, compareVersions
58
59########################################################################
60# Code for adding source files of various types
61#
62# When specifying a source file of some type, a set of tags can be
63# specified for that file.
64
65class SourceList(list):
66    def with_tags_that(self, predicate):
67        '''Return a list of sources with tags that satisfy a predicate.'''
68        def match(source):
69            return predicate(source.tags)
70        return SourceList(filter(match, self))
71
72    def with_any_tags(self, *tags):
73        '''Return a list of sources with any of the supplied tags.'''
74        return self.with_tags_that(lambda stags: len(tags & stags) > 0)
75
76    def with_all_tags(self, *tags):
77        '''Return a list of sources with all of the supplied tags.'''
78        return self.with_tags_that(lambda stags: tags <= stags)
79
80    def with_tag(self, tag):
81        '''Return a list of sources with the supplied tag.'''
82        return self.with_tags_that(lambda stags: tag in stags)
83
84    def without_tags(self, *tags):
85        '''Return a list of sources without any of the supplied tags.'''
86        return self.with_tags_that(lambda stags: len(tags & stags) == 0)
87
88    def without_tag(self, tag):
89        '''Return a list of sources with the supplied tag.'''
90        return self.with_tags_that(lambda stags: tag not in stags)
91
92class SourceMeta(type):
93    '''Meta class for source files that keeps track of all files of a
94    particular type.'''
95    def __init__(cls, name, bases, dict):
96        super(SourceMeta, cls).__init__(name, bases, dict)
97        cls.all = SourceList()
98
99class SourceFile(object):
100    '''Base object that encapsulates the notion of a source file.
101    This includes, the source node, target node, various manipulations
102    of those.  A source file also specifies a set of tags which
103    describing arbitrary properties of the source file.'''
104    __metaclass__ = SourceMeta
105    def __init__(self, source, tags=None, add_tags=None):
106        if tags is None:
107            tags='gem5 lib'
108        if isinstance(tags, basestring):
109            tags = set([tags])
110        if isinstance(add_tags, basestring):
111            add_tags = set([add_tags])
112        if add_tags:
113            tags = tags | add_tags
114        self.tags = set(tags)
115
116        tnode = source
117        if not isinstance(source, SCons.Node.FS.File):
118            tnode = File(source)
119
120        self.tnode = tnode
121        self.snode = tnode.srcnode()
122
123        for base in type(self).__mro__:
124            if issubclass(base, SourceFile):
125                base.all.append(self)
126
127    @property
128    def filename(self):
129        return str(self.tnode)
130
131    @property
132    def dirname(self):
133        return dirname(self.filename)
134
135    @property
136    def basename(self):
137        return basename(self.filename)
138
139    @property
140    def extname(self):
141        index = self.basename.rfind('.')
142        if index <= 0:
143            # dot files aren't extensions
144            return self.basename, None
145
146        return self.basename[:index], self.basename[index+1:]
147
148    def __lt__(self, other): return self.filename < other.filename
149    def __le__(self, other): return self.filename <= other.filename
150    def __gt__(self, other): return self.filename > other.filename
151    def __ge__(self, other): return self.filename >= other.filename
152    def __eq__(self, other): return self.filename == other.filename
153    def __ne__(self, other): return self.filename != other.filename
154
155class Source(SourceFile):
156    ungrouped_tag = 'No link group'
157    source_groups = set()
158
159    _current_group_tag = ungrouped_tag
160
161    @staticmethod
162    def link_group_tag(group):
163        return 'link group: %s' % group
164
165    @classmethod
166    def set_group(cls, group):
167        new_tag = Source.link_group_tag(group)
168        Source._current_group_tag = new_tag
169        Source.source_groups.add(group)
170
171    def _add_link_group_tag(self):
172        self.tags.add(Source._current_group_tag)
173
174    '''Add a c/c++ source file to the build'''
175    def __init__(self, source, tags=None, add_tags=None):
176        '''specify the source file, and any tags'''
177        super(Source, self).__init__(source, tags, add_tags)
178        self._add_link_group_tag()
179
180class PySource(SourceFile):
181    '''Add a python source file to the named package'''
182    invalid_sym_char = re.compile('[^A-z0-9_]')
183    modules = {}
184    tnodes = {}
185    symnames = {}
186
187    def __init__(self, package, source, tags=None, add_tags=None):
188        '''specify the python package, the source file, and any tags'''
189        super(PySource, self).__init__(source, tags, add_tags)
190
191        modname,ext = self.extname
192        assert ext == 'py'
193
194        if package:
195            path = package.split('.')
196        else:
197            path = []
198
199        modpath = path[:]
200        if modname != '__init__':
201            modpath += [ modname ]
202        modpath = '.'.join(modpath)
203
204        arcpath = path + [ self.basename ]
205        abspath = self.snode.abspath
206        if not exists(abspath):
207            abspath = self.tnode.abspath
208
209        self.package = package
210        self.modname = modname
211        self.modpath = modpath
212        self.arcname = joinpath(*arcpath)
213        self.abspath = abspath
214        self.compiled = File(self.filename + 'c')
215        self.cpp = File(self.filename + '.cc')
216        self.symname = PySource.invalid_sym_char.sub('_', modpath)
217
218        PySource.modules[modpath] = self
219        PySource.tnodes[self.tnode] = self
220        PySource.symnames[self.symname] = self
221
222class SimObject(PySource):
223    '''Add a SimObject python file as a python source object and add
224    it to a list of sim object modules'''
225
226    fixed = False
227    modnames = []
228
229    def __init__(self, source, tags=None, add_tags=None):
230        '''Specify the source file and any tags (automatically in
231        the m5.objects package)'''
232        super(SimObject, self).__init__('m5.objects', source, tags, add_tags)
233        if self.fixed:
234            raise AttributeError, "Too late to call SimObject now."
235
236        bisect.insort_right(SimObject.modnames, self.modname)
237
238class ProtoBuf(SourceFile):
239    '''Add a Protocol Buffer to build'''
240
241    def __init__(self, source, tags=None, add_tags=None):
242        '''Specify the source file, and any tags'''
243        super(ProtoBuf, self).__init__(source, tags, add_tags)
244
245        # Get the file name and the extension
246        modname,ext = self.extname
247        assert ext == 'proto'
248
249        # Currently, we stick to generating the C++ headers, so we
250        # only need to track the source and header.
251        self.cc_file = File(modname + '.pb.cc')
252        self.hh_file = File(modname + '.pb.h')
253
254class UnitTest(object):
255    '''Create a UnitTest'''
256
257    all = []
258    def __init__(self, target, *sources, **kwargs):
259        '''Specify the target name and any sources.  Sources that are
260        not SourceFiles are evalued with Source().  All files are
261        tagged with the name of the UnitTest target.'''
262
263        srcs = SourceList()
264        for src in sources:
265            if not isinstance(src, SourceFile):
266                src = Source(src, tags=str(target))
267            srcs.append(src)
268
269        self.sources = srcs
270        self.target = target
271        self.main = kwargs.get('main', False)
272        UnitTest.all.append(self)
273
274# Children should have access
275Export('Source')
276Export('PySource')
277Export('SimObject')
278Export('ProtoBuf')
279Export('UnitTest')
280
281########################################################################
282#
283# Debug Flags
284#
285debug_flags = {}
286def DebugFlag(name, desc=None):
287    if name in debug_flags:
288        raise AttributeError, "Flag %s already specified" % name
289    debug_flags[name] = (name, (), desc)
290
291def CompoundFlag(name, flags, desc=None):
292    if name in debug_flags:
293        raise AttributeError, "Flag %s already specified" % name
294
295    compound = tuple(flags)
296    debug_flags[name] = (name, compound, desc)
297
298Export('DebugFlag')
299Export('CompoundFlag')
300
301########################################################################
302#
303# Set some compiler variables
304#
305
306# Include file paths are rooted in this directory.  SCons will
307# automatically expand '.' to refer to both the source directory and
308# the corresponding build directory to pick up generated include
309# files.
310env.Append(CPPPATH=Dir('.'))
311
312for extra_dir in extras_dir_list:
313    env.Append(CPPPATH=Dir(extra_dir))
314
315# Workaround for bug in SCons version > 0.97d20071212
316# Scons bug id: 2006 gem5 Bug id: 308
317for root, dirs, files in os.walk(base_dir, topdown=True):
318    Dir(root[len(base_dir) + 1:])
319
320########################################################################
321#
322# Walk the tree and execute all SConscripts in subdirectories
323#
324
325here = Dir('.').srcnode().abspath
326for root, dirs, files in os.walk(base_dir, topdown=True):
327    if root == here:
328        # we don't want to recurse back into this SConscript
329        continue
330
331    if 'SConscript' in files:
332        build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
333        Source.set_group(build_dir)
334        SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
335
336for extra_dir in extras_dir_list:
337    prefix_len = len(dirname(extra_dir)) + 1
338
339    # Also add the corresponding build directory to pick up generated
340    # include files.
341    env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
342
343    for root, dirs, files in os.walk(extra_dir, topdown=True):
344        # if build lives in the extras directory, don't walk down it
345        if 'build' in dirs:
346            dirs.remove('build')
347
348        if 'SConscript' in files:
349            build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
350            SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
351
352for opt in export_vars:
353    env.ConfigFile(opt)
354
355def makeTheISA(source, target, env):
356    isas = [ src.get_contents() for src in source ]
357    target_isa = env['TARGET_ISA']
358    def define(isa):
359        return isa.upper() + '_ISA'
360
361    def namespace(isa):
362        return isa[0].upper() + isa[1:].lower() + 'ISA'
363
364
365    code = code_formatter()
366    code('''\
367#ifndef __CONFIG_THE_ISA_HH__
368#define __CONFIG_THE_ISA_HH__
369
370''')
371
372    # create defines for the preprocessing and compile-time determination
373    for i,isa in enumerate(isas):
374        code('#define $0 $1', define(isa), i + 1)
375    code()
376
377    # create an enum for any run-time determination of the ISA, we
378    # reuse the same name as the namespaces
379    code('enum class Arch {')
380    for i,isa in enumerate(isas):
381        if i + 1 == len(isas):
382            code('  $0 = $1', namespace(isa), define(isa))
383        else:
384            code('  $0 = $1,', namespace(isa), define(isa))
385    code('};')
386
387    code('''
388
389#define THE_ISA ${{define(target_isa)}}
390#define TheISA ${{namespace(target_isa)}}
391#define THE_ISA_STR "${{target_isa}}"
392
393#endif // __CONFIG_THE_ISA_HH__''')
394
395    code.write(str(target[0]))
396
397env.Command('config/the_isa.hh', map(Value, all_isa_list),
398            MakeAction(makeTheISA, Transform("CFG ISA", 0)))
399
400def makeTheGPUISA(source, target, env):
401    isas = [ src.get_contents() for src in source ]
402    target_gpu_isa = env['TARGET_GPU_ISA']
403    def define(isa):
404        return isa.upper() + '_ISA'
405
406    def namespace(isa):
407        return isa[0].upper() + isa[1:].lower() + 'ISA'
408
409
410    code = code_formatter()
411    code('''\
412#ifndef __CONFIG_THE_GPU_ISA_HH__
413#define __CONFIG_THE_GPU_ISA_HH__
414
415''')
416
417    # create defines for the preprocessing and compile-time determination
418    for i,isa in enumerate(isas):
419        code('#define $0 $1', define(isa), i + 1)
420    code()
421
422    # create an enum for any run-time determination of the ISA, we
423    # reuse the same name as the namespaces
424    code('enum class GPUArch {')
425    for i,isa in enumerate(isas):
426        if i + 1 == len(isas):
427            code('  $0 = $1', namespace(isa), define(isa))
428        else:
429            code('  $0 = $1,', namespace(isa), define(isa))
430    code('};')
431
432    code('''
433
434#define THE_GPU_ISA ${{define(target_gpu_isa)}}
435#define TheGpuISA ${{namespace(target_gpu_isa)}}
436#define THE_GPU_ISA_STR "${{target_gpu_isa}}"
437
438#endif // __CONFIG_THE_GPU_ISA_HH__''')
439
440    code.write(str(target[0]))
441
442env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
443            MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
444
445########################################################################
446#
447# Prevent any SimObjects from being added after this point, they
448# should all have been added in the SConscripts above
449#
450SimObject.fixed = True
451
452class DictImporter(object):
453    '''This importer takes a dictionary of arbitrary module names that
454    map to arbitrary filenames.'''
455    def __init__(self, modules):
456        self.modules = modules
457        self.installed = set()
458
459    def __del__(self):
460        self.unload()
461
462    def unload(self):
463        import sys
464        for module in self.installed:
465            del sys.modules[module]
466        self.installed = set()
467
468    def find_module(self, fullname, path):
469        if fullname == 'm5.defines':
470            return self
471
472        if fullname == 'm5.objects':
473            return self
474
475        if fullname.startswith('_m5'):
476            return None
477
478        source = self.modules.get(fullname, None)
479        if source is not None and fullname.startswith('m5.objects'):
480            return self
481
482        return None
483
484    def load_module(self, fullname):
485        mod = imp.new_module(fullname)
486        sys.modules[fullname] = mod
487        self.installed.add(fullname)
488
489        mod.__loader__ = self
490        if fullname == 'm5.objects':
491            mod.__path__ = fullname.split('.')
492            return mod
493
494        if fullname == 'm5.defines':
495            mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
496            return mod
497
498        source = self.modules[fullname]
499        if source.modname == '__init__':
500            mod.__path__ = source.modpath
501        mod.__file__ = source.abspath
502
503        exec file(source.abspath, 'r') in mod.__dict__
504
505        return mod
506
507import m5.SimObject
508import m5.params
509from m5.util import code_formatter
510
511m5.SimObject.clear()
512m5.params.clear()
513
514# install the python importer so we can grab stuff from the source
515# tree itself.  We can't have SimObjects added after this point or
516# else we won't know about them for the rest of the stuff.
517importer = DictImporter(PySource.modules)
518sys.meta_path[0:0] = [ importer ]
519
520# import all sim objects so we can populate the all_objects list
521# make sure that we're working with a list, then let's sort it
522for modname in SimObject.modnames:
523    exec('from m5.objects import %s' % modname)
524
525# we need to unload all of the currently imported modules so that they
526# will be re-imported the next time the sconscript is run
527importer.unload()
528sys.meta_path.remove(importer)
529
530sim_objects = m5.SimObject.allClasses
531all_enums = m5.params.allEnums
532
533for name,obj in sorted(sim_objects.iteritems()):
534    for param in obj._params.local.values():
535        # load the ptype attribute now because it depends on the
536        # current version of SimObject.allClasses, but when scons
537        # actually uses the value, all versions of
538        # SimObject.allClasses will have been loaded
539        param.ptype
540
541########################################################################
542#
543# calculate extra dependencies
544#
545module_depends = ["m5", "m5.SimObject", "m5.params"]
546depends = [ PySource.modules[dep].snode for dep in module_depends ]
547depends.sort(key = lambda x: x.name)
548
549########################################################################
550#
551# Commands for the basic automatically generated python files
552#
553
554# Generate Python file containing a dict specifying the current
555# buildEnv flags.
556def makeDefinesPyFile(target, source, env):
557    build_env = source[0].get_contents()
558
559    code = code_formatter()
560    code("""
561import _m5.core
562import m5.util
563
564buildEnv = m5.util.SmartDict($build_env)
565
566compileDate = _m5.core.compileDate
567_globals = globals()
568for key,val in _m5.core.__dict__.iteritems():
569    if key.startswith('flag_'):
570        flag = key[5:]
571        _globals[flag] = val
572del _globals
573""")
574    code.write(target[0].abspath)
575
576defines_info = Value(build_env)
577# Generate a file with all of the compile options in it
578env.Command('python/m5/defines.py', defines_info,
579            MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
580PySource('m5', 'python/m5/defines.py')
581
582# Generate python file containing info about the M5 source code
583def makeInfoPyFile(target, source, env):
584    code = code_formatter()
585    for src in source:
586        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
587        code('$src = ${{repr(data)}}')
588    code.write(str(target[0]))
589
590# Generate a file that wraps the basic top level files
591env.Command('python/m5/info.py',
592            [ '#/COPYING', '#/LICENSE', '#/README', ],
593            MakeAction(makeInfoPyFile, Transform("INFO")))
594PySource('m5', 'python/m5/info.py')
595
596########################################################################
597#
598# Create all of the SimObject param headers and enum headers
599#
600
601def createSimObjectParamStruct(target, source, env):
602    assert len(target) == 1 and len(source) == 1
603
604    name = source[0].get_text_contents()
605    obj = sim_objects[name]
606
607    code = code_formatter()
608    obj.cxx_param_decl(code)
609    code.write(target[0].abspath)
610
611def createSimObjectCxxConfig(is_header):
612    def body(target, source, env):
613        assert len(target) == 1 and len(source) == 1
614
615        name = str(source[0].get_contents())
616        obj = sim_objects[name]
617
618        code = code_formatter()
619        obj.cxx_config_param_file(code, is_header)
620        code.write(target[0].abspath)
621    return body
622
623def createEnumStrings(target, source, env):
624    assert len(target) == 1 and len(source) == 2
625
626    name = source[0].get_text_contents()
627    use_python = source[1].read()
628    obj = all_enums[name]
629
630    code = code_formatter()
631    obj.cxx_def(code)
632    if use_python:
633        obj.pybind_def(code)
634    code.write(target[0].abspath)
635
636def createEnumDecls(target, source, env):
637    assert len(target) == 1 and len(source) == 1
638
639    name = source[0].get_text_contents()
640    obj = all_enums[name]
641
642    code = code_formatter()
643    obj.cxx_decl(code)
644    code.write(target[0].abspath)
645
646def createSimObjectPyBindWrapper(target, source, env):
647    name = source[0].get_text_contents()
648    obj = sim_objects[name]
649
650    code = code_formatter()
651    obj.pybind_decl(code)
652    code.write(target[0].abspath)
653
654# Generate all of the SimObject param C++ struct header files
655params_hh_files = []
656for name,simobj in sorted(sim_objects.iteritems()):
657    py_source = PySource.modules[simobj.__module__]
658    extra_deps = [ py_source.tnode ]
659
660    hh_file = File('params/%s.hh' % name)
661    params_hh_files.append(hh_file)
662    env.Command(hh_file, Value(name),
663                MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
664    env.Depends(hh_file, depends + extra_deps)
665
666# C++ parameter description files
667if GetOption('with_cxx_config'):
668    for name,simobj in sorted(sim_objects.iteritems()):
669        py_source = PySource.modules[simobj.__module__]
670        extra_deps = [ py_source.tnode ]
671
672        cxx_config_hh_file = File('cxx_config/%s.hh' % name)
673        cxx_config_cc_file = File('cxx_config/%s.cc' % name)
674        env.Command(cxx_config_hh_file, Value(name),
675                    MakeAction(createSimObjectCxxConfig(True),
676                    Transform("CXXCPRHH")))
677        env.Command(cxx_config_cc_file, Value(name),
678                    MakeAction(createSimObjectCxxConfig(False),
679                    Transform("CXXCPRCC")))
680        env.Depends(cxx_config_hh_file, depends + extra_deps +
681                    [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
682        env.Depends(cxx_config_cc_file, depends + extra_deps +
683                    [cxx_config_hh_file])
684        Source(cxx_config_cc_file)
685
686    cxx_config_init_cc_file = File('cxx_config/init.cc')
687
688    def createCxxConfigInitCC(target, source, env):
689        assert len(target) == 1 and len(source) == 1
690
691        code = code_formatter()
692
693        for name,simobj in sorted(sim_objects.iteritems()):
694            if not hasattr(simobj, 'abstract') or not simobj.abstract:
695                code('#include "cxx_config/${name}.hh"')
696        code()
697        code('void cxxConfigInit()')
698        code('{')
699        code.indent()
700        for name,simobj in sorted(sim_objects.iteritems()):
701            not_abstract = not hasattr(simobj, 'abstract') or \
702                not simobj.abstract
703            if not_abstract and 'type' in simobj.__dict__:
704                code('cxx_config_directory["${name}"] = '
705                     '${name}CxxConfigParams::makeDirectoryEntry();')
706        code.dedent()
707        code('}')
708        code.write(target[0].abspath)
709
710    py_source = PySource.modules[simobj.__module__]
711    extra_deps = [ py_source.tnode ]
712    env.Command(cxx_config_init_cc_file, Value(name),
713        MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
714    cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
715        for name,simobj in sorted(sim_objects.iteritems())
716        if not hasattr(simobj, 'abstract') or not simobj.abstract]
717    Depends(cxx_config_init_cc_file, cxx_param_hh_files +
718            [File('sim/cxx_config.hh')])
719    Source(cxx_config_init_cc_file)
720
721# Generate all enum header files
722for name,enum in sorted(all_enums.iteritems()):
723    py_source = PySource.modules[enum.__module__]
724    extra_deps = [ py_source.tnode ]
725
726    cc_file = File('enums/%s.cc' % name)
727    env.Command(cc_file, [Value(name), Value(env['USE_PYTHON'])],
728                MakeAction(createEnumStrings, Transform("ENUM STR")))
729    env.Depends(cc_file, depends + extra_deps)
730    Source(cc_file)
731
732    hh_file = File('enums/%s.hh' % name)
733    env.Command(hh_file, Value(name),
734                MakeAction(createEnumDecls, Transform("ENUMDECL")))
735    env.Depends(hh_file, depends + extra_deps)
736
737# Generate SimObject Python bindings wrapper files
738if env['USE_PYTHON']:
739    for name,simobj in sorted(sim_objects.iteritems()):
740        py_source = PySource.modules[simobj.__module__]
741        extra_deps = [ py_source.tnode ]
742        cc_file = File('python/_m5/param_%s.cc' % name)
743        env.Command(cc_file, Value(name),
744                    MakeAction(createSimObjectPyBindWrapper,
745                               Transform("SO PyBind")))
746        env.Depends(cc_file, depends + extra_deps)
747        Source(cc_file)
748
749# Build all protocol buffers if we have got protoc and protobuf available
750if env['HAVE_PROTOBUF']:
751    for proto in ProtoBuf.all:
752        # Use both the source and header as the target, and the .proto
753        # file as the source. When executing the protoc compiler, also
754        # specify the proto_path to avoid having the generated files
755        # include the path.
756        env.Command([proto.cc_file, proto.hh_file], proto.tnode,
757                    MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
758                               '--proto_path ${SOURCE.dir} $SOURCE',
759                               Transform("PROTOC")))
760
761        # Add the C++ source file
762        Source(proto.cc_file, tags=proto.tags)
763elif ProtoBuf.all:
764    print 'Got protobuf to build, but lacks support!'
765    Exit(1)
766
767#
768# Handle debug flags
769#
770def makeDebugFlagCC(target, source, env):
771    assert(len(target) == 1 and len(source) == 1)
772
773    code = code_formatter()
774
775    # delay definition of CompoundFlags until after all the definition
776    # of all constituent SimpleFlags
777    comp_code = code_formatter()
778
779    # file header
780    code('''
781/*
782 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
783 */
784
785#include "base/debug.hh"
786
787namespace Debug {
788
789''')
790
791    for name, flag in sorted(source[0].read().iteritems()):
792        n, compound, desc = flag
793        assert n == name
794
795        if not compound:
796            code('SimpleFlag $name("$name", "$desc");')
797        else:
798            comp_code('CompoundFlag $name("$name", "$desc",')
799            comp_code.indent()
800            last = len(compound) - 1
801            for i,flag in enumerate(compound):
802                if i != last:
803                    comp_code('&$flag,')
804                else:
805                    comp_code('&$flag);')
806            comp_code.dedent()
807
808    code.append(comp_code)
809    code()
810    code('} // namespace Debug')
811
812    code.write(str(target[0]))
813
814def makeDebugFlagHH(target, source, env):
815    assert(len(target) == 1 and len(source) == 1)
816
817    val = eval(source[0].get_contents())
818    name, compound, desc = val
819
820    code = code_formatter()
821
822    # file header boilerplate
823    code('''\
824/*
825 * DO NOT EDIT THIS FILE! Automatically generated by SCons.
826 */
827
828#ifndef __DEBUG_${name}_HH__
829#define __DEBUG_${name}_HH__
830
831namespace Debug {
832''')
833
834    if compound:
835        code('class CompoundFlag;')
836    code('class SimpleFlag;')
837
838    if compound:
839        code('extern CompoundFlag $name;')
840        for flag in compound:
841            code('extern SimpleFlag $flag;')
842    else:
843        code('extern SimpleFlag $name;')
844
845    code('''
846}
847
848#endif // __DEBUG_${name}_HH__
849''')
850
851    code.write(str(target[0]))
852
853for name,flag in sorted(debug_flags.iteritems()):
854    n, compound, desc = flag
855    assert n == name
856
857    hh_file = 'debug/%s.hh' % name
858    env.Command(hh_file, Value(flag),
859                MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
860
861env.Command('debug/flags.cc', Value(debug_flags),
862            MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
863Source('debug/flags.cc')
864
865# version tags
866tags = \
867env.Command('sim/tags.cc', None,
868            MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
869                       Transform("VER TAGS")))
870env.AlwaysBuild(tags)
871
872# Embed python files.  All .py files that have been indicated by a
873# PySource() call in a SConscript need to be embedded into the M5
874# library.  To do that, we compile the file to byte code, marshal the
875# byte code, compress it, and then generate a c++ file that
876# inserts the result into an array.
877def embedPyFile(target, source, env):
878    def c_str(string):
879        if string is None:
880            return "0"
881        return '"%s"' % string
882
883    '''Action function to compile a .py into a code object, marshal
884    it, compress it, and stick it into an asm file so the code appears
885    as just bytes with a label in the data section'''
886
887    src = file(str(source[0]), 'r').read()
888
889    pysource = PySource.tnodes[source[0]]
890    compiled = compile(src, pysource.abspath, 'exec')
891    marshalled = marshal.dumps(compiled)
892    compressed = zlib.compress(marshalled)
893    data = compressed
894    sym = pysource.symname
895
896    code = code_formatter()
897    code('''\
898#include "sim/init.hh"
899
900namespace {
901
902const uint8_t data_${sym}[] = {
903''')
904    code.indent()
905    step = 16
906    for i in xrange(0, len(data), step):
907        x = array.array('B', data[i:i+step])
908        code(''.join('%d,' % d for d in x))
909    code.dedent()
910
911    code('''};
912
913EmbeddedPython embedded_${sym}(
914    ${{c_str(pysource.arcname)}},
915    ${{c_str(pysource.abspath)}},
916    ${{c_str(pysource.modpath)}},
917    data_${sym},
918    ${{len(data)}},
919    ${{len(marshalled)}});
920
921} // anonymous namespace
922''')
923    code.write(str(target[0]))
924
925for source in PySource.all:
926    env.Command(source.cpp, source.tnode,
927                MakeAction(embedPyFile, Transform("EMBED PY")))
928    Source(source.cpp, tags=source.tags, add_tags='python')
929
930########################################################################
931#
932# Define binaries.  Each different build type (debug, opt, etc.) gets
933# a slightly different build environment.
934#
935
936# List of constructed environments to pass back to SConstruct
937date_source = Source('base/date.cc', tags=[])
938
939# Function to create a new build environment as clone of current
940# environment 'env' with modified object suffix and optional stripped
941# binary.  Additional keyword arguments are appended to corresponding
942# build environment vars.
943def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
944    # SCons doesn't know to append a library suffix when there is a '.' in the
945    # name.  Use '_' instead.
946    libname = 'gem5_' + label
947    exename = 'gem5.' + label
948    secondary_exename = 'm5.' + label
949
950    new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
951    new_env.Label = label
952    new_env.Append(**kwargs)
953
954    make_static = lambda source: new_env.StaticObject(source.tnode)
955    make_shared = lambda source: new_env.SharedObject(source.tnode)
956
957    lib_sources = Source.all.with_tag('gem5 lib')
958
959    # Without Python, leave out all Python content from the library
960    # builds.  The option doesn't affect gem5 built as a program
961    if GetOption('without_python'):
962        lib_sources = lib_sources.without_tag('python')
963
964    static_objs = []
965    shared_objs = []
966
967    for s in lib_sources.with_tag(Source.ungrouped_tag):
968        static_objs.append(make_static(s))
969        shared_objs.append(make_shared(s))
970
971    partial_objs = []
972
973    for group in Source.source_groups:
974        srcs = lib_sources.with_tag(Source.link_group_tag(group))
975        if not srcs:
976            continue
977
978        # If partial linking is disabled, add these sources to the build
979        # directly, and short circuit this loop.
980        if disable_partial:
981            for s in srcs:
982                static_objs.append(make_static(s))
983                shared_objs.append(make_shared(s))
984            continue
985
986        # Set up the static partially linked objects.
987        source_objs = [ make_static(s) for s in srcs ]
988        file_name = new_env.subst("${OBJPREFIX}lib${OBJSUFFIX}.partial")
989        target = File(joinpath(group, file_name))
990        partial = env.PartialStatic(target=target, source=source_objs)
991        static_objs.append(partial)
992
993        # Set up the shared partially linked objects.
994        source_objs = [ make_shared(s) for s in srcs ]
995        file_name = new_env.subst("${SHOBJPREFIX}lib${SHOBJSUFFIX}.partial")
996        target = File(joinpath(group, file_name))
997        partial = env.PartialShared(target=target, source=source_objs)
998        shared_objs.append(partial)
999
1000    static_date = make_static(date_source)
1001    new_env.Depends(static_date, static_objs)
1002    static_objs.append(static_date)
1003
1004    shared_date = make_shared(date_source)
1005    new_env.Depends(shared_date, shared_objs)
1006    shared_objs.append(shared_date)
1007
1008    # First make a library of everything but main() so other programs can
1009    # link against m5.
1010    static_lib = new_env.StaticLibrary(libname, static_objs)
1011    shared_lib = new_env.SharedLibrary(libname, shared_objs)
1012
1013    # Now link a stub with main() and the static library.
1014    main_objs = [ make_static(s) for s in Source.all.with_tag('main') ]
1015
1016    for test in UnitTest.all:
1017        test_sources = Source.all.with_tag(str(test.target))
1018        test_objs = [ make_static(s) for s in test_sources ]
1019        if test.main:
1020            test_objs += main_objs
1021        path = 'unittest/%s.%s' % (test.target, label)
1022        new_env.Program(path, test_objs + static_objs)
1023
1024    progname = exename
1025    if strip:
1026        progname += '.unstripped'
1027
1028    targets = new_env.Program(progname, main_objs + static_objs)
1029
1030    if strip:
1031        if sys.platform == 'sunos5':
1032            cmd = 'cp $SOURCE $TARGET; strip $TARGET'
1033        else:
1034            cmd = 'strip $SOURCE -o $TARGET'
1035        targets = new_env.Command(exename, progname,
1036                    MakeAction(cmd, Transform("STRIP")))
1037
1038    new_env.Command(secondary_exename, exename,
1039            MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1040
1041    new_env.M5Binary = targets[0]
1042
1043    # Set up regression tests.
1044    SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1045               variant_dir=Dir('tests').Dir(new_env.Label),
1046               exports={ 'env' : new_env }, duplicate=False)
1047
1048# Start out with the compiler flags common to all compilers,
1049# i.e. they all use -g for opt and -g -pg for prof
1050ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1051           'perf' : ['-g']}
1052
1053# Start out with the linker flags common to all linkers, i.e. -pg for
1054# prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1055# no-as-needed and as-needed as the binutils linker is too clever and
1056# simply doesn't link to the library otherwise.
1057ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1058           'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1059
1060# For Link Time Optimization, the optimisation flags used to compile
1061# individual files are decoupled from those used at link time
1062# (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1063# to also update the linker flags based on the target.
1064if env['GCC']:
1065    if sys.platform == 'sunos5':
1066        ccflags['debug'] += ['-gstabs+']
1067    else:
1068        ccflags['debug'] += ['-ggdb3']
1069    ldflags['debug'] += ['-O0']
1070    # opt, fast, prof and perf all share the same cc flags, also add
1071    # the optimization to the ldflags as LTO defers the optimization
1072    # to link time
1073    for target in ['opt', 'fast', 'prof', 'perf']:
1074        ccflags[target] += ['-O3']
1075        ldflags[target] += ['-O3']
1076
1077    ccflags['fast'] += env['LTO_CCFLAGS']
1078    ldflags['fast'] += env['LTO_LDFLAGS']
1079elif env['CLANG']:
1080    ccflags['debug'] += ['-g', '-O0']
1081    # opt, fast, prof and perf all share the same cc flags
1082    for target in ['opt', 'fast', 'prof', 'perf']:
1083        ccflags[target] += ['-O3']
1084else:
1085    print 'Unknown compiler, please fix compiler options'
1086    Exit(1)
1087
1088
1089# To speed things up, we only instantiate the build environments we
1090# need.  We try to identify the needed environment for each target; if
1091# we can't, we fall back on instantiating all the environments just to
1092# be safe.
1093target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1094obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1095              'gpo' : 'perf'}
1096
1097def identifyTarget(t):
1098    ext = t.split('.')[-1]
1099    if ext in target_types:
1100        return ext
1101    if obj2target.has_key(ext):
1102        return obj2target[ext]
1103    match = re.search(r'/tests/([^/]+)/', t)
1104    if match and match.group(1) in target_types:
1105        return match.group(1)
1106    return 'all'
1107
1108needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1109if 'all' in needed_envs:
1110    needed_envs += target_types
1111
1112# Debug binary
1113if 'debug' in needed_envs:
1114    makeEnv(env, 'debug', '.do',
1115            CCFLAGS = Split(ccflags['debug']),
1116            CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1117            LINKFLAGS = Split(ldflags['debug']))
1118
1119# Optimized binary
1120if 'opt' in needed_envs:
1121    makeEnv(env, 'opt', '.o',
1122            CCFLAGS = Split(ccflags['opt']),
1123            CPPDEFINES = ['TRACING_ON=1'],
1124            LINKFLAGS = Split(ldflags['opt']))
1125
1126# "Fast" binary
1127if 'fast' in needed_envs:
1128    disable_partial = \
1129            env.get('BROKEN_INCREMENTAL_LTO', False) and \
1130            GetOption('force_lto')
1131    makeEnv(env, 'fast', '.fo', strip = True,
1132            CCFLAGS = Split(ccflags['fast']),
1133            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1134            LINKFLAGS = Split(ldflags['fast']),
1135            disable_partial=disable_partial)
1136
1137# Profiled binary using gprof
1138if 'prof' in needed_envs:
1139    makeEnv(env, 'prof', '.po',
1140            CCFLAGS = Split(ccflags['prof']),
1141            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1142            LINKFLAGS = Split(ldflags['prof']))
1143
1144# Profiled binary using google-pprof
1145if 'perf' in needed_envs:
1146    makeEnv(env, 'perf', '.gpo',
1147            CCFLAGS = Split(ccflags['perf']),
1148            CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1149            LINKFLAGS = Split(ldflags['perf']))
1150