1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Global configuration module which exposes two types of configuration
31variables:
32
331. config
342. constants (Also attached to the config variable as an attribute)
35
36The main motivation for this module is to have a centralized location for
37defaults and configuration by command line and files for the test framework.
38
39A secondary goal is to reduce programming errors by providing common constant
40strings and values as python attributes to simplify detection of typos.
41A simple typo in a string can take a lot of debugging to uncover the issue,
42attribute errors are easier to notice and most autocompletion systems detect
43them.
44
45The config variable is initialzed by calling :func:`initialize_config`.
46Before this point only ``constants`` will be availaible. This is to ensure
47that library function writers never accidentally get stale config attributes.
48
49Program arguments/flag arguments are available from the config as attributes.
50If an attribute was not set by the command line or the optional config file,
51then it will fallback to the `_defaults` value, if still the value is not
52found an AttributeError will be raised.
53
54:func define_defaults:
55    Provided by the config if the attribute is not found in the config or
56    commandline. For instance, if we are using the list command fixtures might
57    not be able to count on the build_dir being provided since we aren't going
58    to build anything.
59
60:var constants:
61    Values not directly exposed by the config, but are attached to the object
62    for centralized access. I.E. you can reach them with
63    :code:`config.constants.attribute`. These should be used for setting
64    common string names used across the test framework.
65    :code:`_defaults.build_dir = None` Once this module has been imported
66    constants should not be modified and their base attributes are frozen.
67'''
68import abc
69import argparse
70import copy
71import os
72import re
73
74from ConfigParser import ConfigParser
75from pickle import HIGHEST_PROTOCOL as highest_pickle_protocol
76
77from helper import absdirpath, AttrDict, FrozenAttrDict
78
79class UninitialzedAttributeException(Exception):
80    '''
81    Signals that an attribute in the config file was not initialized.
82    '''
83    pass
84
85class UninitializedConfigException(Exception):
86    '''
87    Signals that the config was not initialized before trying to access an
88    attribute.
89    '''
90    pass
91
92class TagRegex(object):
93    def __init__(self, include, regex):
94        self.include = include
95        self.regex = re.compile(regex)
96
97    def __str__(self):
98        type_ = 'Include' if self.include else 'Remove'
99        return '%10s: %s' % (type_, self.regex.pattern)
100
101class _Config(object):
102    _initialized = False
103
104    __shared_dict = {}
105
106    constants = AttrDict()
107    _defaults = AttrDict()
108    _config = {}
109
110    _cli_args = {}
111    _post_processors = {}
112
113    def __init__(self):
114        # This object will act as if it were a singleton.
115        self.__dict__ = self.__shared_dict
116
117    def _init(self, parser):
118        self._parse_commandline_args(parser)
119        self._run_post_processors()
120        self._initialized = True
121
122    def _init_with_dicts(self, config, defaults):
123        self._config = config
124        self._defaults = defaults
125        self._initialized = True
126
127    def _add_post_processor(self, attr, post_processor):
128        '''
129        :param attr: Attribute to pass to and recieve from the
130        :func:`post_processor`.
131
132        :param post_processor: A callback functions called in a chain to
133            perform additional setup for a config argument. Should return a
134            tuple containing the new value for the config attr.
135        '''
136        if attr not in self._post_processors:
137            self._post_processors[attr] = []
138        self._post_processors[attr].append(post_processor)
139
140    def _set(self, name, value):
141        self._config[name] = value
142
143    def _parse_commandline_args(self, parser):
144        args = parser.parse_args()
145
146        self._config_file_args = {}
147
148        for attr in dir(args):
149            # Ignore non-argument attributes.
150            if not attr.startswith('_'):
151                self._config_file_args[attr] = getattr(args, attr)
152        self._config.update(self._config_file_args)
153
154    def _run_post_processors(self):
155        for attr, callbacks in self._post_processors.items():
156            newval = self._lookup_val(attr)
157            for callback in callbacks:
158                newval = callback(newval)
159            if newval is not None:
160                newval = newval[0]
161            self._set(attr, newval)
162
163
164    def _lookup_val(self, attr):
165        '''
166        Get the attribute from the config or fallback to defaults.
167
168        :returns: If the value is not stored return None. Otherwise a tuple
169            containing the value.
170        '''
171        if attr in self._config:
172            return (self._config[attr],)
173        elif hasattr(self._defaults, attr):
174            return (getattr(self._defaults, attr),)
175
176    def __getattr__(self, attr):
177        if attr in dir(super(_Config, self)):
178            return getattr(super(_Config, self), attr)
179        elif not self._initialized:
180            raise UninitializedConfigException(
181                'Cannot directly access elements from the config before it is'
182                ' initialized')
183        else:
184            val = self._lookup_val(attr)
185            if val is not None:
186                return val[0]
187            else:
188                raise UninitialzedAttributeException(
189                    '%s was not initialzed in the config.' % attr)
190
191    def get_tags(self):
192        d = {typ: set(self.__getattr__(typ))
193            for typ in self.constants.supported_tags}
194        if any(map(lambda vals: bool(vals), d.values())):
195            return d
196        else:
197            return {}
198
199def define_defaults(defaults):
200    '''
201    Defaults are provided by the config if the attribute is not found in the
202    config or commandline. For instance, if we are using the list command
203    fixtures might not be able to count on the build_dir being provided since
204    we aren't going to build anything.
205    '''
206    defaults.base_dir = os.path.abspath(os.path.join(absdirpath(__file__),
207                                                      os.pardir,
208                                                      os.pardir))
209    defaults.result_path = os.path.join(os.getcwd(), '.testing-results')
210    defaults.list_only_failed = False
211
212def define_constants(constants):
213    '''
214    'constants' are values not directly exposed by the config, but are attached
215    to the object for centralized access. These should be used for setting
216    common string names used across the test framework. A simple typo in
217    a string can take a lot of debugging to uncover the issue, attribute errors
218    are easier to notice and most autocompletion systems detect them.
219    '''
220    constants.system_out_name = 'system-out'
221    constants.system_err_name = 'system-err'
222
223    constants.isa_tag_type = 'isa'
224    constants.x86_tag = 'X86'
225    constants.sparc_tag = 'SPARC'
226    constants.alpha_tag = 'ALPHA'
227    constants.riscv_tag = 'RISCV'
228    constants.arm_tag = 'ARM'
229    constants.mips_tag = 'MIPS'
230    constants.power_tag = 'POWER'
231    constants.null_tag = 'NULL'
232
233    constants.variant_tag_type = 'variant'
234    constants.opt_tag = 'opt'
235    constants.debug_tag = 'debug'
236    constants.fast_tag = 'fast'
237
238    constants.length_tag_type = 'length'
239    constants.quick_tag = 'quick'
240    constants.long_tag = 'long'
241
242    constants.supported_tags = {
243        constants.isa_tag_type : (
244            constants.x86_tag,
245            constants.sparc_tag,
246            constants.alpha_tag,
247            constants.riscv_tag,
248            constants.arm_tag,
249            constants.mips_tag,
250            constants.power_tag,
251            constants.null_tag,
252            ),
253        constants.variant_tag_type: (
254            constants.opt_tag,
255            constants.debug_tag,
256            constants.fast_tag,
257        ),
258        constants.length_tag_type: (
259            constants.quick_tag,
260            constants.long_tag,
261        ),
262    }
263
264    constants.supported_isas = constants.supported_tags['isa']
265    constants.supported_variants = constants.supported_tags['variant']
266    constants.supported_lengths = constants.supported_tags['length']
267
268    constants.tempdir_fixture_name = 'tempdir'
269    constants.gem5_simulation_stderr = 'simerr'
270    constants.gem5_simulation_stdout = 'simout'
271    constants.gem5_simulation_stats = 'stats.txt'
272    constants.gem5_simulation_config_ini = 'config.ini'
273    constants.gem5_simulation_config_json = 'config.json'
274    constants.gem5_returncode_fixture_name = 'gem5-returncode'
275    constants.gem5_binary_fixture_name = 'gem5'
276    constants.xml_filename = 'results.xml'
277    constants.pickle_filename = 'results.pickle'
278    constants.pickle_protocol = highest_pickle_protocol
279
280    # The root directory which all test names will be based off of.
281    constants.testing_base = absdirpath(os.path.join(absdirpath(__file__),
282                                                     os.pardir))
283
284def define_post_processors(config):
285    '''
286    post_processors are used to do final configuration of variables. This is
287    useful if there is a dynamically set default, or some function that needs
288    to be applied after parsing in order to set a configration value.
289
290    Post processors must accept a single argument that will either be a tuple
291    containing the already set config value or ``None`` if the config value
292    has not been set to anything. They must return the modified value in the
293    same format.
294    '''
295
296    def set_default_build_dir(build_dir):
297        '''
298        Post-processor to set the default build_dir based on the base_dir.
299
300        .. seealso :func:`~_Config._add_post_processor`
301        '''
302        if not build_dir or build_dir[0] is None:
303            base_dir = config._lookup_val('base_dir')[0]
304            build_dir = (os.path.join(base_dir, 'build'),)
305        return build_dir
306
307    def fix_verbosity_hack(verbose):
308        return (verbose[0].val,)
309
310    def threads_as_int(threads):
311        if threads is not None:
312            return (int(threads[0]),)
313
314    def test_threads_as_int(test_threads):
315        if test_threads is not None:
316            return (int(test_threads[0]),)
317
318    def default_isa(isa):
319        if not isa[0]:
320            return [constants.supported_tags[constants.isa_tag_type]]
321        else:
322            return isa
323
324    def default_variant(variant):
325        if not variant[0]:
326            # Default variant is only opt. No need to run tests with multiple
327            # different compilation targets
328            return [[constants.opt_tag]]
329        else:
330            return variant
331
332    def default_length(length):
333        if not length[0]:
334            return [[constants.quick_tag]]
335        else:
336            return length
337
338    def compile_tag_regex(positional_tags):
339        if not positional_tags:
340            return positional_tags
341        else:
342            new_positional_tags_list = []
343            positional_tags = positional_tags[0]
344
345            for flag, regex in positional_tags:
346                if flag == 'exclude_tags':
347                    tag_regex = TagRegex(False, regex)
348                elif flag  == 'include_tags':
349                    tag_regex = TagRegex(True, regex)
350                else:
351                    raise ValueError('Unsupported flag.')
352                new_positional_tags_list.append(tag_regex)
353
354            return (new_positional_tags_list,)
355
356    config._add_post_processor('build_dir', set_default_build_dir)
357    config._add_post_processor('verbose', fix_verbosity_hack)
358    config._add_post_processor('isa', default_isa)
359    config._add_post_processor('variant', default_variant)
360    config._add_post_processor('length', default_length)
361    config._add_post_processor('threads', threads_as_int)
362    config._add_post_processor('test_threads', test_threads_as_int)
363    config._add_post_processor(StorePositionalTagsAction.position_kword,
364                               compile_tag_regex)
365class Argument(object):
366    '''
367    Class represents a cli argument/flag for a argparse parser.
368
369    :attr name: The long name of this object that will be stored in the arg
370        output by the final parser.
371    '''
372    def __init__(self, *flags, **kwargs):
373        self.flags = flags
374        self.kwargs = kwargs
375
376        if len(flags) == 0:
377            raise ValueError("Need at least one argument.")
378        elif 'dest' in kwargs:
379            self.name = kwargs['dest']
380        elif len(flags) > 1 or flags[0].startswith('-'):
381            for flag in flags:
382                if not flag.startswith('-'):
383                    raise ValueError("invalid option string %s: must start"
384                    "with a character '-'" % flag)
385
386                if flag.startswith('--'):
387                    if not hasattr(self, 'name'):
388                        self.name = flag.lstrip('-')
389
390        if not hasattr(self, 'name'):
391            self.name = flags[0].lstrip('-')
392        self.name = self.name.replace('-', '_')
393
394    def add_to(self, parser):
395        '''Add this argument to the given parser.'''
396        parser.add_argument(*self.flags, **self.kwargs)
397
398    def copy(self):
399        '''Copy this argument so you might modify any of its kwargs.'''
400        return copy.deepcopy(self)
401
402
403class _StickyInt:
404    '''
405    A class that is used to cheat the verbosity count incrementer by
406    pretending to be an int. This makes the int stay on the heap and eat other
407    real numbers when they are added to it.
408
409    We use this so we can allow the verbose flag to be provided before or after
410    the subcommand. This likely has no utility outside of this use case.
411    '''
412    def __init__(self, val=0):
413        self.val = val
414        self.type = int
415    def __add__(self, other):
416        self.val += other
417        return self
418
419common_args = NotImplemented
420
421class StorePositionAction(argparse.Action):
422    '''Base class for classes wishing to create namespaces where
423    arguments are stored in the order provided via the command line.
424    '''
425    position_kword = 'positional'
426
427    def __call__(self, parser, namespace, values, option_string=None):
428        if not self.position_kword in namespace:
429            setattr(namespace, self.position_kword, [])
430        previous = getattr(namespace, self.position_kword)
431        previous.append((self.dest, values))
432        setattr(namespace, self.position_kword, previous)
433
434class StorePositionalTagsAction(StorePositionAction):
435    position_kword = 'tag_filters'
436
437def define_common_args(config):
438    '''
439    Common args are arguments which are likely to be simular between different
440    subcommands, so they are available to all by placing their definitions
441    here.
442    '''
443    global common_args
444
445    # A list of common arguments/flags used across cli parsers.
446    common_args = [
447        Argument(
448            'directory',
449            nargs='?',
450            default=os.getcwd(),
451            help='Directory to start searching for tests in'),
452        Argument(
453            '--exclude-tags',
454            action=StorePositionalTagsAction,
455            help='A tag comparison used to select tests.'),
456        Argument(
457            '--include-tags',
458            action=StorePositionalTagsAction,
459            help='A tag comparison used to select tests.'),
460        Argument(
461            '--isa',
462            action='append',
463            default=[],
464            help="Only tests that are valid with one of these ISAs. "
465                 "Comma separated."),
466        Argument(
467            '--variant',
468            action='append',
469            default=[],
470            help="Only tests that are valid with one of these binary variants"
471                 "(e.g., opt, debug). Comma separated."),
472        Argument(
473            '--length',
474            action='append',
475            default=[],
476            help="Only tests that are one of these lengths. Comma separated."),
477        Argument(
478            '--uid',
479            action='store',
480            default=None,
481            help='UID of a specific test item to run.'),
482        Argument(
483            '--build-dir',
484            action='store',
485            help='Build directory for SCons'),
486        Argument(
487            '--base-dir',
488            action='store',
489            default=config._defaults.base_dir,
490            help='Directory to change to in order to exec scons.'),
491        Argument(
492            '-j', '--threads',
493            action='store',
494            default=1,
495            help='Number of threads to run SCons with.'),
496        Argument(
497            '-t', '--test-threads',
498            action='store',
499            default=1,
500            help='Number of threads to spawn to run concurrent tests with.'),
501        Argument(
502            '-v',
503            action='count',
504            dest='verbose',
505            default=_StickyInt(),
506            help='Increase verbosity'),
507        Argument(
508            '--config-path',
509            action='store',
510            default=os.getcwd(),
511            help='Path to read a testing.ini config in'
512        ),
513        Argument(
514            '--skip-build',
515            action='store_true',
516            default=False,
517            help='Skip the building component of SCons targets.'
518        ),
519        Argument(
520            '--result-path',
521            action='store',
522            help='The path to store results in.'
523        ),
524    ]
525
526    # NOTE: There is a limitation which arises due to this format. If you have
527    # multiple arguments with the same name only the final one in the list
528    # will be saved.
529    #
530    # e.g. if you have a -v argument which increments verbosity level and
531    # a separate --verbose flag which 'store's verbosity level. the final
532    # one in the list will be saved.
533    common_args = AttrDict({arg.name:arg for arg in common_args})
534
535
536class ArgParser(object):
537    __metaclass__ = abc.ABCMeta
538
539    def __init__(self, parser):
540        # Copy public methods of the parser.
541        for attr in dir(parser):
542            if not attr.startswith('_'):
543                setattr(self, attr, getattr(parser, attr))
544        self.parser = parser
545        self.add_argument = self.parser.add_argument
546
547        # Argument will be added to all parsers and subparsers.
548        common_args.verbose.add_to(parser)
549
550
551class CommandParser(ArgParser):
552    '''
553    Main parser which parses command strings and uses those to direct to
554    a subparser.
555    '''
556    def __init__(self):
557        parser = argparse.ArgumentParser()
558        super(CommandParser, self).__init__(parser)
559        self.subparser = self.add_subparsers(dest='command')
560
561
562class RunParser(ArgParser):
563    '''
564    Parser for the \'run\' command.
565    '''
566    def __init__(self, subparser):
567        parser = subparser.add_parser(
568            'run',
569            help='''Run Tests.'''
570        )
571
572        super(RunParser, self).__init__(parser)
573
574        common_args.uid.add_to(parser)
575        common_args.skip_build.add_to(parser)
576        common_args.directory.add_to(parser)
577        common_args.build_dir.add_to(parser)
578        common_args.base_dir.add_to(parser)
579        common_args.threads.add_to(parser)
580        common_args.test_threads.add_to(parser)
581        common_args.isa.add_to(parser)
582        common_args.variant.add_to(parser)
583        common_args.length.add_to(parser)
584        common_args.include_tags.add_to(parser)
585        common_args.exclude_tags.add_to(parser)
586
587
588class ListParser(ArgParser):
589    '''
590    Parser for the \'list\' command.
591    '''
592    def __init__(self, subparser):
593        parser = subparser.add_parser(
594            'list',
595            help='''List and query test metadata.'''
596        )
597        super(ListParser, self).__init__(parser)
598
599        Argument(
600            '--suites',
601            action='store_true',
602            default=False,
603            help='List all test suites.'
604        ).add_to(parser)
605        Argument(
606            '--tests',
607            action='store_true',
608            default=False,
609            help='List all test cases.'
610        ).add_to(parser)
611        Argument(
612            '--fixtures',
613            action='store_true',
614            default=False,
615            help='List all fixtures.'
616        ).add_to(parser)
617        Argument(
618            '--all-tags',
619            action='store_true',
620            default=False,
621            help='List all tags.'
622        ).add_to(parser)
623        Argument(
624            '-q',
625            dest='quiet',
626            action='store_true',
627            default=False,
628            help='Quiet output (machine readable).'
629        ).add_to(parser)
630
631        common_args.directory.add_to(parser)
632        common_args.isa.add_to(parser)
633        common_args.variant.add_to(parser)
634        common_args.length.add_to(parser)
635        common_args.include_tags.add_to(parser)
636        common_args.exclude_tags.add_to(parser)
637
638
639class RerunParser(ArgParser):
640    def __init__(self, subparser):
641        parser = subparser.add_parser(
642            'rerun',
643            help='''Rerun failed tests.'''
644        )
645        super(RerunParser, self).__init__(parser)
646
647        common_args.skip_build.add_to(parser)
648        common_args.directory.add_to(parser)
649        common_args.build_dir.add_to(parser)
650        common_args.base_dir.add_to(parser)
651        common_args.threads.add_to(parser)
652        common_args.test_threads.add_to(parser)
653        common_args.isa.add_to(parser)
654        common_args.variant.add_to(parser)
655        common_args.length.add_to(parser)
656
657config = _Config()
658define_constants(config.constants)
659
660# Constants are directly exposed and available once this module is created.
661# All constants MUST be defined before this point.
662config.constants = FrozenAttrDict(config.constants.__dict__)
663constants = config.constants
664
665'''
666This config object is the singleton config object available throughout the
667framework.
668'''
669def initialize_config():
670    '''
671    Parse the commandline arguments and setup the config varibles.
672    '''
673    global config
674
675    # Setup constants and defaults
676    define_defaults(config._defaults)
677    define_post_processors(config)
678    define_common_args(config)
679
680    # Setup parser and subcommands
681    baseparser = CommandParser()
682    runparser = RunParser(baseparser.subparser)
683    listparser = ListParser(baseparser.subparser)
684    rerunparser = RerunParser(baseparser.subparser)
685
686    # Initialize the config by parsing args and running callbacks.
687    config._init(baseparser)
688