jobfile.py revision 2142 
1# Copyright (c) 2005-2006 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Nathan Binkert
28
29import sys
30
31class ternary(object):
32    def __new__(cls, *args):
33        if len(args) > 1:
34            raise TypeError, \
35                  '%s() takes at most 1 argument (%d given)' % \
36                  (cls.__name__, len(args))
37
38        if args:
39            if not isinstance(args[0], (bool, ternary)):
40                raise TypeError, \
41                      '%s() argument must be True, False, or Any' % \
42                      cls.__name__
43            return args[0]
44        return super(ternary, cls).__new__(cls)
45
46    def __bool__(self):
47        return True
48
49    def __neg__(self):
50        return self
51
52    def __eq__(self, other):
53        return True
54
55    def __ne__(self, other):
56        return False
57
58    def __str__(self):
59        return 'Any'
60
61    def __repr__(self):
62        return 'Any'
63
64Any = ternary()
65
66class Flags(dict):
67    def __init__(self, *args, **kwargs):
68        super(Flags, self).__init__()
69        self.update(*args, **kwargs)
70
71    def __getattr__(self, attr):
72        return self[attr]
73
74    def __setattr__(self, attr, value):
75        self[attr] = value
76
77    def __setitem__(self, item, value):
78        return super(Flags, self).__setitem__(item, ternary(value))
79
80    def __getitem__(self, item):
81        if item not in self:
82            return False
83        return super(Flags, self).__getitem__(item)
84
85    def update(self, *args, **kwargs):
86        for arg in args:
87            if isinstance(arg, Flags):
88                super(Flags, self).update(arg)
89            elif isinstance(arg, dict):
90                for key,val in kwargs.iteritems():
91                    self[key] = val
92            else:
93                raise AttributeError, \
94                      'flags not of type %s or %s, but %s' % \
95                      (Flags, dict, type(arg))
96
97        for key,val in kwargs.iteritems():
98            self[key] = val
99
100    def match(self, *args, **kwargs):
101        match = Flags(*args, **kwargs)
102
103        for key,value in match.iteritems():
104            if self[key] != value:
105                return False
106
107        return True
108
109def crossproduct(items):
110    if not isinstance(items, (list, tuple)):
111        raise AttributeError, 'crossproduct works only on sequences'
112
113    if not items:
114        yield None
115        return
116
117    current = items[0]
118    remainder = items[1:]
119
120    if not hasattr(current, '__iter__'):
121        current = [ current ]
122
123    for item in current:
124        for rem in crossproduct(remainder):
125            data = [ item ]
126            if rem:
127                data += rem
128            yield data
129
130def flatten(items):
131    if not isinstance(items, (list, tuple)):
132        yield items
133        return
134
135    for item in items:
136        for flat in flatten(item):
137            yield flat
138
139class Data(object):
140    def __init__(self, name, desc, **kwargs):
141        self.name = name
142        self.desc = desc
143        self.system = None
144        self.flags = Flags()
145        self.env = {}
146        for k,v in kwargs.iteritems():
147            setattr(self, k, v)
148
149    def update(self, obj):
150        if not isinstance(obj, Data):
151            raise AttributeError, "can only update from Data object"
152
153        self.env.update(obj.env)
154        self.flags.update(obj.flags)
155        if obj.system:
156            if self.system and self.system != obj.system:
157                raise AttributeError, \
158                      "conflicting values for system: '%s'/'%s'" % \
159                      (self.system, obj.system)
160            self.system = obj.system
161
162    def printinfo(self):
163        if self.name:
164            print 'name: %s' % self.name
165        if self.desc:
166            print 'desc: %s' % self.desc
167        if self.system:
168            print 'system: %s' % self.system
169
170    def printverbose(self):
171        print 'flags:'
172        keys = self.flags.keys()
173        keys.sort()
174        for key in keys:
175            print '    %s = %s' % (key, self.flags[key])
176        print 'env:'
177        keys = self.env.keys()
178        keys.sort()
179        for key in keys:
180            print '    %s = %s' % (key, self.env[key])
181        print
182
183    def __str__(self):
184        return self.name
185
186class Job(Data):
187    def __init__(self, options):
188        super(Job, self).__init__('', '')
189        self.setoptions(options)
190
191        self.checkpoint = False
192        opts = []
193        for opt in options:
194            cpt = opt.group.checkpoint
195            if not cpt:
196                self.checkpoint = True
197                continue
198            if isinstance(cpt, Option):
199                opt = cpt.clone(suboptions=False)
200            else:
201                opt = opt.clone(suboptions=False)
202
203            opts.append(opt)
204
205        if not opts:
206            self.checkpoint = False
207
208        if self.checkpoint:
209            self.checkpoint = Job(opts)
210
211    def clone(self):
212        return Job(self.options)
213
214    def __getattribute__(self, attr):
215        if attr == 'name':
216            names = [ ]
217            for opt in self.options:
218                if opt.name:
219                    names.append(opt.name)
220            return ':'.join(names)
221
222        if attr == 'desc':
223            descs = [ ]
224            for opt in self.options:
225                if opt.desc:
226                    descs.append(opt.desc)
227            return ', '.join(descs)
228
229        return super(Job, self).__getattribute__(attr)
230
231    def setoptions(self, options):
232        config = options[0].config
233        for opt in options:
234            if opt.config != config:
235                raise AttributeError, \
236                      "All options are not from the same Configuration"
237
238        self.config = config
239        self.groups = [ opt.group for opt in options ]
240        self.options = options
241
242        self.update(self.config)
243        for group in self.groups:
244            self.update(group)
245
246        for option in self.options:
247            self.update(option)
248            if option._suboption:
249                self.update(option._suboption)
250
251    def printinfo(self):
252        super(Job, self).printinfo()
253        if self.checkpoint:
254            print 'checkpoint: %s' % self.checkpoint.name
255        print 'config: %s' % self.config.name
256        print 'groups: %s' % [ g.name for g in self.groups ]
257        print 'options: %s' % [ o.name for o in self.options ]
258        super(Job, self).printverbose()
259
260class SubOption(Data):
261    def __init__(self, name, desc, **kwargs):
262        super(SubOption, self).__init__(name, desc, **kwargs)
263        self.number = None
264
265class Option(Data):
266    def __init__(self, name, desc, **kwargs):
267        super(Option, self).__init__(name, desc, **kwargs)
268        self._suboptions = []
269        self._suboption = None
270        self.number = None
271
272    def __getattribute__(self, attr):
273        if attr == 'name':
274            name = self.__dict__[attr]
275            if self._suboption is not None:
276                name = '%s:%s' % (name, self._suboption.name)
277            return name
278
279        if attr == 'desc':
280            desc = [ self.__dict__[attr] ]
281            if self._suboption is not None and self._suboption.desc:
282                desc.append(self._suboption.desc)
283            return ', '.join(desc)
284
285
286        return super(Option, self).__getattribute__(attr)
287
288    def suboption(self, name, desc, **kwargs):
289        subo = SubOption(name, desc, **kwargs)
290        subo.config = self.config
291        subo.group = self.group
292        subo.option = self
293        subo.number = len(self._suboptions)
294        self._suboptions.append(subo)
295        return subo
296
297    def clone(self, suboptions=True):
298        option = Option(self.__dict__['name'], self.__dict__['desc'])
299        option.update(self)
300        option.group = self.group
301        option.config = self.config
302        option.number = self.number
303        if suboptions:
304            option._suboptions.extend(self._suboptions)
305            option._suboption = self._suboption
306        return option
307
308    def subopts(self):
309        if not self._suboptions:
310            return [ self ]
311
312        subopts = []
313        for subo in self._suboptions:
314            option = self.clone()
315            option._suboption = subo
316            subopts.append(option)
317
318        return subopts
319
320    def printinfo(self):
321        super(Option, self).printinfo()
322        print 'config: %s' % self.config.name
323        super(Option, self).printverbose()
324
325class Group(Data):
326    def __init__(self, name, desc, **kwargs):
327        super(Group, self).__init__(name, desc, **kwargs)
328        self._options = []
329        self.checkpoint = False
330        self.number = None
331
332    def option(self, name, desc, **kwargs):
333        opt = Option(name, desc, **kwargs)
334        opt.config = self.config
335        opt.group = self
336        opt.number = len(self._options)
337        self._options.append(opt)
338        return opt
339
340    def options(self):
341        return self._options
342
343    def subopts(self):
344        subopts = []
345        for opt in self._options:
346            for subo in opt.subopts():
347                subopts.append(subo)
348        return subopts
349
350    def printinfo(self):
351        super(Group, self).printinfo()
352        print 'config: %s' % self.config.name
353        print 'options: %s' % [ o.name for o in self._options ]
354        super(Group, self).printverbose()
355
356class Configuration(Data):
357    def __init__(self, name, desc, **kwargs):
358        super(Configuration, self).__init__(name, desc, **kwargs)
359        self._groups = []
360        self._posfilters = []
361        self._negfilters = []
362
363    def group(self, name, desc, **kwargs):
364        grp = Group(name, desc, **kwargs)
365        grp.config = self
366        grp.number = len(self._groups)
367        self._groups.append(grp)
368        return grp
369
370    def groups(self, flags=Flags(), sign=True):
371        if not flags:
372            return self._groups
373
374        return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ]
375
376    def checkchildren(self, kids):
377        for kid in kids:
378            if kid.config != self:
379                raise AttributeError, "child from the wrong configuration"
380
381    def sortgroups(self, groups):
382        groups = [ (grp.number, grp) for grp in groups ]
383        groups.sort()
384        return [ grp[1] for grp in groups ]
385
386    def options(self, groups = None, checkpoint = False):
387        if groups is None:
388            groups = self._groups
389        self.checkchildren(groups)
390        groups = self.sortgroups(groups)
391        if checkpoint:
392            groups = [ grp for grp in groups if grp.checkpoint ]
393            optgroups = [ g.options() for g in groups ]
394        else:
395            optgroups = [ g.subopts() for g in groups ]
396        for options in crossproduct(optgroups):
397            for opt in options:
398                cpt = opt.group.checkpoint
399                if not isinstance(cpt, bool) and cpt != opt:
400                    if checkpoint:
401                        break
402                    else:
403                        yield options
404            else:
405                if checkpoint:
406                    yield options
407
408    def addfilter(self, filt, pos=True):
409        import re
410        filt = re.compile(filt)
411        if pos:
412            self._posfilters.append(filt)
413        else:
414            self._negfilters.append(filt)
415
416    def jobfilter(self, job):
417        for filt in self._negfilters:
418            if filt.match(job.name):
419                return False
420
421        if not self._posfilters:
422            return True
423
424        for filt in self._posfilters:
425            if filt.match(job.name):
426                return True
427
428        return False
429
430    def checkpoints(self, groups = None):
431        for options in self.options(groups, True):
432            job = Job(options)
433            if self.jobfilter(job):
434                yield job
435
436    def jobs(self, groups = None):
437        for options in self.options(groups, False):
438            job = Job(options)
439            if self.jobfilter(job):
440                yield job
441
442    def alljobs(self, groups = None):
443        for options in self.options(groups, True):
444            yield Job(options)
445        for options in self.options(groups, False):
446            yield Job(options)
447
448    def find(self, jobname):
449        for job in self.alljobs():
450            if job.name == jobname:
451                return job
452        else:
453            raise AttributeError, "job '%s' not found" % jobname
454
455    def job(self, options):
456        self.checkchildren(options)
457        options = [ (opt.group.number, opt) for opt in options ]
458        options.sort()
459        options = [ opt[1] for opt in options ]
460        job = Job(options)
461        return job
462
463    def printinfo(self):
464        super(Configuration, self).printinfo()
465        print 'groups: %s' % [ g.name for g in self._grouips ]
466        super(Configuration, self).printverbose()
467
468def JobFile(jobfile):
469    from os.path import expanduser, isfile, join as joinpath
470    filename = expanduser(jobfile)
471
472    # Can't find filename in the current path, search sys.path
473    if not isfile(filename):
474        for path in sys.path:
475            testname = joinpath(path, filename)
476            if isfile(testname):
477                filename = testname
478                break
479        else:
480            raise AttributeError, \
481                  "Could not find file '%s'" % jobfile
482
483    data = {}
484    execfile(filename, data)
485    if 'conf' not in data:
486        raise ImportError, 'cannot import name conf from %s' % jobfile
487    conf = data['conf']
488    import jobfile
489    if not isinstance(conf, Configuration):
490        raise AttributeError, \
491              'conf in jobfile: %s (%s) is not type %s' % \
492              (jobfile, type(conf), Configuration)
493    return conf
494
495if __name__ == '__main__':
496    from jobfile import *
497    import sys
498
499    usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0]
500
501    try:
502        import getopt
503        opts, args = getopt.getopt(sys.argv[1:], '-bcv')
504    except getopt.GetoptError:
505        sys.exit(usage)
506
507    if len(args) != 1:
508        raise AttributeError, usage
509
510    both = False
511    checkpoint = False
512    verbose = False
513    for opt,arg in opts:
514        if opt == '-b':
515            both = True
516            checkpoint = True
517        if opt == '-c':
518            checkpoint = True
519        if opt == '-v':
520            verbose = True
521
522    jobfile = args[0]
523    conf = JobFile(jobfile)
524
525    if both:
526        gen = conf.alljobs()
527    elif checkpoint:
528        gen = conf.checkpoints()
529    else:
530        gen = conf.jobs()
531
532    for job in gen:
533        if not verbose:
534            cpt = ''
535            if job.checkpoint:
536                cpt = job.checkpoint.name
537            print job.name, cpt
538        else:
539            job.printinfo()
540