jobfile.py revision 2142
1# Copyright (c) 2005-2006 The Regents of The University of Michigan 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Nathan Binkert 28 29import sys 30 31class ternary(object): 32 def __new__(cls, *args): 33 if len(args) > 1: 34 raise TypeError, \ 35 '%s() takes at most 1 argument (%d given)' % \ 36 (cls.__name__, len(args)) 37 38 if args: 39 if not isinstance(args[0], (bool, ternary)): 40 raise TypeError, \ 41 '%s() argument must be True, False, or Any' % \ 42 cls.__name__ 43 return args[0] 44 return super(ternary, cls).__new__(cls) 45 46 def __bool__(self): 47 return True 48 49 def __neg__(self): 50 return self 51 52 def __eq__(self, other): 53 return True 54 55 def __ne__(self, other): 56 return False 57 58 def __str__(self): 59 return 'Any' 60 61 def __repr__(self): 62 return 'Any' 63 64Any = ternary() 65 66class Flags(dict): 67 def __init__(self, *args, **kwargs): 68 super(Flags, self).__init__() 69 self.update(*args, **kwargs) 70 71 def __getattr__(self, attr): 72 return self[attr] 73 74 def __setattr__(self, attr, value): 75 self[attr] = value 76 77 def __setitem__(self, item, value): 78 return super(Flags, self).__setitem__(item, ternary(value)) 79 80 def __getitem__(self, item): 81 if item not in self: 82 return False 83 return super(Flags, self).__getitem__(item) 84 85 def update(self, *args, **kwargs): 86 for arg in args: 87 if isinstance(arg, Flags): 88 super(Flags, self).update(arg) 89 elif isinstance(arg, dict): 90 for key,val in kwargs.iteritems(): 91 self[key] = val 92 else: 93 raise AttributeError, \ 94 'flags not of type %s or %s, but %s' % \ 95 (Flags, dict, type(arg)) 96 97 for key,val in kwargs.iteritems(): 98 self[key] = val 99 100 def match(self, *args, **kwargs): 101 match = Flags(*args, **kwargs) 102 103 for key,value in match.iteritems(): 104 if self[key] != value: 105 return False 106 107 return True 108 109def crossproduct(items): 110 if not isinstance(items, (list, tuple)): 111 raise AttributeError, 'crossproduct works only on sequences' 112 113 if not items: 114 yield None 115 return 116 117 current = items[0] 118 remainder = items[1:] 119 120 if not hasattr(current, '__iter__'): 121 current = [ current ] 122 123 for item in current: 124 for rem in crossproduct(remainder): 125 data = [ item ] 126 if rem: 127 data += rem 128 yield data 129 130def flatten(items): 131 if not isinstance(items, (list, tuple)): 132 yield items 133 return 134 135 for item in items: 136 for flat in flatten(item): 137 yield flat 138 139class Data(object): 140 def __init__(self, name, desc, **kwargs): 141 self.name = name 142 self.desc = desc 143 self.system = None 144 self.flags = Flags() 145 self.env = {} 146 for k,v in kwargs.iteritems(): 147 setattr(self, k, v) 148 149 def update(self, obj): 150 if not isinstance(obj, Data): 151 raise AttributeError, "can only update from Data object" 152 153 self.env.update(obj.env) 154 self.flags.update(obj.flags) 155 if obj.system: 156 if self.system and self.system != obj.system: 157 raise AttributeError, \ 158 "conflicting values for system: '%s'/'%s'" % \ 159 (self.system, obj.system) 160 self.system = obj.system 161 162 def printinfo(self): 163 if self.name: 164 print 'name: %s' % self.name 165 if self.desc: 166 print 'desc: %s' % self.desc 167 if self.system: 168 print 'system: %s' % self.system 169 170 def printverbose(self): 171 print 'flags:' 172 keys = self.flags.keys() 173 keys.sort() 174 for key in keys: 175 print ' %s = %s' % (key, self.flags[key]) 176 print 'env:' 177 keys = self.env.keys() 178 keys.sort() 179 for key in keys: 180 print ' %s = %s' % (key, self.env[key]) 181 print 182 183 def __str__(self): 184 return self.name 185 186class Job(Data): 187 def __init__(self, options): 188 super(Job, self).__init__('', '') 189 self.setoptions(options) 190 191 self.checkpoint = False 192 opts = [] 193 for opt in options: 194 cpt = opt.group.checkpoint 195 if not cpt: 196 self.checkpoint = True 197 continue 198 if isinstance(cpt, Option): 199 opt = cpt.clone(suboptions=False) 200 else: 201 opt = opt.clone(suboptions=False) 202 203 opts.append(opt) 204 205 if not opts: 206 self.checkpoint = False 207 208 if self.checkpoint: 209 self.checkpoint = Job(opts) 210 211 def clone(self): 212 return Job(self.options) 213 214 def __getattribute__(self, attr): 215 if attr == 'name': 216 names = [ ] 217 for opt in self.options: 218 if opt.name: 219 names.append(opt.name) 220 return ':'.join(names) 221 222 if attr == 'desc': 223 descs = [ ] 224 for opt in self.options: 225 if opt.desc: 226 descs.append(opt.desc) 227 return ', '.join(descs) 228 229 return super(Job, self).__getattribute__(attr) 230 231 def setoptions(self, options): 232 config = options[0].config 233 for opt in options: 234 if opt.config != config: 235 raise AttributeError, \ 236 "All options are not from the same Configuration" 237 238 self.config = config 239 self.groups = [ opt.group for opt in options ] 240 self.options = options 241 242 self.update(self.config) 243 for group in self.groups: 244 self.update(group) 245 246 for option in self.options: 247 self.update(option) 248 if option._suboption: 249 self.update(option._suboption) 250 251 def printinfo(self): 252 super(Job, self).printinfo() 253 if self.checkpoint: 254 print 'checkpoint: %s' % self.checkpoint.name 255 print 'config: %s' % self.config.name 256 print 'groups: %s' % [ g.name for g in self.groups ] 257 print 'options: %s' % [ o.name for o in self.options ] 258 super(Job, self).printverbose() 259 260class SubOption(Data): 261 def __init__(self, name, desc, **kwargs): 262 super(SubOption, self).__init__(name, desc, **kwargs) 263 self.number = None 264 265class Option(Data): 266 def __init__(self, name, desc, **kwargs): 267 super(Option, self).__init__(name, desc, **kwargs) 268 self._suboptions = [] 269 self._suboption = None 270 self.number = None 271 272 def __getattribute__(self, attr): 273 if attr == 'name': 274 name = self.__dict__[attr] 275 if self._suboption is not None: 276 name = '%s:%s' % (name, self._suboption.name) 277 return name 278 279 if attr == 'desc': 280 desc = [ self.__dict__[attr] ] 281 if self._suboption is not None and self._suboption.desc: 282 desc.append(self._suboption.desc) 283 return ', '.join(desc) 284 285 286 return super(Option, self).__getattribute__(attr) 287 288 def suboption(self, name, desc, **kwargs): 289 subo = SubOption(name, desc, **kwargs) 290 subo.config = self.config 291 subo.group = self.group 292 subo.option = self 293 subo.number = len(self._suboptions) 294 self._suboptions.append(subo) 295 return subo 296 297 def clone(self, suboptions=True): 298 option = Option(self.__dict__['name'], self.__dict__['desc']) 299 option.update(self) 300 option.group = self.group 301 option.config = self.config 302 option.number = self.number 303 if suboptions: 304 option._suboptions.extend(self._suboptions) 305 option._suboption = self._suboption 306 return option 307 308 def subopts(self): 309 if not self._suboptions: 310 return [ self ] 311 312 subopts = [] 313 for subo in self._suboptions: 314 option = self.clone() 315 option._suboption = subo 316 subopts.append(option) 317 318 return subopts 319 320 def printinfo(self): 321 super(Option, self).printinfo() 322 print 'config: %s' % self.config.name 323 super(Option, self).printverbose() 324 325class Group(Data): 326 def __init__(self, name, desc, **kwargs): 327 super(Group, self).__init__(name, desc, **kwargs) 328 self._options = [] 329 self.checkpoint = False 330 self.number = None 331 332 def option(self, name, desc, **kwargs): 333 opt = Option(name, desc, **kwargs) 334 opt.config = self.config 335 opt.group = self 336 opt.number = len(self._options) 337 self._options.append(opt) 338 return opt 339 340 def options(self): 341 return self._options 342 343 def subopts(self): 344 subopts = [] 345 for opt in self._options: 346 for subo in opt.subopts(): 347 subopts.append(subo) 348 return subopts 349 350 def printinfo(self): 351 super(Group, self).printinfo() 352 print 'config: %s' % self.config.name 353 print 'options: %s' % [ o.name for o in self._options ] 354 super(Group, self).printverbose() 355 356class Configuration(Data): 357 def __init__(self, name, desc, **kwargs): 358 super(Configuration, self).__init__(name, desc, **kwargs) 359 self._groups = [] 360 self._posfilters = [] 361 self._negfilters = [] 362 363 def group(self, name, desc, **kwargs): 364 grp = Group(name, desc, **kwargs) 365 grp.config = self 366 grp.number = len(self._groups) 367 self._groups.append(grp) 368 return grp 369 370 def groups(self, flags=Flags(), sign=True): 371 if not flags: 372 return self._groups 373 374 return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ] 375 376 def checkchildren(self, kids): 377 for kid in kids: 378 if kid.config != self: 379 raise AttributeError, "child from the wrong configuration" 380 381 def sortgroups(self, groups): 382 groups = [ (grp.number, grp) for grp in groups ] 383 groups.sort() 384 return [ grp[1] for grp in groups ] 385 386 def options(self, groups = None, checkpoint = False): 387 if groups is None: 388 groups = self._groups 389 self.checkchildren(groups) 390 groups = self.sortgroups(groups) 391 if checkpoint: 392 groups = [ grp for grp in groups if grp.checkpoint ] 393 optgroups = [ g.options() for g in groups ] 394 else: 395 optgroups = [ g.subopts() for g in groups ] 396 for options in crossproduct(optgroups): 397 for opt in options: 398 cpt = opt.group.checkpoint 399 if not isinstance(cpt, bool) and cpt != opt: 400 if checkpoint: 401 break 402 else: 403 yield options 404 else: 405 if checkpoint: 406 yield options 407 408 def addfilter(self, filt, pos=True): 409 import re 410 filt = re.compile(filt) 411 if pos: 412 self._posfilters.append(filt) 413 else: 414 self._negfilters.append(filt) 415 416 def jobfilter(self, job): 417 for filt in self._negfilters: 418 if filt.match(job.name): 419 return False 420 421 if not self._posfilters: 422 return True 423 424 for filt in self._posfilters: 425 if filt.match(job.name): 426 return True 427 428 return False 429 430 def checkpoints(self, groups = None): 431 for options in self.options(groups, True): 432 job = Job(options) 433 if self.jobfilter(job): 434 yield job 435 436 def jobs(self, groups = None): 437 for options in self.options(groups, False): 438 job = Job(options) 439 if self.jobfilter(job): 440 yield job 441 442 def alljobs(self, groups = None): 443 for options in self.options(groups, True): 444 yield Job(options) 445 for options in self.options(groups, False): 446 yield Job(options) 447 448 def find(self, jobname): 449 for job in self.alljobs(): 450 if job.name == jobname: 451 return job 452 else: 453 raise AttributeError, "job '%s' not found" % jobname 454 455 def job(self, options): 456 self.checkchildren(options) 457 options = [ (opt.group.number, opt) for opt in options ] 458 options.sort() 459 options = [ opt[1] for opt in options ] 460 job = Job(options) 461 return job 462 463 def printinfo(self): 464 super(Configuration, self).printinfo() 465 print 'groups: %s' % [ g.name for g in self._grouips ] 466 super(Configuration, self).printverbose() 467 468def JobFile(jobfile): 469 from os.path import expanduser, isfile, join as joinpath 470 filename = expanduser(jobfile) 471 472 # Can't find filename in the current path, search sys.path 473 if not isfile(filename): 474 for path in sys.path: 475 testname = joinpath(path, filename) 476 if isfile(testname): 477 filename = testname 478 break 479 else: 480 raise AttributeError, \ 481 "Could not find file '%s'" % jobfile 482 483 data = {} 484 execfile(filename, data) 485 if 'conf' not in data: 486 raise ImportError, 'cannot import name conf from %s' % jobfile 487 conf = data['conf'] 488 import jobfile 489 if not isinstance(conf, Configuration): 490 raise AttributeError, \ 491 'conf in jobfile: %s (%s) is not type %s' % \ 492 (jobfile, type(conf), Configuration) 493 return conf 494 495if __name__ == '__main__': 496 from jobfile import * 497 import sys 498 499 usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0] 500 501 try: 502 import getopt 503 opts, args = getopt.getopt(sys.argv[1:], '-bcv') 504 except getopt.GetoptError: 505 sys.exit(usage) 506 507 if len(args) != 1: 508 raise AttributeError, usage 509 510 both = False 511 checkpoint = False 512 verbose = False 513 for opt,arg in opts: 514 if opt == '-b': 515 both = True 516 checkpoint = True 517 if opt == '-c': 518 checkpoint = True 519 if opt == '-v': 520 verbose = True 521 522 jobfile = args[0] 523 conf = JobFile(jobfile) 524 525 if both: 526 gen = conf.alljobs() 527 elif checkpoint: 528 gen = conf.checkpoints() 529 else: 530 gen = conf.jobs() 531 532 for job in gen: 533 if not verbose: 534 cpt = '' 535 if job.checkpoint: 536 cpt = job.checkpoint.name 537 print job.name, cpt 538 else: 539 job.printinfo() 540