jobfile.py revision 1881
1# Copyright (c) 2005 The Regents of The University of Michigan 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Nathan Binkert 28 29import sys 30 31class ternary(object): 32 def __new__(cls, *args): 33 if len(args) > 1: 34 raise TypeError, \ 35 '%s() takes at most 1 argument (%d given)' % \ 36 (cls.__name__, len(args)) 37 38 if args: 39 if not isinstance(args[0], (bool, ternary)): 40 raise TypeError, \ 41 '%s() argument must be True, False, or Any' % \ 42 cls.__name__ 43 return args[0] 44 return super(ternary, cls).__new__(cls) 45 46 def __bool__(self): 47 return True 48 49 def __neg__(self): 50 return self 51 52 def __eq__(self, other): 53 return True 54 55 def __ne__(self, other): 56 return False 57 58 def __str__(self): 59 return 'Any' 60 61 def __repr__(self): 62 return 'Any' 63 64Any = ternary() 65 66class Flags(dict): 67 def __init__(self, *args, **kwargs): 68 super(Flags, self).__init__() 69 self.update(*args, **kwargs) 70 71 def __getattr__(self, attr): 72 return self[attr] 73 74 def __setattr__(self, attr, value): 75 self[attr] = value 76 77 def __setitem__(self, item, value): 78 return super(Flags, self).__setitem__(item, ternary(value)) 79 80 def __getitem__(self, item): 81 if item not in self: 82 return False 83 return super(Flags, self).__getitem__(item) 84 85 def update(self, *args, **kwargs): 86 for arg in args: 87 if isinstance(arg, Flags): 88 super(Flags, self).update(arg) 89 elif isinstance(arg, dict): 90 for key,val in kwargs.iteritems(): 91 self[key] = val 92 else: 93 raise AttributeError, \ 94 'flags not of type %s or %s, but %s' % \ 95 (Flags, dict, type(arg)) 96 97 for key,val in kwargs.iteritems(): 98 self[key] = val 99 100 def match(self, *args, **kwargs): 101 match = Flags(*args, **kwargs) 102 103 for key,value in match.iteritems(): 104 if self[key] != value: 105 return False 106 107 return True 108 109def crossproduct(items): 110 if not isinstance(items, (list, tuple)): 111 raise AttributeError, 'crossproduct works only on sequences' 112 113 if not items: 114 yield None 115 return 116 117 current = items[0] 118 remainder = items[1:] 119 120 if not hasattr(current, '__iter__'): 121 current = [ current ] 122 123 for item in current: 124 for rem in crossproduct(remainder): 125 data = [ item ] 126 if rem: 127 data += rem 128 yield data 129 130def flatten(items): 131 if not isinstance(items, (list, tuple)): 132 yield items 133 return 134 135 for item in items: 136 for flat in flatten(item): 137 yield flat 138 139class Data(object): 140 def __init__(self, name, desc, **kwargs): 141 self.name = name 142 self.desc = desc 143 self.system = None 144 self.flags = Flags() 145 self.env = {} 146 for k,v in kwargs.iteritems(): 147 setattr(self, k, v) 148 149 def update(self, obj): 150 if not isinstance(obj, Data): 151 raise AttributeError, "can only update from Data object" 152 153 self.env.update(obj.env) 154 self.flags.update(obj.flags) 155 if obj.system: 156 if self.system and self.system != obj.system: 157 raise AttributeError, \ 158 "conflicting values for system: '%s'/'%s'" % \ 159 (self.system, obj.system) 160 self.system = obj.system 161 162 def printinfo(self): 163 if self.name: 164 print 'name: %s' % self.name 165 if self.desc: 166 print 'desc: %s' % self.desc 167 if self.system: 168 print 'system: %s' % self.system 169 170 def printverbose(self): 171 print 'flags:' 172 keys = self.flags.keys() 173 keys.sort() 174 for key in keys: 175 print ' %s = %s' % (key, self.flags[key]) 176 print 'env:' 177 keys = self.env.keys() 178 keys.sort() 179 for key in keys: 180 print ' %s = %s' % (key, self.env[key]) 181 print 182 183 def __str__(self): 184 return self.name 185 186class Job(Data): 187 def __init__(self, options): 188 super(Job, self).__init__('', '') 189 self.setoptions(options) 190 191 self.checkpoint = False 192 opts = [] 193 for opt in options: 194 cpt = opt.group.checkpoint 195 if not cpt: 196 self.checkpoint = True 197 continue 198 if isinstance(cpt, Option): 199 opt = cpt.clone(suboptions=False) 200 else: 201 opt = opt.clone(suboptions=False) 202 203 opts.append(opt) 204 205 if not opts: 206 self.checkpoint = False 207 208 if self.checkpoint: 209 self.checkpoint = Job(opts) 210 211 def clone(self): 212 return Job(self.options) 213 214 def __getattribute__(self, attr): 215 if attr == 'name': 216 names = [ ] 217 for opt in self.options: 218 if opt.name: 219 names.append(opt.name) 220 return ':'.join(names) 221 222 if attr == 'desc': 223 descs = [ ] 224 for opt in self.options: 225 if opt.desc: 226 descs.append(opt.desc) 227 return ', '.join(descs) 228 229 return super(Job, self).__getattribute__(attr) 230 231 def setoptions(self, options): 232 config = options[0].config 233 for opt in options: 234 if opt.config != config: 235 raise AttributeError, \ 236 "All options are not from the same Configuration" 237 238 self.config = config 239 self.groups = [ opt.group for opt in options ] 240 self.options = options 241 242 self.update(self.config) 243 for group in self.groups: 244 self.update(group) 245 246 for option in self.options: 247 self.update(option) 248 if option._suboption: 249 self.update(option._suboption) 250 251 def printinfo(self): 252 super(Job, self).printinfo() 253 if self.checkpoint: 254 print 'checkpoint: %s' % self.checkpoint.name 255 print 'config: %s' % self.config.name 256 print 'groups: %s' % [ g.name for g in self.groups ] 257 print 'options: %s' % [ o.name for o in self.options ] 258 super(Job, self).printverbose() 259 260class SubOption(Data): 261 def __init__(self, name, desc, **kwargs): 262 super(SubOption, self).__init__(name, desc, **kwargs) 263 self.number = None 264 265class Option(Data): 266 def __init__(self, name, desc, **kwargs): 267 super(Option, self).__init__(name, desc, **kwargs) 268 self._suboptions = [] 269 self._suboption = None 270 self.number = None 271 272 def __getattribute__(self, attr): 273 if attr == 'name': 274 name = self.__dict__[attr] 275 if self._suboption is not None: 276 name = '%s:%s' % (name, self._suboption.name) 277 return name 278 279 if attr == 'desc': 280 desc = self.__dict__[attr] 281 if self._suboption is not None: 282 desc = '%s, %s' % (desc, self._suboption.desc) 283 return desc 284 285 return super(Option, self).__getattribute__(attr) 286 287 def suboption(self, name, desc, **kwargs): 288 subo = SubOption(name, desc, **kwargs) 289 subo.config = self.config 290 subo.group = self.group 291 subo.option = self 292 subo.number = len(self._suboptions) 293 self._suboptions.append(subo) 294 return subo 295 296 def clone(self, suboptions=True): 297 option = Option(self.__dict__['name'], self.__dict__['desc']) 298 option.update(self) 299 option.group = self.group 300 option.config = self.config 301 option.number = self.number 302 if suboptions: 303 option._suboptions.extend(self._suboptions) 304 option._suboption = self._suboption 305 return option 306 307 def subopts(self): 308 if not self._suboptions: 309 return [ self ] 310 311 subopts = [] 312 for subo in self._suboptions: 313 option = self.clone() 314 option._suboption = subo 315 subopts.append(option) 316 317 return subopts 318 319 def printinfo(self): 320 super(Option, self).printinfo() 321 print 'config: %s' % self.config.name 322 super(Option, self).printverbose() 323 324class Group(Data): 325 def __init__(self, name, desc, **kwargs): 326 super(Group, self).__init__(name, desc, **kwargs) 327 self._options = [] 328 self.checkpoint = False 329 self.number = None 330 331 def option(self, name, desc, **kwargs): 332 opt = Option(name, desc, **kwargs) 333 opt.config = self.config 334 opt.group = self 335 opt.number = len(self._options) 336 self._options.append(opt) 337 return opt 338 339 def options(self): 340 return self._options 341 342 def subopts(self): 343 subopts = [] 344 for opt in self._options: 345 for subo in opt.subopts(): 346 subopts.append(subo) 347 return subopts 348 349 def printinfo(self): 350 super(Group, self).printinfo() 351 print 'config: %s' % self.config.name 352 print 'options: %s' % [ o.name for o in self._options ] 353 super(Group, self).printverbose() 354 355class Configuration(Data): 356 def __init__(self, name, desc, **kwargs): 357 super(Configuration, self).__init__(name, desc, **kwargs) 358 self._groups = [] 359 360 def group(self, name, desc, **kwargs): 361 grp = Group(name, desc, **kwargs) 362 grp.config = self 363 grp.number = len(self._groups) 364 self._groups.append(grp) 365 return grp 366 367 def groups(self, flags=Flags(), sign=True): 368 if not flags: 369 return self._groups 370 371 return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ] 372 373 def checkchildren(self, kids): 374 for kid in kids: 375 if kid.config != self: 376 raise AttributeError, "child from the wrong configuration" 377 378 def sortgroups(self, groups): 379 groups = [ (grp.number, grp) for grp in groups ] 380 groups.sort() 381 return [ grp[1] for grp in groups ] 382 383 def options(self, groups = None, checkpoint = False): 384 if groups is None: 385 groups = self._groups 386 self.checkchildren(groups) 387 groups = self.sortgroups(groups) 388 if checkpoint: 389 groups = [ grp for grp in groups if grp.checkpoint ] 390 optgroups = [ g.options() for g in groups ] 391 else: 392 optgroups = [ g.subopts() for g in groups ] 393 for options in crossproduct(optgroups): 394 for opt in options: 395 cpt = opt.group.checkpoint 396 if not isinstance(cpt, bool) and cpt != opt: 397 if checkpoint: 398 break 399 else: 400 yield options 401 else: 402 if checkpoint: 403 yield options 404 405 def checkpoints(self, groups = None): 406 for options in self.options(groups, True): 407 yield Job(options) 408 409 def jobs(self, groups = None): 410 for options in self.options(groups, False): 411 yield Job(options) 412 413 def alljobs(self, groups = None): 414 for options in self.options(groups, True): 415 yield Job(options) 416 for options in self.options(groups, False): 417 yield Job(options) 418 419 def find(self, jobname): 420 for job in self.alljobs(): 421 if job.name == jobname: 422 return job 423 else: 424 raise AttributeError, "job '%s' not found" % jobname 425 426 def job(self, options): 427 self.checkchildren(options) 428 options = [ (opt.group.number, opt) for opt in options ] 429 options.sort() 430 options = [ opt[1] for opt in options ] 431 job = Job(options) 432 return job 433 434 def printinfo(self): 435 super(Configuration, self).printinfo() 436 print 'groups: %s' % [ g.name for g in self._grouips ] 437 super(Configuration, self).printverbose() 438 439def JobFile(jobfile): 440 from os.path import expanduser, isfile, join as joinpath 441 filename = expanduser(jobfile) 442 443 # Can't find filename in the current path, search sys.path 444 if not isfile(filename): 445 for path in sys.path: 446 testname = joinpath(path, filename) 447 if isfile(testname): 448 filename = testname 449 break 450 else: 451 raise AttributeError, \ 452 "Could not find file '%s'" % jobfile 453 454 data = {} 455 execfile(filename, data) 456 if 'conf' not in data: 457 raise ImportError, 'cannot import name conf from %s' % jobfile 458 conf = data['conf'] 459 import jobfile 460 if not isinstance(conf, Configuration): 461 raise AttributeError, \ 462 'conf in jobfile: %s (%s) is not type %s' % \ 463 (jobfile, type(conf), Configuration) 464 return conf 465 466if __name__ == '__main__': 467 from jobfile import * 468 import sys 469 470 usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0] 471 472 try: 473 import getopt 474 opts, args = getopt.getopt(sys.argv[1:], '-bcv') 475 except getopt.GetoptError: 476 sys.exit(usage) 477 478 if len(args) != 1: 479 raise AttributeError, usage 480 481 both = False 482 checkpoint = False 483 verbose = False 484 for opt,arg in opts: 485 if opt == '-b': 486 both = True 487 checkpoint = True 488 if opt == '-c': 489 checkpoint = True 490 if opt == '-v': 491 verbose = True 492 493 jobfile = args[0] 494 conf = JobFile(jobfile) 495 496 if both: 497 gen = conf.alljobs() 498 elif checkpoint: 499 gen = conf.checkpoints() 500 else: 501 gen = conf.jobs() 502 503 for job in gen: 504 if not verbose: 505 cpt = '' 506 if job.checkpoint: 507 cpt = job.checkpoint.name 508 print job.name, cpt 509 else: 510 job.printinfo() 511