jobfile.py revision 5467
1# Copyright (c) 2005-2006 The Regents of The University of Michigan 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Nathan Binkert 28 29import sys 30 31from attrdict import attrdict, optiondict 32from misc import crossproduct, flatten 33 34class Data(object): 35 def __init__(self, name, desc, **kwargs): 36 self.name = name 37 self.desc = desc 38 self.__dict__.update(kwargs) 39 40 def update(self, obj): 41 if not isinstance(obj, Data): 42 raise AttributeError, "can only update from Data object" 43 44 for k,v in obj.__dict__.iteritems(): 45 if not k.startswith('_'): 46 self.__dict__[k] = v 47 if hasattr(self, 'system') and hasattr(obj, 'system'): 48 if self.system != obj.system: 49 raise AttributeError, \ 50 "conflicting values for system: '%s'/'%s'" % \ 51 (self.system, obj.system) 52 53 def printinfo(self): 54 if self.name: 55 print 'name: %s' % self.name 56 if self.desc: 57 print 'desc: %s' % self.desc 58 try: 59 if self.system: 60 print 'system: %s' % self.system 61 except AttributeError: 62 pass 63 64 def printverbose(self): 65 for key in self: 66 val = self[key] 67 if isinstance(val, dict): 68 import pprint 69 val = pprint.pformat(val) 70 print '%-20s = %s' % (key, val) 71 print 72 73 def __contains__(self, attr): 74 if attr.startswith('_'): 75 return False 76 return attr in self.__dict__ 77 78 def __getitem__(self, key): 79 if key.startswith('_'): 80 raise KeyError, "Key '%s' not found" % attr 81 return self.__dict__[key] 82 83 def __iter__(self): 84 keys = self.__dict__.keys() 85 keys.sort() 86 for key in keys: 87 if not key.startswith('_'): 88 yield key 89 90 def optiondict(self): 91 result = optiondict() 92 for key in self: 93 result[key] = self[key] 94 return result 95 96 def __str__(self): 97 return self.name 98 99class Job(Data): 100 def __init__(self, options): 101 super(Job, self).__init__('', '') 102 103 config = options[0]._config 104 for opt in options: 105 if opt._config != config: 106 raise AttributeError, \ 107 "All options are not from the same Configuration" 108 109 self._config = config 110 self._groups = [ opt._group for opt in options ] 111 self._options = options 112 113 self.update(self._config) 114 for group in self._groups: 115 self.update(group) 116 117 self._is_checkpoint = True 118 119 for option in self._options: 120 self.update(option) 121 if not option._group._checkpoint: 122 self._is_checkpoint = False 123 124 if option._suboption: 125 self.update(option._suboption) 126 self._is_checkpoint = False 127 128 names = [ ] 129 for opt in self._options: 130 if opt.name: 131 names.append(opt.name) 132 self.name = ':'.join(names) 133 134 descs = [ ] 135 for opt in self._options: 136 if opt.desc: 137 descs.append(opt.desc) 138 self.desc = ', '.join(descs) 139 140 self._checkpoint = None 141 if not self._is_checkpoint: 142 opts = [] 143 for opt in options: 144 cpt = opt._group._checkpoint 145 if not cpt: 146 continue 147 if isinstance(cpt, Option): 148 opt = cpt.clone(suboptions=False) 149 else: 150 opt = opt.clone(suboptions=False) 151 152 opts.append(opt) 153 154 if opts: 155 self._checkpoint = Job(opts) 156 157 def clone(self): 158 return Job(self._options) 159 160 def printinfo(self): 161 super(Job, self).printinfo() 162 if self._checkpoint: 163 print 'checkpoint: %s' % self._checkpoint.name 164 print 'config: %s' % self._config.name 165 print 'groups: %s' % [ g.name for g in self._groups ] 166 print 'options: %s' % [ o.name for o in self._options ] 167 super(Job, self).printverbose() 168 169class SubOption(Data): 170 def __init__(self, name, desc, **kwargs): 171 super(SubOption, self).__init__(name, desc, **kwargs) 172 self._number = None 173 174class Option(Data): 175 def __init__(self, name, desc, **kwargs): 176 super(Option, self).__init__(name, desc, **kwargs) 177 self._suboptions = [] 178 self._suboption = None 179 self._number = None 180 181 def __getattribute__(self, attr): 182 if attr == 'name': 183 name = self.__dict__[attr] 184 if self._suboption is not None: 185 name = '%s:%s' % (name, self._suboption.name) 186 return name 187 188 if attr == 'desc': 189 desc = [ self.__dict__[attr] ] 190 if self._suboption is not None and self._suboption.desc: 191 desc.append(self._suboption.desc) 192 return ', '.join(desc) 193 194 return super(Option, self).__getattribute__(attr) 195 196 def suboption(self, name, desc, **kwargs): 197 subo = SubOption(name, desc, **kwargs) 198 subo._config = self._config 199 subo._group = self._group 200 subo._option = self 201 subo._number = len(self._suboptions) 202 self._suboptions.append(subo) 203 return subo 204 205 def clone(self, suboptions=True): 206 option = Option(self.__dict__['name'], self.__dict__['desc']) 207 option.update(self) 208 option._group = self._group 209 option._config = self._config 210 option._number = self._number 211 if suboptions: 212 option._suboptions.extend(self._suboptions) 213 option._suboption = self._suboption 214 return option 215 216 def subopts(self): 217 if not self._suboptions: 218 return [ self ] 219 220 subopts = [] 221 for subo in self._suboptions: 222 option = self.clone() 223 option._suboption = subo 224 subopts.append(option) 225 226 return subopts 227 228 def printinfo(self): 229 super(Option, self).printinfo() 230 print 'config: %s' % self._config.name 231 super(Option, self).printverbose() 232 233class Group(Data): 234 def __init__(self, name, desc, **kwargs): 235 super(Group, self).__init__(name, desc, **kwargs) 236 self._options = [] 237 self._number = None 238 self._checkpoint = False 239 240 def option(self, name, desc, **kwargs): 241 opt = Option(name, desc, **kwargs) 242 opt._config = self._config 243 opt._group = self 244 opt._number = len(self._options) 245 self._options.append(opt) 246 return opt 247 248 def options(self): 249 return self._options 250 251 def subopts(self): 252 subopts = [] 253 for opt in self._options: 254 for subo in opt.subopts(): 255 subopts.append(subo) 256 return subopts 257 258 def printinfo(self): 259 super(Group, self).printinfo() 260 print 'config: %s' % self._config.name 261 print 'options: %s' % [ o.name for o in self._options ] 262 super(Group, self).printverbose() 263 264class Configuration(Data): 265 def __init__(self, name, desc, **kwargs): 266 super(Configuration, self).__init__(name, desc, **kwargs) 267 self._groups = [] 268 self._posfilters = [] 269 self._negfilters = [] 270 271 def group(self, name, desc, **kwargs): 272 grp = Group(name, desc, **kwargs) 273 grp._config = self 274 grp._number = len(self._groups) 275 self._groups.append(grp) 276 return grp 277 278 def groups(self): 279 return self._groups 280 281 def checkchildren(self, kids): 282 for kid in kids: 283 if kid._config != self: 284 raise AttributeError, "child from the wrong configuration" 285 286 def sortgroups(self, groups): 287 groups = [ (grp._number, grp) for grp in groups ] 288 groups.sort() 289 return [ grp[1] for grp in groups ] 290 291 def options(self, groups=None, checkpoint=False): 292 if groups is None: 293 groups = self._groups 294 self.checkchildren(groups) 295 groups = self.sortgroups(groups) 296 if checkpoint: 297 groups = [ grp for grp in groups if grp._checkpoint ] 298 optgroups = [ g.options() for g in groups ] 299 else: 300 optgroups = [ g.subopts() for g in groups ] 301 if not optgroups: 302 return 303 for options in crossproduct(optgroups): 304 for opt in options: 305 cpt = opt._group._checkpoint 306 if not isinstance(cpt, bool) and cpt != opt: 307 if checkpoint: 308 break 309 else: 310 yield options 311 else: 312 if checkpoint: 313 yield options 314 315 def addfilter(self, filt, pos=True): 316 import re 317 filt = re.compile(filt) 318 if pos: 319 self._posfilters.append(filt) 320 else: 321 self._negfilters.append(filt) 322 323 def jobfilter(self, job): 324 for filt in self._negfilters: 325 if filt.match(job.name): 326 return False 327 328 if not self._posfilters: 329 return True 330 331 for filt in self._posfilters: 332 if filt.match(job.name): 333 return True 334 335 return False 336 337 def checkpoints(self, groups=None): 338 for options in self.options(groups, True): 339 job = Job(options) 340 if self.jobfilter(job): 341 yield job 342 343 def jobs(self, groups=None): 344 for options in self.options(groups, False): 345 job = Job(options) 346 if self.jobfilter(job): 347 yield job 348 349 def alljobs(self, groups=None): 350 for options in self.options(groups, True): 351 yield Job(options) 352 for options in self.options(groups, False): 353 yield Job(options) 354 355 def find(self, jobname): 356 for job in self.alljobs(): 357 if job.name == jobname: 358 return job 359 else: 360 raise AttributeError, "job '%s' not found" % jobname 361 362 def job(self, options): 363 self.checkchildren(options) 364 options = [ (opt._group._number, opt) for opt in options ] 365 options.sort() 366 options = [ opt[1] for opt in options ] 367 job = Job(options) 368 return job 369 370 def printinfo(self): 371 super(Configuration, self).printinfo() 372 print 'groups: %s' % [ g.name for g in self._groups ] 373 super(Configuration, self).printverbose() 374 375def JobFile(jobfile): 376 from os.path import expanduser, isfile, join as joinpath 377 filename = expanduser(jobfile) 378 379 # Can't find filename in the current path, search sys.path 380 if not isfile(filename): 381 for path in sys.path: 382 testname = joinpath(path, filename) 383 if isfile(testname): 384 filename = testname 385 break 386 else: 387 raise AttributeError, \ 388 "Could not find file '%s'" % jobfile 389 390 data = {} 391 execfile(filename, data) 392 if 'conf' not in data: 393 raise ImportError, 'cannot import name conf from %s' % jobfile 394 conf = data['conf'] 395 import jobfile 396 if not isinstance(conf, Configuration): 397 raise AttributeError, \ 398 'conf in jobfile: %s (%s) is not type %s' % \ 399 (jobfile, type(conf), Configuration) 400 return conf 401 402def main(conf=None): 403 import sys 404 405 usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0] 406 407 try: 408 import getopt 409 opts, args = getopt.getopt(sys.argv[1:], '-bcv') 410 except getopt.GetoptError: 411 sys.exit(usage) 412 413 both = False 414 checkpoint = False 415 verbose = False 416 for opt,arg in opts: 417 if opt == '-b': 418 both = True 419 checkpoint = True 420 if opt == '-c': 421 checkpoint = True 422 if opt == '-v': 423 verbose = True 424 425 if conf is None: 426 if len(args) != 1: 427 raise AttributeError, usage 428 conf = JobFile(args[0]) 429 else: 430 if len(args) != 0: 431 raise AttributeError, usage 432 433 if both: 434 jobs = conf.alljobs() 435 elif checkpoint: 436 jobs = conf.checkpoints() 437 else: 438 jobs = conf.jobs() 439 440 for job in jobs: 441 if verbose: 442 job.printinfo() 443 else: 444 cpt = '' 445 if job._checkpoint: 446 cpt = job._checkpoint.name 447 print job.name, cpt 448 449if __name__ == '__main__': 450 main() 451