db.py (1758:74acd5b23964) | db.py (1772:a3a83e812a5e) |
---|---|
1 | |
2# Copyright (c) 2003-2004 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 1# Copyright (c) 2003-2004 The Regents of The University of Michigan 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 |
28#Permission is granted to use, copy, create derivative works and 29#redistribute this software and such derivative works for any purpose, 30#so long as the copyright notice above, this grant of permission, and 31#the disclaimer below appear in all copies made; and so long as the 32#name of The University of Michigan is not used in any advertising or 33#publicity pertaining to the use or distribution of this software 34#without specific, written prior authorization. 35# 36#THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION FROM THE 37#UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY PURPOSE, AND WITHOUT 38#WARRANTY BY THE UNIVERSITY OF MICHIGAN OF ANY KIND, EITHER EXPRESS OR 39#IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF 40#MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE REGENTS OF 41#THE UNIVERSITY OF MICHIGAN SHALL NOT BE LIABLE FOR ANY DAMAGES, 42#INCLUDING DIRECT, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 43#DAMAGES, WITH RESPECT TO ANY CLAIM ARISING OUT OF OR IN CONNECTION 44#WITH THE USE OF THE SOFTWARE, EVEN IF IT HAS BEEN OR IS HEREAFTER 45#ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 46 | |
47import MySQLdb, re, string 48 49def statcmp(a, b): 50 v1 = a.split('.') 51 v2 = b.split('.') 52 53 last = min(len(v1), len(v2)) - 1 54 for i,j in zip(v1[0:last], v2[0:last]): 55 if i != j: 56 return cmp(i, j) 57 58 # Special compare for last element. 59 if len(v1) == len(v2): 60 return cmp(v1[last], v2[last]) 61 else: 62 return cmp(len(v1), len(v2)) 63 64class RunData: 65 def __init__(self, row): 66 self.run = int(row[0]) 67 self.name = row[1] 68 self.user = row[2] 69 self.project = row[3] 70 71class SubData: 72 def __init__(self, row): 73 self.stat = int(row[0]) 74 self.x = int(row[1]) 75 self.y = int(row[2]) 76 self.name = row[3] 77 self.descr = row[4] 78 79class Data: 80 def __init__(self, row): 81 if len(row) != 5: 82 raise 'stat db error' 83 self.stat = int(row[0]) 84 self.run = int(row[1]) 85 self.x = int(row[2]) 86 self.y = int(row[3]) 87 self.data = float(row[4]) 88 89 def __repr__(self): 90 return '''Data(['%d', '%d', '%d', '%d', '%f'])''' % ( self.stat, 91 self.run, self.x, self.y, self.data) 92 93class StatData(object): 94 def __init__(self, row): 95 self.stat = int(row[0]) 96 self.name = row[1] 97 self.desc = row[2] 98 self.type = row[3] 99 self.prereq = int(row[5]) 100 self.precision = int(row[6]) 101 102 import flags 103 self.flags = 0 104 if int(row[4]): self.flags |= flags.printable 105 if int(row[7]): self.flags |= flags.nozero 106 if int(row[8]): self.flags |= flags.nonan 107 if int(row[9]): self.flags |= flags.total 108 if int(row[10]): self.flags |= flags.pdf 109 if int(row[11]): self.flags |= flags.cdf 110 111 if self.type == 'DIST' or self.type == 'VECTORDIST': 112 self.min = float(row[12]) 113 self.max = float(row[13]) 114 self.bktsize = float(row[14]) 115 self.size = int(row[15]) 116 117 if self.type == 'FORMULA': 118 self.formula = self.db.allFormulas[self.stat] 119 120class Node(object): 121 def __init__(self, name): 122 self.name = name 123 def __str__(self): 124 return name 125 126class Database(object): 127 def __init__(self): 128 self.host = 'zizzer.pool' 129 self.user = '' 130 self.passwd = '' 131 self.db = 'm5stats' 132 self.cursor = None 133 134 self.allStats = [] 135 self.allStatIds = {} 136 self.allStatNames = {} 137 138 self.allSubData = {} 139 140 self.allRuns = [] 141 self.allRunIds = {} 142 self.allRunNames = {} 143 144 self.allBins = [] 145 self.allBinIds = {} 146 self.allBinNames = {} 147 148 self.allFormulas = {} 149 150 self.stattop = {} 151 self.statdict = {} 152 self.statlist = [] 153 154 self.mode = 'sum'; 155 self.runs = None 156 self.bins = None 157 self.ticks = None 158 self.__dict__['get'] = type(self).sum 159 160 def query(self, sql): 161 self.cursor.execute(sql) 162 163 def update_dict(self, dict): 164 dict.update(self.stattop) 165 166 def append(self, stat): 167 statname = re.sub(':', '__', stat.name) 168 path = string.split(statname, '.') 169 pathtop = path[0] 170 fullname = '' 171 172 x = self 173 while len(path) > 1: 174 name = path.pop(0) 175 if not x.__dict__.has_key(name): 176 x.__dict__[name] = Node(fullname + name) 177 x = x.__dict__[name] 178 fullname = '%s%s.' % (fullname, name) 179 180 name = path.pop(0) 181 x.__dict__[name] = stat 182 183 self.stattop[pathtop] = self.__dict__[pathtop] 184 self.statdict[statname] = stat 185 self.statlist.append(statname) 186 187 def connect(self): 188 # connect 189 self.thedb = MySQLdb.connect(db=self.db, 190 host=self.host, 191 user=self.user, 192 passwd=self.passwd) 193 194 # create a cursor 195 self.cursor = self.thedb.cursor() 196 197 self.query('''select rn_id,rn_name,rn_sample,rn_user,rn_project 198 from runs''') 199 for result in self.cursor.fetchall(): 200 run = RunData(result); 201 self.allRuns.append(run) 202 self.allRunIds[run.run] = run 203 self.allRunNames[run.name] = run 204 205 self.query('select * from bins') 206 for id,name in self.cursor.fetchall(): 207 self.allBinIds[int(id)] = name 208 self.allBinNames[name] = int(id) 209 210 self.query('select sd_stat,sd_x,sd_y,sd_name,sd_descr from subdata') 211 for result in self.cursor.fetchall(): 212 subdata = SubData(result) 213 if self.allSubData.has_key(subdata.stat): 214 self.allSubData[subdata.stat].append(subdata) 215 else: 216 self.allSubData[subdata.stat] = [ subdata ] 217 218 self.query('select * from formulas') 219 for id,formula in self.cursor.fetchall(): 220 self.allFormulas[int(id)] = formula.tostring() 221 222 StatData.db = self 223 self.query('select * from stats') 224 import info 225 for result in self.cursor.fetchall(): 226 stat = info.NewStat(StatData(result)) 227 self.append(stat) 228 self.allStats.append(stat) 229 self.allStatIds[stat.stat] = stat 230 self.allStatNames[stat.name] = stat 231 232 # Name: listbins 233 # Desc: Prints all bins matching regex argument, if no argument 234 # is given all bins are returned 235 def listBins(self, regex='.*'): 236 print '%-50s %-10s' % ('bin name', 'id') 237 print '-' * 61 238 names = self.allBinNames.keys() 239 names.sort() 240 for name in names: 241 id = self.allBinNames[name] 242 print '%-50s %-10d' % (name, id) 243 244 # Name: listruns 245 # Desc: Prints all runs matching a given user, if no argument 246 # is given all runs are returned 247 def listRuns(self, user=None): 248 print '%-40s %-10s %-5s' % ('run name', 'user', 'id') 249 print '-' * 62 250 for run in self.allRuns: 251 if user == None or user == run.user: 252 print '%-40s %-10s %-10d' % (run.name, run.user, run.run) 253 254 # Name: listTicks 255 # Desc: Prints all samples for a given run 256 def listTicks(self, runs=None): 257 print "tick" 258 print "----------------------------------------" 259 sql = 'select distinct dt_tick from data where dt_stat=1180 and (' 260 if runs != None: 261 first = True 262 for run in runs: 263 if first: 264 # sql += ' where' 265 first = False 266 else: 267 sql += ' or' 268 sql += ' dt_run=%s' % run.run 269 sql += ')' 270 self.query(sql) 271 for r in self.cursor.fetchall(): 272 print r[0] 273 274 # Name: retTicks 275 # Desc: Prints all samples for a given run 276 def retTicks(self, runs=None): 277 sql = 'select distinct dt_tick from data where dt_stat=1180 and (' 278 if runs != None: 279 first = True 280 for run in runs: 281 if first: 282 first = False 283 else: 284 sql += ' or' 285 sql += ' dt_run=%s' % run.run 286 sql += ')' 287 self.query(sql) 288 ret = [] 289 for r in self.cursor.fetchall(): 290 ret.append(r[0]) 291 return ret 292 293 # Name: liststats 294 # Desc: Prints all statistics that appear in the database, 295 # the optional argument is a regular expression that can 296 # be used to prune the result set 297 def listStats(self, regex=None): 298 print '%-60s %-8s %-10s' % ('stat name', 'id', 'type') 299 print '-' * 80 300 301 rx = None 302 if regex != None: 303 rx = re.compile(regex) 304 305 stats = [ stat.name for stat in self.allStats ] 306 stats.sort(statcmp) 307 for stat in stats: 308 stat = self.allStatNames[stat] 309 if rx == None or rx.match(stat.name): 310 print '%-60s %-8s %-10s' % (stat.name, stat.stat, stat.type) 311 312 # Name: liststats 313 # Desc: Prints all statistics that appear in the database, 314 # the optional argument is a regular expression that can 315 # be used to prune the result set 316 def listFormulas(self, regex=None): 317 print '%-60s %s' % ('formula name', 'formula') 318 print '-' * 80 319 320 rx = None 321 if regex != None: 322 rx = re.compile(regex) 323 324 stats = [ stat.name for stat in self.allStats ] 325 stats.sort(statcmp) 326 for stat in stats: 327 stat = self.allStatNames[stat] 328 if stat.type == 'FORMULA' and (rx == None or rx.match(stat.name)): 329 print '%-60s %s' % (stat.name, self.allFormulas[stat.stat]) 330 331 def getStat(self, stats): 332 if type(stats) is not list: 333 stats = [ stats ] 334 335 ret = [] 336 for stat in stats: 337 if type(stat) is int: 338 ret.append(self.allStatIds[stat]) 339 340 if type(stat) is str: 341 rx = re.compile(stat) 342 for stat in self.allStats: 343 if rx.match(stat.name): 344 ret.append(stat) 345 return ret 346 347 def getBin(self, bins): 348 if type(bins) is not list: 349 bins = [ bins ] 350 351 ret = [] 352 for bin in bins: 353 if type(bin) is int: 354 ret.append(bin) 355 elif type(bin) is str: 356 ret.append(self.allBinNames[bin]) 357 else: 358 for name,id in self.allBinNames.items(): 359 if bin.match(name): 360 ret.append(id) 361 362 return ret 363 364 def getNotBin(self, bin): 365 map = {} 366 for bin in getBin(bin): 367 map[bin] = 1 368 369 ret = [] 370 for bin in self.allBinIds.keys(): 371 if not map.has_key(bin): 372 ret.append(bin) 373 374 return ret 375 376 ######################################### 377 # get the data 378 # 379 def inner(self, op, stat, bins, ticks, group=False): 380 sql = 'select ' 381 sql += 'dt_stat as stat, ' 382 sql += 'dt_run as run, ' 383 sql += 'dt_x as x, ' 384 sql += 'dt_y as y, ' 385 if group: 386 sql += 'dt_tick as tick, ' 387 sql += '%s(dt_data) as data ' % op 388 sql += 'from data ' 389 sql += 'where ' 390 391 if isinstance(stat, list): 392 val = ' or '.join([ 'dt_stat=%d' % s.stat for s in stat ]) 393 sql += ' (%s)' % val 394 else: 395 sql += ' dt_stat=%d' % stat.stat 396 397 if self.runs != None and len(self.runs): 398 val = ' or '.join([ 'dt_run=%d' % r for r in self.runs ]) 399 sql += ' and (%s)' % val 400 401 if bins != None and len(bins): 402 val = ' or '.join([ 'dt_bin=%d' % b for b in bins ]) 403 sql += ' and (%s)' % val 404 405 if ticks != None and len(ticks): 406 val = ' or '.join([ 'dt_tick=%d' % s for s in ticks ]) 407 sql += ' and (%s)' % val 408 409 sql += ' group by dt_stat,dt_run,dt_x,dt_y' 410 if group: 411 sql += ',dt_tick' 412 return sql 413 414 def outer(self, op_out, op_in, stat, bins, ticks): 415 sql = self.inner(op_in, stat, bins, ticks, True) 416 sql = 'select stat,run,x,y,%s(data) from (%s) as tb ' % (op_out, sql) 417 sql += 'group by stat,run,x,y' 418 return sql 419 420 # Name: sum 421 # Desc: given a run, a stat and an array of samples and bins, 422 # sum all the bins and then get the standard deviation of the 423 # samples for non-binned runs. This will just return the average 424 # of samples, however a bin array still must be passed 425 def sum(self, stat, bins, ticks): 426 return self.inner('sum', stat, bins, ticks) 427 428 # Name: avg 429 # Desc: given a run, a stat and an array of samples and bins, 430 # sum all the bins and then average the samples for non-binned 431 # runs this will just return the average of samples, however 432 # a bin array still must be passed 433 def avg(self, stat, bins, ticks): 434 return self.outer('avg', 'sum', stat, bins, ticks) 435 436 # Name: stdev 437 # Desc: given a run, a stat and an array of samples and bins, 438 # sum all the bins and then get the standard deviation of the 439 # samples for non-binned runs. This will just return the average 440 # of samples, however a bin array still must be passed 441 def stdev(self, stat, bins, ticks): 442 return self.outer('stddev', 'sum', stat, bins, ticks) 443 444 def __getattribute__(self, attr): 445 if attr != 'get': 446 return super(Database, self).__getattribute__(attr) 447 448 if self.__dict__['get'] == type(self).sum: 449 return 'sum' 450 elif self.__dict__['get'] == type(self).avg: 451 return 'avg' 452 elif self.__dict__['get'] == type(self).stdev: 453 return 'stdev' 454 else: 455 return '' 456 457 def __setattr__(self, attr, value): 458 if attr != 'get': 459 super(Database, self).__setattr__(attr, value) 460 return 461 462 if value == 'sum': 463 self.__dict__['get'] = type(self).sum 464 elif value == 'avg': 465 self.__dict__['get'] = type(self).avg 466 elif value == 'stdev': 467 self.__dict__['get'] = type(self).stdev 468 else: 469 raise AttributeError, "can only set get to: sum | avg | stdev" 470 471 def data(self, stat, bins=None, ticks=None): 472 if bins is None: 473 bins = self.bins 474 if ticks is None: 475 ticks = self.ticks 476 sql = self.__dict__['get'](self, stat, bins, ticks) 477 self.query(sql) 478 479 runs = {} 480 for x in self.cursor.fetchall(): 481 data = Data(x) 482 if not runs.has_key(data.run): 483 runs[data.run] = {} 484 if not runs[data.run].has_key(data.x): 485 runs[data.run][data.x] = {} 486 487 runs[data.run][data.x][data.y] = data.data 488 return runs | 27import MySQLdb, re, string 28 29def statcmp(a, b): 30 v1 = a.split('.') 31 v2 = b.split('.') 32 33 last = min(len(v1), len(v2)) - 1 34 for i,j in zip(v1[0:last], v2[0:last]): 35 if i != j: 36 return cmp(i, j) 37 38 # Special compare for last element. 39 if len(v1) == len(v2): 40 return cmp(v1[last], v2[last]) 41 else: 42 return cmp(len(v1), len(v2)) 43 44class RunData: 45 def __init__(self, row): 46 self.run = int(row[0]) 47 self.name = row[1] 48 self.user = row[2] 49 self.project = row[3] 50 51class SubData: 52 def __init__(self, row): 53 self.stat = int(row[0]) 54 self.x = int(row[1]) 55 self.y = int(row[2]) 56 self.name = row[3] 57 self.descr = row[4] 58 59class Data: 60 def __init__(self, row): 61 if len(row) != 5: 62 raise 'stat db error' 63 self.stat = int(row[0]) 64 self.run = int(row[1]) 65 self.x = int(row[2]) 66 self.y = int(row[3]) 67 self.data = float(row[4]) 68 69 def __repr__(self): 70 return '''Data(['%d', '%d', '%d', '%d', '%f'])''' % ( self.stat, 71 self.run, self.x, self.y, self.data) 72 73class StatData(object): 74 def __init__(self, row): 75 self.stat = int(row[0]) 76 self.name = row[1] 77 self.desc = row[2] 78 self.type = row[3] 79 self.prereq = int(row[5]) 80 self.precision = int(row[6]) 81 82 import flags 83 self.flags = 0 84 if int(row[4]): self.flags |= flags.printable 85 if int(row[7]): self.flags |= flags.nozero 86 if int(row[8]): self.flags |= flags.nonan 87 if int(row[9]): self.flags |= flags.total 88 if int(row[10]): self.flags |= flags.pdf 89 if int(row[11]): self.flags |= flags.cdf 90 91 if self.type == 'DIST' or self.type == 'VECTORDIST': 92 self.min = float(row[12]) 93 self.max = float(row[13]) 94 self.bktsize = float(row[14]) 95 self.size = int(row[15]) 96 97 if self.type == 'FORMULA': 98 self.formula = self.db.allFormulas[self.stat] 99 100class Node(object): 101 def __init__(self, name): 102 self.name = name 103 def __str__(self): 104 return name 105 106class Database(object): 107 def __init__(self): 108 self.host = 'zizzer.pool' 109 self.user = '' 110 self.passwd = '' 111 self.db = 'm5stats' 112 self.cursor = None 113 114 self.allStats = [] 115 self.allStatIds = {} 116 self.allStatNames = {} 117 118 self.allSubData = {} 119 120 self.allRuns = [] 121 self.allRunIds = {} 122 self.allRunNames = {} 123 124 self.allBins = [] 125 self.allBinIds = {} 126 self.allBinNames = {} 127 128 self.allFormulas = {} 129 130 self.stattop = {} 131 self.statdict = {} 132 self.statlist = [] 133 134 self.mode = 'sum'; 135 self.runs = None 136 self.bins = None 137 self.ticks = None 138 self.__dict__['get'] = type(self).sum 139 140 def query(self, sql): 141 self.cursor.execute(sql) 142 143 def update_dict(self, dict): 144 dict.update(self.stattop) 145 146 def append(self, stat): 147 statname = re.sub(':', '__', stat.name) 148 path = string.split(statname, '.') 149 pathtop = path[0] 150 fullname = '' 151 152 x = self 153 while len(path) > 1: 154 name = path.pop(0) 155 if not x.__dict__.has_key(name): 156 x.__dict__[name] = Node(fullname + name) 157 x = x.__dict__[name] 158 fullname = '%s%s.' % (fullname, name) 159 160 name = path.pop(0) 161 x.__dict__[name] = stat 162 163 self.stattop[pathtop] = self.__dict__[pathtop] 164 self.statdict[statname] = stat 165 self.statlist.append(statname) 166 167 def connect(self): 168 # connect 169 self.thedb = MySQLdb.connect(db=self.db, 170 host=self.host, 171 user=self.user, 172 passwd=self.passwd) 173 174 # create a cursor 175 self.cursor = self.thedb.cursor() 176 177 self.query('''select rn_id,rn_name,rn_sample,rn_user,rn_project 178 from runs''') 179 for result in self.cursor.fetchall(): 180 run = RunData(result); 181 self.allRuns.append(run) 182 self.allRunIds[run.run] = run 183 self.allRunNames[run.name] = run 184 185 self.query('select * from bins') 186 for id,name in self.cursor.fetchall(): 187 self.allBinIds[int(id)] = name 188 self.allBinNames[name] = int(id) 189 190 self.query('select sd_stat,sd_x,sd_y,sd_name,sd_descr from subdata') 191 for result in self.cursor.fetchall(): 192 subdata = SubData(result) 193 if self.allSubData.has_key(subdata.stat): 194 self.allSubData[subdata.stat].append(subdata) 195 else: 196 self.allSubData[subdata.stat] = [ subdata ] 197 198 self.query('select * from formulas') 199 for id,formula in self.cursor.fetchall(): 200 self.allFormulas[int(id)] = formula.tostring() 201 202 StatData.db = self 203 self.query('select * from stats') 204 import info 205 for result in self.cursor.fetchall(): 206 stat = info.NewStat(StatData(result)) 207 self.append(stat) 208 self.allStats.append(stat) 209 self.allStatIds[stat.stat] = stat 210 self.allStatNames[stat.name] = stat 211 212 # Name: listbins 213 # Desc: Prints all bins matching regex argument, if no argument 214 # is given all bins are returned 215 def listBins(self, regex='.*'): 216 print '%-50s %-10s' % ('bin name', 'id') 217 print '-' * 61 218 names = self.allBinNames.keys() 219 names.sort() 220 for name in names: 221 id = self.allBinNames[name] 222 print '%-50s %-10d' % (name, id) 223 224 # Name: listruns 225 # Desc: Prints all runs matching a given user, if no argument 226 # is given all runs are returned 227 def listRuns(self, user=None): 228 print '%-40s %-10s %-5s' % ('run name', 'user', 'id') 229 print '-' * 62 230 for run in self.allRuns: 231 if user == None or user == run.user: 232 print '%-40s %-10s %-10d' % (run.name, run.user, run.run) 233 234 # Name: listTicks 235 # Desc: Prints all samples for a given run 236 def listTicks(self, runs=None): 237 print "tick" 238 print "----------------------------------------" 239 sql = 'select distinct dt_tick from data where dt_stat=1180 and (' 240 if runs != None: 241 first = True 242 for run in runs: 243 if first: 244 # sql += ' where' 245 first = False 246 else: 247 sql += ' or' 248 sql += ' dt_run=%s' % run.run 249 sql += ')' 250 self.query(sql) 251 for r in self.cursor.fetchall(): 252 print r[0] 253 254 # Name: retTicks 255 # Desc: Prints all samples for a given run 256 def retTicks(self, runs=None): 257 sql = 'select distinct dt_tick from data where dt_stat=1180 and (' 258 if runs != None: 259 first = True 260 for run in runs: 261 if first: 262 first = False 263 else: 264 sql += ' or' 265 sql += ' dt_run=%s' % run.run 266 sql += ')' 267 self.query(sql) 268 ret = [] 269 for r in self.cursor.fetchall(): 270 ret.append(r[0]) 271 return ret 272 273 # Name: liststats 274 # Desc: Prints all statistics that appear in the database, 275 # the optional argument is a regular expression that can 276 # be used to prune the result set 277 def listStats(self, regex=None): 278 print '%-60s %-8s %-10s' % ('stat name', 'id', 'type') 279 print '-' * 80 280 281 rx = None 282 if regex != None: 283 rx = re.compile(regex) 284 285 stats = [ stat.name for stat in self.allStats ] 286 stats.sort(statcmp) 287 for stat in stats: 288 stat = self.allStatNames[stat] 289 if rx == None or rx.match(stat.name): 290 print '%-60s %-8s %-10s' % (stat.name, stat.stat, stat.type) 291 292 # Name: liststats 293 # Desc: Prints all statistics that appear in the database, 294 # the optional argument is a regular expression that can 295 # be used to prune the result set 296 def listFormulas(self, regex=None): 297 print '%-60s %s' % ('formula name', 'formula') 298 print '-' * 80 299 300 rx = None 301 if regex != None: 302 rx = re.compile(regex) 303 304 stats = [ stat.name for stat in self.allStats ] 305 stats.sort(statcmp) 306 for stat in stats: 307 stat = self.allStatNames[stat] 308 if stat.type == 'FORMULA' and (rx == None or rx.match(stat.name)): 309 print '%-60s %s' % (stat.name, self.allFormulas[stat.stat]) 310 311 def getStat(self, stats): 312 if type(stats) is not list: 313 stats = [ stats ] 314 315 ret = [] 316 for stat in stats: 317 if type(stat) is int: 318 ret.append(self.allStatIds[stat]) 319 320 if type(stat) is str: 321 rx = re.compile(stat) 322 for stat in self.allStats: 323 if rx.match(stat.name): 324 ret.append(stat) 325 return ret 326 327 def getBin(self, bins): 328 if type(bins) is not list: 329 bins = [ bins ] 330 331 ret = [] 332 for bin in bins: 333 if type(bin) is int: 334 ret.append(bin) 335 elif type(bin) is str: 336 ret.append(self.allBinNames[bin]) 337 else: 338 for name,id in self.allBinNames.items(): 339 if bin.match(name): 340 ret.append(id) 341 342 return ret 343 344 def getNotBin(self, bin): 345 map = {} 346 for bin in getBin(bin): 347 map[bin] = 1 348 349 ret = [] 350 for bin in self.allBinIds.keys(): 351 if not map.has_key(bin): 352 ret.append(bin) 353 354 return ret 355 356 ######################################### 357 # get the data 358 # 359 def inner(self, op, stat, bins, ticks, group=False): 360 sql = 'select ' 361 sql += 'dt_stat as stat, ' 362 sql += 'dt_run as run, ' 363 sql += 'dt_x as x, ' 364 sql += 'dt_y as y, ' 365 if group: 366 sql += 'dt_tick as tick, ' 367 sql += '%s(dt_data) as data ' % op 368 sql += 'from data ' 369 sql += 'where ' 370 371 if isinstance(stat, list): 372 val = ' or '.join([ 'dt_stat=%d' % s.stat for s in stat ]) 373 sql += ' (%s)' % val 374 else: 375 sql += ' dt_stat=%d' % stat.stat 376 377 if self.runs != None and len(self.runs): 378 val = ' or '.join([ 'dt_run=%d' % r for r in self.runs ]) 379 sql += ' and (%s)' % val 380 381 if bins != None and len(bins): 382 val = ' or '.join([ 'dt_bin=%d' % b for b in bins ]) 383 sql += ' and (%s)' % val 384 385 if ticks != None and len(ticks): 386 val = ' or '.join([ 'dt_tick=%d' % s for s in ticks ]) 387 sql += ' and (%s)' % val 388 389 sql += ' group by dt_stat,dt_run,dt_x,dt_y' 390 if group: 391 sql += ',dt_tick' 392 return sql 393 394 def outer(self, op_out, op_in, stat, bins, ticks): 395 sql = self.inner(op_in, stat, bins, ticks, True) 396 sql = 'select stat,run,x,y,%s(data) from (%s) as tb ' % (op_out, sql) 397 sql += 'group by stat,run,x,y' 398 return sql 399 400 # Name: sum 401 # Desc: given a run, a stat and an array of samples and bins, 402 # sum all the bins and then get the standard deviation of the 403 # samples for non-binned runs. This will just return the average 404 # of samples, however a bin array still must be passed 405 def sum(self, stat, bins, ticks): 406 return self.inner('sum', stat, bins, ticks) 407 408 # Name: avg 409 # Desc: given a run, a stat and an array of samples and bins, 410 # sum all the bins and then average the samples for non-binned 411 # runs this will just return the average of samples, however 412 # a bin array still must be passed 413 def avg(self, stat, bins, ticks): 414 return self.outer('avg', 'sum', stat, bins, ticks) 415 416 # Name: stdev 417 # Desc: given a run, a stat and an array of samples and bins, 418 # sum all the bins and then get the standard deviation of the 419 # samples for non-binned runs. This will just return the average 420 # of samples, however a bin array still must be passed 421 def stdev(self, stat, bins, ticks): 422 return self.outer('stddev', 'sum', stat, bins, ticks) 423 424 def __getattribute__(self, attr): 425 if attr != 'get': 426 return super(Database, self).__getattribute__(attr) 427 428 if self.__dict__['get'] == type(self).sum: 429 return 'sum' 430 elif self.__dict__['get'] == type(self).avg: 431 return 'avg' 432 elif self.__dict__['get'] == type(self).stdev: 433 return 'stdev' 434 else: 435 return '' 436 437 def __setattr__(self, attr, value): 438 if attr != 'get': 439 super(Database, self).__setattr__(attr, value) 440 return 441 442 if value == 'sum': 443 self.__dict__['get'] = type(self).sum 444 elif value == 'avg': 445 self.__dict__['get'] = type(self).avg 446 elif value == 'stdev': 447 self.__dict__['get'] = type(self).stdev 448 else: 449 raise AttributeError, "can only set get to: sum | avg | stdev" 450 451 def data(self, stat, bins=None, ticks=None): 452 if bins is None: 453 bins = self.bins 454 if ticks is None: 455 ticks = self.ticks 456 sql = self.__dict__['get'](self, stat, bins, ticks) 457 self.query(sql) 458 459 runs = {} 460 for x in self.cursor.fetchall(): 461 data = Data(x) 462 if not runs.has_key(data.run): 463 runs[data.run] = {} 464 if not runs[data.run].has_key(data.x): 465 runs[data.run][data.x] = {} 466 467 runs[data.run][data.x][data.y] = data.data 468 return runs |