profile.py (2004:cdf8e86d1983) profile.py (2006:3ca085495c69)
1# Copyright (c) 2005 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright

--- 13 unchanged lines hidden (view full) ---

22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27from orderdict import orderdict
28import output
29
1# Copyright (c) 2005 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright

--- 13 unchanged lines hidden (view full) ---

22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27from orderdict import orderdict
28import output
29
30class FileData(dict):
31 def __init__(self, filename):
32 self.filename = filename
33 fd = file(filename)
34 current = []
35 for line in fd:
36 line = line.strip()
37 if line.startswith('>>>'):
38 current = []
39 self[line[3:]] = current
40 else:
41 current.append(line)
42 fd.close()
43
30class RunData(dict):
44class RunData(dict):
31 def __init__(self, filename=None):
45 def __init__(self, filename):
32 self.filename = filename
33
46 self.filename = filename
47
34 def __getattr__(self, attr):
48 def __getattribute__(self, attr):
35 if attr == 'total':
36 total = 0.0
37 for value in self.itervalues():
38 total += value
39 return total
49 if attr == 'total':
50 total = 0.0
51 for value in self.itervalues():
52 total += value
53 return total
54
55 if attr == 'filedata':
56 return FileData(self.filename)
57
40 if attr == 'maxsymlen':
41 return max([ len(sym) for sym in self.iterkeys() ])
42
58 if attr == 'maxsymlen':
59 return max([ len(sym) for sym in self.iterkeys() ])
60
61 return super(RunData, self).__getattribute__(attr)
62
43 def display(self, output=None, limit=None, maxsymlen=None):
44 if not output:
45 import sys
46 output = sys.stdout
47 elif isinstance(output, str):
48 output = file(output, 'w')
49
50 total = float(self.total)

--- 6 unchanged lines hidden (view full) ---

57
58 if not maxsymlen:
59 maxsymlen = self.maxsymlen
60
61 symbolf = "%-" + str(maxsymlen + 1) + "s %.2f%%"
62 for number,name in symbols:
63 print >>output, symbolf % (name, 100.0 * (float(number) / total))
64
63 def display(self, output=None, limit=None, maxsymlen=None):
64 if not output:
65 import sys
66 output = sys.stdout
67 elif isinstance(output, str):
68 output = file(output, 'w')
69
70 total = float(self.total)

--- 6 unchanged lines hidden (view full) ---

77
78 if not maxsymlen:
79 maxsymlen = self.maxsymlen
80
81 symbolf = "%-" + str(maxsymlen + 1) + "s %.2f%%"
82 for number,name in symbols:
83 print >>output, symbolf % (name, 100.0 * (float(number) / total))
84
65
66
67class PCData(RunData):
68 def __init__(self, filename=None, categorize=None, showidle=True):
69 super(PCData, self).__init__(self, filename)
85class PCData(RunData):
86 def __init__(self, filename=None, categorize=None, showidle=True):
87 super(PCData, self).__init__(self, filename)
70 if filename is None:
71 return
72
88
73 fd = file(filename)
74
75 for line in fd:
76 if line.strip() == '>>>PC data':
77 break
78
79 for line in fd:
80 if line.startswith('>>>'):
81 break
82
89 filedata = self.filedata['PC data']
90 for line in filedata:
83 (symbol, count) = line.split()
84 if symbol == "0x0":
85 continue
86 count = int(count)
87
88 if categorize is not None:
89 category = categorize(symbol)
90 if category is None:
91 category = 'other'
92 elif category == 'idle' and not showidle:
93 continue
94
95 self[category] = count
96
91 (symbol, count) = line.split()
92 if symbol == "0x0":
93 continue
94 count = int(count)
95
96 if categorize is not None:
97 category = categorize(symbol)
98 if category is None:
99 category = 'other'
100 elif category == 'idle' and not showidle:
101 continue
102
103 self[category] = count
104
97 fd.close()
98
99class FuncNode(object):
105class FuncNode(object):
100 def __new__(cls, filename = None):
101 if filename is None:
106 def __new__(cls, filedata=None):
107 if filedata is None:
102 return super(FuncNode, cls).__new__(cls)
103
108 return super(FuncNode, cls).__new__(cls)
109
104 fd = file(filename, 'r')
105 fditer = iter(fd)
106 nodes = {}
110 nodes = {}
107 for line in fditer:
108 if line.strip() == '>>>function data':
109 break
110
111 for line in fditer:
112 if line.startswith('>>>'):
113 break
114
115 data = line.split()
116 node_id = int(data[0], 16)
111 for line in filedata['function data']:
112 data = line.split(' ')
113 node_id = long(data[0], 16)
117 node = FuncNode()
118 node.symbol = data[1]
114 node = FuncNode()
115 node.symbol = data[1]
119 node.count = int(data[2])
120 node.children = [ int(child, 16) for child in data[3:] ]
116 if node.symbol == '':
117 node.symbol = 'unknown'
118 node.count = long(data[2])
119 node.children = [ long(child, 16) for child in data[3:] ]
121 nodes[node_id] = node
122
123 for node in nodes.itervalues():
124 children = []
125 for cid in node.children:
126 child = nodes[cid]
127 children.append(child)
128 child.parent = node
129 node.children = tuple(children)
130 if not nodes:
120 nodes[node_id] = node
121
122 for node in nodes.itervalues():
123 children = []
124 for cid in node.children:
125 child = nodes[cid]
126 children.append(child)
127 child.parent = node
128 node.children = tuple(children)
129 if not nodes:
131 print filename
130 print filedata.filename
132 print nodes
133 return nodes[0]
134
131 print nodes
132 return nodes[0]
133
135 def __init__(self, filename=None):
136 pass
137
138 def total(self):
139 total = self.count
140 for child in self.children:
141 total += child.total()
142
143 return total
144
145 def aggregate(self, dict, categorize, incategory):

--- 47 unchanged lines hidden (view full) ---

193
194 def dot(self, dot, threshold=0.1, categorize=None):
195 self._dot(dot, threshold, categorize, self.total())
196 self._cleandot()
197
198class FuncData(RunData):
199 def __init__(self, filename, categorize=None):
200 super(FuncData, self).__init__(filename)
134 def total(self):
135 total = self.count
136 for child in self.children:
137 total += child.total()
138
139 return total
140
141 def aggregate(self, dict, categorize, incategory):

--- 47 unchanged lines hidden (view full) ---

189
190 def dot(self, dot, threshold=0.1, categorize=None):
191 self._dot(dot, threshold, categorize, self.total())
192 self._cleandot()
193
194class FuncData(RunData):
195 def __init__(self, filename, categorize=None):
196 super(FuncData, self).__init__(filename)
201 self.tree = FuncNode(filename)
202 self.tree.aggregate(self, categorize, incategory=False)
203 self.total = self.tree.total()
197 tree = self.tree
198 tree.aggregate(self, categorize, incategory=False)
199 self.total = tree.total()
204
200
201 def __getattribute__(self, attr):
202 if attr == 'tree':
203 return FuncNode(self.filedata)
204 return super(FuncData, self).__getattribute__(attr)
205
205 def displayx(self, output=None, maxcount=None):
206 if output is None:
207 import sys
208 output = sys.stdout
209
210 items = [ (val,key) for key,val in self.iteritems() ]
211 items.sort(reverse=True)
212 for val,key in items:

--- 56 unchanged lines hidden (view full) ---

269 'data already stored for run %s and cpu %s' % (run, cpu)
270
271 self.data[run][cpu] = data
272
273 def getdata(self, run, cpu):
274 try:
275 return self.data[run][cpu]
276 except KeyError:
206 def displayx(self, output=None, maxcount=None):
207 if output is None:
208 import sys
209 output = sys.stdout
210
211 items = [ (val,key) for key,val in self.iteritems() ]
212 items.sort(reverse=True)
213 for val,key in items:

--- 56 unchanged lines hidden (view full) ---

270 'data already stored for run %s and cpu %s' % (run, cpu)
271
272 self.data[run][cpu] = data
273
274 def getdata(self, run, cpu):
275 try:
276 return self.data[run][cpu]
277 except KeyError:
278 print run, cpu
277 return None
278
279 def alldata(self):
280 for run,cpus in self.data.iteritems():
281 for cpu,data in cpus.iteritems():
282 yield run,cpu,data
283
284 def get(self, job, stat):
285 if job.system is None:
286 raise AttributeError, 'The job must have a system set'
287
288 run = job.name
289 cpu = '%s.run%d' % (job.system, self.cpu)
290 data = self.getdata(run, cpu)
291 if not data:
279 return None
280
281 def alldata(self):
282 for run,cpus in self.data.iteritems():
283 for cpu,data in cpus.iteritems():
284 yield run,cpu,data
285
286 def get(self, job, stat):
287 if job.system is None:
288 raise AttributeError, 'The job must have a system set'
289
290 run = job.name
291 cpu = '%s.run%d' % (job.system, self.cpu)
292 data = self.getdata(run, cpu)
293 if not data:
292 return [ 0.0 for c in self.categories ]
294 return None
293
294 values = []
295 for category in self.categories:
295
296 values = []
297 for category in self.categories:
296 values.append(data.get(category, 0.0))
297 return values
298 val = float(data.get(category, 0.0))
299 if val < 0.0:
300 raise ValueError, 'value is %f' % val
301 values.append(val)
302 total = sum(values)
303 return [ v / total * 100.0 for v in values ]
298
299 def dump(self):
300 for run,cpu,data in self.alldata():
301 print 'run %s, cpu %s' % (run, cpu)
302 data.dump()
303 print
304
305 def write_dot(self, threshold, jobfile=None, jobs=None):

--- 71 unchanged lines hidden (view full) ---

377
378 if exitcode is not None:
379 sys.exit(exitcode)
380
381if __name__ == '__main__':
382 import getopt, re, sys
383 from os.path import expanduser
384 from output import StatOutput
304
305 def dump(self):
306 for run,cpu,data in self.alldata():
307 print 'run %s, cpu %s' % (run, cpu)
308 data.dump()
309 print
310
311 def write_dot(self, threshold, jobfile=None, jobs=None):

--- 71 unchanged lines hidden (view full) ---

383
384 if exitcode is not None:
385 sys.exit(exitcode)
386
387if __name__ == '__main__':
388 import getopt, re, sys
389 from os.path import expanduser
390 from output import StatOutput
385 from jobfile import JobFile
386
387 # default option values
388 numsyms = 10
389 graph = None
390 cpus = [ 0 ]
391 categorize = False
392 showidle = True
393 funcdata = True
394 jobfilename = 'Test.py'
395 dodot = False
391
392 # default option values
393 numsyms = 10
394 graph = None
395 cpus = [ 0 ]
396 categorize = False
397 showidle = True
398 funcdata = True
399 jobfilename = 'Test.py'
400 dodot = False
396 dotformat = 'raw'
401 dotfile = None
397 textout = False
398 threshold = 0.01
399 inputfile = None
400
401 try:
402 opts, args = getopt.getopt(sys.argv[1:], 'C:cdD:f:g:ij:n:pT:t')
403 except getopt.GetoptError:
404 usage(2)
405
406 for o,a in opts:
407 if o == '-C':
408 cpus = [ int(x) for x in a.split(',') ]
409 elif o == '-c':
410 categorize = True
411 elif o == '-D':
402 textout = False
403 threshold = 0.01
404 inputfile = None
405
406 try:
407 opts, args = getopt.getopt(sys.argv[1:], 'C:cdD:f:g:ij:n:pT:t')
408 except getopt.GetoptError:
409 usage(2)
410
411 for o,a in opts:
412 if o == '-C':
413 cpus = [ int(x) for x in a.split(',') ]
414 elif o == '-c':
415 categorize = True
416 elif o == '-D':
412 dotformat = a
417 dotfile = a
413 elif o == '-d':
414 dodot = True
415 elif o == '-f':
416 inputfile = expanduser(a)
417 elif o == '-g':
418 graph = a
419 elif o == '-i':
420 showidle = False

--- 8 unchanged lines hidden (view full) ---

429 elif o == '-t':
430 textout = True
431
432 if args:
433 print "'%s'" % args, len(args)
434 usage(1)
435
436 if inputfile:
418 elif o == '-d':
419 dodot = True
420 elif o == '-f':
421 inputfile = expanduser(a)
422 elif o == '-g':
423 graph = a
424 elif o == '-i':
425 showidle = False

--- 8 unchanged lines hidden (view full) ---

434 elif o == '-t':
435 textout = True
436
437 if args:
438 print "'%s'" % args, len(args)
439 usage(1)
440
441 if inputfile:
437 data = FuncData(inputfile)
442 catfunc = None
443 if categorize:
444 catfunc = func_categorize
445 data = FuncData(inputfile, categorize=catfunc)
438
439 if dodot:
440 import pydot
441 dot = pydot.Dot()
446
447 if dodot:
448 import pydot
449 dot = pydot.Dot()
442 data.dot(dot, threshold=threshold)
450 data.tree.dot(dot, threshold=threshold)
443 #dot.orientation = 'landscape'
444 #dot.ranksep='equally'
445 #dot.rank='samerank'
451 #dot.orientation = 'landscape'
452 #dot.ranksep='equally'
453 #dot.rank='samerank'
446 dot.write(dotfile, format=dotformat)
454 dot.write(dotfile, format='png')
447 else:
448 data.display(limit=numsyms)
449
450 else:
455 else:
456 data.display(limit=numsyms)
457
458 else:
459 from jobfile import JobFile
451 jobfile = JobFile(jobfilename)
452
453 if funcdata:
454 profile = FuncProfile()
455 else:
456 profile = PCProfile()
457
458 if not categorize:
459 profile.categorize = None
460 profile.inputdir(jobfile.rootdir)
461
462 if graph:
463 for cpu in cpus:
464 profile.cpu = cpu
465 if funcdata:
466 name = 'funcstacks%d' % cpu
467 else:
468 name = 'stacks%d' % cpu
460 jobfile = JobFile(jobfilename)
461
462 if funcdata:
463 profile = FuncProfile()
464 else:
465 profile = PCProfile()
466
467 if not categorize:
468 profile.categorize = None
469 profile.inputdir(jobfile.rootdir)
470
471 if graph:
472 for cpu in cpus:
473 profile.cpu = cpu
474 if funcdata:
475 name = 'funcstacks%d' % cpu
476 else:
477 name = 'stacks%d' % cpu
469 output = StatOutput(name, jobfile, info=profile)
470 output.graph(graph)
478 output = StatOutput(jobfile, info=profile)
479 output.xlabel = 'System Configuration'
480 output.ylabel = '% CPU utilization'
481 output.stat = name
482 output.graph(name, graph)
471
472 if dodot:
473 for cpu in cpus:
474 profile.cpu = cpu
475 profile.write_dot(jobfile=jobfile, threshold=threshold)
476
477 if textout:
478 for cpu in cpus:
479 profile.cpu = cpu
480 profile.write_txt(jobfile=jobfile)
481
482 if not graph and not textout and not dodot:
483 for cpu in cpus:
484 if not categorize:
485 profile.categorize = None
486 profile.cpu = cpu
487 profile.display(jobfile=jobfile, limit=numsyms)
483
484 if dodot:
485 for cpu in cpus:
486 profile.cpu = cpu
487 profile.write_dot(jobfile=jobfile, threshold=threshold)
488
489 if textout:
490 for cpu in cpus:
491 profile.cpu = cpu
492 profile.write_txt(jobfile=jobfile)
493
494 if not graph and not textout and not dodot:
495 for cpu in cpus:
496 if not categorize:
497 profile.categorize = None
498 profile.cpu = cpu
499 profile.display(jobfile=jobfile, limit=numsyms)