profile.py (1881:fc205a7edd58) profile.py (1917:2d0b0bd256ab)
1# Copyright (c) 2005 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright

--- 13 unchanged lines hidden (view full) ---

22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27from orderdict import orderdict
28import output
29
1# Copyright (c) 2005 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright

--- 13 unchanged lines hidden (view full) ---

22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27from orderdict import orderdict
28import output
29
30class ProfileData(object):
31 def __init__(self):
32 self.data = {}
33 self.total = {}
34 self.runs = orderdict()
35 self.runlist = []
30class RunData(dict):
31 def __init__(self, filename=None):
32 self.filename = filename
36
33
37 def addvalue(self, run, cpu, symbol, value):
38 value = float(value)
39 self.data[run, cpu, symbol] = self.getvalue(run, cpu, symbol) + value
40 self.total[run, cpu] = self.gettotal(run, cpu) + value
41 if run not in self.runs:
42 self.runs[run] = orderdict()
34 def __getattr__(self, attr):
35 if attr == 'total':
36 total = 0.0
37 for value in self.itervalues():
38 total += value
39 return total
40 if attr == 'maxsymlen':
41 return max([ len(sym) for sym in self.iterkeys() ])
43
42
44 if cpu not in self.runs[run]:
45 self.runs[run][cpu] = {}
43 def display(self, output=None, limit=None, maxsymlen=None):
44 if not output:
45 import sys
46 output = sys.stdout
47 elif isinstance(output, str):
48 output = file(output, 'w')
46
49
47 if symbol not in self.runs[run][cpu]:
48 self.runs[run][cpu][symbol] = 0
50 total = float(self.total)
49
51
50 self.runs[run][cpu][symbol] += value
52 # swap (string,count) order so we can sort on count
53 symbols = [ (count,name) for name,count in self.iteritems() ]
54 symbols.sort(reverse=True)
55 if limit is not None:
56 symbols = symbols[:limit]
51
57
52 def getvalue(self, run, cpu, symbol):
53 return self.data.get((run, cpu, symbol), 0)
58 if not maxsymlen:
59 maxsymlen = self.maxsymlen
54
60
55 def gettotal(self, run, cpu):
56 return self.total.get((run, cpu), 0)
61 symbolf = "%-" + str(maxsymlen + 1) + "s %.2f%%"
62 for number,name in symbols:
63 print >>output, symbolf % (name, 100.0 * (float(number) / total))
57
64
58class Profile(object):
59 default_order = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp']
60
65
61 # This list controls the order of values in stacked bar data output
62 default_categories = [ 'interrupt',
63 'driver',
64 'stack',
65 'bufmgt',
66 'copy',
67 'user',
68 'other',
69 'idle']
70
66
71 def __init__(self, run_order=[], categories=[], stacknames=[]):
72 if not run_order:
73 run_order = Profile.default_order
74 if not categories:
75 categories = Profile.default_categories
67class PCData(RunData):
68 def __init__(self, filename=None, categorize=None, showidle=True):
69 super(PCData, self).__init__(self, filename)
70 if filename is None:
71 return
76
72
77 self.run_order = run_order
78 self.categories = categories
79 self.rcategories = []
80 self.rcategories.extend(categories)
81 self.rcategories.reverse()
82 self.stacknames = stacknames
83 self.prof = ProfileData()
84 self.categorize = True
85 self.showidle = True
86 self.maxsymlen = 0
87
88 def category(self, symbol):
89 from categories import categories, categories_re
90 if categories.has_key(symbol):
91 return categories[symbol]
92 for regexp, cat in categories_re:
93 if regexp.match(symbol):
94 return cat
95 return 'other'
96
97 # Parse input file and put the results in the given run and cpu
98 def parsefile(self, run, cpu, filename):
99 fd = file(filename)
100
101 for line in fd:
73 fd = file(filename)
74
75 for line in fd:
76 if line.strip() == '>>>PC data':
77 break
78
79 for line in fd:
80 if line.startswith('>>>'):
81 break
82
102 (symbol, count) = line.split()
103 if symbol == "0x0":
104 continue
105 count = int(count)
106
83 (symbol, count) = line.split()
84 if symbol == "0x0":
85 continue
86 count = int(count)
87
107 if self.categorize:
108 symbol = self.category(symbol)
109 if symbol == 'idle' and not self.showidle:
88 if categorize is not None:
89 category = categorize(symbol)
90 if category is None:
91 category = 'other'
92 elif category == 'idle' and not showidle:
110 continue
111
93 continue
94
112 if symbol not in self.categories:
113 symbol = 'other'
95 self[category] = count
114
96
115 self.maxsymlen = max(self.maxsymlen, len(symbol))
116 self.prof.addvalue(run, cpu, symbol, count)
117
118 fd.close()
119
97 fd.close()
98
99class FuncNode(object):
100 def __new__(cls, filename = None):
101 if filename is None:
102 return super(FuncNode, cls).__new__(cls)
103
104 fd = file(filename, 'r')
105 fditer = iter(fd)
106 nodes = {}
107 for line in fditer:
108 if line.strip() == '>>>function data':
109 break
110
111 for line in fditer:
112 if line.startswith('>>>'):
113 break
114
115 data = line.split()
116 node_id = int(data[0], 16)
117 node = FuncNode()
118 node.symbol = data[1]
119 node.count = int(data[2])
120 node.children = [ int(child, 16) for child in data[3:] ]
121 nodes[node_id] = node
122
123 for node in nodes.itervalues():
124 children = []
125 for cid in node.children:
126 child = nodes[cid]
127 children.append(child)
128 child.parent = node
129 node.children = tuple(children)
130 if not nodes:
131 print filename
132 print nodes
133 return nodes[0]
134
135 def __init__(self, filename=None):
136 pass
137
138 def total(self):
139 total = self.count
140 for child in self.children:
141 total += child.total()
142
143 return total
144
145 def aggregate(self, dict, categorize, incategory):
146 category = None
147 if categorize:
148 category = categorize(self.symbol)
149
150 total = self.count
151 for child in self.children:
152 total += child.aggregate(dict, categorize, category or incategory)
153
154 if category:
155 dict[category] = dict.get(category, 0) + total
156 return 0
157 elif not incategory:
158 dict[self.symbol] = dict.get(self.symbol, 0) + total
159
160 return total
161
162 def dump(self):
163 kids = [ child.symbol for child in self.children]
164 print '%s %d <%s>' % (self.symbol, self.count, ', '.join(kids))
165 for child in self.children:
166 child.dump()
167
168 def _dot(self, dot, threshold, categorize, total):
169 from pydot import Dot, Edge, Node
170 self.dot_node = None
171
172 value = self.total() * 100.0 / total
173 if value < threshold:
174 return
175 if categorize:
176 category = categorize(self.symbol)
177 if category and category != 'other':
178 return
179 label = '%s %.2f%%' % (self.symbol, value)
180 self.dot_node = Node(self, label=label)
181 dot.add_node(self.dot_node)
182
183 for child in self.children:
184 child._dot(dot, threshold, categorize, total)
185 if child.dot_node is not None:
186 dot.add_edge(Edge(self, child))
187
188 def _cleandot(self):
189 for child in self.children:
190 child._cleandot()
191 self.dot_node = None
192 del self.__dict__['dot_node']
193
194 def dot(self, dot, threshold=0.1, categorize=None):
195 self._dot(dot, threshold, categorize, self.total())
196 self._cleandot()
197
198class FuncData(RunData):
199 def __init__(self, filename, categorize=None):
200 super(FuncData, self).__init__(filename)
201 self.tree = FuncNode(filename)
202 self.tree.aggregate(self, categorize, incategory=False)
203 self.total = self.tree.total()
204
205 def displayx(self, output=None, maxcount=None):
206 if output is None:
207 import sys
208 output = sys.stdout
209
210 items = [ (val,key) for key,val in self.iteritems() ]
211 items.sort(reverse=True)
212 for val,key in items:
213 if maxcount is not None:
214 if maxcount == 0:
215 return
216 maxcount -= 1
217
218 percent = val * 100.0 / self.total
219 print >>output, '%-30s %8s' % (key, '%3.2f%%' % percent)
220
221class Profile(object):
222 # This list controls the order of values in stacked bar data output
223 default_categories = [ 'interrupt',
224 'driver',
225 'stack',
226 'buffer',
227 'copy',
228 'syscall',
229 'user',
230 'other',
231 'idle']
232
233 def __init__(self, datatype, categorize=None):
234 categories = Profile.default_categories
235
236 self.datatype = datatype
237 self.categorize = categorize
238 self.data = {}
239 self.categories = categories[:]
240 self.rcategories = categories[:]
241 self.rcategories.reverse()
242 self.cpu = 0
243
120 # Read in files
121 def inputdir(self, directory):
122 import os, os.path, re
123 from os.path import expanduser, join as joinpath
124
125 directory = expanduser(directory)
244 # Read in files
245 def inputdir(self, directory):
246 import os, os.path, re
247 from os.path import expanduser, join as joinpath
248
249 directory = expanduser(directory)
126 label_ex = re.compile(r'm5prof\.(.*)')
250 label_ex = re.compile(r'profile\.(.*).dat')
127 for root,dirs,files in os.walk(directory):
128 for name in files:
129 match = label_ex.match(name)
130 if not match:
131 continue
132
133 filename = joinpath(root, name)
134 prefix = os.path.commonprefix([root, directory])
135 dirname = root[len(prefix)+1:]
251 for root,dirs,files in os.walk(directory):
252 for name in files:
253 match = label_ex.match(name)
254 if not match:
255 continue
256
257 filename = joinpath(root, name)
258 prefix = os.path.commonprefix([root, directory])
259 dirname = root[len(prefix)+1:]
136 self.parsefile(dirname, match.group(1), filename)
260 data = self.datatype(filename, self.categorize)
261 self.setdata(dirname, match.group(1), data)
137
262
263 def setdata(self, run, cpu, data):
264 if run not in self.data:
265 self.data[run] = {}
266
267 if cpu in self.data[run]:
268 raise AttributeError, \
269 'data already stored for run %s and cpu %s' % (run, cpu)
270
271 self.data[run][cpu] = data
272
273 def getdata(self, run, cpu):
274 try:
275 return self.data[run][cpu]
276 except KeyError:
277 return None
278
279 def alldata(self):
280 for run,cpus in self.data.iteritems():
281 for cpu,data in cpus.iteritems():
282 yield run,cpu,data
283
138 def get(self, job, stat):
139 if job.system is None:
140 raise AttributeError, 'The job must have a system set'
141
284 def get(self, job, stat):
285 if job.system is None:
286 raise AttributeError, 'The job must have a system set'
287
142 cpu = '%s.full0' % job.system
288 data = self.getdata(job.name, '%s.full%d' % (job.system, self.cpu))
289 if not data:
290 return [ 0.0 for c in self.categories ]
291
143 values = []
292 values = []
144 for cat in self.categories:
145 values.append(self.prof.getvalue(job.name, cpu, cat))
293 for category in self.categories:
294 values.append(data.get(category, 0.0))
146 return values
295 return values
296
297 def dump(self):
298 for run,cpu,data in self.alldata():
299 print 'run %s, cpu %s' % (run, cpu)
300 data.dump()
301 print
302
303 def write_dot(self, threshold, jobfile=None, jobs=None):
304 import pydot
305
306 if jobs is None:
307 jobs = [ job for job in jobfile.jobs() ]
308
309 for job in jobs:
310 cpu = '%s.full%d' % (job.system, self.cpu)
311 symbols = self.getdata(job.name, cpu)
312 if not symbols:
313 continue
314
315 dot = pydot.Dot()
316 symbols.tree.dot(dot, threshold=threshold)
317 dot.write(symbols.filename[:-3] + 'dot')
318
319 def write_txt(self, jobfile=None, jobs=None):
320 if jobs is None:
321 jobs = [ job for job in jobfile.jobs() ]
322
323 for job in jobs:
324 cpu = '%s.full%d' % (job.system, self.cpu)
325 symbols = self.getdata(job.name, cpu)
326 if not symbols:
327 continue
328
329 output = file(symbols.filename[:-3] + 'txt', 'w')
330 symbols.display(output)
331
332 def display(self, jobfile=None, jobs=None, limit=None):
333 if jobs is None:
334 jobs = [ job for job in jobfile.jobs() ]
335
336 maxsymlen = 0
337
338 thejobs = []
339 for job in jobs:
340 cpu = '%s.full%d' % (job.system, self.cpu)
341 symbols = self.getdata(job.name, cpu)
342 if symbols:
343 thejobs.append(job)
344 maxsymlen = max(maxsymlen, symbols.maxsymlen)
345
346 for job in thejobs:
347 cpu = '%s.full%d' % (job.system, self.cpu)
348 symbols = self.getdata(job.name, cpu)
349 print job.name
350 symbols.display(limit=limit, maxsymlen=maxsymlen)
351 print
352
353
354from categories import func_categorize, pc_categorize
355class PCProfile(Profile):
356 def __init__(self, categorize=pc_categorize):
357 super(PCProfile, self).__init__(PCData, categorize)
358
359
360class FuncProfile(Profile):
361 def __init__(self, categorize=func_categorize):
362 super(FuncProfile, self).__init__(FuncData, categorize)
363
364def usage(exitcode = None):
365 print '''\
366Usage: %s [-bc] [-g <dir>] [-j <jobfile>] [-n <num>]
367
368 -c groups symbols into categories
369 -b dumps data for bar charts
370 -d generate dot output
371 -g <d> draw graphs and send output to <d>
372 -j <jobfile> specify a different jobfile (default is Test.py)
373 -n <n> selects number of top symbols to print (default 5)
374''' % sys.argv[0]
375
376 if exitcode is not None:
377 sys.exit(exitcode)
378
379if __name__ == '__main__':
380 import getopt, re, sys
381 from os.path import expanduser
382 from output import StatOutput
383 from jobfile import JobFile
384
385 # default option values
386 numsyms = 10
387 graph = None
388 cpus = [ 0 ]
389 categorize = False
390 showidle = True
391 funcdata = True
392 jobfilename = 'Test.py'
393 dodot = False
394 dotformat = 'raw'
395 textout = False
396 threshold = 0.01
397 inputfile = None
398
399 try:
400 opts, args = getopt.getopt(sys.argv[1:], 'C:cdD:f:g:ij:n:pT:t')
401 except getopt.GetoptError:
402 usage(2)
403
404 for o,a in opts:
405 if o == '-C':
406 cpus = [ int(x) for x in a.split(',') ]
407 elif o == '-c':
408 categorize = True
409 elif o == '-D':
410 dotformat = a
411 elif o == '-d':
412 dodot = True
413 elif o == '-f':
414 inputfile = expanduser(a)
415 elif o == '-g':
416 graph = a
417 elif o == '-i':
418 showidle = False
419 elif o == '-j':
420 jobfilename = a
421 elif o == '-n':
422 numsyms = int(a)
423 elif o == '-p':
424 funcdata = False
425 elif o == '-T':
426 threshold = float(a)
427 elif o == '-t':
428 textout = True
429
430 if args:
431 print "'%s'" % args, len(args)
432 usage(1)
433
434 if inputfile:
435 data = FuncData(inputfile)
436
437 if dodot:
438 import pydot
439 dot = pydot.Dot()
440 data.dot(dot, threshold=threshold)
441 #dot.orientation = 'landscape'
442 #dot.ranksep='equally'
443 #dot.rank='samerank'
444 dot.write(dotfile, format=dotformat)
445 else:
446 data.display(limit=numsyms)
447
448 else:
449 jobfile = JobFile(jobfilename)
450
451 if funcdata:
452 profile = FuncProfile()
453 else:
454 profile = PCProfile()
455
456 profile.inputdir(jobfile.rootdir)
457
458 if graph:
459 for cpu in cpus:
460 profile.cpu = cpu
461 if funcdata:
462 name = 'funcstacks%d' % cpu
463 else:
464 name = 'stacks%d' % cpu
465 output = StatOutput(name, jobfile, info=profile)
466 output.graph(graph)
467
468 if dodot:
469 for cpu in cpus:
470 profile.cpu = cpu
471 profile.write_dot(jobfile=jobfile, threshold=threshold)
472
473 if not categorize:
474 for cpu in cpus:
475 profile.cpu = cpu
476 profile.categorize = None
477
478 if textout:
479 for cpu in cpus:
480 profile.cpu = cpu
481 profile.write_txt(jobfile=jobfile)
482
483 if not graph and not textout and not dodot:
484 for cpu in cpus:
485 profile.cpu = cpu
486 profile.display(jobfile=jobfile, limit=numsyms)