profile.py revision 1933:83716112740d 
1# Copyright (c) 2005 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27from orderdict import orderdict
28import output
29
30class RunData(dict):
31    def __init__(self, filename=None):
32        self.filename = filename
33
34    def __getattr__(self, attr):
35        if attr == 'total':
36            total = 0.0
37            for value in self.itervalues():
38                total += value
39            return total
40        if attr == 'maxsymlen':
41            return max([ len(sym) for sym in self.iterkeys() ])
42
43    def display(self, output=None, limit=None, maxsymlen=None):
44        if not output:
45            import sys
46            output = sys.stdout
47        elif isinstance(output, str):
48            output = file(output, 'w')
49
50        total = float(self.total)
51
52        # swap (string,count) order so we can sort on count
53        symbols = [ (count,name) for name,count in self.iteritems() ]
54        symbols.sort(reverse=True)
55        if limit is not None:
56            symbols = symbols[:limit]
57
58        if not maxsymlen:
59            maxsymlen = self.maxsymlen
60
61        symbolf = "%-" + str(maxsymlen + 1) + "s %.2f%%"
62        for number,name in symbols:
63            print >>output, symbolf % (name, 100.0 * (float(number) / total))
64
65
66
67class PCData(RunData):
68    def __init__(self, filename=None, categorize=None, showidle=True):
69        super(PCData, self).__init__(self, filename)
70        if filename is None:
71            return
72
73        fd = file(filename)
74
75        for line in fd:
76            if line.strip() == '>>>PC data':
77                break
78
79        for line in fd:
80            if line.startswith('>>>'):
81                break
82
83            (symbol, count) = line.split()
84            if symbol == "0x0":
85                continue
86            count = int(count)
87
88            if categorize is not None:
89                category = categorize(symbol)
90                if category is None:
91                    category = 'other'
92                elif category == 'idle' and not showidle:
93                    continue
94
95                self[category] = count
96
97        fd.close()
98
99class FuncNode(object):
100    def __new__(cls, filename = None):
101        if filename is None:
102            return super(FuncNode, cls).__new__(cls)
103
104        fd = file(filename, 'r')
105        fditer = iter(fd)
106        nodes = {}
107        for line in fditer:
108            if line.strip() == '>>>function data':
109                break
110
111        for line in fditer:
112            if line.startswith('>>>'):
113                break
114
115            data = line.split()
116            node_id = int(data[0], 16)
117            node = FuncNode()
118            node.symbol = data[1]
119            node.count = int(data[2])
120            node.children = [ int(child, 16) for child in data[3:] ]
121            nodes[node_id] = node
122
123        for node in nodes.itervalues():
124            children = []
125            for cid in node.children:
126                child = nodes[cid]
127                children.append(child)
128                child.parent = node
129            node.children = tuple(children)
130        if not nodes:
131            print filename
132            print nodes
133        return nodes[0]
134
135    def __init__(self, filename=None):
136        pass
137
138    def total(self):
139        total = self.count
140        for child in self.children:
141            total += child.total()
142
143        return total
144
145    def aggregate(self, dict, categorize, incategory):
146        category = None
147        if categorize:
148            category = categorize(self.symbol)
149
150        total = self.count
151        for child in self.children:
152            total += child.aggregate(dict, categorize, category or incategory)
153
154        if category:
155            dict[category] = dict.get(category, 0) + total
156            return 0
157        elif not incategory:
158            dict[self.symbol] = dict.get(self.symbol, 0) + total
159
160        return total
161
162    def dump(self):
163        kids = [ child.symbol for child in self.children]
164        print '%s %d <%s>' % (self.symbol, self.count, ', '.join(kids))
165        for child in self.children:
166            child.dump()
167
168    def _dot(self, dot, threshold, categorize, total):
169        from pydot import Dot, Edge, Node
170        self.dot_node = None
171
172        value = self.total() * 100.0 / total
173        if value < threshold:
174            return
175        if categorize:
176            category = categorize(self.symbol)
177            if category and category != 'other':
178                return
179        label = '%s %.2f%%' % (self.symbol, value)
180        self.dot_node = Node(self, label=label)
181        dot.add_node(self.dot_node)
182
183        for child in self.children:
184            child._dot(dot, threshold, categorize, total)
185            if child.dot_node is not None:
186                dot.add_edge(Edge(self, child))
187
188    def _cleandot(self):
189        for child in self.children:
190            child._cleandot()
191            self.dot_node = None
192            del self.__dict__['dot_node']
193
194    def dot(self, dot, threshold=0.1, categorize=None):
195        self._dot(dot, threshold, categorize, self.total())
196        self._cleandot()
197
198class FuncData(RunData):
199    def __init__(self, filename, categorize=None):
200        super(FuncData, self).__init__(filename)
201        self.tree = FuncNode(filename)
202        self.tree.aggregate(self, categorize, incategory=False)
203        self.total = self.tree.total()
204
205    def displayx(self, output=None, maxcount=None):
206        if output is None:
207            import sys
208            output = sys.stdout
209
210        items = [ (val,key) for key,val in self.iteritems() ]
211        items.sort(reverse=True)
212        for val,key in items:
213            if maxcount is not None:
214                if maxcount == 0:
215                    return
216                maxcount -= 1
217
218            percent = val * 100.0 / self.total
219            print >>output, '%-30s %8s' % (key, '%3.2f%%' % percent)
220
221class Profile(object):
222    # This list controls the order of values in stacked bar data output
223    default_categories = [ 'interrupt',
224                           'driver',
225                           'stack',
226                           'buffer',
227                           'copy',
228                           'syscall',
229                           'user',
230                           'other',
231                           'idle']
232
233    def __init__(self, datatype, categorize=None):
234        categories = Profile.default_categories
235
236        self.datatype = datatype
237        self.categorize = categorize
238        self.data = {}
239        self.categories = categories[:]
240        self.rcategories = categories[:]
241        self.rcategories.reverse()
242        self.cpu = 0
243
244    # Read in files
245    def inputdir(self, directory):
246        import os, os.path, re
247        from os.path import expanduser, join as joinpath
248
249        directory = expanduser(directory)
250        label_ex = re.compile(r'profile\.(.*).dat')
251        for root,dirs,files in os.walk(directory):
252            for name in files:
253                match = label_ex.match(name)
254                if not match:
255                    continue
256
257                filename = joinpath(root, name)
258                prefix = os.path.commonprefix([root, directory])
259                dirname = root[len(prefix)+1:]
260                data = self.datatype(filename, self.categorize)
261                self.setdata(dirname, match.group(1), data)
262
263    def setdata(self, run, cpu, data):
264        if run not in self.data:
265            self.data[run] = {}
266
267        if cpu in self.data[run]:
268            raise AttributeError, \
269                  'data already stored for run %s and cpu %s' % (run, cpu)
270
271        self.data[run][cpu] = data
272
273    def getdata(self, run, cpu):
274        try:
275            return self.data[run][cpu]
276        except KeyError:
277            return None
278
279    def alldata(self):
280        for run,cpus in self.data.iteritems():
281            for cpu,data in cpus.iteritems():
282                yield run,cpu,data
283
284    def get(self, job, stat):
285        if job.system is None:
286            raise AttributeError, 'The job must have a system set'
287
288        data = self.getdata(job.name, '%s.full%d' % (job.system, self.cpu))
289        if not data:
290            return [ 0.0 for c in self.categories ]
291
292        values = []
293        for category in self.categories:
294            values.append(data.get(category, 0.0))
295        return values
296
297    def dump(self):
298        for run,cpu,data in self.alldata():
299            print 'run %s, cpu %s' % (run, cpu)
300            data.dump()
301            print
302
303    def write_dot(self, threshold, jobfile=None, jobs=None):
304        import pydot
305
306        if jobs is None:
307            jobs = [ job for job in jobfile.jobs() ]
308
309        for job in jobs:
310            cpu =  '%s.full%d' % (job.system, self.cpu)
311            symbols = self.getdata(job.name, cpu)
312            if not symbols:
313                continue
314
315            dot = pydot.Dot()
316            symbols.tree.dot(dot, threshold=threshold)
317            dot.write(symbols.filename[:-3] + 'dot')
318
319    def write_txt(self, jobfile=None, jobs=None, limit=None):
320        if jobs is None:
321            jobs = [ job for job in jobfile.jobs() ]
322
323        for job in jobs:
324            cpu =  '%s.full%d' % (job.system, self.cpu)
325            symbols = self.getdata(job.name, cpu)
326            if not symbols:
327                continue
328
329            output = file(symbols.filename[:-3] + 'txt', 'w')
330            symbols.display(output, limit)
331
332    def display(self, jobfile=None, jobs=None, limit=None):
333        if jobs is None:
334            jobs = [ job for job in jobfile.jobs() ]
335
336        maxsymlen = 0
337
338        thejobs = []
339        for job in jobs:
340            cpu =  '%s.full%d' % (job.system, self.cpu)
341            symbols = self.getdata(job.name, cpu)
342            if symbols:
343                thejobs.append(job)
344                maxsymlen = max(maxsymlen, symbols.maxsymlen)
345
346        for job in thejobs:
347            cpu =  '%s.full%d' % (job.system, self.cpu)
348            symbols = self.getdata(job.name, cpu)
349            print job.name
350            symbols.display(limit=limit, maxsymlen=maxsymlen)
351            print
352
353
354from categories import func_categorize, pc_categorize
355class PCProfile(Profile):
356    def __init__(self, categorize=pc_categorize):
357        super(PCProfile, self).__init__(PCData, categorize)
358
359
360class FuncProfile(Profile):
361    def __init__(self, categorize=func_categorize):
362        super(FuncProfile, self).__init__(FuncData, categorize)
363
364def usage(exitcode = None):
365    print '''\
366Usage: %s [-bc] [-g <dir>] [-j <jobfile>] [-n <num>]
367
368    -c           groups symbols into categories
369    -b           dumps data for bar charts
370    -d           generate dot output
371    -g <d>       draw graphs and send output to <d>
372    -j <jobfile> specify a different jobfile (default is Test.py)
373    -n <n>       selects number of top symbols to print (default 5)
374''' % sys.argv[0]
375
376    if exitcode is not None:
377        sys.exit(exitcode)
378
379if __name__ == '__main__':
380    import getopt, re, sys
381    from os.path import expanduser
382    from output import StatOutput
383    from jobfile import JobFile
384
385    # default option values
386    numsyms = 10
387    graph = None
388    cpus = [ 0 ]
389    categorize = False
390    showidle = True
391    funcdata = True
392    jobfilename = 'Test.py'
393    dodot = False
394    dotformat = 'raw'
395    textout = False
396    threshold = 0.01
397    inputfile = None
398
399    try:
400        opts, args = getopt.getopt(sys.argv[1:], 'C:cdD:f:g:ij:n:pT:t')
401    except getopt.GetoptError:
402        usage(2)
403
404    for o,a in opts:
405        if o == '-C':
406            cpus = [ int(x) for x in a.split(',') ]
407        elif o == '-c':
408            categorize = True
409        elif o == '-D':
410            dotformat = a
411        elif o == '-d':
412            dodot = True
413        elif o == '-f':
414            inputfile = expanduser(a)
415        elif o == '-g':
416            graph = a
417        elif o == '-i':
418            showidle = False
419        elif o == '-j':
420            jobfilename = a
421        elif o == '-n':
422            numsyms = int(a)
423        elif o == '-p':
424            funcdata = False
425        elif o == '-T':
426            threshold = float(a)
427        elif o == '-t':
428            textout = True
429
430    if args:
431        print "'%s'" % args, len(args)
432        usage(1)
433
434    if inputfile:
435        data = FuncData(inputfile)
436
437        if dodot:
438            import pydot
439            dot = pydot.Dot()
440            data.dot(dot, threshold=threshold)
441            #dot.orientation = 'landscape'
442            #dot.ranksep='equally'
443            #dot.rank='samerank'
444            dot.write(dotfile, format=dotformat)
445        else:
446            data.display(limit=numsyms)
447
448    else:
449        jobfile = JobFile(jobfilename)
450
451        if funcdata:
452            profile = FuncProfile()
453        else:
454            profile = PCProfile()
455
456        if not categorize:
457            profile.categorize = None
458        profile.inputdir(jobfile.rootdir)
459
460        if graph:
461            for cpu in cpus:
462                profile.cpu = cpu
463                if funcdata:
464                    name = 'funcstacks%d' % cpu
465                else:
466                    name = 'stacks%d' % cpu
467                output = StatOutput(name, jobfile, info=profile)
468                output.graph(graph)
469
470        if dodot:
471            for cpu in cpus:
472                profile.cpu = cpu
473                profile.write_dot(jobfile=jobfile, threshold=threshold)
474
475        if textout:
476            for cpu in cpus:
477                profile.cpu = cpu
478                profile.write_txt(jobfile=jobfile)
479
480        if not graph and not textout and not dodot:
481            for cpu in cpus:
482                if not categorize:
483                    profile.categorize = None
484                profile.cpu = cpu
485                profile.display(jobfile=jobfile, limit=numsyms)
486