m5stats2streamline.py revision 9935:cc9dc514036e 
1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50#                    in conversion. Sample .ini files can be found in
51#                    util/streamline.
52#                    NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55#                    stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v12 (DS-5 v5.13)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73        formatter_class=argparse.RawDescriptionHelpFormatter,
74        description="""
75        Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76        (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77        enabled and some patches applied to the target Linux kernel.)
78
79        Visit http://www.gem5.org/Streamline for more details.
80
81        APC project generation based on Gator v12 (DS-5 v5.13)
82        Subsequent versions should be backward compatible
83        """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86                    help=".ini file that describes which stats to be included \
87                    in conversion. Sample .ini files can be found in \
88                    util/streamline. NOTE: this is NOT the gem5 config.ini \
89                    file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92                    help="Path to gem5 run folder (must contain config.ini, \
93                    stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96                    help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99                    default=1000000,
100                    help="Maximum number of scheduling (context switch) \
101                    events to be processed. Set to truncate early. \
102                    Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105                    help="Do not use gzipped .bmp files for visual annotations. \
106                    This option is only required when using Streamline versions \
107                    older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110                    help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115    print "ERROR: <dest .apc folder> should end with '.apc'!"
116    sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134    global num_cpus, num_l2
135
136    print "\n==============================="
137    print "Parsing gem5 config.ini file..."
138    print config_file
139    print "===============================\n"
140    config = ConfigParser()
141    if not config.read(config_file):
142        print "ERROR: config file '", config_file, "' not found"
143        sys.exit(1)
144
145    if config.has_section("system.cpu"):
146        num_cpus = 1
147    else:
148        num_cpus = 0
149        while config.has_section("system.cpu" + str(num_cpus)):
150            num_cpus += 1
151
152    if config.has_section("system.l2"):
153        num_l2 = 1
154    else:
155        num_l2 = 0
156        while config.has_section("system.l2" + str(num_l2)):
157            num_l2 += 1
158
159    print "Num CPUs:", num_cpus
160    print "Num L2s:", num_l2
161    print ""
162
163    return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175    def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176        if pid == 0: # Idle
177            self.uid = 0
178        elif pid == -1: # Kernel
179            self.uid = 0
180        else:
181            self.uid = uid
182        self.pid = pid
183        self.tgid = tgid
184        self.is_process = is_process
185        self.task_name = task_name
186        self.children = []
187        self.tick = tick # time this task first appeared
188
189class Event(object):
190    def __init__(self, tick, task):
191        self.tick = tick
192        self.task = task
193
194############################################################
195# Types used in APC Protocol
196#  - packed32, packed64
197#  - int32
198#  - string
199############################################################
200
201#  variable length packed 4-byte signed value
202def packed32(x):
203    ret = []
204    if ((x & 0xffffff80) == 0):
205        ret.append(x & 0x7f)
206    elif ((x & 0xffffc000) == 0):
207        ret.append((x | 0x80) & 0xff)
208        ret.append((x >> 7) & 0x7f)
209    elif ((x & 0xffe00000) == 0):
210        ret.append((x | 0x80) & 0xff)
211        ret.append(((x >> 7) | 0x80) & 0xff)
212        ret.append((x >> 14) & 0x7f)
213    elif ((x & 0xf0000000) == 0):
214        ret.append((x | 0x80) & 0xff)
215        ret.append(((x >> 7) | 0x80) & 0xff)
216        ret.append(((x >> 14) | 0x80) & 0xff)
217        ret.append((x >> 21) & 0x7f)
218    else:
219        ret.append((x | 0x80) & 0xff)
220        ret.append(((x >> 7) | 0x80) & 0xff)
221        ret.append(((x >> 14) | 0x80) & 0xff)
222        ret.append(((x >> 21) | 0x80) & 0xff)
223        ret.append((x >> 28) & 0x0f)
224    return ret
225
226#  variable length packed 8-byte signed value
227def packed64(x):
228    ret = []
229    if ((x & 0xffffffffffffff80) == 0):
230        ret.append(x & 0x7f)
231    elif ((x & 0xffffffffffffc000) == 0):
232        ret.append((x | 0x80) & 0xff)
233        ret.append((x >> 7) & 0x7f)
234    elif ((x & 0xffffffffffe00000) == 0):
235        ret.append((x | 0x80) & 0xff)
236        ret.append(((x >> 7) | 0x80) & 0xff)
237        ret.append((x >> 14) & 0x7f)
238    elif ((x & 0xfffffffff0000000) == 0):
239        ret.append((x | 0x80) & 0xff)
240        ret.append(((x >> 7) | 0x80) & 0xff)
241        ret.append(((x >> 14) | 0x80) & 0xff)
242        ret.append((x >> 21) & 0x7f)
243    elif ((x & 0xfffffff800000000) == 0):
244        ret.append((x | 0x80) & 0xff)
245        ret.append(((x >> 7) | 0x80) & 0xff)
246        ret.append(((x >> 14) | 0x80) & 0xff)
247        ret.append(((x >> 21) | 0x80) & 0xff)
248        ret.append((x >> 28) & 0x7f)
249    elif ((x & 0xfffffc0000000000) == 0):
250        ret.append((x | 0x80) & 0xff)
251        ret.append(((x >> 7) | 0x80) & 0xff)
252        ret.append(((x >> 14) | 0x80) & 0xff)
253        ret.append(((x >> 21) | 0x80) & 0xff)
254        ret.append(((x >> 28) | 0x80) & 0xff)
255        ret.append((x >> 35) & 0x7f)
256    elif ((x & 0xfffe000000000000) == 0):
257        ret.append((x | 0x80) & 0xff)
258        ret.append(((x >> 7) | 0x80) & 0xff)
259        ret.append(((x >> 14) | 0x80) & 0xff)
260        ret.append(((x >> 21) | 0x80) & 0xff)
261        ret.append(((x >> 28) | 0x80) & 0xff)
262        ret.append(((x >> 35) | 0x80) & 0xff)
263        ret.append((x >> 42) & 0x7f)
264    elif ((x & 0xff00000000000000) == 0):
265        ret.append((x | 0x80) & 0xff)
266        ret.append(((x >> 7) | 0x80) & 0xff)
267        ret.append(((x >> 14) | 0x80) & 0xff)
268        ret.append(((x >> 21) | 0x80) & 0xff)
269        ret.append(((x >> 28) | 0x80) & 0xff)
270        ret.append(((x >> 35) | 0x80) & 0xff)
271        ret.append(((x >> 42) | 0x80) & 0xff)
272        ret.append((x >> 49) & 0x7f)
273    elif ((x & 0x8000000000000000) == 0):
274        ret.append((x | 0x80) & 0xff)
275        ret.append(((x >> 7) | 0x80) & 0xff)
276        ret.append(((x >> 14) | 0x80) & 0xff)
277        ret.append(((x >> 21) | 0x80) & 0xff)
278        ret.append(((x >> 28) | 0x80) & 0xff)
279        ret.append(((x >> 35) | 0x80) & 0xff)
280        ret.append(((x >> 42) | 0x80) & 0xff)
281        ret.append(((x >> 49) | 0x80) & 0xff)
282        ret.append((x >> 56) & 0x7f)
283    else:
284        ret.append((x | 0x80) & 0xff)
285        ret.append(((x >> 7) | 0x80) & 0xff)
286        ret.append(((x >> 14) | 0x80) & 0xff)
287        ret.append(((x >> 21) | 0x80) & 0xff)
288        ret.append(((x >> 28) | 0x80) & 0xff)
289        ret.append(((x >> 35) | 0x80) & 0xff)
290        ret.append(((x >> 42) | 0x80) & 0xff)
291        ret.append(((x >> 49) | 0x80) & 0xff)
292        ret.append(((x >> 56) | 0x80) & 0xff)
293        ret.append((x >> 63) & 0x7f)
294    return ret
295
296# 4-byte signed little endian
297def int32(x):
298    ret = []
299    ret.append(x & 0xff)
300    ret.append((x >> 8) & 0xff)
301    ret.append((x >> 16) & 0xff)
302    ret.append((x >> 24) & 0xff)
303    return ret
304
305# 2-byte signed little endian
306def int16(x):
307    ret = []
308    ret.append(x & 0xff)
309    ret.append((x >> 8) & 0xff)
310    return ret
311
312# a packed32 length followed by the specified number of characters
313def stringList(x):
314    ret = []
315    ret += packed32(len(x))
316    for i in x:
317        ret.append(i)
318    return ret
319
320def utf8StringList(x):
321    ret = []
322    for i in x:
323        ret.append(ord(i))
324    return ret
325
326# packed64 time value in nanoseconds relative to the uptime from the
327# Summary message.
328def timestampList(x):
329    ret = packed64(x)
330    return ret
331
332
333############################################################
334# Write binary
335############################################################
336
337def writeBinary(outfile, binary_list):
338    for i in binary_list:
339        outfile.write("%c" % i)
340
341############################################################
342# APC Protocol Frame Types
343############################################################
344
345def addFrameHeader(frame_type, body, core):
346    ret = []
347
348    if frame_type == "Summary":
349        code = 1
350    elif frame_type == "Backtrace":
351        code = 2
352    elif frame_type == "Name":
353        code = 3
354    elif frame_type == "Counter":
355        code = 4
356    elif frame_type == "Block Counter":
357        code = 5
358    elif frame_type == "Annotate":
359        code = 6
360    elif frame_type == "Sched Trace":
361        code = 7
362    elif frame_type == "GPU Trace":
363        code = 8
364    elif frame_type == "Idle":
365        code = 9
366    else:
367        print "ERROR: Unknown frame type:", frame_type
368        sys.exit(1)
369
370    packed_code = packed32(code)
371
372    packed_core = packed32(core)
373
374    length = int32(len(packed_code) + len(packed_core) + len(body))
375
376    ret = length + packed_code + packed_core + body
377    return ret
378
379
380# Summary frame
381#  - timestamp: packed64
382#  - uptime: packed64
383def summaryFrame(timestamp, uptime):
384    frame_type = "Summary"
385    body = packed64(timestamp) + packed64(uptime)
386    ret = addFrameHeader(frame_type, body, 0)
387    return ret
388
389# Backtrace frame
390#  - not implemented yet
391def backtraceFrame():
392    pass
393
394# Cookie name message
395#  - cookie: packed32
396#  - name: string
397def cookieNameFrame(cookie, name):
398    frame_type = "Name"
399    packed_code = packed32(1)
400    body = packed_code + packed32(cookie) + stringList(name)
401    ret = addFrameHeader(frame_type, body, 0)
402    return ret
403
404# Thread name message
405#  - timestamp: timestamp
406#  - thread id: packed32
407#  - name: string
408def threadNameFrame(timestamp, thread_id, name):
409    frame_type = "Name"
410    packed_code = packed32(2)
411    body = packed_code + timestampList(timestamp) + \
412            packed32(thread_id) + stringList(name)
413    ret = addFrameHeader(frame_type, body, 0)
414    return ret
415
416# Core name message
417#  - name: string
418def coreNameFrame(name):
419    frame_type = "Name"
420    packed_code = packed32(3)
421    body = packed_code + stringList(name)
422    ret = addFrameHeader(frame_type, body, 0)
423    return ret
424
425# Counter frame message
426#  - timestamp: timestamp
427#  - core: packed32
428#  - key: packed32
429#  - value: packed64
430def counterFrame(timestamp, core, key, value):
431    frame_type = "Counter"
432    body = timestampList(timestamp) + packed32(core) + packed32(key) + \
433            packed64(value)
434    ret = addFrameHeader(frame_type, body, core)
435    return ret
436
437# Block Counter frame message
438#  - key: packed32
439#  - value: packed64
440def blockCounterFrame(core, key, value):
441    frame_type = "Block Counter"
442    body = packed32(key) + packed64(value)
443    ret = addFrameHeader(frame_type, body, core)
444    return ret
445
446# Annotate frame messages
447#  - core: packed32
448#  - tid: packed32
449#  - timestamp: timestamp
450#  - size: packed32
451#  - body
452def annotateFrame(core, tid, timestamp, size, userspace_body):
453    frame_type = "Annotate"
454    body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
455            packed32(size) + userspace_body
456    ret = addFrameHeader(frame_type, body, core)
457    return ret
458
459# Scheduler Trace frame messages
460# Sched Switch
461#  - Code: 1
462#  - timestamp: timestamp
463#  - pid: packed32
464#  - tid: packed32
465#  - cookie: packed32
466#  - state: packed32
467def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
468    frame_type = "Sched Trace"
469    body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
470            packed32(tid) + packed32(cookie) + packed32(state)
471    ret = addFrameHeader(frame_type, body, core)
472    return ret
473
474# Sched Thread Exit
475#  - Code: 2
476#  - timestamp: timestamp
477#  - tid: packed32
478def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
479    frame_type = "Sched Trace"
480    body = packed32(2) + timestampList(timestamp) + packed32(tid)
481    ret = addFrameHeader(frame_type, body, core)
482    return ret
483
484# GPU Trace frame messages
485#  - Not implemented yet
486def gpuTraceFrame():
487    pass
488
489# Idle frame messages
490# Enter Idle
491#  - code: 1
492#  - timestamp: timestamp
493#  - core: packed32
494def enterIdleFrame(timestamp, core):
495    frame_type = "Idle"
496    body = packed32(1) + timestampList(timestamp) + packed32(core)
497    ret = addFrameHeader(frame_type, body, core)
498    return ret
499
500# Exit Idle
501#  - code: 2
502#  - timestamp: timestamp
503#  - core: packed32
504def exitIdleFrame(timestamp, core):
505    frame_type = "Idle"
506    body = packed32(2) + timestampList(timestamp) + packed32(core)
507    ret = addFrameHeader(frame_type, body, core)
508    return ret
509
510
511####################################################################
512def parseProcessInfo(task_file):
513    print "\n==============================="
514    print "Parsing Task file..."
515    print task_file
516    print "===============================\n"
517
518    global start_tick, end_tick, num_cpus
519    global process_dict, thread_dict, process_list
520    global event_list, unified_event_list
521    global idle_uid, kernel_uid
522
523    event_list = []
524    unified_event_list = []
525    for cpu in range(num_cpus):
526        event_list.append([])
527
528    uid = 1 # uid 0 is reserved for idle
529
530    # Dummy Tasks for frame buffers and system diagrams
531    process = Task(uid, 9999, 9999, "framebuffer", True, 0)
532    process_list.append(process)
533    uid += 1
534    thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
535    process.children.append(thread)
536    uid += 1
537    process = Task(uid, 9998, 9998, "System", True, 0)
538    process_list.append(process)
539    # if we don't find the real kernel, use this to keep things going
540    kernel_uid = uid
541    uid += 1
542    thread = Task(uid, 9998, 9998, "System", False, 0)
543    process.children.append(thread)
544    uid += 1
545
546    ext = os.path.splitext(task_file)[1]
547
548    try:
549        if ext == ".gz":
550            process_file = gzip.open(task_file, 'rb')
551        else:
552            process_file = open(task_file, 'rb')
553    except:
554        print "ERROR opening task file:", task_file
555        print "Make sure context switch task dumping is enabled in gem5."
556        sys.exit(1)
557
558    process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
559        "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
560
561    task_name_failure_warned = False
562
563    for line in process_file:
564        match = re.match(process_re, line)
565        if match:
566            tick = int(match.group(1))
567            if (start_tick < 0):
568                start_tick = tick
569            cpu_id = int(match.group(3))
570            pid = int(match.group(4))
571            tgid = int(match.group(5))
572            task_name = match.group(6)
573
574            if not task_name_failure_warned:
575                if task_name == "FailureIn_curTaskName":
576                    print "-------------------------------------------------"
577                    print "WARNING: Task name not set correctly!"
578                    print "Process/Thread info will not be displayed correctly"
579                    print "Perhaps forgot to apply m5struct.patch to kernel?"
580                    print "-------------------------------------------------"
581                    task_name_failure_warned = True
582
583            if not tgid in process_dict:
584                if tgid == pid:
585                    # new task is parent as well
586                    if args.verbose:
587                        print "new process", uid, pid, tgid, task_name
588                    if tgid == 0:
589                        # new process is the "idle" task
590                        process = Task(uid, pid, tgid, "idle", True, tick)
591                        idle_uid = 0
592                    else:
593                        process = Task(uid, pid, tgid, task_name, True, tick)
594                else:
595                    if tgid == 0:
596                        process = Task(uid, tgid, tgid, "idle", True, tick)
597                        idle_uid = 0
598                    else:
599                        # parent process name not known yet
600                        process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
601                if tgid == -1: # kernel
602                    kernel_uid = 0
603                uid += 1
604                process_dict[tgid] = process
605                process_list.append(process)
606            else:
607                if tgid == pid:
608                    if process_dict[tgid].task_name == "_Unknown_":
609                        if args.verbose:
610                            print "new process", \
611                                process_dict[tgid].uid, pid, tgid, task_name
612                        process_dict[tgid].task_name = task_name
613                    if process_dict[tgid].task_name != task_name and tgid != 0:
614                        process_dict[tgid].task_name = task_name
615
616            if not pid in thread_dict:
617                if args.verbose:
618                    print "new thread", \
619                       uid, process_dict[tgid].uid, pid, tgid, task_name
620                thread = Task(uid, pid, tgid, task_name, False, tick)
621                uid += 1
622                thread_dict[pid] = thread
623                process_dict[tgid].children.append(thread)
624            else:
625                if thread_dict[pid].task_name != task_name:
626                    thread_dict[pid].task_name = task_name
627
628            if args.verbose:
629                print tick, uid, cpu_id, pid, tgid, task_name
630
631            task = thread_dict[pid]
632            event = Event(tick, task)
633            event_list[cpu_id].append(event)
634            unified_event_list.append(event)
635
636            if len(unified_event_list) == num_events:
637                print "Truncating at", num_events, "events!"
638                break
639    print "Found %d events." % len(unified_event_list)
640
641    for process in process_list:
642        if process.pid > 9990: # fix up framebuffer ticks
643            process.tick = start_tick
644        print process.uid, process.pid, process.tgid, \
645            process.task_name, str(process.tick)
646        for thread in process.children:
647            if thread.pid > 9990:
648                thread.tick = start_tick
649            print "\t", thread.uid, thread.pid, thread.tgid, \
650                thread.task_name, str(thread.tick)
651
652    end_tick = tick
653
654    print "Start tick:", start_tick
655    print "End tick:  ", end_tick
656    print ""
657
658    return
659
660
661def initOutput(output_path):
662    if not os.path.exists(output_path):
663        os.mkdir(output_path)
664
665def ticksToNs(tick):
666    if ticks_in_ns < 0:
667        print "ticks_in_ns not set properly!"
668        sys.exit(1)
669
670    return tick / ticks_in_ns
671
672def writeXmlFile(xml, filename):
673    f = open(filename, "w")
674    txt = ET.tostring(xml)
675    f.write(minidom.parseString(txt).toprettyxml())
676    f.close()
677
678
679# StatsEntry that contains individual statistics
680class StatsEntry(object):
681    def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
682
683        # Full name of statistics
684        self.name = name
685
686        # Streamline group name that statistic will belong to
687        self.group = group
688
689        # Index of statistics within group (used to change colors within groups)
690        self.group_index = group_index
691
692        # Shorter name with "system" stripped off
693        # and symbols converted to alphanumerics
694        self.short_name = re.sub("system\.", "", name)
695        self.short_name = re.sub(":", "_", name)
696
697        # Regex for this stat (string version used to construct union regex)
698        self.regex_string = "^" + name + "\s+([\d\.]+)"
699        self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
700        self.description = ""
701
702        # Whether this stat is use per CPU or not
703        self.per_cpu = per_cpu
704        self.per_switchcpu = per_switchcpu
705
706        # Key used in .apc protocol (as described in captured.xml)
707        self.key = key
708
709        # List of values of stat per timestamp
710        self.values = []
711
712        # Whether this stat has been found for the current timestamp
713        self.found = False
714
715        # Whether this stat has been found at least once
716        # (to suppress too many warnings)
717        self.not_found_at_least_once = False
718
719        # Field used to hold ElementTree subelement for this stat
720        self.ET_element = None
721
722        # Create per-CPU stat name and regex, etc.
723        if self.per_cpu:
724            self.per_cpu_regex_string = []
725            self.per_cpu_regex = []
726            self.per_cpu_name = []
727            self.per_cpu_found = []
728            for i in range(num_cpus):
729                # Resuming from checkpoints results in using "switch_cpus"
730                if per_switchcpu:
731                    per_cpu_name = "system.switch_cpus"
732                else:
733                    per_cpu_name = "system.cpu"
734
735                # No CPU number appends if num_cpus == 1
736                if num_cpus > 1:
737                    per_cpu_name += str(i)
738                per_cpu_name += "." + self.name
739                self.per_cpu_name.append(per_cpu_name)
740                print "\t", per_cpu_name
741
742                self.per_cpu_regex_string.\
743                    append("^" + per_cpu_name + "\s+[\d\.]+")
744                self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
745                    "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
746                self.values.append([])
747                self.per_cpu_found.append(False)
748
749    def append_value(self, val, per_cpu_index = None):
750        if self.per_cpu:
751            self.values[per_cpu_index].append(str(val))
752        else:
753            self.values.append(str(val))
754
755# Global stats object that contains the list of stats entries
756# and other utility functions
757class Stats(object):
758    def __init__(self):
759        self.stats_list = []
760        self.tick_list = []
761        self.next_key = 1
762
763    def register(self, name, group, group_index, per_cpu, per_switchcpu):
764        print "registering stat:", name, "group:", group, group_index
765        self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
766            per_switchcpu, self.next_key))
767        self.next_key += 1
768
769    # Union of all stats to accelerate parsing speed
770    def createStatsRegex(self):
771        regex_strings = [];
772        print "\nnum entries in stats_list", len(self.stats_list)
773        for entry in self.stats_list:
774            if entry.per_cpu:
775                for i in range(num_cpus):
776                    regex_strings.append(entry.per_cpu_regex_string[i])
777            else:
778                regex_strings.append(entry.regex_string)
779
780        self.regex = re.compile('|'.join(regex_strings))
781
782
783def registerStats(config_file):
784    print "==============================="
785    print "Parsing stats config.ini file..."
786    print config_file
787    print "==============================="
788
789    config = ConfigParser()
790    if not config.read(config_file):
791        print "ERROR: config file '", config_file, "' not found!"
792        sys.exit(1)
793
794    print "\nRegistering Stats..."
795
796    stats = Stats()
797
798    per_cpu_stat_groups = config.options('PER_CPU_STATS')
799    for group in per_cpu_stat_groups:
800        i = 0
801        per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
802        for item in per_cpu_stats_list:
803            if item:
804                stats.register(item, group, i, True, False)
805                i += 1
806
807    per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
808    for group in per_cpu_stat_groups:
809        i = 0
810        per_cpu_stats_list = \
811            config.get('PER_SWITCHCPU_STATS', group).split('\n')
812        for item in per_cpu_stats_list:
813            if item:
814                stats.register(item, group, i, True, True)
815                i += 1
816
817    per_l2_stat_groups = config.options('PER_L2_STATS')
818    for group in per_l2_stat_groups:
819        i = 0
820        per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
821        for item in per_l2_stats_list:
822            if item:
823                for l2 in range(num_l2):
824                    name = item
825                    prefix = "system.l2"
826                    if num_l2 > 1:
827                        prefix += str(l2)
828                    prefix += "."
829                    name = prefix + name
830                    stats.register(name, group, i, False, False)
831                i += 1
832
833    other_stat_groups = config.options('OTHER_STATS')
834    for group in other_stat_groups:
835        i = 0
836        other_stats_list = config.get('OTHER_STATS', group).split('\n')
837        for item in other_stats_list:
838            if item:
839                stats.register(item, group, i, False, False)
840                i += 1
841
842    stats.createStatsRegex()
843
844    return stats
845
846# Parse and read in gem5 stats file
847# Streamline counters are organized per CPU
848def readGem5Stats(stats, gem5_stats_file):
849    print "\n==============================="
850    print "Parsing gem5 stats file..."
851    print gem5_stats_file
852    print "===============================\n"
853    ext = os.path.splitext(gem5_stats_file)[1]
854
855    window_start_regex = \
856        re.compile("^---------- Begin Simulation Statistics ----------")
857    window_end_regex = \
858        re.compile("^---------- End Simulation Statistics   ----------")
859    final_tick_regex = re.compile("^final_tick\s+(\d+)")
860
861    global ticks_in_ns
862    sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
863    sim_freq = -1
864
865    try:
866        if ext == ".gz":
867            f = gzip.open(gem5_stats_file, "r")
868        else:
869            f = open(gem5_stats_file, "r")
870    except:
871        print "ERROR opening stats file", gem5_stats_file, "!"
872        sys.exit(1)
873
874    stats_not_found_list = stats.stats_list[:]
875    window_num = 0
876
877    while (True):
878        error = False
879        try:
880            line = f.readline()
881        except IOError:
882            print ""
883            print "WARNING: IO error in stats file"
884            print "(gzip stream not closed properly?)...continuing for now"
885            error = True
886        if not line:
887            break
888
889        # Find out how many gem5 ticks in 1ns
890        if sim_freq < 0:
891            m = sim_freq_regex.match(line)
892            if m:
893                sim_freq = int(m.group(1)) # ticks in 1 sec
894                ticks_in_ns = int(sim_freq / 1e9)
895                print "Simulation frequency found! 1 tick == %e sec\n" \
896                        % (1.0 / sim_freq)
897
898        # Final tick in gem5 stats: current absolute timestamp
899        m = final_tick_regex.match(line)
900        if m:
901            tick = int(m.group(1))
902            if tick > end_tick:
903                break
904            stats.tick_list.append(tick)
905
906
907        if (window_end_regex.match(line) or error):
908            if args.verbose:
909                print "new window"
910            for stat in stats.stats_list:
911                if stat.per_cpu:
912                    for i in range(num_cpus):
913                        if not stat.per_cpu_found[i]:
914                            if not stat.not_found_at_least_once:
915                                print "WARNING: stat not found in window #", \
916                                    window_num, ":", stat.per_cpu_name[i]
917                                print "suppressing further warnings for " + \
918                                    "this stat"
919                                stat.not_found_at_least_once = True
920                            stat.values[i].append(str(0))
921                        stat.per_cpu_found[i] = False
922                else:
923                    if not stat.found:
924                        if not stat.not_found_at_least_once:
925                            print "WARNING: stat not found in window #", \
926                                window_num, ":", stat.name
927                            print "suppressing further warnings for this stat"
928                            stat.not_found_at_least_once = True
929                        stat.values.append(str(0))
930                    stat.found = False
931            stats_not_found_list = stats.stats_list[:]
932            window_num += 1
933            if error:
934                break
935
936        # Do a single regex of the union of all stats first for speed
937        if stats.regex.match(line):
938            # Then loop through only the stats we haven't seen in this window
939            for stat in stats_not_found_list[:]:
940                if stat.per_cpu:
941                    for i in range(num_cpus):
942                        m = stat.per_cpu_regex[i].match(line)
943                        if m:
944                            if stat.name == "ipc":
945                                value = str(int(float(m.group(1)) * 1000))
946                            else:
947                                value = str(int(float(m.group(1))))
948                            if args.verbose:
949                                print stat.per_cpu_name[i], value
950                            stat.values[i].append(value)
951                            stat.per_cpu_found[i] = True
952                            all_found = True
953                            for j in range(num_cpus):
954                                if not stat.per_cpu_found[j]:
955                                    all_found = False
956                            if all_found:
957                                stats_not_found_list.remove(stat)
958                            if stat.description == "":
959                                stat.description = m.group(2)
960                else:
961                    m = stat.regex.match(line)
962                    if m:
963                        value = str(int(float(m.group(1))))
964                        if args.verbose:
965                            print stat.name, value
966                        stat.values.append(value)
967                        stat.found = True
968                        stats_not_found_list.remove(stat)
969                        if stat.description == "":
970                            stat.description = m.group(2)
971    f.close()
972
973
974# Create session.xml file in .apc folder
975def doSessionXML(output_path):
976    session_file = output_path + "/session.xml"
977
978    xml = ET.Element("session")
979
980    xml.set("version", "1")
981    xml.set("call_stack_unwinding", "no")
982    xml.set("parse_debug_info", "no")
983    xml.set("high_resolution", "yes")
984    xml.set("buffer_mode", "streaming")
985    xml.set("sample_rate", "low")
986
987    # Setting duration to zero for now. Doesn't affect visualization.
988    xml.set("duration", "0")
989
990    xml.set("target_host", "")
991    xml.set("target_port", "8080")
992
993    writeXmlFile(xml, session_file)
994
995
996# Create captured.xml file in .apc folder
997def doCapturedXML(output_path, stats):
998    captured_file = output_path + "/captured.xml"
999
1000    xml = ET.Element("captured")
1001    xml.set("version", "1")
1002    xml.set("protocol", "12")
1003
1004    target = ET.SubElement(xml, "target")
1005    target.set("name", "gem5")
1006    target.set("sample_rate", "1000")
1007    target.set("cores", str(num_cpus))
1008
1009    counters = ET.SubElement(xml, "counters")
1010    for stat in stats.stats_list:
1011        s = ET.SubElement(counters, "counter")
1012        stat_name = re.sub("\.", "_", stat.short_name)
1013        s.set("title", stat.group)
1014        s.set("name", stat_name)
1015        s.set("color", "0x00000000")
1016        s.set("key", "0x%08x" % stat.key)
1017        s.set("type", stat_name)
1018        s.set("event", "0x00000000")
1019        if stat.per_cpu:
1020            s.set("per_cpu", "yes")
1021        else:
1022            s.set("per_cpu", "no")
1023        s.set("display", "")
1024        s.set("units", "")
1025        s.set("average_selection", "no")
1026        s.set("description", stat.description)
1027
1028    writeXmlFile(xml, captured_file)
1029
1030# Writes out Streamline cookies (unique IDs per process/thread)
1031def writeCookiesThreads(blob):
1032    thread_list = []
1033    for process in process_list:
1034        if process.uid > 0:
1035            print "cookie", process.task_name, process.uid
1036            writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1037
1038        # pid and tgid need to be positive values -- no longer true?
1039        for thread in process.children:
1040            thread_list.append(thread)
1041
1042    # Threads need to be sorted in timestamp order
1043    thread_list.sort(key = lambda x: x.tick)
1044    for thread in thread_list:
1045        print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1046                thread.tgid, thread.pid
1047        writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1048                thread.pid, thread.task_name))
1049
1050# Writes context switch info as Streamline scheduling events
1051def writeSchedEvents(blob):
1052    for cpu in range(num_cpus):
1053        for event in event_list[cpu]:
1054            timestamp = ticksToNs(event.tick)
1055            pid = event.task.tgid
1056            tid = event.task.pid
1057            if process_dict.has_key(event.task.tgid):
1058                cookie = process_dict[event.task.tgid].uid
1059            else:
1060                cookie = 0
1061
1062            # State:
1063            #   0: waiting on other event besides I/O
1064            #   1: Contention/pre-emption
1065            #   2: Waiting on I/O
1066            #   3: Waiting on mutex
1067            # Hardcoding to 0 for now. Other states not implemented yet.
1068            state = 0
1069
1070            if args.verbose:
1071                print cpu, timestamp, pid, tid, cookie
1072
1073            writeBinary(blob,\
1074                schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1075
1076# Writes selected gem5 statistics as Streamline counters
1077def writeCounters(blob, stats):
1078    timestamp_list = []
1079    for tick in stats.tick_list:
1080        if tick > end_tick:
1081            break
1082        timestamp_list.append(ticksToNs(tick))
1083
1084    for stat in stats.stats_list:
1085        if stat.per_cpu:
1086            stat_length = len(stat.values[0])
1087        else:
1088            stat_length = len(stat.values)
1089
1090    for n in range(len(timestamp_list)):
1091        for stat in stats.stats_list:
1092            if stat.per_cpu:
1093                for i in range(num_cpus):
1094                    writeBinary(blob, counterFrame(timestamp_list[n], i, \
1095                                    stat.key, int(float(stat.values[i][n]))))
1096            else:
1097                writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1098                                    stat.key, int(float(stat.values[n]))))
1099
1100# Streamline can display LCD frame buffer dumps (gzipped bmp)
1101# This function converts the frame buffer dumps to the Streamline format
1102def writeVisualAnnotations(blob, input_path, output_path):
1103    frame_path = input_path + "/frames_system.vncserver"
1104    if not os.path.exists(frame_path):
1105        return
1106
1107    frame_count = 0
1108    file_list = os.listdir(frame_path)
1109    file_list.sort()
1110    re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1111
1112    # Use first non-negative pid to tag visual annotations
1113    annotate_pid = -1
1114    for e in unified_event_list:
1115        pid = e.task.pid
1116        if pid >= 0:
1117            annotate_pid = pid
1118            break
1119
1120    for fn in file_list:
1121        m = re_fb.match(fn)
1122        if m:
1123            seq = m.group(1)
1124            tick = int(m.group(2))
1125            if tick > end_tick:
1126                break
1127            frame_count += 1
1128
1129            userspace_body = []
1130            userspace_body += packed32(0x1C) # escape code
1131            userspace_body += packed32(0x04) # visual code
1132
1133            text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1134            userspace_body += int16(len(text_annotation))
1135            userspace_body += utf8StringList(text_annotation)
1136
1137            if gzipped_bmp_supported:
1138                # copy gzipped bmp directly
1139                bytes_read = open(frame_path + "/" + fn, "rb").read()
1140            else:
1141                # copy uncompressed bmp
1142                bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1143
1144            userspace_body += int32(len(bytes_read))
1145            userspace_body += bytes_read
1146
1147            writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1148                                len(userspace_body), userspace_body))
1149
1150    print "\nfound", frame_count, "frames for visual annotation.\n"
1151
1152
1153def createApcProject(input_path, output_path, stats):
1154    initOutput(output_path)
1155
1156    blob = open(output_path + "/0000000000", "wb")
1157
1158    # Summary frame takes current system time and system uptime.
1159    # Filling in with random values for now.
1160    writeBinary(blob, summaryFrame(1234, 5678))
1161
1162    writeCookiesThreads(blob)
1163
1164    print "writing Events"
1165    writeSchedEvents(blob)
1166
1167    print "writing Counters"
1168    writeCounters(blob, stats)
1169
1170    print "writing Visual Annotations"
1171    writeVisualAnnotations(blob, input_path, output_path)
1172
1173    doSessionXML(output_path)
1174    doCapturedXML(output_path, stats)
1175
1176    blob.close()
1177
1178
1179
1180#######################
1181# Main Routine
1182
1183input_path = args.input_path
1184output_path = args.output_path
1185
1186####
1187# Make sure input path exists
1188####
1189if not os.path.exists(input_path):
1190    print "ERROR: Input path %s does not exist!" % input_path
1191    sys.exit(1)
1192
1193####
1194# Parse gem5 configuration file to find # of CPUs and L2s
1195####
1196(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1197
1198####
1199# Parse task file to find process/thread info
1200####
1201parseProcessInfo(input_path + "/system.tasks.txt")
1202
1203####
1204# Parse stat config file and register stats
1205####
1206stat_config_file = args.stat_config_file
1207stats = registerStats(stat_config_file)
1208
1209####
1210# Parse gem5 stats
1211####
1212# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1213if os.path.exists(input_path + "/stats.txt") and \
1214    os.path.exists(input_path + "/stats.txt.gz"):
1215    print "WARNING: Both stats.txt.gz and stats.txt exist. \
1216            Using stats.txt.gz by default."
1217
1218gem5_stats_file = input_path + "/stats.txt.gz"
1219if not os.path.exists(gem5_stats_file):
1220    gem5_stats_file = input_path + "/stats.txt"
1221if not os.path.exists(gem5_stats_file):
1222    print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1223    sys.exit(1)
1224
1225readGem5Stats(stats, gem5_stats_file)
1226
1227####
1228# Create Streamline .apc project folder
1229####
1230createApcProject(input_path, output_path, stats)
1231
1232print "All done!"
1233