1#!/usr/bin/env python2.7
2
3# Copyright (c) 2012, 2014 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50#                    in conversion. Sample .ini files can be found in
51#                    util/streamline.
52#                    NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55#                    stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v17 (DS-5 v5.17)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73        formatter_class=argparse.RawDescriptionHelpFormatter,
74        description="""
75        Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76        (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77        enabled and some patches applied to the target Linux kernel.)
78
79        Visit http://www.gem5.org/Streamline for more details.
80
81        APC project generation based on Gator v17 (DS-5 v5.17)
82        Subsequent versions should be backward compatible
83        """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86                    help=".ini file that describes which stats to be included \
87                    in conversion. Sample .ini files can be found in \
88                    util/streamline. NOTE: this is NOT the gem5 config.ini \
89                    file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92                    help="Path to gem5 run folder (must contain config.ini, \
93                    stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96                    help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99                    default=1000000,
100                    help="Maximum number of scheduling (context switch) \
101                    events to be processed. Set to truncate early. \
102                    Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105                    help="Do not use gzipped .bmp files for visual annotations. \
106                    This option is only required when using Streamline versions \
107                    older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110                    help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115    print "ERROR: <dest .apc folder> should end with '.apc'!"
116    sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134    global num_cpus, num_l2
135
136    print "\n==============================="
137    print "Parsing gem5 config.ini file..."
138    print config_file
139    print "===============================\n"
140    config = ConfigParser()
141    if not config.read(config_file):
142        print "ERROR: config file '", config_file, "' not found"
143        sys.exit(1)
144
145    if config.has_section("system.cpu"):
146        num_cpus = 1
147    else:
148        num_cpus = 0
149        while config.has_section("system.cpu" + str(num_cpus)):
150            num_cpus += 1
151
152    if config.has_section("system.l2_cache"):
153        num_l2 = 1
154    else:
155        num_l2 = 0
156        while config.has_section("system.l2_cache" + str(num_l2)):
157            num_l2 += 1
158
159    print "Num CPUs:", num_cpus
160    print "Num L2s:", num_l2
161    print ""
162
163    return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175    def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176        if pid == 0: # Idle
177            self.uid = 0
178        elif pid == -1: # Kernel
179            self.uid = 0
180        else:
181            self.uid = uid
182        self.pid = pid
183        self.tgid = tgid
184        self.is_process = is_process
185        self.task_name = task_name
186        self.children = []
187        self.tick = tick # time this task first appeared
188
189class Event(object):
190    def __init__(self, tick, task):
191        self.tick = tick
192        self.task = task
193
194############################################################
195# Types used in APC Protocol
196#  - packed32, packed64
197#  - int32
198#  - string
199############################################################
200
201def packed32(x):
202    ret = []
203    more = True
204    while more:
205        b = x & 0x7f
206        x = x >> 7
207        if (((x == 0) and ((b & 0x40) == 0)) or \
208            ((x == -1) and ((b & 0x40) != 0))):
209            more = False
210        else:
211            b = b | 0x80
212        ret.append(b)
213    return ret
214
215# For historical reasons, 32/64-bit versions of functions are presevered
216def packed64(x):
217    return packed32(x)
218
219#  variable length packed 4-byte signed value
220def unsigned_packed32(x):
221    ret = []
222    if ((x & 0xffffff80) == 0):
223        ret.append(x & 0x7f)
224    elif ((x & 0xffffc000) == 0):
225        ret.append((x | 0x80) & 0xff)
226        ret.append((x >> 7) & 0x7f)
227    elif ((x & 0xffe00000) == 0):
228        ret.append((x | 0x80) & 0xff)
229        ret.append(((x >> 7) | 0x80) & 0xff)
230        ret.append((x >> 14) & 0x7f)
231    elif ((x & 0xf0000000) == 0):
232        ret.append((x | 0x80) & 0xff)
233        ret.append(((x >> 7) | 0x80) & 0xff)
234        ret.append(((x >> 14) | 0x80) & 0xff)
235        ret.append((x >> 21) & 0x7f)
236    else:
237        ret.append((x | 0x80) & 0xff)
238        ret.append(((x >> 7) | 0x80) & 0xff)
239        ret.append(((x >> 14) | 0x80) & 0xff)
240        ret.append(((x >> 21) | 0x80) & 0xff)
241        ret.append((x >> 28) & 0x0f)
242    return ret
243
244#  variable length packed 8-byte signed value
245def unsigned_packed64(x):
246    ret = []
247    if ((x & 0xffffffffffffff80) == 0):
248        ret.append(x & 0x7f)
249    elif ((x & 0xffffffffffffc000) == 0):
250        ret.append((x | 0x80) & 0xff)
251        ret.append((x >> 7) & 0x7f)
252    elif ((x & 0xffffffffffe00000) == 0):
253        ret.append((x | 0x80) & 0xff)
254        ret.append(((x >> 7) | 0x80) & 0xff)
255        ret.append((x >> 14) & 0x7f)
256    elif ((x & 0xfffffffff0000000) == 0):
257        ret.append((x | 0x80) & 0xff)
258        ret.append(((x >> 7) | 0x80) & 0xff)
259        ret.append(((x >> 14) | 0x80) & 0xff)
260        ret.append((x >> 21) & 0x7f)
261    elif ((x & 0xfffffff800000000) == 0):
262        ret.append((x | 0x80) & 0xff)
263        ret.append(((x >> 7) | 0x80) & 0xff)
264        ret.append(((x >> 14) | 0x80) & 0xff)
265        ret.append(((x >> 21) | 0x80) & 0xff)
266        ret.append((x >> 28) & 0x7f)
267    elif ((x & 0xfffffc0000000000) == 0):
268        ret.append((x | 0x80) & 0xff)
269        ret.append(((x >> 7) | 0x80) & 0xff)
270        ret.append(((x >> 14) | 0x80) & 0xff)
271        ret.append(((x >> 21) | 0x80) & 0xff)
272        ret.append(((x >> 28) | 0x80) & 0xff)
273        ret.append((x >> 35) & 0x7f)
274    elif ((x & 0xfffe000000000000) == 0):
275        ret.append((x | 0x80) & 0xff)
276        ret.append(((x >> 7) | 0x80) & 0xff)
277        ret.append(((x >> 14) | 0x80) & 0xff)
278        ret.append(((x >> 21) | 0x80) & 0xff)
279        ret.append(((x >> 28) | 0x80) & 0xff)
280        ret.append(((x >> 35) | 0x80) & 0xff)
281        ret.append((x >> 42) & 0x7f)
282    elif ((x & 0xff00000000000000) == 0):
283        ret.append((x | 0x80) & 0xff)
284        ret.append(((x >> 7) | 0x80) & 0xff)
285        ret.append(((x >> 14) | 0x80) & 0xff)
286        ret.append(((x >> 21) | 0x80) & 0xff)
287        ret.append(((x >> 28) | 0x80) & 0xff)
288        ret.append(((x >> 35) | 0x80) & 0xff)
289        ret.append(((x >> 42) | 0x80) & 0xff)
290        ret.append((x >> 49) & 0x7f)
291    elif ((x & 0x8000000000000000) == 0):
292        ret.append((x | 0x80) & 0xff)
293        ret.append(((x >> 7) | 0x80) & 0xff)
294        ret.append(((x >> 14) | 0x80) & 0xff)
295        ret.append(((x >> 21) | 0x80) & 0xff)
296        ret.append(((x >> 28) | 0x80) & 0xff)
297        ret.append(((x >> 35) | 0x80) & 0xff)
298        ret.append(((x >> 42) | 0x80) & 0xff)
299        ret.append(((x >> 49) | 0x80) & 0xff)
300        ret.append((x >> 56) & 0x7f)
301    else:
302        ret.append((x | 0x80) & 0xff)
303        ret.append(((x >> 7) | 0x80) & 0xff)
304        ret.append(((x >> 14) | 0x80) & 0xff)
305        ret.append(((x >> 21) | 0x80) & 0xff)
306        ret.append(((x >> 28) | 0x80) & 0xff)
307        ret.append(((x >> 35) | 0x80) & 0xff)
308        ret.append(((x >> 42) | 0x80) & 0xff)
309        ret.append(((x >> 49) | 0x80) & 0xff)
310        ret.append(((x >> 56) | 0x80) & 0xff)
311        ret.append((x >> 63) & 0x7f)
312    return ret
313
314# 4-byte signed little endian
315def int32(x):
316    ret = []
317    ret.append(x & 0xff)
318    ret.append((x >> 8) & 0xff)
319    ret.append((x >> 16) & 0xff)
320    ret.append((x >> 24) & 0xff)
321    return ret
322
323# 2-byte signed little endian
324def int16(x):
325    ret = []
326    ret.append(x & 0xff)
327    ret.append((x >> 8) & 0xff)
328    return ret
329
330# a packed32 length followed by the specified number of characters
331def stringList(x):
332    ret = []
333    ret += packed32(len(x))
334    for i in x:
335        ret.append(i)
336    return ret
337
338def utf8StringList(x):
339    ret = []
340    for i in x:
341        ret.append(ord(i))
342    return ret
343
344# packed64 time value in nanoseconds relative to the uptime from the
345# Summary message.
346def timestampList(x):
347    ret = packed64(x)
348    return ret
349
350
351############################################################
352# Write binary
353############################################################
354
355def writeBinary(outfile, binary_list):
356    for i in binary_list:
357        outfile.write("%c" % i)
358
359############################################################
360# APC Protocol Frame Types
361############################################################
362
363def addFrameHeader(frame_type, body, core):
364    ret = []
365
366    if frame_type == "Summary":
367        code = 1
368    elif frame_type == "Backtrace":
369        code = 2
370    elif frame_type == "Name":
371        code = 3
372    elif frame_type == "Counter":
373        code = 4
374    elif frame_type == "Block Counter":
375        code = 5
376    elif frame_type == "Annotate":
377        code = 6
378    elif frame_type == "Sched Trace":
379        code = 7
380    elif frame_type == "GPU Trace":
381        code = 8
382    elif frame_type == "Idle":
383        code = 9
384    else:
385        print "ERROR: Unknown frame type:", frame_type
386        sys.exit(1)
387
388    packed_code = packed32(code)
389
390    packed_core = packed32(core)
391
392    length = int32(len(packed_code) + len(packed_core) + len(body))
393
394    ret = length + packed_code + packed_core + body
395    return ret
396
397
398# Summary frame
399#  - timestamp: packed64
400#  - uptime: packed64
401def summaryFrame(timestamp, uptime):
402    frame_type = "Summary"
403    newline_canary = stringList("1\n2\r\n3\r4\n\r5")
404    monotonic_delta = packed64(0)
405    end_of_attr = stringList("")
406    body = newline_canary + packed64(timestamp) + packed64(uptime)
407    body += monotonic_delta + end_of_attr
408    ret = addFrameHeader(frame_type, body, 0)
409    return ret
410
411# Backtrace frame
412#  - not implemented yet
413def backtraceFrame():
414    pass
415
416# Cookie name message
417#  - cookie: packed32
418#  - name: string
419def cookieNameFrame(cookie, name):
420    frame_type = "Name"
421    packed_code = packed32(1)
422    body = packed_code + packed32(cookie) + stringList(name)
423    ret = addFrameHeader(frame_type, body, 0)
424    return ret
425
426# Thread name message
427#  - timestamp: timestamp
428#  - thread id: packed32
429#  - name: string
430def threadNameFrame(timestamp, thread_id, name):
431    frame_type = "Name"
432    packed_code = packed32(2)
433    body = packed_code + timestampList(timestamp) + \
434            packed32(thread_id) + stringList(name)
435    ret = addFrameHeader(frame_type, body, 0)
436    return ret
437
438# Core name message
439#  - name: string
440#  - core_id: packed32
441#  - cpuid: packed32
442def coreNameFrame(name, core_id, cpuid):
443    frame_type = "Name"
444    packed_code = packed32(3)
445    body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
446    ret = addFrameHeader(frame_type, body, 0)
447    return ret
448
449# IRQ Cookie name message
450#  - cookie: packed32
451#  - name: string
452#  - irq: packed32
453def irqCookieNameFrame(cookie, name, irq):
454    frame_type = "Name"
455    packed_code = packed32(5)
456    body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
457    ret = addFrameHeader(frame_type, body, 0)
458    return ret
459
460# Counter frame message
461#  - timestamp: timestamp
462#  - core: packed32
463#  - key: packed32
464#  - value: packed64
465def counterFrame(timestamp, core, key, value):
466    frame_type = "Counter"
467    body = timestampList(timestamp) + packed32(core) + packed32(key) + \
468            packed64(value)
469    ret = addFrameHeader(frame_type, body, core)
470    return ret
471
472# Block Counter frame message
473#  - key: packed32
474#  - value: packed64
475def blockCounterFrame(core, key, value):
476    frame_type = "Block Counter"
477    body = packed32(key) + packed64(value)
478    ret = addFrameHeader(frame_type, body, core)
479    return ret
480
481# Annotate frame messages
482#  - core: packed32
483#  - tid: packed32
484#  - timestamp: timestamp
485#  - size: packed32
486#  - body
487def annotateFrame(core, tid, timestamp, size, userspace_body):
488    frame_type = "Annotate"
489    body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
490            packed32(size) + userspace_body
491    ret = addFrameHeader(frame_type, body, core)
492    return ret
493
494# Scheduler Trace frame messages
495# Sched Switch
496#  - Code: 1
497#  - timestamp: timestamp
498#  - pid: packed32
499#  - tid: packed32
500#  - cookie: packed32
501#  - state: packed32
502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
503    frame_type = "Sched Trace"
504    body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
505            packed32(tid) + packed32(cookie) + packed32(state)
506    ret = addFrameHeader(frame_type, body, core)
507    return ret
508
509# Sched Thread Exit
510#  - Code: 2
511#  - timestamp: timestamp
512#  - tid: packed32
513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
514    frame_type = "Sched Trace"
515    body = packed32(2) + timestampList(timestamp) + packed32(tid)
516    ret = addFrameHeader(frame_type, body, core)
517    return ret
518
519# GPU Trace frame messages
520#  - Not implemented yet
521def gpuTraceFrame():
522    pass
523
524# Idle frame messages
525# Enter Idle
526#  - code: 1
527#  - timestamp: timestamp
528#  - core: packed32
529def enterIdleFrame(timestamp, core):
530    frame_type = "Idle"
531    body = packed32(1) + timestampList(timestamp) + packed32(core)
532    ret = addFrameHeader(frame_type, body, core)
533    return ret
534
535# Exit Idle
536#  - code: 2
537#  - timestamp: timestamp
538#  - core: packed32
539def exitIdleFrame(timestamp, core):
540    frame_type = "Idle"
541    body = packed32(2) + timestampList(timestamp) + packed32(core)
542    ret = addFrameHeader(frame_type, body, core)
543    return ret
544
545
546####################################################################
547def parseProcessInfo(task_file):
548    print "\n==============================="
549    print "Parsing Task file..."
550    print task_file
551    print "===============================\n"
552
553    global start_tick, end_tick, num_cpus
554    global process_dict, thread_dict, process_list
555    global event_list, unified_event_list
556    global idle_uid, kernel_uid
557
558    event_list = []
559    unified_event_list = []
560    for cpu in range(num_cpus):
561        event_list.append([])
562
563    uid = 1 # uid 0 is reserved for idle
564
565    # Dummy Tasks for frame buffers and system diagrams
566    process = Task(uid, 9999, 9999, "framebuffer", True, 0)
567    process_list.append(process)
568    uid += 1
569    thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
570    process.children.append(thread)
571    uid += 1
572    process = Task(uid, 9998, 9998, "System", True, 0)
573    process_list.append(process)
574    # if we don't find the real kernel, use this to keep things going
575    kernel_uid = uid
576    uid += 1
577    thread = Task(uid, 9998, 9998, "System", False, 0)
578    process.children.append(thread)
579    uid += 1
580
581    ext = os.path.splitext(task_file)[1]
582
583    try:
584        if ext == ".gz":
585            process_file = gzip.open(task_file, 'rb')
586        else:
587            process_file = open(task_file, 'rb')
588    except:
589        print "ERROR opening task file:", task_file
590        print "Make sure context switch task dumping is enabled in gem5."
591        sys.exit(1)
592
593    process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
594        "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
595
596    task_name_failure_warned = False
597
598    for line in process_file:
599        match = re.match(process_re, line)
600        if match:
601            tick = int(match.group(1))
602            if (start_tick < 0):
603                start_tick = tick
604            cpu_id = int(match.group(3))
605            pid = int(match.group(4))
606            tgid = int(match.group(5))
607            task_name = match.group(6)
608
609            if not task_name_failure_warned:
610                if task_name == "FailureIn_curTaskName":
611                    print "-------------------------------------------------"
612                    print "WARNING: Task name not set correctly!"
613                    print "Process/Thread info will not be displayed correctly"
614                    print "Perhaps forgot to apply m5struct.patch to kernel?"
615                    print "-------------------------------------------------"
616                    task_name_failure_warned = True
617
618            if not tgid in process_dict:
619                if tgid == pid:
620                    # new task is parent as well
621                    if args.verbose:
622                        print "new process", uid, pid, tgid, task_name
623                    if tgid == 0:
624                        # new process is the "idle" task
625                        process = Task(uid, pid, tgid, "idle", True, tick)
626                        idle_uid = 0
627                    else:
628                        process = Task(uid, pid, tgid, task_name, True, tick)
629                else:
630                    if tgid == 0:
631                        process = Task(uid, tgid, tgid, "idle", True, tick)
632                        idle_uid = 0
633                    else:
634                        # parent process name not known yet
635                        process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
636                if tgid == -1: # kernel
637                    kernel_uid = 0
638                uid += 1
639                process_dict[tgid] = process
640                process_list.append(process)
641            else:
642                if tgid == pid:
643                    if process_dict[tgid].task_name == "_Unknown_":
644                        if args.verbose:
645                            print "new process", \
646                                process_dict[tgid].uid, pid, tgid, task_name
647                        process_dict[tgid].task_name = task_name
648                    if process_dict[tgid].task_name != task_name and tgid != 0:
649                        process_dict[tgid].task_name = task_name
650
651            if not pid in thread_dict:
652                if args.verbose:
653                    print "new thread", \
654                       uid, process_dict[tgid].uid, pid, tgid, task_name
655                thread = Task(uid, pid, tgid, task_name, False, tick)
656                uid += 1
657                thread_dict[pid] = thread
658                process_dict[tgid].children.append(thread)
659            else:
660                if thread_dict[pid].task_name != task_name:
661                    thread_dict[pid].task_name = task_name
662
663            if args.verbose:
664                print tick, uid, cpu_id, pid, tgid, task_name
665
666            task = thread_dict[pid]
667            event = Event(tick, task)
668            event_list[cpu_id].append(event)
669            unified_event_list.append(event)
670
671            if len(unified_event_list) == num_events:
672                print "Truncating at", num_events, "events!"
673                break
674    print "Found %d events." % len(unified_event_list)
675
676    for process in process_list:
677        if process.pid > 9990: # fix up framebuffer ticks
678            process.tick = start_tick
679        print process.uid, process.pid, process.tgid, \
680            process.task_name, str(process.tick)
681        for thread in process.children:
682            if thread.pid > 9990:
683                thread.tick = start_tick
684            print "\t", thread.uid, thread.pid, thread.tgid, \
685                thread.task_name, str(thread.tick)
686
687    end_tick = tick
688
689    print "Start tick:", start_tick
690    print "End tick:  ", end_tick
691    print ""
692
693    return
694
695
696def initOutput(output_path):
697    if not os.path.exists(output_path):
698        os.mkdir(output_path)
699
700def ticksToNs(tick):
701    if ticks_in_ns < 0:
702        print "ticks_in_ns not set properly!"
703        sys.exit(1)
704
705    return tick / ticks_in_ns
706
707def writeXmlFile(xml, filename):
708    f = open(filename, "w")
709    txt = ET.tostring(xml)
710    f.write(minidom.parseString(txt).toprettyxml())
711    f.close()
712
713
714# StatsEntry that contains individual statistics
715class StatsEntry(object):
716    def __init__(self, name, group, group_index, per_cpu, key):
717
718        # Full name of statistics
719        self.name = name
720
721        # Streamline group name that statistic will belong to
722        self.group = group
723
724        # Index of statistics within group (used to change colors within groups)
725        self.group_index = group_index
726
727        # Shorter name with "system" stripped off
728        # and symbols converted to alphanumerics
729        self.short_name = re.sub("system\.", "", name)
730        self.short_name = re.sub(":", "_", name)
731
732        # Regex for this stat (string version used to construct union regex)
733        self.regex_string = "^" + name + "\s+([\d\.]+)"
734        self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
735        self.description = ""
736
737        # Whether this stat is use per CPU or not
738        self.per_cpu = per_cpu
739
740        # Key used in .apc protocol (as described in captured.xml)
741        self.key = key
742
743        # List of values of stat per timestamp
744        self.values = []
745
746        # Whether this stat has been found for the current timestamp
747        self.found = False
748
749        # Whether this stat has been found at least once
750        # (to suppress too many warnings)
751        self.not_found_at_least_once = False
752
753        # Field used to hold ElementTree subelement for this stat
754        self.ET_element = None
755
756        # Create per-CPU stat name and regex, etc.
757        if self.per_cpu:
758            self.per_cpu_regex_string = []
759            self.per_cpu_regex = []
760            self.per_cpu_name = []
761            self.per_cpu_found = []
762            for i in range(num_cpus):
763                if num_cpus > 1:
764                    per_cpu_name = re.sub("#", str(i), self.name)
765                else:
766                    per_cpu_name = re.sub("#", "", self.name)
767
768                self.per_cpu_name.append(per_cpu_name)
769                print "\t", per_cpu_name
770
771                self.per_cpu_regex_string.\
772                    append("^" + per_cpu_name + "\s+[\d\.]+")
773                self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
774                    "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
775                self.values.append([])
776                self.per_cpu_found.append(False)
777
778    def append_value(self, val, per_cpu_index = None):
779        if self.per_cpu:
780            self.values[per_cpu_index].append(str(val))
781        else:
782            self.values.append(str(val))
783
784# Global stats object that contains the list of stats entries
785# and other utility functions
786class Stats(object):
787    def __init__(self):
788        self.stats_list = []
789        self.tick_list = []
790        self.next_key = 1
791
792    def register(self, name, group, group_index, per_cpu):
793        print "registering stat:", name, "group:", group, group_index
794        self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
795            self.next_key))
796        self.next_key += 1
797
798    # Union of all stats to accelerate parsing speed
799    def createStatsRegex(self):
800        regex_strings = [];
801        print "\nnum entries in stats_list", len(self.stats_list)
802        for entry in self.stats_list:
803            if entry.per_cpu:
804                for i in range(num_cpus):
805                    regex_strings.append(entry.per_cpu_regex_string[i])
806            else:
807                regex_strings.append(entry.regex_string)
808
809        self.regex = re.compile('|'.join(regex_strings))
810
811
812def registerStats(config_file):
813    print "==============================="
814    print "Parsing stats config.ini file..."
815    print config_file
816    print "==============================="
817
818    config = ConfigParser()
819    if not config.read(config_file):
820        print "ERROR: config file '", config_file, "' not found!"
821        sys.exit(1)
822
823    print "\nRegistering Stats..."
824
825    stats = Stats()
826
827    per_cpu_stat_groups = config.options('PER_CPU_STATS')
828    for group in per_cpu_stat_groups:
829        i = 0
830        per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
831        for item in per_cpu_stats_list:
832            if item:
833                stats.register(item, group, i, True)
834                i += 1
835
836    per_l2_stat_groups = config.options('PER_L2_STATS')
837    for group in per_l2_stat_groups:
838        i = 0
839        per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
840        for item in per_l2_stats_list:
841            if item:
842                for l2 in range(num_l2):
843                    if num_l2 > 1:
844                        name = re.sub("#", str(l2), item)
845                    else:
846                        name = re.sub("#", "", item)
847                    stats.register(name, group, i, False)
848                i += 1
849
850    other_stat_groups = config.options('OTHER_STATS')
851    for group in other_stat_groups:
852        i = 0
853        other_stats_list = config.get('OTHER_STATS', group).split('\n')
854        for item in other_stats_list:
855            if item:
856                stats.register(item, group, i, False)
857                i += 1
858
859    stats.createStatsRegex()
860
861    return stats
862
863# Parse and read in gem5 stats file
864# Streamline counters are organized per CPU
865def readGem5Stats(stats, gem5_stats_file):
866    print "\n==============================="
867    print "Parsing gem5 stats file..."
868    print gem5_stats_file
869    print "===============================\n"
870    ext = os.path.splitext(gem5_stats_file)[1]
871
872    window_start_regex = \
873        re.compile("^---------- Begin Simulation Statistics ----------")
874    window_end_regex = \
875        re.compile("^---------- End Simulation Statistics   ----------")
876    final_tick_regex = re.compile("^final_tick\s+(\d+)")
877
878    global ticks_in_ns
879    sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
880    sim_freq = -1
881
882    try:
883        if ext == ".gz":
884            f = gzip.open(gem5_stats_file, "r")
885        else:
886            f = open(gem5_stats_file, "r")
887    except:
888        print "ERROR opening stats file", gem5_stats_file, "!"
889        sys.exit(1)
890
891    stats_not_found_list = stats.stats_list[:]
892    window_num = 0
893
894    while (True):
895        error = False
896        try:
897            line = f.readline()
898        except IOError:
899            print ""
900            print "WARNING: IO error in stats file"
901            print "(gzip stream not closed properly?)...continuing for now"
902            error = True
903        if not line:
904            break
905
906        # Find out how many gem5 ticks in 1ns
907        if sim_freq < 0:
908            m = sim_freq_regex.match(line)
909            if m:
910                sim_freq = int(m.group(1)) # ticks in 1 sec
911                ticks_in_ns = int(sim_freq / 1e9)
912                print "Simulation frequency found! 1 tick == %e sec\n" \
913                        % (1.0 / sim_freq)
914
915        # Final tick in gem5 stats: current absolute timestamp
916        m = final_tick_regex.match(line)
917        if m:
918            tick = int(m.group(1))
919            if tick > end_tick:
920                break
921            stats.tick_list.append(tick)
922
923
924        if (window_end_regex.match(line) or error):
925            if args.verbose:
926                print "new window"
927            for stat in stats.stats_list:
928                if stat.per_cpu:
929                    for i in range(num_cpus):
930                        if not stat.per_cpu_found[i]:
931                            if not stat.not_found_at_least_once:
932                                print "WARNING: stat not found in window #", \
933                                    window_num, ":", stat.per_cpu_name[i]
934                                print "suppressing further warnings for " + \
935                                    "this stat"
936                                stat.not_found_at_least_once = True
937                            stat.values[i].append(str(0))
938                        stat.per_cpu_found[i] = False
939                else:
940                    if not stat.found:
941                        if not stat.not_found_at_least_once:
942                            print "WARNING: stat not found in window #", \
943                                window_num, ":", stat.name
944                            print "suppressing further warnings for this stat"
945                            stat.not_found_at_least_once = True
946                        stat.values.append(str(0))
947                    stat.found = False
948            stats_not_found_list = stats.stats_list[:]
949            window_num += 1
950            if error:
951                break
952
953        # Do a single regex of the union of all stats first for speed
954        if stats.regex.match(line):
955            # Then loop through only the stats we haven't seen in this window
956            for stat in stats_not_found_list[:]:
957                if stat.per_cpu:
958                    for i in range(num_cpus):
959                        m = stat.per_cpu_regex[i].match(line)
960                        if m:
961                            if stat.name == "ipc":
962                                value = str(int(float(m.group(1)) * 1000))
963                            else:
964                                value = str(int(float(m.group(1))))
965                            if args.verbose:
966                                print stat.per_cpu_name[i], value
967                            stat.values[i].append(value)
968                            stat.per_cpu_found[i] = True
969                            all_found = True
970                            for j in range(num_cpus):
971                                if not stat.per_cpu_found[j]:
972                                    all_found = False
973                            if all_found:
974                                stats_not_found_list.remove(stat)
975                            if stat.description == "":
976                                stat.description = m.group(2)
977                else:
978                    m = stat.regex.match(line)
979                    if m:
980                        value = str(int(float(m.group(1))))
981                        if args.verbose:
982                            print stat.name, value
983                        stat.values.append(value)
984                        stat.found = True
985                        stats_not_found_list.remove(stat)
986                        if stat.description == "":
987                            stat.description = m.group(2)
988    f.close()
989
990
991# Create session.xml file in .apc folder
992def doSessionXML(output_path):
993    session_file = output_path + "/session.xml"
994
995    xml = ET.Element("session")
996
997    xml.set("version", "1")
998    xml.set("call_stack_unwinding", "no")
999    xml.set("parse_debug_info", "no")
1000    xml.set("high_resolution", "yes")
1001    xml.set("buffer_mode", "streaming")
1002    xml.set("sample_rate", "low")
1003
1004    # Setting duration to zero for now. Doesn't affect visualization.
1005    xml.set("duration", "0")
1006
1007    xml.set("target_host", "")
1008    xml.set("target_port", "8080")
1009
1010    writeXmlFile(xml, session_file)
1011
1012
1013# Create captured.xml file in .apc folder
1014def doCapturedXML(output_path, stats):
1015    captured_file = output_path + "/captured.xml"
1016
1017    xml = ET.Element("captured")
1018    xml.set("version", "1")
1019    xml.set("protocol", "17")
1020    xml.set("backtrace_processing", "none")
1021
1022    target = ET.SubElement(xml, "target")
1023    target.set("name", "gem5")
1024    target.set("sample_rate", "1000")
1025    target.set("cores", str(num_cpus))
1026
1027    counters = ET.SubElement(xml, "counters")
1028    for stat in stats.stats_list:
1029        s = ET.SubElement(counters, "counter")
1030        stat_name = re.sub("\.", "_", stat.short_name)
1031        stat_name = re.sub("#", "", stat_name)
1032        s.set("title", stat.group)
1033        s.set("name", stat_name)
1034        s.set("color", "0x00000000")
1035        s.set("key", "0x%08x" % stat.key)
1036        s.set("type", stat_name)
1037        s.set("event", "0x00000000")
1038        if stat.per_cpu:
1039            s.set("per_cpu", "yes")
1040        else:
1041            s.set("per_cpu", "no")
1042        s.set("display", "")
1043        s.set("units", "")
1044        s.set("average_selection", "no")
1045        s.set("description", stat.description)
1046
1047    writeXmlFile(xml, captured_file)
1048
1049# Writes out Streamline cookies (unique IDs per process/thread)
1050def writeCookiesThreads(blob):
1051    thread_list = []
1052    for process in process_list:
1053        if process.uid > 0:
1054            print "cookie", process.task_name, process.uid
1055            writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1056
1057        # pid and tgid need to be positive values -- no longer true?
1058        for thread in process.children:
1059            thread_list.append(thread)
1060
1061    # Threads need to be sorted in timestamp order
1062    thread_list.sort(key = lambda x: x.tick)
1063    for thread in thread_list:
1064        print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1065                thread.tgid, thread.pid
1066        writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1067                thread.pid, thread.task_name))
1068
1069# Writes context switch info as Streamline scheduling events
1070def writeSchedEvents(blob):
1071    for cpu in range(num_cpus):
1072        for event in event_list[cpu]:
1073            timestamp = ticksToNs(event.tick)
1074            pid = event.task.tgid
1075            tid = event.task.pid
1076            if process_dict.has_key(event.task.tgid):
1077                cookie = process_dict[event.task.tgid].uid
1078            else:
1079                cookie = 0
1080
1081            # State:
1082            #   0: waiting on other event besides I/O
1083            #   1: Contention/pre-emption
1084            #   2: Waiting on I/O
1085            #   3: Waiting on mutex
1086            # Hardcoding to 0 for now. Other states not implemented yet.
1087            state = 0
1088
1089            if args.verbose:
1090                print cpu, timestamp, pid, tid, cookie
1091
1092            writeBinary(blob,\
1093                schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1094
1095# Writes selected gem5 statistics as Streamline counters
1096def writeCounters(blob, stats):
1097    timestamp_list = []
1098    for tick in stats.tick_list:
1099        if tick > end_tick:
1100            break
1101        timestamp_list.append(ticksToNs(tick))
1102
1103    for stat in stats.stats_list:
1104        if stat.per_cpu:
1105            stat_length = len(stat.values[0])
1106        else:
1107            stat_length = len(stat.values)
1108
1109    for n in range(len(timestamp_list)):
1110        for stat in stats.stats_list:
1111            if stat.per_cpu:
1112                for i in range(num_cpus):
1113                    writeBinary(blob, counterFrame(timestamp_list[n], i, \
1114                                    stat.key, int(float(stat.values[i][n]))))
1115            else:
1116                writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1117                                    stat.key, int(float(stat.values[n]))))
1118
1119# Streamline can display LCD frame buffer dumps (gzipped bmp)
1120# This function converts the frame buffer dumps to the Streamline format
1121def writeVisualAnnotations(blob, input_path, output_path):
1122    frame_path = input_path + "/frames_system.vncserver"
1123    if not os.path.exists(frame_path):
1124        return
1125
1126    frame_count = 0
1127    file_list = os.listdir(frame_path)
1128    file_list.sort()
1129    re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1130
1131    # Use first non-negative pid to tag visual annotations
1132    annotate_pid = -1
1133    for e in unified_event_list:
1134        pid = e.task.pid
1135        if pid >= 0:
1136            annotate_pid = pid
1137            break
1138
1139    for fn in file_list:
1140        m = re_fb.match(fn)
1141        if m:
1142            seq = m.group(1)
1143            tick = int(m.group(2))
1144            if tick > end_tick:
1145                break
1146            frame_count += 1
1147
1148            userspace_body = []
1149            userspace_body += packed32(0x1C) # escape code
1150            userspace_body += packed32(0x04) # visual code
1151
1152            text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1153            userspace_body += int16(len(text_annotation))
1154            userspace_body += utf8StringList(text_annotation)
1155
1156            if gzipped_bmp_supported:
1157                # copy gzipped bmp directly
1158                bytes_read = open(frame_path + "/" + fn, "rb").read()
1159            else:
1160                # copy uncompressed bmp
1161                bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1162
1163            userspace_body += int32(len(bytes_read))
1164            userspace_body += bytes_read
1165
1166            writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1167                                len(userspace_body), userspace_body))
1168
1169    print "\nfound", frame_count, "frames for visual annotation.\n"
1170
1171
1172def createApcProject(input_path, output_path, stats):
1173    initOutput(output_path)
1174
1175    blob = open(output_path + "/0000000000", "wb")
1176
1177    # Summary frame takes current system time and system uptime.
1178    # Filling in with random values for now.
1179    writeBinary(blob, summaryFrame(1234, 5678))
1180
1181    writeCookiesThreads(blob)
1182
1183    print "writing Events"
1184    writeSchedEvents(blob)
1185
1186    print "writing Counters"
1187    writeCounters(blob, stats)
1188
1189    print "writing Visual Annotations"
1190    writeVisualAnnotations(blob, input_path, output_path)
1191
1192    doSessionXML(output_path)
1193    doCapturedXML(output_path, stats)
1194
1195    blob.close()
1196
1197
1198
1199#######################
1200# Main Routine
1201
1202input_path = args.input_path
1203output_path = args.output_path
1204
1205####
1206# Make sure input path exists
1207####
1208if not os.path.exists(input_path):
1209    print "ERROR: Input path %s does not exist!" % input_path
1210    sys.exit(1)
1211
1212####
1213# Parse gem5 configuration file to find # of CPUs and L2s
1214####
1215(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1216
1217####
1218# Parse task file to find process/thread info
1219####
1220parseProcessInfo(input_path + "/system.tasks.txt")
1221
1222####
1223# Parse stat config file and register stats
1224####
1225stat_config_file = args.stat_config_file
1226stats = registerStats(stat_config_file)
1227
1228####
1229# Parse gem5 stats
1230####
1231# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1232if os.path.exists(input_path + "/stats.txt") and \
1233    os.path.exists(input_path + "/stats.txt.gz"):
1234    print "WARNING: Both stats.txt.gz and stats.txt exist. \
1235            Using stats.txt.gz by default."
1236
1237gem5_stats_file = input_path + "/stats.txt.gz"
1238if not os.path.exists(gem5_stats_file):
1239    gem5_stats_file = input_path + "/stats.txt"
1240if not os.path.exists(gem5_stats_file):
1241    print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1242    sys.exit(1)
1243
1244readGem5Stats(stats, gem5_stats_file)
1245
1246####
1247# Create Streamline .apc project folder
1248####
1249createApcProject(input_path, output_path, stats)
1250
1251print "All done!"
1252