m5stats2streamline.py revision 10016:dffa80408656 
1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50#                    in conversion. Sample .ini files can be found in
51#                    util/streamline.
52#                    NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55#                    stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v17 (DS-5 v5.17)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73        formatter_class=argparse.RawDescriptionHelpFormatter,
74        description="""
75        Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76        (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77        enabled and some patches applied to the target Linux kernel.)
78
79        Visit http://www.gem5.org/Streamline for more details.
80
81        APC project generation based on Gator v17 (DS-5 v5.17)
82        Subsequent versions should be backward compatible
83        """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86                    help=".ini file that describes which stats to be included \
87                    in conversion. Sample .ini files can be found in \
88                    util/streamline. NOTE: this is NOT the gem5 config.ini \
89                    file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92                    help="Path to gem5 run folder (must contain config.ini, \
93                    stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96                    help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99                    default=1000000,
100                    help="Maximum number of scheduling (context switch) \
101                    events to be processed. Set to truncate early. \
102                    Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105                    help="Do not use gzipped .bmp files for visual annotations. \
106                    This option is only required when using Streamline versions \
107                    older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110                    help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115    print "ERROR: <dest .apc folder> should end with '.apc'!"
116    sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134    global num_cpus, num_l2
135
136    print "\n==============================="
137    print "Parsing gem5 config.ini file..."
138    print config_file
139    print "===============================\n"
140    config = ConfigParser()
141    if not config.read(config_file):
142        print "ERROR: config file '", config_file, "' not found"
143        sys.exit(1)
144
145    if config.has_section("system.cpu"):
146        num_cpus = 1
147    else:
148        num_cpus = 0
149        while config.has_section("system.cpu" + str(num_cpus)):
150            num_cpus += 1
151
152    if config.has_section("system.l2"):
153        num_l2 = 1
154    else:
155        num_l2 = 0
156        while config.has_section("system.l2" + str(num_l2)):
157            num_l2 += 1
158
159    print "Num CPUs:", num_cpus
160    print "Num L2s:", num_l2
161    print ""
162
163    return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175    def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176        if pid == 0: # Idle
177            self.uid = 0
178        elif pid == -1: # Kernel
179            self.uid = 0
180        else:
181            self.uid = uid
182        self.pid = pid
183        self.tgid = tgid
184        self.is_process = is_process
185        self.task_name = task_name
186        self.children = []
187        self.tick = tick # time this task first appeared
188
189class Event(object):
190    def __init__(self, tick, task):
191        self.tick = tick
192        self.task = task
193
194############################################################
195# Types used in APC Protocol
196#  - packed32, packed64
197#  - int32
198#  - string
199############################################################
200
201def packed32(x):
202    ret = []
203    more = True
204    while more:
205        b = x & 0x7f
206        x = x >> 7
207        if (((x == 0) and ((b & 0x40) == 0)) or \
208            ((x == -1) and ((b & 0x40) != 0))):
209            more = False
210        else:
211            b = b | 0x80
212        ret.append(b)
213    return ret
214
215# For historical reasons, 32/64-bit versions of functions are presevered
216def packed64(x):
217    return packed32(x)
218
219#  variable length packed 4-byte signed value
220def unsigned_packed32(x):
221    ret = []
222    if ((x & 0xffffff80) == 0):
223        ret.append(x & 0x7f)
224    elif ((x & 0xffffc000) == 0):
225        ret.append((x | 0x80) & 0xff)
226        ret.append((x >> 7) & 0x7f)
227    elif ((x & 0xffe00000) == 0):
228        ret.append((x | 0x80) & 0xff)
229        ret.append(((x >> 7) | 0x80) & 0xff)
230        ret.append((x >> 14) & 0x7f)
231    elif ((x & 0xf0000000) == 0):
232        ret.append((x | 0x80) & 0xff)
233        ret.append(((x >> 7) | 0x80) & 0xff)
234        ret.append(((x >> 14) | 0x80) & 0xff)
235        ret.append((x >> 21) & 0x7f)
236    else:
237        ret.append((x | 0x80) & 0xff)
238        ret.append(((x >> 7) | 0x80) & 0xff)
239        ret.append(((x >> 14) | 0x80) & 0xff)
240        ret.append(((x >> 21) | 0x80) & 0xff)
241        ret.append((x >> 28) & 0x0f)
242    return ret
243
244#  variable length packed 8-byte signed value
245def unsigned_packed64(x):
246    ret = []
247    if ((x & 0xffffffffffffff80) == 0):
248        ret.append(x & 0x7f)
249    elif ((x & 0xffffffffffffc000) == 0):
250        ret.append((x | 0x80) & 0xff)
251        ret.append((x >> 7) & 0x7f)
252    elif ((x & 0xffffffffffe00000) == 0):
253        ret.append((x | 0x80) & 0xff)
254        ret.append(((x >> 7) | 0x80) & 0xff)
255        ret.append((x >> 14) & 0x7f)
256    elif ((x & 0xfffffffff0000000) == 0):
257        ret.append((x | 0x80) & 0xff)
258        ret.append(((x >> 7) | 0x80) & 0xff)
259        ret.append(((x >> 14) | 0x80) & 0xff)
260        ret.append((x >> 21) & 0x7f)
261    elif ((x & 0xfffffff800000000) == 0):
262        ret.append((x | 0x80) & 0xff)
263        ret.append(((x >> 7) | 0x80) & 0xff)
264        ret.append(((x >> 14) | 0x80) & 0xff)
265        ret.append(((x >> 21) | 0x80) & 0xff)
266        ret.append((x >> 28) & 0x7f)
267    elif ((x & 0xfffffc0000000000) == 0):
268        ret.append((x | 0x80) & 0xff)
269        ret.append(((x >> 7) | 0x80) & 0xff)
270        ret.append(((x >> 14) | 0x80) & 0xff)
271        ret.append(((x >> 21) | 0x80) & 0xff)
272        ret.append(((x >> 28) | 0x80) & 0xff)
273        ret.append((x >> 35) & 0x7f)
274    elif ((x & 0xfffe000000000000) == 0):
275        ret.append((x | 0x80) & 0xff)
276        ret.append(((x >> 7) | 0x80) & 0xff)
277        ret.append(((x >> 14) | 0x80) & 0xff)
278        ret.append(((x >> 21) | 0x80) & 0xff)
279        ret.append(((x >> 28) | 0x80) & 0xff)
280        ret.append(((x >> 35) | 0x80) & 0xff)
281        ret.append((x >> 42) & 0x7f)
282    elif ((x & 0xff00000000000000) == 0):
283        ret.append((x | 0x80) & 0xff)
284        ret.append(((x >> 7) | 0x80) & 0xff)
285        ret.append(((x >> 14) | 0x80) & 0xff)
286        ret.append(((x >> 21) | 0x80) & 0xff)
287        ret.append(((x >> 28) | 0x80) & 0xff)
288        ret.append(((x >> 35) | 0x80) & 0xff)
289        ret.append(((x >> 42) | 0x80) & 0xff)
290        ret.append((x >> 49) & 0x7f)
291    elif ((x & 0x8000000000000000) == 0):
292        ret.append((x | 0x80) & 0xff)
293        ret.append(((x >> 7) | 0x80) & 0xff)
294        ret.append(((x >> 14) | 0x80) & 0xff)
295        ret.append(((x >> 21) | 0x80) & 0xff)
296        ret.append(((x >> 28) | 0x80) & 0xff)
297        ret.append(((x >> 35) | 0x80) & 0xff)
298        ret.append(((x >> 42) | 0x80) & 0xff)
299        ret.append(((x >> 49) | 0x80) & 0xff)
300        ret.append((x >> 56) & 0x7f)
301    else:
302        ret.append((x | 0x80) & 0xff)
303        ret.append(((x >> 7) | 0x80) & 0xff)
304        ret.append(((x >> 14) | 0x80) & 0xff)
305        ret.append(((x >> 21) | 0x80) & 0xff)
306        ret.append(((x >> 28) | 0x80) & 0xff)
307        ret.append(((x >> 35) | 0x80) & 0xff)
308        ret.append(((x >> 42) | 0x80) & 0xff)
309        ret.append(((x >> 49) | 0x80) & 0xff)
310        ret.append(((x >> 56) | 0x80) & 0xff)
311        ret.append((x >> 63) & 0x7f)
312    return ret
313
314# 4-byte signed little endian
315def int32(x):
316    ret = []
317    ret.append(x & 0xff)
318    ret.append((x >> 8) & 0xff)
319    ret.append((x >> 16) & 0xff)
320    ret.append((x >> 24) & 0xff)
321    return ret
322
323# 2-byte signed little endian
324def int16(x):
325    ret = []
326    ret.append(x & 0xff)
327    ret.append((x >> 8) & 0xff)
328    return ret
329
330# a packed32 length followed by the specified number of characters
331def stringList(x):
332    ret = []
333    ret += packed32(len(x))
334    for i in x:
335        ret.append(i)
336    return ret
337
338def utf8StringList(x):
339    ret = []
340    for i in x:
341        ret.append(ord(i))
342    return ret
343
344# packed64 time value in nanoseconds relative to the uptime from the
345# Summary message.
346def timestampList(x):
347    ret = packed64(x)
348    return ret
349
350
351############################################################
352# Write binary
353############################################################
354
355def writeBinary(outfile, binary_list):
356    for i in binary_list:
357        outfile.write("%c" % i)
358
359############################################################
360# APC Protocol Frame Types
361############################################################
362
363def addFrameHeader(frame_type, body, core):
364    ret = []
365
366    if frame_type == "Summary":
367        code = 1
368    elif frame_type == "Backtrace":
369        code = 2
370    elif frame_type == "Name":
371        code = 3
372    elif frame_type == "Counter":
373        code = 4
374    elif frame_type == "Block Counter":
375        code = 5
376    elif frame_type == "Annotate":
377        code = 6
378    elif frame_type == "Sched Trace":
379        code = 7
380    elif frame_type == "GPU Trace":
381        code = 8
382    elif frame_type == "Idle":
383        code = 9
384    else:
385        print "ERROR: Unknown frame type:", frame_type
386        sys.exit(1)
387
388    packed_code = packed32(code)
389
390    packed_core = packed32(core)
391
392    length = int32(len(packed_code) + len(packed_core) + len(body))
393
394    ret = length + packed_code + packed_core + body
395    return ret
396
397
398# Summary frame
399#  - timestamp: packed64
400#  - uptime: packed64
401def summaryFrame(timestamp, uptime):
402    frame_type = "Summary"
403    newline_canary = stringList("1\n2\r\n3\r4\n\r5")
404    monotonic_delta = packed64(0)
405    end_of_attr = stringList("")
406    body = newline_canary + packed64(timestamp) + packed64(uptime)
407    body += monotonic_delta + end_of_attr
408    ret = addFrameHeader(frame_type, body, 0)
409    return ret
410
411# Backtrace frame
412#  - not implemented yet
413def backtraceFrame():
414    pass
415
416# Cookie name message
417#  - cookie: packed32
418#  - name: string
419def cookieNameFrame(cookie, name):
420    frame_type = "Name"
421    packed_code = packed32(1)
422    body = packed_code + packed32(cookie) + stringList(name)
423    ret = addFrameHeader(frame_type, body, 0)
424    return ret
425
426# Thread name message
427#  - timestamp: timestamp
428#  - thread id: packed32
429#  - name: string
430def threadNameFrame(timestamp, thread_id, name):
431    frame_type = "Name"
432    packed_code = packed32(2)
433    body = packed_code + timestampList(timestamp) + \
434            packed32(thread_id) + stringList(name)
435    ret = addFrameHeader(frame_type, body, 0)
436    return ret
437
438# Core name message
439#  - name: string
440#  - core_id: packed32
441#  - cpuid: packed32
442def coreNameFrame(name, core_id, cpuid):
443    frame_type = "Name"
444    packed_code = packed32(3)
445    body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
446    ret = addFrameHeader(frame_type, body, 0)
447    return ret
448
449# IRQ Cookie name message
450#  - cookie: packed32
451#  - name: string
452#  - irq: packed32
453def irqCookieNameFrame(cookie, name, irq):
454    frame_type = "Name"
455    packed_code = packed32(5)
456    body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
457    ret = addFrameHeader(frame_type, body, 0)
458    return ret
459
460# Counter frame message
461#  - timestamp: timestamp
462#  - core: packed32
463#  - key: packed32
464#  - value: packed64
465def counterFrame(timestamp, core, key, value):
466    frame_type = "Counter"
467    body = timestampList(timestamp) + packed32(core) + packed32(key) + \
468            packed64(value)
469    ret = addFrameHeader(frame_type, body, core)
470    return ret
471
472# Block Counter frame message
473#  - key: packed32
474#  - value: packed64
475def blockCounterFrame(core, key, value):
476    frame_type = "Block Counter"
477    body = packed32(key) + packed64(value)
478    ret = addFrameHeader(frame_type, body, core)
479    return ret
480
481# Annotate frame messages
482#  - core: packed32
483#  - tid: packed32
484#  - timestamp: timestamp
485#  - size: packed32
486#  - body
487def annotateFrame(core, tid, timestamp, size, userspace_body):
488    frame_type = "Annotate"
489    body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
490            packed32(size) + userspace_body
491    ret = addFrameHeader(frame_type, body, core)
492    return ret
493
494# Scheduler Trace frame messages
495# Sched Switch
496#  - Code: 1
497#  - timestamp: timestamp
498#  - pid: packed32
499#  - tid: packed32
500#  - cookie: packed32
501#  - state: packed32
502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
503    frame_type = "Sched Trace"
504    body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
505            packed32(tid) + packed32(cookie) + packed32(state)
506    ret = addFrameHeader(frame_type, body, core)
507    return ret
508
509# Sched Thread Exit
510#  - Code: 2
511#  - timestamp: timestamp
512#  - tid: packed32
513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
514    frame_type = "Sched Trace"
515    body = packed32(2) + timestampList(timestamp) + packed32(tid)
516    ret = addFrameHeader(frame_type, body, core)
517    return ret
518
519# GPU Trace frame messages
520#  - Not implemented yet
521def gpuTraceFrame():
522    pass
523
524# Idle frame messages
525# Enter Idle
526#  - code: 1
527#  - timestamp: timestamp
528#  - core: packed32
529def enterIdleFrame(timestamp, core):
530    frame_type = "Idle"
531    body = packed32(1) + timestampList(timestamp) + packed32(core)
532    ret = addFrameHeader(frame_type, body, core)
533    return ret
534
535# Exit Idle
536#  - code: 2
537#  - timestamp: timestamp
538#  - core: packed32
539def exitIdleFrame(timestamp, core):
540    frame_type = "Idle"
541    body = packed32(2) + timestampList(timestamp) + packed32(core)
542    ret = addFrameHeader(frame_type, body, core)
543    return ret
544
545
546####################################################################
547def parseProcessInfo(task_file):
548    print "\n==============================="
549    print "Parsing Task file..."
550    print task_file
551    print "===============================\n"
552
553    global start_tick, end_tick, num_cpus
554    global process_dict, thread_dict, process_list
555    global event_list, unified_event_list
556    global idle_uid, kernel_uid
557
558    event_list = []
559    unified_event_list = []
560    for cpu in range(num_cpus):
561        event_list.append([])
562
563    uid = 1 # uid 0 is reserved for idle
564
565    # Dummy Tasks for frame buffers and system diagrams
566    process = Task(uid, 9999, 9999, "framebuffer", True, 0)
567    process_list.append(process)
568    uid += 1
569    thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
570    process.children.append(thread)
571    uid += 1
572    process = Task(uid, 9998, 9998, "System", True, 0)
573    process_list.append(process)
574    # if we don't find the real kernel, use this to keep things going
575    kernel_uid = uid
576    uid += 1
577    thread = Task(uid, 9998, 9998, "System", False, 0)
578    process.children.append(thread)
579    uid += 1
580
581    ext = os.path.splitext(task_file)[1]
582
583    try:
584        if ext == ".gz":
585            process_file = gzip.open(task_file, 'rb')
586        else:
587            process_file = open(task_file, 'rb')
588    except:
589        print "ERROR opening task file:", task_file
590        print "Make sure context switch task dumping is enabled in gem5."
591        sys.exit(1)
592
593    process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
594        "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
595
596    task_name_failure_warned = False
597
598    for line in process_file:
599        match = re.match(process_re, line)
600        if match:
601            tick = int(match.group(1))
602            if (start_tick < 0):
603                start_tick = tick
604            cpu_id = int(match.group(3))
605            pid = int(match.group(4))
606            tgid = int(match.group(5))
607            task_name = match.group(6)
608
609            if not task_name_failure_warned:
610                if task_name == "FailureIn_curTaskName":
611                    print "-------------------------------------------------"
612                    print "WARNING: Task name not set correctly!"
613                    print "Process/Thread info will not be displayed correctly"
614                    print "Perhaps forgot to apply m5struct.patch to kernel?"
615                    print "-------------------------------------------------"
616                    task_name_failure_warned = True
617
618            if not tgid in process_dict:
619                if tgid == pid:
620                    # new task is parent as well
621                    if args.verbose:
622                        print "new process", uid, pid, tgid, task_name
623                    if tgid == 0:
624                        # new process is the "idle" task
625                        process = Task(uid, pid, tgid, "idle", True, tick)
626                        idle_uid = 0
627                    else:
628                        process = Task(uid, pid, tgid, task_name, True, tick)
629                else:
630                    if tgid == 0:
631                        process = Task(uid, tgid, tgid, "idle", True, tick)
632                        idle_uid = 0
633                    else:
634                        # parent process name not known yet
635                        process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
636                if tgid == -1: # kernel
637                    kernel_uid = 0
638                uid += 1
639                process_dict[tgid] = process
640                process_list.append(process)
641            else:
642                if tgid == pid:
643                    if process_dict[tgid].task_name == "_Unknown_":
644                        if args.verbose:
645                            print "new process", \
646                                process_dict[tgid].uid, pid, tgid, task_name
647                        process_dict[tgid].task_name = task_name
648                    if process_dict[tgid].task_name != task_name and tgid != 0:
649                        process_dict[tgid].task_name = task_name
650
651            if not pid in thread_dict:
652                if args.verbose:
653                    print "new thread", \
654                       uid, process_dict[tgid].uid, pid, tgid, task_name
655                thread = Task(uid, pid, tgid, task_name, False, tick)
656                uid += 1
657                thread_dict[pid] = thread
658                process_dict[tgid].children.append(thread)
659            else:
660                if thread_dict[pid].task_name != task_name:
661                    thread_dict[pid].task_name = task_name
662
663            if args.verbose:
664                print tick, uid, cpu_id, pid, tgid, task_name
665
666            task = thread_dict[pid]
667            event = Event(tick, task)
668            event_list[cpu_id].append(event)
669            unified_event_list.append(event)
670
671            if len(unified_event_list) == num_events:
672                print "Truncating at", num_events, "events!"
673                break
674    print "Found %d events." % len(unified_event_list)
675
676    for process in process_list:
677        if process.pid > 9990: # fix up framebuffer ticks
678            process.tick = start_tick
679        print process.uid, process.pid, process.tgid, \
680            process.task_name, str(process.tick)
681        for thread in process.children:
682            if thread.pid > 9990:
683                thread.tick = start_tick
684            print "\t", thread.uid, thread.pid, thread.tgid, \
685                thread.task_name, str(thread.tick)
686
687    end_tick = tick
688
689    print "Start tick:", start_tick
690    print "End tick:  ", end_tick
691    print ""
692
693    return
694
695
696def initOutput(output_path):
697    if not os.path.exists(output_path):
698        os.mkdir(output_path)
699
700def ticksToNs(tick):
701    if ticks_in_ns < 0:
702        print "ticks_in_ns not set properly!"
703        sys.exit(1)
704
705    return tick / ticks_in_ns
706
707def writeXmlFile(xml, filename):
708    f = open(filename, "w")
709    txt = ET.tostring(xml)
710    f.write(minidom.parseString(txt).toprettyxml())
711    f.close()
712
713
714# StatsEntry that contains individual statistics
715class StatsEntry(object):
716    def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
717
718        # Full name of statistics
719        self.name = name
720
721        # Streamline group name that statistic will belong to
722        self.group = group
723
724        # Index of statistics within group (used to change colors within groups)
725        self.group_index = group_index
726
727        # Shorter name with "system" stripped off
728        # and symbols converted to alphanumerics
729        self.short_name = re.sub("system\.", "", name)
730        self.short_name = re.sub(":", "_", name)
731
732        # Regex for this stat (string version used to construct union regex)
733        self.regex_string = "^" + name + "\s+([\d\.]+)"
734        self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
735        self.description = ""
736
737        # Whether this stat is use per CPU or not
738        self.per_cpu = per_cpu
739        self.per_switchcpu = per_switchcpu
740
741        # Key used in .apc protocol (as described in captured.xml)
742        self.key = key
743
744        # List of values of stat per timestamp
745        self.values = []
746
747        # Whether this stat has been found for the current timestamp
748        self.found = False
749
750        # Whether this stat has been found at least once
751        # (to suppress too many warnings)
752        self.not_found_at_least_once = False
753
754        # Field used to hold ElementTree subelement for this stat
755        self.ET_element = None
756
757        # Create per-CPU stat name and regex, etc.
758        if self.per_cpu:
759            self.per_cpu_regex_string = []
760            self.per_cpu_regex = []
761            self.per_cpu_name = []
762            self.per_cpu_found = []
763            for i in range(num_cpus):
764                # Resuming from checkpoints results in using "switch_cpus"
765                if per_switchcpu:
766                    per_cpu_name = "system.switch_cpus"
767                else:
768                    per_cpu_name = "system.cpu"
769
770                # No CPU number appends if num_cpus == 1
771                if num_cpus > 1:
772                    per_cpu_name += str(i)
773                per_cpu_name += "." + self.name
774                self.per_cpu_name.append(per_cpu_name)
775                print "\t", per_cpu_name
776
777                self.per_cpu_regex_string.\
778                    append("^" + per_cpu_name + "\s+[\d\.]+")
779                self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
780                    "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
781                self.values.append([])
782                self.per_cpu_found.append(False)
783
784    def append_value(self, val, per_cpu_index = None):
785        if self.per_cpu:
786            self.values[per_cpu_index].append(str(val))
787        else:
788            self.values.append(str(val))
789
790# Global stats object that contains the list of stats entries
791# and other utility functions
792class Stats(object):
793    def __init__(self):
794        self.stats_list = []
795        self.tick_list = []
796        self.next_key = 1
797
798    def register(self, name, group, group_index, per_cpu, per_switchcpu):
799        print "registering stat:", name, "group:", group, group_index
800        self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
801            per_switchcpu, self.next_key))
802        self.next_key += 1
803
804    # Union of all stats to accelerate parsing speed
805    def createStatsRegex(self):
806        regex_strings = [];
807        print "\nnum entries in stats_list", len(self.stats_list)
808        for entry in self.stats_list:
809            if entry.per_cpu:
810                for i in range(num_cpus):
811                    regex_strings.append(entry.per_cpu_regex_string[i])
812            else:
813                regex_strings.append(entry.regex_string)
814
815        self.regex = re.compile('|'.join(regex_strings))
816
817
818def registerStats(config_file):
819    print "==============================="
820    print "Parsing stats config.ini file..."
821    print config_file
822    print "==============================="
823
824    config = ConfigParser()
825    if not config.read(config_file):
826        print "ERROR: config file '", config_file, "' not found!"
827        sys.exit(1)
828
829    print "\nRegistering Stats..."
830
831    stats = Stats()
832
833    per_cpu_stat_groups = config.options('PER_CPU_STATS')
834    for group in per_cpu_stat_groups:
835        i = 0
836        per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
837        for item in per_cpu_stats_list:
838            if item:
839                stats.register(item, group, i, True, False)
840                i += 1
841
842    per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
843    for group in per_cpu_stat_groups:
844        i = 0
845        per_cpu_stats_list = \
846            config.get('PER_SWITCHCPU_STATS', group).split('\n')
847        for item in per_cpu_stats_list:
848            if item:
849                stats.register(item, group, i, True, True)
850                i += 1
851
852    per_l2_stat_groups = config.options('PER_L2_STATS')
853    for group in per_l2_stat_groups:
854        i = 0
855        per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
856        for item in per_l2_stats_list:
857            if item:
858                for l2 in range(num_l2):
859                    name = item
860                    prefix = "system.l2"
861                    if num_l2 > 1:
862                        prefix += str(l2)
863                    prefix += "."
864                    name = prefix + name
865                    stats.register(name, group, i, False, False)
866                i += 1
867
868    other_stat_groups = config.options('OTHER_STATS')
869    for group in other_stat_groups:
870        i = 0
871        other_stats_list = config.get('OTHER_STATS', group).split('\n')
872        for item in other_stats_list:
873            if item:
874                stats.register(item, group, i, False, False)
875                i += 1
876
877    stats.createStatsRegex()
878
879    return stats
880
881# Parse and read in gem5 stats file
882# Streamline counters are organized per CPU
883def readGem5Stats(stats, gem5_stats_file):
884    print "\n==============================="
885    print "Parsing gem5 stats file..."
886    print gem5_stats_file
887    print "===============================\n"
888    ext = os.path.splitext(gem5_stats_file)[1]
889
890    window_start_regex = \
891        re.compile("^---------- Begin Simulation Statistics ----------")
892    window_end_regex = \
893        re.compile("^---------- End Simulation Statistics   ----------")
894    final_tick_regex = re.compile("^final_tick\s+(\d+)")
895
896    global ticks_in_ns
897    sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
898    sim_freq = -1
899
900    try:
901        if ext == ".gz":
902            f = gzip.open(gem5_stats_file, "r")
903        else:
904            f = open(gem5_stats_file, "r")
905    except:
906        print "ERROR opening stats file", gem5_stats_file, "!"
907        sys.exit(1)
908
909    stats_not_found_list = stats.stats_list[:]
910    window_num = 0
911
912    while (True):
913        error = False
914        try:
915            line = f.readline()
916        except IOError:
917            print ""
918            print "WARNING: IO error in stats file"
919            print "(gzip stream not closed properly?)...continuing for now"
920            error = True
921        if not line:
922            break
923
924        # Find out how many gem5 ticks in 1ns
925        if sim_freq < 0:
926            m = sim_freq_regex.match(line)
927            if m:
928                sim_freq = int(m.group(1)) # ticks in 1 sec
929                ticks_in_ns = int(sim_freq / 1e9)
930                print "Simulation frequency found! 1 tick == %e sec\n" \
931                        % (1.0 / sim_freq)
932
933        # Final tick in gem5 stats: current absolute timestamp
934        m = final_tick_regex.match(line)
935        if m:
936            tick = int(m.group(1))
937            if tick > end_tick:
938                break
939            stats.tick_list.append(tick)
940
941
942        if (window_end_regex.match(line) or error):
943            if args.verbose:
944                print "new window"
945            for stat in stats.stats_list:
946                if stat.per_cpu:
947                    for i in range(num_cpus):
948                        if not stat.per_cpu_found[i]:
949                            if not stat.not_found_at_least_once:
950                                print "WARNING: stat not found in window #", \
951                                    window_num, ":", stat.per_cpu_name[i]
952                                print "suppressing further warnings for " + \
953                                    "this stat"
954                                stat.not_found_at_least_once = True
955                            stat.values[i].append(str(0))
956                        stat.per_cpu_found[i] = False
957                else:
958                    if not stat.found:
959                        if not stat.not_found_at_least_once:
960                            print "WARNING: stat not found in window #", \
961                                window_num, ":", stat.name
962                            print "suppressing further warnings for this stat"
963                            stat.not_found_at_least_once = True
964                        stat.values.append(str(0))
965                    stat.found = False
966            stats_not_found_list = stats.stats_list[:]
967            window_num += 1
968            if error:
969                break
970
971        # Do a single regex of the union of all stats first for speed
972        if stats.regex.match(line):
973            # Then loop through only the stats we haven't seen in this window
974            for stat in stats_not_found_list[:]:
975                if stat.per_cpu:
976                    for i in range(num_cpus):
977                        m = stat.per_cpu_regex[i].match(line)
978                        if m:
979                            if stat.name == "ipc":
980                                value = str(int(float(m.group(1)) * 1000))
981                            else:
982                                value = str(int(float(m.group(1))))
983                            if args.verbose:
984                                print stat.per_cpu_name[i], value
985                            stat.values[i].append(value)
986                            stat.per_cpu_found[i] = True
987                            all_found = True
988                            for j in range(num_cpus):
989                                if not stat.per_cpu_found[j]:
990                                    all_found = False
991                            if all_found:
992                                stats_not_found_list.remove(stat)
993                            if stat.description == "":
994                                stat.description = m.group(2)
995                else:
996                    m = stat.regex.match(line)
997                    if m:
998                        value = str(int(float(m.group(1))))
999                        if args.verbose:
1000                            print stat.name, value
1001                        stat.values.append(value)
1002                        stat.found = True
1003                        stats_not_found_list.remove(stat)
1004                        if stat.description == "":
1005                            stat.description = m.group(2)
1006    f.close()
1007
1008
1009# Create session.xml file in .apc folder
1010def doSessionXML(output_path):
1011    session_file = output_path + "/session.xml"
1012
1013    xml = ET.Element("session")
1014
1015    xml.set("version", "1")
1016    xml.set("call_stack_unwinding", "no")
1017    xml.set("parse_debug_info", "no")
1018    xml.set("high_resolution", "yes")
1019    xml.set("buffer_mode", "streaming")
1020    xml.set("sample_rate", "low")
1021
1022    # Setting duration to zero for now. Doesn't affect visualization.
1023    xml.set("duration", "0")
1024
1025    xml.set("target_host", "")
1026    xml.set("target_port", "8080")
1027
1028    writeXmlFile(xml, session_file)
1029
1030
1031# Create captured.xml file in .apc folder
1032def doCapturedXML(output_path, stats):
1033    captured_file = output_path + "/captured.xml"
1034
1035    xml = ET.Element("captured")
1036    xml.set("version", "1")
1037    xml.set("protocol", "17")
1038    xml.set("backtrace_processing", "none")
1039
1040    target = ET.SubElement(xml, "target")
1041    target.set("name", "gem5")
1042    target.set("sample_rate", "1000")
1043    target.set("cores", str(num_cpus))
1044
1045    counters = ET.SubElement(xml, "counters")
1046    for stat in stats.stats_list:
1047        s = ET.SubElement(counters, "counter")
1048        stat_name = re.sub("\.", "_", stat.short_name)
1049        s.set("title", stat.group)
1050        s.set("name", stat_name)
1051        s.set("color", "0x00000000")
1052        s.set("key", "0x%08x" % stat.key)
1053        s.set("type", stat_name)
1054        s.set("event", "0x00000000")
1055        if stat.per_cpu:
1056            s.set("per_cpu", "yes")
1057        else:
1058            s.set("per_cpu", "no")
1059        s.set("display", "")
1060        s.set("units", "")
1061        s.set("average_selection", "no")
1062        s.set("description", stat.description)
1063
1064    writeXmlFile(xml, captured_file)
1065
1066# Writes out Streamline cookies (unique IDs per process/thread)
1067def writeCookiesThreads(blob):
1068    thread_list = []
1069    for process in process_list:
1070        if process.uid > 0:
1071            print "cookie", process.task_name, process.uid
1072            writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1073
1074        # pid and tgid need to be positive values -- no longer true?
1075        for thread in process.children:
1076            thread_list.append(thread)
1077
1078    # Threads need to be sorted in timestamp order
1079    thread_list.sort(key = lambda x: x.tick)
1080    for thread in thread_list:
1081        print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1082                thread.tgid, thread.pid
1083        writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1084                thread.pid, thread.task_name))
1085
1086# Writes context switch info as Streamline scheduling events
1087def writeSchedEvents(blob):
1088    for cpu in range(num_cpus):
1089        for event in event_list[cpu]:
1090            timestamp = ticksToNs(event.tick)
1091            pid = event.task.tgid
1092            tid = event.task.pid
1093            if process_dict.has_key(event.task.tgid):
1094                cookie = process_dict[event.task.tgid].uid
1095            else:
1096                cookie = 0
1097
1098            # State:
1099            #   0: waiting on other event besides I/O
1100            #   1: Contention/pre-emption
1101            #   2: Waiting on I/O
1102            #   3: Waiting on mutex
1103            # Hardcoding to 0 for now. Other states not implemented yet.
1104            state = 0
1105
1106            if args.verbose:
1107                print cpu, timestamp, pid, tid, cookie
1108
1109            writeBinary(blob,\
1110                schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1111
1112# Writes selected gem5 statistics as Streamline counters
1113def writeCounters(blob, stats):
1114    timestamp_list = []
1115    for tick in stats.tick_list:
1116        if tick > end_tick:
1117            break
1118        timestamp_list.append(ticksToNs(tick))
1119
1120    for stat in stats.stats_list:
1121        if stat.per_cpu:
1122            stat_length = len(stat.values[0])
1123        else:
1124            stat_length = len(stat.values)
1125
1126    for n in range(len(timestamp_list)):
1127        for stat in stats.stats_list:
1128            if stat.per_cpu:
1129                for i in range(num_cpus):
1130                    writeBinary(blob, counterFrame(timestamp_list[n], i, \
1131                                    stat.key, int(float(stat.values[i][n]))))
1132            else:
1133                writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1134                                    stat.key, int(float(stat.values[n]))))
1135
1136# Streamline can display LCD frame buffer dumps (gzipped bmp)
1137# This function converts the frame buffer dumps to the Streamline format
1138def writeVisualAnnotations(blob, input_path, output_path):
1139    frame_path = input_path + "/frames_system.vncserver"
1140    if not os.path.exists(frame_path):
1141        return
1142
1143    frame_count = 0
1144    file_list = os.listdir(frame_path)
1145    file_list.sort()
1146    re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1147
1148    # Use first non-negative pid to tag visual annotations
1149    annotate_pid = -1
1150    for e in unified_event_list:
1151        pid = e.task.pid
1152        if pid >= 0:
1153            annotate_pid = pid
1154            break
1155
1156    for fn in file_list:
1157        m = re_fb.match(fn)
1158        if m:
1159            seq = m.group(1)
1160            tick = int(m.group(2))
1161            if tick > end_tick:
1162                break
1163            frame_count += 1
1164
1165            userspace_body = []
1166            userspace_body += packed32(0x1C) # escape code
1167            userspace_body += packed32(0x04) # visual code
1168
1169            text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1170            userspace_body += int16(len(text_annotation))
1171            userspace_body += utf8StringList(text_annotation)
1172
1173            if gzipped_bmp_supported:
1174                # copy gzipped bmp directly
1175                bytes_read = open(frame_path + "/" + fn, "rb").read()
1176            else:
1177                # copy uncompressed bmp
1178                bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1179
1180            userspace_body += int32(len(bytes_read))
1181            userspace_body += bytes_read
1182
1183            writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1184                                len(userspace_body), userspace_body))
1185
1186    print "\nfound", frame_count, "frames for visual annotation.\n"
1187
1188
1189def createApcProject(input_path, output_path, stats):
1190    initOutput(output_path)
1191
1192    blob = open(output_path + "/0000000000", "wb")
1193
1194    # Summary frame takes current system time and system uptime.
1195    # Filling in with random values for now.
1196    writeBinary(blob, summaryFrame(1234, 5678))
1197
1198    writeCookiesThreads(blob)
1199
1200    print "writing Events"
1201    writeSchedEvents(blob)
1202
1203    print "writing Counters"
1204    writeCounters(blob, stats)
1205
1206    print "writing Visual Annotations"
1207    writeVisualAnnotations(blob, input_path, output_path)
1208
1209    doSessionXML(output_path)
1210    doCapturedXML(output_path, stats)
1211
1212    blob.close()
1213
1214
1215
1216#######################
1217# Main Routine
1218
1219input_path = args.input_path
1220output_path = args.output_path
1221
1222####
1223# Make sure input path exists
1224####
1225if not os.path.exists(input_path):
1226    print "ERROR: Input path %s does not exist!" % input_path
1227    sys.exit(1)
1228
1229####
1230# Parse gem5 configuration file to find # of CPUs and L2s
1231####
1232(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1233
1234####
1235# Parse task file to find process/thread info
1236####
1237parseProcessInfo(input_path + "/system.tasks.txt")
1238
1239####
1240# Parse stat config file and register stats
1241####
1242stat_config_file = args.stat_config_file
1243stats = registerStats(stat_config_file)
1244
1245####
1246# Parse gem5 stats
1247####
1248# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1249if os.path.exists(input_path + "/stats.txt") and \
1250    os.path.exists(input_path + "/stats.txt.gz"):
1251    print "WARNING: Both stats.txt.gz and stats.txt exist. \
1252            Using stats.txt.gz by default."
1253
1254gem5_stats_file = input_path + "/stats.txt.gz"
1255if not os.path.exists(gem5_stats_file):
1256    gem5_stats_file = input_path + "/stats.txt"
1257if not os.path.exists(gem5_stats_file):
1258    print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1259    sys.exit(1)
1260
1261readGem5Stats(stats, gem5_stats_file)
1262
1263####
1264# Create Streamline .apc project folder
1265####
1266createApcProject(input_path, output_path, stats)
1267
1268print "All done!"
1269