1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v12 (DS-5 v5.13)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
81 APC project generation based on Gator v12 (DS-5 v5.13)
82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
201# variable length packed 4-byte signed value
202def packed32(x):
203 ret = []
204 if ((x & 0xffffff80) == 0):
205 ret.append(x & 0x7f)
206 elif ((x & 0xffffc000) == 0):
207 ret.append((x | 0x80) & 0xff)
208 ret.append((x >> 7) & 0x7f)
209 elif ((x & 0xffe00000) == 0):
210 ret.append((x | 0x80) & 0xff)
211 ret.append(((x >> 7) | 0x80) & 0xff)
212 ret.append((x >> 14) & 0x7f)
213 elif ((x & 0xf0000000) == 0):
214 ret.append((x | 0x80) & 0xff)
215 ret.append(((x >> 7) | 0x80) & 0xff)
216 ret.append(((x >> 14) | 0x80) & 0xff)
217 ret.append((x >> 21) & 0x7f)
218 else:
219 ret.append((x | 0x80) & 0xff)
220 ret.append(((x >> 7) | 0x80) & 0xff)
221 ret.append(((x >> 14) | 0x80) & 0xff)
222 ret.append(((x >> 21) | 0x80) & 0xff)
223 ret.append((x >> 28) & 0x0f)
224 return ret
225
226# variable length packed 8-byte signed value
227def packed64(x):
228 ret = []
229 if ((x & 0xffffffffffffff80) == 0):
230 ret.append(x & 0x7f)
231 elif ((x & 0xffffffffffffc000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append((x >> 7) & 0x7f)
234 elif ((x & 0xffffffffffe00000) == 0):
235 ret.append((x | 0x80) & 0xff)
236 ret.append(((x >> 7) | 0x80) & 0xff)
237 ret.append((x >> 14) & 0x7f)
238 elif ((x & 0xfffffffff0000000) == 0):
239 ret.append((x | 0x80) & 0xff)
240 ret.append(((x >> 7) | 0x80) & 0xff)
241 ret.append(((x >> 14) | 0x80) & 0xff)
242 ret.append((x >> 21) & 0x7f)
243 elif ((x & 0xfffffff800000000) == 0):
244 ret.append((x | 0x80) & 0xff)
245 ret.append(((x >> 7) | 0x80) & 0xff)
246 ret.append(((x >> 14) | 0x80) & 0xff)
247 ret.append(((x >> 21) | 0x80) & 0xff)
248 ret.append((x >> 28) & 0x7f)
249 elif ((x & 0xfffffc0000000000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append(((x >> 7) | 0x80) & 0xff)
252 ret.append(((x >> 14) | 0x80) & 0xff)
253 ret.append(((x >> 21) | 0x80) & 0xff)
254 ret.append(((x >> 28) | 0x80) & 0xff)
255 ret.append((x >> 35) & 0x7f)
256 elif ((x & 0xfffe000000000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append(((x >> 21) | 0x80) & 0xff)
261 ret.append(((x >> 28) | 0x80) & 0xff)
262 ret.append(((x >> 35) | 0x80) & 0xff)
263 ret.append((x >> 42) & 0x7f)
264 elif ((x & 0xff00000000000000) == 0):
265 ret.append((x | 0x80) & 0xff)
266 ret.append(((x >> 7) | 0x80) & 0xff)
267 ret.append(((x >> 14) | 0x80) & 0xff)
268 ret.append(((x >> 21) | 0x80) & 0xff)
269 ret.append(((x >> 28) | 0x80) & 0xff)
270 ret.append(((x >> 35) | 0x80) & 0xff)
271 ret.append(((x >> 42) | 0x80) & 0xff)
272 ret.append((x >> 49) & 0x7f)
273 elif ((x & 0x8000000000000000) == 0):
274 ret.append((x | 0x80) & 0xff)
275 ret.append(((x >> 7) | 0x80) & 0xff)
276 ret.append(((x >> 14) | 0x80) & 0xff)
277 ret.append(((x >> 21) | 0x80) & 0xff)
278 ret.append(((x >> 28) | 0x80) & 0xff)
279 ret.append(((x >> 35) | 0x80) & 0xff)
280 ret.append(((x >> 42) | 0x80) & 0xff)
281 ret.append(((x >> 49) | 0x80) & 0xff)
282 ret.append((x >> 56) & 0x7f)
283 else:
284 ret.append((x | 0x80) & 0xff)
285 ret.append(((x >> 7) | 0x80) & 0xff)
286 ret.append(((x >> 14) | 0x80) & 0xff)
287 ret.append(((x >> 21) | 0x80) & 0xff)
288 ret.append(((x >> 28) | 0x80) & 0xff)
289 ret.append(((x >> 35) | 0x80) & 0xff)
290 ret.append(((x >> 42) | 0x80) & 0xff)
291 ret.append(((x >> 49) | 0x80) & 0xff)
292 ret.append(((x >> 56) | 0x80) & 0xff)
293 ret.append((x >> 63) & 0x7f)
294 return ret
295
296# 4-byte signed little endian
297def int32(x):
298 ret = []
299 ret.append(x & 0xff)
300 ret.append((x >> 8) & 0xff)
301 ret.append((x >> 16) & 0xff)
302 ret.append((x >> 24) & 0xff)
303 return ret
304
305# 2-byte signed little endian
306def int16(x):
307 ret = []
308 ret.append(x & 0xff)
309 ret.append((x >> 8) & 0xff)
310 return ret
311
312# a packed32 length followed by the specified number of characters
313def stringList(x):
314 ret = []
315 ret += packed32(len(x))
316 for i in x:
317 ret.append(i)
318 return ret
319
320def utf8StringList(x):
321 ret = []
322 for i in x:
323 ret.append(ord(i))
324 return ret
325
326# packed64 time value in nanoseconds relative to the uptime from the
327# Summary message.
328def timestampList(x):
329 ret = packed64(x)
330 return ret
331
332
333############################################################
334# Write binary
335############################################################
336
337def writeBinary(outfile, binary_list):
338 for i in binary_list:
339 outfile.write("%c" % i)
340
341############################################################
342# APC Protocol Frame Types
343############################################################
344
345def addFrameHeader(frame_type, body, core):
346 ret = []
347
348 if frame_type == "Summary":
349 code = 1
350 elif frame_type == "Backtrace":
351 code = 2
352 elif frame_type == "Name":
353 code = 3
354 elif frame_type == "Counter":
355 code = 4
356 elif frame_type == "Block Counter":
357 code = 5
358 elif frame_type == "Annotate":
359 code = 6
360 elif frame_type == "Sched Trace":
361 code = 7
362 elif frame_type == "GPU Trace":
363 code = 8
364 elif frame_type == "Idle":
365 code = 9
366 else:
367 print "ERROR: Unknown frame type:", frame_type
368 sys.exit(1)
369
370 packed_code = packed32(code)
371
372 packed_core = packed32(core)
373
374 length = int32(len(packed_code) + len(packed_core) + len(body))
375
376 ret = length + packed_code + packed_core + body
377 return ret
378
379
380# Summary frame
381# - timestamp: packed64
382# - uptime: packed64
383def summaryFrame(timestamp, uptime):
384 frame_type = "Summary"
385 body = packed64(timestamp) + packed64(uptime)
386 ret = addFrameHeader(frame_type, body, 0)
387 return ret
388
389# Backtrace frame
390# - not implemented yet
391def backtraceFrame():
392 pass
393
394# Cookie name message
395# - cookie: packed32
396# - name: string
397def cookieNameFrame(cookie, name):
398 frame_type = "Name"
399 packed_code = packed32(1)
400 body = packed_code + packed32(cookie) + stringList(name)
401 ret = addFrameHeader(frame_type, body, 0)
402 return ret
403
404# Thread name message
405# - timestamp: timestamp
406# - thread id: packed32
407# - name: string
408def threadNameFrame(timestamp, thread_id, name):
409 frame_type = "Name"
410 packed_code = packed32(2)
411 body = packed_code + timestampList(timestamp) + \
412 packed32(thread_id) + stringList(name)
413 ret = addFrameHeader(frame_type, body, 0)
414 return ret
415
416# Core name message
417# - name: string
418def coreNameFrame(name):
419 frame_type = "Name"
420 packed_code = packed32(3)
421 body = packed_code + stringList(name)
422 ret = addFrameHeader(frame_type, body, 0)
423 return ret
424
425# Counter frame message
426# - timestamp: timestamp
427# - core: packed32
428# - key: packed32
429# - value: packed64
430def counterFrame(timestamp, core, key, value):
431 frame_type = "Counter"
432 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
433 packed64(value)
434 ret = addFrameHeader(frame_type, body, core)
435 return ret
436
437# Block Counter frame message
438# - key: packed32
439# - value: packed64
440def blockCounterFrame(core, key, value):
441 frame_type = "Block Counter"
442 body = packed32(key) + packed64(value)
443 ret = addFrameHeader(frame_type, body, core)
444 return ret
445
446# Annotate frame messages
447# - core: packed32
448# - tid: packed32
449# - timestamp: timestamp
450# - size: packed32
451# - body
452def annotateFrame(core, tid, timestamp, size, userspace_body):
453 frame_type = "Annotate"
454 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
455 packed32(size) + userspace_body
456 ret = addFrameHeader(frame_type, body, core)
457 return ret
458
459# Scheduler Trace frame messages
460# Sched Switch
461# - Code: 1
462# - timestamp: timestamp
463# - pid: packed32
464# - tid: packed32
465# - cookie: packed32
466# - state: packed32
467def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
468 frame_type = "Sched Trace"
469 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
470 packed32(tid) + packed32(cookie) + packed32(state)
471 ret = addFrameHeader(frame_type, body, core)
472 return ret
473
474# Sched Thread Exit
475# - Code: 2
476# - timestamp: timestamp
477# - tid: packed32
478def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
479 frame_type = "Sched Trace"
480 body = packed32(2) + timestampList(timestamp) + packed32(tid)
481 ret = addFrameHeader(frame_type, body, core)
482 return ret
483
484# GPU Trace frame messages
485# - Not implemented yet
486def gpuTraceFrame():
487 pass
488
489# Idle frame messages
490# Enter Idle
491# - code: 1
492# - timestamp: timestamp
493# - core: packed32
494def enterIdleFrame(timestamp, core):
495 frame_type = "Idle"
496 body = packed32(1) + timestampList(timestamp) + packed32(core)
497 ret = addFrameHeader(frame_type, body, core)
498 return ret
499
500# Exit Idle
501# - code: 2
502# - timestamp: timestamp
503# - core: packed32
504def exitIdleFrame(timestamp, core):
505 frame_type = "Idle"
506 body = packed32(2) + timestampList(timestamp) + packed32(core)
507 ret = addFrameHeader(frame_type, body, core)
508 return ret
509
510
511####################################################################
512def parseProcessInfo(task_file):
513 print "\n==============================="
514 print "Parsing Task file..."
515 print task_file
516 print "===============================\n"
517
518 global start_tick, end_tick, num_cpus
519 global process_dict, thread_dict, process_list
520 global event_list, unified_event_list
521 global idle_uid, kernel_uid
522
523 event_list = []
524 unified_event_list = []
525 for cpu in range(num_cpus):
526 event_list.append([])
527
528 uid = 1 # uid 0 is reserved for idle
529
530 # Dummy Tasks for frame buffers and system diagrams
531 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
532 process_list.append(process)
533 uid += 1
534 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
535 process.children.append(thread)
536 uid += 1
537 process = Task(uid, 9998, 9998, "System", True, 0)
538 process_list.append(process)
539 # if we don't find the real kernel, use this to keep things going
540 kernel_uid = uid
541 uid += 1
542 thread = Task(uid, 9998, 9998, "System", False, 0)
543 process.children.append(thread)
544 uid += 1
545
546 ext = os.path.splitext(task_file)[1]
547
548 try:
549 if ext == ".gz":
550 process_file = gzip.open(task_file, 'rb')
551 else:
552 process_file = open(task_file, 'rb')
553 except:
554 print "ERROR opening task file:", task_file
555 print "Make sure context switch task dumping is enabled in gem5."
556 sys.exit(1)
557
558 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
559 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
560
561 task_name_failure_warned = False
562
563 for line in process_file:
564 match = re.match(process_re, line)
565 if match:
566 tick = int(match.group(1))
567 if (start_tick < 0):
568 start_tick = tick
569 cpu_id = int(match.group(3))
570 pid = int(match.group(4))
571 tgid = int(match.group(5))
572 task_name = match.group(6)
573
574 if not task_name_failure_warned:
575 if task_name == "FailureIn_curTaskName":
576 print "-------------------------------------------------"
577 print "WARNING: Task name not set correctly!"
578 print "Process/Thread info will not be displayed correctly"
579 print "Perhaps forgot to apply m5struct.patch to kernel?"
580 print "-------------------------------------------------"
581 task_name_failure_warned = True
582
583 if not tgid in process_dict:
584 if tgid == pid:
585 # new task is parent as well
586 if args.verbose:
587 print "new process", uid, pid, tgid, task_name
588 if tgid == 0:
589 # new process is the "idle" task
590 process = Task(uid, pid, tgid, "idle", True, tick)
591 idle_uid = 0
592 else:
593 process = Task(uid, pid, tgid, task_name, True, tick)
594 else:
595 if tgid == 0:
596 process = Task(uid, tgid, tgid, "idle", True, tick)
597 idle_uid = 0
598 else:
599 # parent process name not known yet
600 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
601 if tgid == -1: # kernel
602 kernel_uid = 0
603 uid += 1
604 process_dict[tgid] = process
605 process_list.append(process)
606 else:
607 if tgid == pid:
608 if process_dict[tgid].task_name == "_Unknown_":
609 if args.verbose:
610 print "new process", \
611 process_dict[tgid].uid, pid, tgid, task_name
612 process_dict[tgid].task_name = task_name
613 if process_dict[tgid].task_name != task_name and tgid != 0:
614 process_dict[tgid].task_name = task_name
615
616 if not pid in thread_dict:
617 if args.verbose:
618 print "new thread", \
619 uid, process_dict[tgid].uid, pid, tgid, task_name
620 thread = Task(uid, pid, tgid, task_name, False, tick)
621 uid += 1
622 thread_dict[pid] = thread
623 process_dict[tgid].children.append(thread)
624 else:
625 if thread_dict[pid].task_name != task_name:
626 thread_dict[pid].task_name = task_name
627
628 if args.verbose:
629 print tick, uid, cpu_id, pid, tgid, task_name
630
631 task = thread_dict[pid]
632 event = Event(tick, task)
633 event_list[cpu_id].append(event)
634 unified_event_list.append(event)
635
636 if len(unified_event_list) == num_events:
637 print "Truncating at", num_events, "events!"
638 break
639 print "Found %d events." % len(unified_event_list)
640
641 for process in process_list:
642 if process.pid > 9990: # fix up framebuffer ticks
643 process.tick = start_tick
644 print process.uid, process.pid, process.tgid, \
645 process.task_name, str(process.tick)
646 for thread in process.children:
647 if thread.pid > 9990:
648 thread.tick = start_tick
649 print "\t", thread.uid, thread.pid, thread.tgid, \
650 thread.task_name, str(thread.tick)
651
652 end_tick = tick
653
654 print "Start tick:", start_tick
655 print "End tick: ", end_tick
656 print ""
657
658 return
659
660
661def initOutput(output_path):
662 if not os.path.exists(output_path):
663 os.mkdir(output_path)
664
665def ticksToNs(tick):
666 if ticks_in_ns < 0:
667 print "ticks_in_ns not set properly!"
668 sys.exit(1)
669
670 return tick / ticks_in_ns
671
672def writeXmlFile(xml, filename):
673 f = open(filename, "w")
674 txt = ET.tostring(xml)
675 f.write(minidom.parseString(txt).toprettyxml())
676 f.close()
677
678
679# StatsEntry that contains individual statistics
680class StatsEntry(object):
681 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
682
683 # Full name of statistics
684 self.name = name
685
686 # Streamline group name that statistic will belong to
687 self.group = group
688
689 # Index of statistics within group (used to change colors within groups)
690 self.group_index = group_index
691
692 # Shorter name with "system" stripped off
693 # and symbols converted to alphanumerics
694 self.short_name = re.sub("system\.", "", name)
695 self.short_name = re.sub(":", "_", name)
696
697 # Regex for this stat (string version used to construct union regex)
698 self.regex_string = "^" + name + "\s+([\d\.]+)"
699 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
700 self.description = ""
701
702 # Whether this stat is use per CPU or not
703 self.per_cpu = per_cpu
704 self.per_switchcpu = per_switchcpu
705
706 # Key used in .apc protocol (as described in captured.xml)
707 self.key = key
708
709 # List of values of stat per timestamp
710 self.values = []
711
712 # Whether this stat has been found for the current timestamp
713 self.found = False
714
715 # Whether this stat has been found at least once
716 # (to suppress too many warnings)
717 self.not_found_at_least_once = False
718
719 # Field used to hold ElementTree subelement for this stat
720 self.ET_element = None
721
722 # Create per-CPU stat name and regex, etc.
723 if self.per_cpu:
724 self.per_cpu_regex_string = []
725 self.per_cpu_regex = []
726 self.per_cpu_name = []
727 self.per_cpu_found = []
728 for i in range(num_cpus):
729 # Resuming from checkpoints results in using "switch_cpus"
730 if per_switchcpu:
731 per_cpu_name = "system.switch_cpus"
732 else:
733 per_cpu_name = "system.cpu"
734
735 # No CPU number appends if num_cpus == 1
736 if num_cpus > 1:
737 per_cpu_name += str(i)
738 per_cpu_name += "." + self.name
739 self.per_cpu_name.append(per_cpu_name)
740 print "\t", per_cpu_name
741
742 self.per_cpu_regex_string.\
743 append("^" + per_cpu_name + "\s+[\d\.]+")
744 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
745 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
746 self.values.append([])
747 self.per_cpu_found.append(False)
748
749 def append_value(self, val, per_cpu_index = None):
750 if self.per_cpu:
751 self.values[per_cpu_index].append(str(val))
752 else:
753 self.values.append(str(val))
754
755# Global stats object that contains the list of stats entries
756# and other utility functions
757class Stats(object):
758 def __init__(self):
759 self.stats_list = []
760 self.tick_list = []
761 self.next_key = 1
762
763 def register(self, name, group, group_index, per_cpu, per_switchcpu):
764 print "registering stat:", name, "group:", group, group_index
765 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
766 per_switchcpu, self.next_key))
767 self.next_key += 1
768
769 # Union of all stats to accelerate parsing speed
770 def createStatsRegex(self):
771 regex_strings = [];
772 print "\nnum entries in stats_list", len(self.stats_list)
773 for entry in self.stats_list:
774 if entry.per_cpu:
775 for i in range(num_cpus):
776 regex_strings.append(entry.per_cpu_regex_string[i])
777 else:
778 regex_strings.append(entry.regex_string)
779
780 self.regex = re.compile('|'.join(regex_strings))
781
782
783def registerStats(config_file):
784 print "==============================="
785 print "Parsing stats config.ini file..."
786 print config_file
787 print "==============================="
788
789 config = ConfigParser()
790 if not config.read(config_file):
791 print "ERROR: config file '", config_file, "' not found!"
792 sys.exit(1)
793
794 print "\nRegistering Stats..."
795
796 stats = Stats()
797
798 per_cpu_stat_groups = config.options('PER_CPU_STATS')
799 for group in per_cpu_stat_groups:
800 i = 0
801 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
802 for item in per_cpu_stats_list:
803 if item:
804 stats.register(item, group, i, True, False)
805 i += 1
806
807 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
808 for group in per_cpu_stat_groups:
809 i = 0
810 per_cpu_stats_list = \
811 config.get('PER_SWITCHCPU_STATS', group).split('\n')
812 for item in per_cpu_stats_list:
813 if item:
814 stats.register(item, group, i, True, True)
815 i += 1
816
817 per_l2_stat_groups = config.options('PER_L2_STATS')
818 for group in per_l2_stat_groups:
819 i = 0
820 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
821 for item in per_l2_stats_list:
822 if item:
823 for l2 in range(num_l2):
824 name = item
825 prefix = "system.l2"
826 if num_l2 > 1:
827 prefix += str(l2)
828 prefix += "."
829 name = prefix + name
830 stats.register(name, group, i, False, False)
831 i += 1
832
833 other_stat_groups = config.options('OTHER_STATS')
834 for group in other_stat_groups:
835 i = 0
836 other_stats_list = config.get('OTHER_STATS', group).split('\n')
837 for item in other_stats_list:
838 if item:
839 stats.register(item, group, i, False, False)
840 i += 1
841
842 stats.createStatsRegex()
843
844 return stats
845
846# Parse and read in gem5 stats file
847# Streamline counters are organized per CPU
848def readGem5Stats(stats, gem5_stats_file):
849 print "\n==============================="
850 print "Parsing gem5 stats file..."
851 print gem5_stats_file
852 print "===============================\n"
853 ext = os.path.splitext(gem5_stats_file)[1]
854
855 window_start_regex = \
856 re.compile("^---------- Begin Simulation Statistics ----------")
857 window_end_regex = \
858 re.compile("^---------- End Simulation Statistics ----------")
859 final_tick_regex = re.compile("^final_tick\s+(\d+)")
860
861 global ticks_in_ns
862 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
863 sim_freq = -1
864
865 try:
866 if ext == ".gz":
867 f = gzip.open(gem5_stats_file, "r")
868 else:
869 f = open(gem5_stats_file, "r")
870 except:
871 print "ERROR opening stats file", gem5_stats_file, "!"
872 sys.exit(1)
873
874 stats_not_found_list = stats.stats_list[:]
875 window_num = 0
876
877 while (True):
878 error = False
879 try:
880 line = f.readline()
881 except IOError:
882 print ""
883 print "WARNING: IO error in stats file"
884 print "(gzip stream not closed properly?)...continuing for now"
885 error = True
886 if not line:
887 break
888
889 # Find out how many gem5 ticks in 1ns
890 if sim_freq < 0:
891 m = sim_freq_regex.match(line)
892 if m:
893 sim_freq = int(m.group(1)) # ticks in 1 sec
894 ticks_in_ns = int(sim_freq / 1e9)
895 print "Simulation frequency found! 1 tick == %e sec\n" \
896 % (1.0 / sim_freq)
897
898 # Final tick in gem5 stats: current absolute timestamp
899 m = final_tick_regex.match(line)
900 if m:
901 tick = int(m.group(1))
902 if tick > end_tick:
903 break
904 stats.tick_list.append(tick)
905
906
907 if (window_end_regex.match(line) or error):
908 if args.verbose:
909 print "new window"
910 for stat in stats.stats_list:
911 if stat.per_cpu:
912 for i in range(num_cpus):
913 if not stat.per_cpu_found[i]:
914 if not stat.not_found_at_least_once:
915 print "WARNING: stat not found in window #", \
916 window_num, ":", stat.per_cpu_name[i]
917 print "suppressing further warnings for " + \
918 "this stat"
919 stat.not_found_at_least_once = True
920 stat.values[i].append(str(0))
921 stat.per_cpu_found[i] = False
922 else:
923 if not stat.found:
924 if not stat.not_found_at_least_once:
925 print "WARNING: stat not found in window #", \
926 window_num, ":", stat.name
927 print "suppressing further warnings for this stat"
928 stat.not_found_at_least_once = True
929 stat.values.append(str(0))
930 stat.found = False
931 stats_not_found_list = stats.stats_list[:]
932 window_num += 1
933 if error:
934 break
935
936 # Do a single regex of the union of all stats first for speed
937 if stats.regex.match(line):
938 # Then loop through only the stats we haven't seen in this window
939 for stat in stats_not_found_list[:]:
940 if stat.per_cpu:
941 for i in range(num_cpus):
942 m = stat.per_cpu_regex[i].match(line)
943 if m:
944 if stat.name == "ipc":
945 value = str(int(float(m.group(1)) * 1000))
946 else:
947 value = str(int(float(m.group(1))))
948 if args.verbose:
949 print stat.per_cpu_name[i], value
950 stat.values[i].append(value)
951 stat.per_cpu_found[i] = True
952 all_found = True
953 for j in range(num_cpus):
954 if not stat.per_cpu_found[j]:
955 all_found = False
956 if all_found:
957 stats_not_found_list.remove(stat)
958 if stat.description == "":
959 stat.description = m.group(2)
960 else:
961 m = stat.regex.match(line)
962 if m:
963 value = str(int(float(m.group(1))))
964 if args.verbose:
965 print stat.name, value
966 stat.values.append(value)
967 stat.found = True
968 stats_not_found_list.remove(stat)
969 if stat.description == "":
970 stat.description = m.group(2)
971 f.close()
972
973
974# Create session.xml file in .apc folder
975def doSessionXML(output_path):
976 session_file = output_path + "/session.xml"
977
978 xml = ET.Element("session")
979
980 xml.set("version", "1")
981 xml.set("call_stack_unwinding", "no")
982 xml.set("parse_debug_info", "no")
983 xml.set("high_resolution", "yes")
984 xml.set("buffer_mode", "streaming")
985 xml.set("sample_rate", "low")
986
987 # Setting duration to zero for now. Doesn't affect visualization.
988 xml.set("duration", "0")
989
990 xml.set("target_host", "")
991 xml.set("target_port", "8080")
992
993 writeXmlFile(xml, session_file)
994
995
996# Create captured.xml file in .apc folder
997def doCapturedXML(output_path, stats):
998 captured_file = output_path + "/captured.xml"
999
1000 xml = ET.Element("captured")
1001 xml.set("version", "1")
1002 xml.set("protocol", "12")
1003
1004 target = ET.SubElement(xml, "target")
1005 target.set("name", "gem5")
1006 target.set("sample_rate", "1000")
1007 target.set("cores", str(num_cpus))
1008
1009 counters = ET.SubElement(xml, "counters")
1010 for stat in stats.stats_list:
1011 s = ET.SubElement(counters, "counter")
1012 stat_name = re.sub("\.", "_", stat.short_name)
1013 s.set("title", stat.group)
1014 s.set("name", stat_name)
1015 s.set("color", "0x00000000")
1016 s.set("key", "0x%08x" % stat.key)
1017 s.set("type", stat_name)
1018 s.set("event", "0x00000000")
1019 if stat.per_cpu:
1020 s.set("per_cpu", "yes")
1021 else:
1022 s.set("per_cpu", "no")
1023 s.set("display", "")
1024 s.set("units", "")
1025 s.set("average_selection", "no")
1026 s.set("description", stat.description)
1027
1028 writeXmlFile(xml, captured_file)
1029
1030# Writes out Streamline cookies (unique IDs per process/thread)
1031def writeCookiesThreads(blob):
1032 thread_list = []
1033 for process in process_list:
1034 if process.uid > 0:
1035 print "cookie", process.task_name, process.uid
1036 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1037
1038 # pid and tgid need to be positive values -- no longer true?
1039 for thread in process.children:
1040 thread_list.append(thread)
1041
1042 # Threads need to be sorted in timestamp order
1043 thread_list.sort(key = lambda x: x.tick)
1044 for thread in thread_list:
1045 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1046 thread.tgid, thread.pid
1047 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1048 thread.pid, thread.task_name))
1049
1050# Writes context switch info as Streamline scheduling events
1051def writeSchedEvents(blob):
1052 for cpu in range(num_cpus):
1053 for event in event_list[cpu]:
1054 timestamp = ticksToNs(event.tick)
1055 pid = event.task.tgid
1056 tid = event.task.pid
1057 if process_dict.has_key(event.task.tgid):
1058 cookie = process_dict[event.task.tgid].uid
1059 else:
1060 cookie = 0
1061
1062 # State:
1063 # 0: waiting on other event besides I/O
1064 # 1: Contention/pre-emption
1065 # 2: Waiting on I/O
1066 # 3: Waiting on mutex
1067 # Hardcoding to 0 for now. Other states not implemented yet.
1068 state = 0
1069
1070 if args.verbose:
1071 print cpu, timestamp, pid, tid, cookie
1072
1073 writeBinary(blob,\
1074 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1075
1076# Writes selected gem5 statistics as Streamline counters
1077def writeCounters(blob, stats):
1078 timestamp_list = []
1079 for tick in stats.tick_list:
1080 if tick > end_tick:
1081 break
1082 timestamp_list.append(ticksToNs(tick))
1083
1084 for stat in stats.stats_list:
1085 if stat.per_cpu:
1086 stat_length = len(stat.values[0])
1087 else:
1088 stat_length = len(stat.values)
1089
1090 for n in range(len(timestamp_list)):
1091 for stat in stats.stats_list:
1092 if stat.per_cpu:
1093 for i in range(num_cpus):
1094 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1095 stat.key, int(float(stat.values[i][n]))))
1096 else:
1097 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1098 stat.key, int(float(stat.values[n]))))
1099
1100# Streamline can display LCD frame buffer dumps (gzipped bmp)
1101# This function converts the frame buffer dumps to the Streamline format
1102def writeVisualAnnotations(blob, input_path, output_path):
1103 frame_path = input_path + "/frames_system.vncserver"
1104 if not os.path.exists(frame_path):
1105 return
1106
1107 frame_count = 0
1108 file_list = os.listdir(frame_path)
1109 file_list.sort()
1110 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1111
1112 # Use first non-negative pid to tag visual annotations
1113 annotate_pid = -1
1114 for e in unified_event_list:
1115 pid = e.task.pid
1116 if pid >= 0:
1117 annotate_pid = pid
1118 break
1119
1120 for fn in file_list:
1121 m = re_fb.match(fn)
1122 if m:
1123 seq = m.group(1)
1124 tick = int(m.group(2))
1125 if tick > end_tick:
1126 break
1127 frame_count += 1
1128
1129 userspace_body = []
1130 userspace_body += packed32(0x1C) # escape code
1131 userspace_body += packed32(0x04) # visual code
1132
1133 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1134 userspace_body += int16(len(text_annotation))
1135 userspace_body += utf8StringList(text_annotation)
1136
1137 if gzipped_bmp_supported:
1138 # copy gzipped bmp directly
1139 bytes_read = open(frame_path + "/" + fn, "rb").read()
1140 else:
1141 # copy uncompressed bmp
1142 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1143
1144 userspace_body += int32(len(bytes_read))
1145 userspace_body += bytes_read
1146
1147 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1148 len(userspace_body), userspace_body))
1149
1150 print "\nfound", frame_count, "frames for visual annotation.\n"
1151
1152
1153def createApcProject(input_path, output_path, stats):
1154 initOutput(output_path)
1155
1156 blob = open(output_path + "/0000000000", "wb")
1157
1158 # Summary frame takes current system time and system uptime.
1159 # Filling in with random values for now.
1160 writeBinary(blob, summaryFrame(1234, 5678))
1161
1162 writeCookiesThreads(blob)
1163
1164 print "writing Events"
1165 writeSchedEvents(blob)
1166
1167 print "writing Counters"
1168 writeCounters(blob, stats)
1169
1170 print "writing Visual Annotations"
1171 writeVisualAnnotations(blob, input_path, output_path)
1172
1173 doSessionXML(output_path)
1174 doCapturedXML(output_path, stats)
1175
1176 blob.close()
1177
1178
1179
1180#######################
1181# Main Routine
1182
1183input_path = args.input_path
1184output_path = args.output_path
1185
1186####
1187# Make sure input path exists
1188####
1189if not os.path.exists(input_path):
1190 print "ERROR: Input path %s does not exist!" % input_path
1191 sys.exit(1)
1192
1193####
1194# Parse gem5 configuration file to find # of CPUs and L2s
1195####
1196(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1197
1198####
1199# Parse task file to find process/thread info
1200####
1201parseProcessInfo(input_path + "/system.tasks.txt")
1202
1203####
1204# Parse stat config file and register stats
1205####
1206stat_config_file = args.stat_config_file
1207stats = registerStats(stat_config_file)
1208
1209####
1210# Parse gem5 stats
1211####
1212# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1213if os.path.exists(input_path + "/stats.txt") and \
1214 os.path.exists(input_path + "/stats.txt.gz"):
1215 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1216 Using stats.txt.gz by default."
1217
1218gem5_stats_file = input_path + "/stats.txt.gz"
1219if not os.path.exists(gem5_stats_file):
1220 gem5_stats_file = input_path + "/stats.txt"
1221if not os.path.exists(gem5_stats_file):
1222 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1223 sys.exit(1)
1224
1225readGem5Stats(stats, gem5_stats_file)
1226
1227####
1228# Create Streamline .apc project folder
1229####
1230createApcProject(input_path, output_path, stats)
1231
1232print "All done!"
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v12 (DS-5 v5.13)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
81 APC project generation based on Gator v12 (DS-5 v5.13)
82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
201# variable length packed 4-byte signed value
202def packed32(x):
203 ret = []
204 if ((x & 0xffffff80) == 0):
205 ret.append(x & 0x7f)
206 elif ((x & 0xffffc000) == 0):
207 ret.append((x | 0x80) & 0xff)
208 ret.append((x >> 7) & 0x7f)
209 elif ((x & 0xffe00000) == 0):
210 ret.append((x | 0x80) & 0xff)
211 ret.append(((x >> 7) | 0x80) & 0xff)
212 ret.append((x >> 14) & 0x7f)
213 elif ((x & 0xf0000000) == 0):
214 ret.append((x | 0x80) & 0xff)
215 ret.append(((x >> 7) | 0x80) & 0xff)
216 ret.append(((x >> 14) | 0x80) & 0xff)
217 ret.append((x >> 21) & 0x7f)
218 else:
219 ret.append((x | 0x80) & 0xff)
220 ret.append(((x >> 7) | 0x80) & 0xff)
221 ret.append(((x >> 14) | 0x80) & 0xff)
222 ret.append(((x >> 21) | 0x80) & 0xff)
223 ret.append((x >> 28) & 0x0f)
224 return ret
225
226# variable length packed 8-byte signed value
227def packed64(x):
228 ret = []
229 if ((x & 0xffffffffffffff80) == 0):
230 ret.append(x & 0x7f)
231 elif ((x & 0xffffffffffffc000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append((x >> 7) & 0x7f)
234 elif ((x & 0xffffffffffe00000) == 0):
235 ret.append((x | 0x80) & 0xff)
236 ret.append(((x >> 7) | 0x80) & 0xff)
237 ret.append((x >> 14) & 0x7f)
238 elif ((x & 0xfffffffff0000000) == 0):
239 ret.append((x | 0x80) & 0xff)
240 ret.append(((x >> 7) | 0x80) & 0xff)
241 ret.append(((x >> 14) | 0x80) & 0xff)
242 ret.append((x >> 21) & 0x7f)
243 elif ((x & 0xfffffff800000000) == 0):
244 ret.append((x | 0x80) & 0xff)
245 ret.append(((x >> 7) | 0x80) & 0xff)
246 ret.append(((x >> 14) | 0x80) & 0xff)
247 ret.append(((x >> 21) | 0x80) & 0xff)
248 ret.append((x >> 28) & 0x7f)
249 elif ((x & 0xfffffc0000000000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append(((x >> 7) | 0x80) & 0xff)
252 ret.append(((x >> 14) | 0x80) & 0xff)
253 ret.append(((x >> 21) | 0x80) & 0xff)
254 ret.append(((x >> 28) | 0x80) & 0xff)
255 ret.append((x >> 35) & 0x7f)
256 elif ((x & 0xfffe000000000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append(((x >> 21) | 0x80) & 0xff)
261 ret.append(((x >> 28) | 0x80) & 0xff)
262 ret.append(((x >> 35) | 0x80) & 0xff)
263 ret.append((x >> 42) & 0x7f)
264 elif ((x & 0xff00000000000000) == 0):
265 ret.append((x | 0x80) & 0xff)
266 ret.append(((x >> 7) | 0x80) & 0xff)
267 ret.append(((x >> 14) | 0x80) & 0xff)
268 ret.append(((x >> 21) | 0x80) & 0xff)
269 ret.append(((x >> 28) | 0x80) & 0xff)
270 ret.append(((x >> 35) | 0x80) & 0xff)
271 ret.append(((x >> 42) | 0x80) & 0xff)
272 ret.append((x >> 49) & 0x7f)
273 elif ((x & 0x8000000000000000) == 0):
274 ret.append((x | 0x80) & 0xff)
275 ret.append(((x >> 7) | 0x80) & 0xff)
276 ret.append(((x >> 14) | 0x80) & 0xff)
277 ret.append(((x >> 21) | 0x80) & 0xff)
278 ret.append(((x >> 28) | 0x80) & 0xff)
279 ret.append(((x >> 35) | 0x80) & 0xff)
280 ret.append(((x >> 42) | 0x80) & 0xff)
281 ret.append(((x >> 49) | 0x80) & 0xff)
282 ret.append((x >> 56) & 0x7f)
283 else:
284 ret.append((x | 0x80) & 0xff)
285 ret.append(((x >> 7) | 0x80) & 0xff)
286 ret.append(((x >> 14) | 0x80) & 0xff)
287 ret.append(((x >> 21) | 0x80) & 0xff)
288 ret.append(((x >> 28) | 0x80) & 0xff)
289 ret.append(((x >> 35) | 0x80) & 0xff)
290 ret.append(((x >> 42) | 0x80) & 0xff)
291 ret.append(((x >> 49) | 0x80) & 0xff)
292 ret.append(((x >> 56) | 0x80) & 0xff)
293 ret.append((x >> 63) & 0x7f)
294 return ret
295
296# 4-byte signed little endian
297def int32(x):
298 ret = []
299 ret.append(x & 0xff)
300 ret.append((x >> 8) & 0xff)
301 ret.append((x >> 16) & 0xff)
302 ret.append((x >> 24) & 0xff)
303 return ret
304
305# 2-byte signed little endian
306def int16(x):
307 ret = []
308 ret.append(x & 0xff)
309 ret.append((x >> 8) & 0xff)
310 return ret
311
312# a packed32 length followed by the specified number of characters
313def stringList(x):
314 ret = []
315 ret += packed32(len(x))
316 for i in x:
317 ret.append(i)
318 return ret
319
320def utf8StringList(x):
321 ret = []
322 for i in x:
323 ret.append(ord(i))
324 return ret
325
326# packed64 time value in nanoseconds relative to the uptime from the
327# Summary message.
328def timestampList(x):
329 ret = packed64(x)
330 return ret
331
332
333############################################################
334# Write binary
335############################################################
336
337def writeBinary(outfile, binary_list):
338 for i in binary_list:
339 outfile.write("%c" % i)
340
341############################################################
342# APC Protocol Frame Types
343############################################################
344
345def addFrameHeader(frame_type, body, core):
346 ret = []
347
348 if frame_type == "Summary":
349 code = 1
350 elif frame_type == "Backtrace":
351 code = 2
352 elif frame_type == "Name":
353 code = 3
354 elif frame_type == "Counter":
355 code = 4
356 elif frame_type == "Block Counter":
357 code = 5
358 elif frame_type == "Annotate":
359 code = 6
360 elif frame_type == "Sched Trace":
361 code = 7
362 elif frame_type == "GPU Trace":
363 code = 8
364 elif frame_type == "Idle":
365 code = 9
366 else:
367 print "ERROR: Unknown frame type:", frame_type
368 sys.exit(1)
369
370 packed_code = packed32(code)
371
372 packed_core = packed32(core)
373
374 length = int32(len(packed_code) + len(packed_core) + len(body))
375
376 ret = length + packed_code + packed_core + body
377 return ret
378
379
380# Summary frame
381# - timestamp: packed64
382# - uptime: packed64
383def summaryFrame(timestamp, uptime):
384 frame_type = "Summary"
385 body = packed64(timestamp) + packed64(uptime)
386 ret = addFrameHeader(frame_type, body, 0)
387 return ret
388
389# Backtrace frame
390# - not implemented yet
391def backtraceFrame():
392 pass
393
394# Cookie name message
395# - cookie: packed32
396# - name: string
397def cookieNameFrame(cookie, name):
398 frame_type = "Name"
399 packed_code = packed32(1)
400 body = packed_code + packed32(cookie) + stringList(name)
401 ret = addFrameHeader(frame_type, body, 0)
402 return ret
403
404# Thread name message
405# - timestamp: timestamp
406# - thread id: packed32
407# - name: string
408def threadNameFrame(timestamp, thread_id, name):
409 frame_type = "Name"
410 packed_code = packed32(2)
411 body = packed_code + timestampList(timestamp) + \
412 packed32(thread_id) + stringList(name)
413 ret = addFrameHeader(frame_type, body, 0)
414 return ret
415
416# Core name message
417# - name: string
418def coreNameFrame(name):
419 frame_type = "Name"
420 packed_code = packed32(3)
421 body = packed_code + stringList(name)
422 ret = addFrameHeader(frame_type, body, 0)
423 return ret
424
425# Counter frame message
426# - timestamp: timestamp
427# - core: packed32
428# - key: packed32
429# - value: packed64
430def counterFrame(timestamp, core, key, value):
431 frame_type = "Counter"
432 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
433 packed64(value)
434 ret = addFrameHeader(frame_type, body, core)
435 return ret
436
437# Block Counter frame message
438# - key: packed32
439# - value: packed64
440def blockCounterFrame(core, key, value):
441 frame_type = "Block Counter"
442 body = packed32(key) + packed64(value)
443 ret = addFrameHeader(frame_type, body, core)
444 return ret
445
446# Annotate frame messages
447# - core: packed32
448# - tid: packed32
449# - timestamp: timestamp
450# - size: packed32
451# - body
452def annotateFrame(core, tid, timestamp, size, userspace_body):
453 frame_type = "Annotate"
454 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
455 packed32(size) + userspace_body
456 ret = addFrameHeader(frame_type, body, core)
457 return ret
458
459# Scheduler Trace frame messages
460# Sched Switch
461# - Code: 1
462# - timestamp: timestamp
463# - pid: packed32
464# - tid: packed32
465# - cookie: packed32
466# - state: packed32
467def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
468 frame_type = "Sched Trace"
469 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
470 packed32(tid) + packed32(cookie) + packed32(state)
471 ret = addFrameHeader(frame_type, body, core)
472 return ret
473
474# Sched Thread Exit
475# - Code: 2
476# - timestamp: timestamp
477# - tid: packed32
478def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
479 frame_type = "Sched Trace"
480 body = packed32(2) + timestampList(timestamp) + packed32(tid)
481 ret = addFrameHeader(frame_type, body, core)
482 return ret
483
484# GPU Trace frame messages
485# - Not implemented yet
486def gpuTraceFrame():
487 pass
488
489# Idle frame messages
490# Enter Idle
491# - code: 1
492# - timestamp: timestamp
493# - core: packed32
494def enterIdleFrame(timestamp, core):
495 frame_type = "Idle"
496 body = packed32(1) + timestampList(timestamp) + packed32(core)
497 ret = addFrameHeader(frame_type, body, core)
498 return ret
499
500# Exit Idle
501# - code: 2
502# - timestamp: timestamp
503# - core: packed32
504def exitIdleFrame(timestamp, core):
505 frame_type = "Idle"
506 body = packed32(2) + timestampList(timestamp) + packed32(core)
507 ret = addFrameHeader(frame_type, body, core)
508 return ret
509
510
511####################################################################
512def parseProcessInfo(task_file):
513 print "\n==============================="
514 print "Parsing Task file..."
515 print task_file
516 print "===============================\n"
517
518 global start_tick, end_tick, num_cpus
519 global process_dict, thread_dict, process_list
520 global event_list, unified_event_list
521 global idle_uid, kernel_uid
522
523 event_list = []
524 unified_event_list = []
525 for cpu in range(num_cpus):
526 event_list.append([])
527
528 uid = 1 # uid 0 is reserved for idle
529
530 # Dummy Tasks for frame buffers and system diagrams
531 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
532 process_list.append(process)
533 uid += 1
534 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
535 process.children.append(thread)
536 uid += 1
537 process = Task(uid, 9998, 9998, "System", True, 0)
538 process_list.append(process)
539 # if we don't find the real kernel, use this to keep things going
540 kernel_uid = uid
541 uid += 1
542 thread = Task(uid, 9998, 9998, "System", False, 0)
543 process.children.append(thread)
544 uid += 1
545
546 ext = os.path.splitext(task_file)[1]
547
548 try:
549 if ext == ".gz":
550 process_file = gzip.open(task_file, 'rb')
551 else:
552 process_file = open(task_file, 'rb')
553 except:
554 print "ERROR opening task file:", task_file
555 print "Make sure context switch task dumping is enabled in gem5."
556 sys.exit(1)
557
558 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
559 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
560
561 task_name_failure_warned = False
562
563 for line in process_file:
564 match = re.match(process_re, line)
565 if match:
566 tick = int(match.group(1))
567 if (start_tick < 0):
568 start_tick = tick
569 cpu_id = int(match.group(3))
570 pid = int(match.group(4))
571 tgid = int(match.group(5))
572 task_name = match.group(6)
573
574 if not task_name_failure_warned:
575 if task_name == "FailureIn_curTaskName":
576 print "-------------------------------------------------"
577 print "WARNING: Task name not set correctly!"
578 print "Process/Thread info will not be displayed correctly"
579 print "Perhaps forgot to apply m5struct.patch to kernel?"
580 print "-------------------------------------------------"
581 task_name_failure_warned = True
582
583 if not tgid in process_dict:
584 if tgid == pid:
585 # new task is parent as well
586 if args.verbose:
587 print "new process", uid, pid, tgid, task_name
588 if tgid == 0:
589 # new process is the "idle" task
590 process = Task(uid, pid, tgid, "idle", True, tick)
591 idle_uid = 0
592 else:
593 process = Task(uid, pid, tgid, task_name, True, tick)
594 else:
595 if tgid == 0:
596 process = Task(uid, tgid, tgid, "idle", True, tick)
597 idle_uid = 0
598 else:
599 # parent process name not known yet
600 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
601 if tgid == -1: # kernel
602 kernel_uid = 0
603 uid += 1
604 process_dict[tgid] = process
605 process_list.append(process)
606 else:
607 if tgid == pid:
608 if process_dict[tgid].task_name == "_Unknown_":
609 if args.verbose:
610 print "new process", \
611 process_dict[tgid].uid, pid, tgid, task_name
612 process_dict[tgid].task_name = task_name
613 if process_dict[tgid].task_name != task_name and tgid != 0:
614 process_dict[tgid].task_name = task_name
615
616 if not pid in thread_dict:
617 if args.verbose:
618 print "new thread", \
619 uid, process_dict[tgid].uid, pid, tgid, task_name
620 thread = Task(uid, pid, tgid, task_name, False, tick)
621 uid += 1
622 thread_dict[pid] = thread
623 process_dict[tgid].children.append(thread)
624 else:
625 if thread_dict[pid].task_name != task_name:
626 thread_dict[pid].task_name = task_name
627
628 if args.verbose:
629 print tick, uid, cpu_id, pid, tgid, task_name
630
631 task = thread_dict[pid]
632 event = Event(tick, task)
633 event_list[cpu_id].append(event)
634 unified_event_list.append(event)
635
636 if len(unified_event_list) == num_events:
637 print "Truncating at", num_events, "events!"
638 break
639 print "Found %d events." % len(unified_event_list)
640
641 for process in process_list:
642 if process.pid > 9990: # fix up framebuffer ticks
643 process.tick = start_tick
644 print process.uid, process.pid, process.tgid, \
645 process.task_name, str(process.tick)
646 for thread in process.children:
647 if thread.pid > 9990:
648 thread.tick = start_tick
649 print "\t", thread.uid, thread.pid, thread.tgid, \
650 thread.task_name, str(thread.tick)
651
652 end_tick = tick
653
654 print "Start tick:", start_tick
655 print "End tick: ", end_tick
656 print ""
657
658 return
659
660
661def initOutput(output_path):
662 if not os.path.exists(output_path):
663 os.mkdir(output_path)
664
665def ticksToNs(tick):
666 if ticks_in_ns < 0:
667 print "ticks_in_ns not set properly!"
668 sys.exit(1)
669
670 return tick / ticks_in_ns
671
672def writeXmlFile(xml, filename):
673 f = open(filename, "w")
674 txt = ET.tostring(xml)
675 f.write(minidom.parseString(txt).toprettyxml())
676 f.close()
677
678
679# StatsEntry that contains individual statistics
680class StatsEntry(object):
681 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
682
683 # Full name of statistics
684 self.name = name
685
686 # Streamline group name that statistic will belong to
687 self.group = group
688
689 # Index of statistics within group (used to change colors within groups)
690 self.group_index = group_index
691
692 # Shorter name with "system" stripped off
693 # and symbols converted to alphanumerics
694 self.short_name = re.sub("system\.", "", name)
695 self.short_name = re.sub(":", "_", name)
696
697 # Regex for this stat (string version used to construct union regex)
698 self.regex_string = "^" + name + "\s+([\d\.]+)"
699 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
700 self.description = ""
701
702 # Whether this stat is use per CPU or not
703 self.per_cpu = per_cpu
704 self.per_switchcpu = per_switchcpu
705
706 # Key used in .apc protocol (as described in captured.xml)
707 self.key = key
708
709 # List of values of stat per timestamp
710 self.values = []
711
712 # Whether this stat has been found for the current timestamp
713 self.found = False
714
715 # Whether this stat has been found at least once
716 # (to suppress too many warnings)
717 self.not_found_at_least_once = False
718
719 # Field used to hold ElementTree subelement for this stat
720 self.ET_element = None
721
722 # Create per-CPU stat name and regex, etc.
723 if self.per_cpu:
724 self.per_cpu_regex_string = []
725 self.per_cpu_regex = []
726 self.per_cpu_name = []
727 self.per_cpu_found = []
728 for i in range(num_cpus):
729 # Resuming from checkpoints results in using "switch_cpus"
730 if per_switchcpu:
731 per_cpu_name = "system.switch_cpus"
732 else:
733 per_cpu_name = "system.cpu"
734
735 # No CPU number appends if num_cpus == 1
736 if num_cpus > 1:
737 per_cpu_name += str(i)
738 per_cpu_name += "." + self.name
739 self.per_cpu_name.append(per_cpu_name)
740 print "\t", per_cpu_name
741
742 self.per_cpu_regex_string.\
743 append("^" + per_cpu_name + "\s+[\d\.]+")
744 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
745 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
746 self.values.append([])
747 self.per_cpu_found.append(False)
748
749 def append_value(self, val, per_cpu_index = None):
750 if self.per_cpu:
751 self.values[per_cpu_index].append(str(val))
752 else:
753 self.values.append(str(val))
754
755# Global stats object that contains the list of stats entries
756# and other utility functions
757class Stats(object):
758 def __init__(self):
759 self.stats_list = []
760 self.tick_list = []
761 self.next_key = 1
762
763 def register(self, name, group, group_index, per_cpu, per_switchcpu):
764 print "registering stat:", name, "group:", group, group_index
765 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
766 per_switchcpu, self.next_key))
767 self.next_key += 1
768
769 # Union of all stats to accelerate parsing speed
770 def createStatsRegex(self):
771 regex_strings = [];
772 print "\nnum entries in stats_list", len(self.stats_list)
773 for entry in self.stats_list:
774 if entry.per_cpu:
775 for i in range(num_cpus):
776 regex_strings.append(entry.per_cpu_regex_string[i])
777 else:
778 regex_strings.append(entry.regex_string)
779
780 self.regex = re.compile('|'.join(regex_strings))
781
782
783def registerStats(config_file):
784 print "==============================="
785 print "Parsing stats config.ini file..."
786 print config_file
787 print "==============================="
788
789 config = ConfigParser()
790 if not config.read(config_file):
791 print "ERROR: config file '", config_file, "' not found!"
792 sys.exit(1)
793
794 print "\nRegistering Stats..."
795
796 stats = Stats()
797
798 per_cpu_stat_groups = config.options('PER_CPU_STATS')
799 for group in per_cpu_stat_groups:
800 i = 0
801 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
802 for item in per_cpu_stats_list:
803 if item:
804 stats.register(item, group, i, True, False)
805 i += 1
806
807 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
808 for group in per_cpu_stat_groups:
809 i = 0
810 per_cpu_stats_list = \
811 config.get('PER_SWITCHCPU_STATS', group).split('\n')
812 for item in per_cpu_stats_list:
813 if item:
814 stats.register(item, group, i, True, True)
815 i += 1
816
817 per_l2_stat_groups = config.options('PER_L2_STATS')
818 for group in per_l2_stat_groups:
819 i = 0
820 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
821 for item in per_l2_stats_list:
822 if item:
823 for l2 in range(num_l2):
824 name = item
825 prefix = "system.l2"
826 if num_l2 > 1:
827 prefix += str(l2)
828 prefix += "."
829 name = prefix + name
830 stats.register(name, group, i, False, False)
831 i += 1
832
833 other_stat_groups = config.options('OTHER_STATS')
834 for group in other_stat_groups:
835 i = 0
836 other_stats_list = config.get('OTHER_STATS', group).split('\n')
837 for item in other_stats_list:
838 if item:
839 stats.register(item, group, i, False, False)
840 i += 1
841
842 stats.createStatsRegex()
843
844 return stats
845
846# Parse and read in gem5 stats file
847# Streamline counters are organized per CPU
848def readGem5Stats(stats, gem5_stats_file):
849 print "\n==============================="
850 print "Parsing gem5 stats file..."
851 print gem5_stats_file
852 print "===============================\n"
853 ext = os.path.splitext(gem5_stats_file)[1]
854
855 window_start_regex = \
856 re.compile("^---------- Begin Simulation Statistics ----------")
857 window_end_regex = \
858 re.compile("^---------- End Simulation Statistics ----------")
859 final_tick_regex = re.compile("^final_tick\s+(\d+)")
860
861 global ticks_in_ns
862 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
863 sim_freq = -1
864
865 try:
866 if ext == ".gz":
867 f = gzip.open(gem5_stats_file, "r")
868 else:
869 f = open(gem5_stats_file, "r")
870 except:
871 print "ERROR opening stats file", gem5_stats_file, "!"
872 sys.exit(1)
873
874 stats_not_found_list = stats.stats_list[:]
875 window_num = 0
876
877 while (True):
878 error = False
879 try:
880 line = f.readline()
881 except IOError:
882 print ""
883 print "WARNING: IO error in stats file"
884 print "(gzip stream not closed properly?)...continuing for now"
885 error = True
886 if not line:
887 break
888
889 # Find out how many gem5 ticks in 1ns
890 if sim_freq < 0:
891 m = sim_freq_regex.match(line)
892 if m:
893 sim_freq = int(m.group(1)) # ticks in 1 sec
894 ticks_in_ns = int(sim_freq / 1e9)
895 print "Simulation frequency found! 1 tick == %e sec\n" \
896 % (1.0 / sim_freq)
897
898 # Final tick in gem5 stats: current absolute timestamp
899 m = final_tick_regex.match(line)
900 if m:
901 tick = int(m.group(1))
902 if tick > end_tick:
903 break
904 stats.tick_list.append(tick)
905
906
907 if (window_end_regex.match(line) or error):
908 if args.verbose:
909 print "new window"
910 for stat in stats.stats_list:
911 if stat.per_cpu:
912 for i in range(num_cpus):
913 if not stat.per_cpu_found[i]:
914 if not stat.not_found_at_least_once:
915 print "WARNING: stat not found in window #", \
916 window_num, ":", stat.per_cpu_name[i]
917 print "suppressing further warnings for " + \
918 "this stat"
919 stat.not_found_at_least_once = True
920 stat.values[i].append(str(0))
921 stat.per_cpu_found[i] = False
922 else:
923 if not stat.found:
924 if not stat.not_found_at_least_once:
925 print "WARNING: stat not found in window #", \
926 window_num, ":", stat.name
927 print "suppressing further warnings for this stat"
928 stat.not_found_at_least_once = True
929 stat.values.append(str(0))
930 stat.found = False
931 stats_not_found_list = stats.stats_list[:]
932 window_num += 1
933 if error:
934 break
935
936 # Do a single regex of the union of all stats first for speed
937 if stats.regex.match(line):
938 # Then loop through only the stats we haven't seen in this window
939 for stat in stats_not_found_list[:]:
940 if stat.per_cpu:
941 for i in range(num_cpus):
942 m = stat.per_cpu_regex[i].match(line)
943 if m:
944 if stat.name == "ipc":
945 value = str(int(float(m.group(1)) * 1000))
946 else:
947 value = str(int(float(m.group(1))))
948 if args.verbose:
949 print stat.per_cpu_name[i], value
950 stat.values[i].append(value)
951 stat.per_cpu_found[i] = True
952 all_found = True
953 for j in range(num_cpus):
954 if not stat.per_cpu_found[j]:
955 all_found = False
956 if all_found:
957 stats_not_found_list.remove(stat)
958 if stat.description == "":
959 stat.description = m.group(2)
960 else:
961 m = stat.regex.match(line)
962 if m:
963 value = str(int(float(m.group(1))))
964 if args.verbose:
965 print stat.name, value
966 stat.values.append(value)
967 stat.found = True
968 stats_not_found_list.remove(stat)
969 if stat.description == "":
970 stat.description = m.group(2)
971 f.close()
972
973
974# Create session.xml file in .apc folder
975def doSessionXML(output_path):
976 session_file = output_path + "/session.xml"
977
978 xml = ET.Element("session")
979
980 xml.set("version", "1")
981 xml.set("call_stack_unwinding", "no")
982 xml.set("parse_debug_info", "no")
983 xml.set("high_resolution", "yes")
984 xml.set("buffer_mode", "streaming")
985 xml.set("sample_rate", "low")
986
987 # Setting duration to zero for now. Doesn't affect visualization.
988 xml.set("duration", "0")
989
990 xml.set("target_host", "")
991 xml.set("target_port", "8080")
992
993 writeXmlFile(xml, session_file)
994
995
996# Create captured.xml file in .apc folder
997def doCapturedXML(output_path, stats):
998 captured_file = output_path + "/captured.xml"
999
1000 xml = ET.Element("captured")
1001 xml.set("version", "1")
1002 xml.set("protocol", "12")
1003
1004 target = ET.SubElement(xml, "target")
1005 target.set("name", "gem5")
1006 target.set("sample_rate", "1000")
1007 target.set("cores", str(num_cpus))
1008
1009 counters = ET.SubElement(xml, "counters")
1010 for stat in stats.stats_list:
1011 s = ET.SubElement(counters, "counter")
1012 stat_name = re.sub("\.", "_", stat.short_name)
1013 s.set("title", stat.group)
1014 s.set("name", stat_name)
1015 s.set("color", "0x00000000")
1016 s.set("key", "0x%08x" % stat.key)
1017 s.set("type", stat_name)
1018 s.set("event", "0x00000000")
1019 if stat.per_cpu:
1020 s.set("per_cpu", "yes")
1021 else:
1022 s.set("per_cpu", "no")
1023 s.set("display", "")
1024 s.set("units", "")
1025 s.set("average_selection", "no")
1026 s.set("description", stat.description)
1027
1028 writeXmlFile(xml, captured_file)
1029
1030# Writes out Streamline cookies (unique IDs per process/thread)
1031def writeCookiesThreads(blob):
1032 thread_list = []
1033 for process in process_list:
1034 if process.uid > 0:
1035 print "cookie", process.task_name, process.uid
1036 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1037
1038 # pid and tgid need to be positive values -- no longer true?
1039 for thread in process.children:
1040 thread_list.append(thread)
1041
1042 # Threads need to be sorted in timestamp order
1043 thread_list.sort(key = lambda x: x.tick)
1044 for thread in thread_list:
1045 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1046 thread.tgid, thread.pid
1047 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1048 thread.pid, thread.task_name))
1049
1050# Writes context switch info as Streamline scheduling events
1051def writeSchedEvents(blob):
1052 for cpu in range(num_cpus):
1053 for event in event_list[cpu]:
1054 timestamp = ticksToNs(event.tick)
1055 pid = event.task.tgid
1056 tid = event.task.pid
1057 if process_dict.has_key(event.task.tgid):
1058 cookie = process_dict[event.task.tgid].uid
1059 else:
1060 cookie = 0
1061
1062 # State:
1063 # 0: waiting on other event besides I/O
1064 # 1: Contention/pre-emption
1065 # 2: Waiting on I/O
1066 # 3: Waiting on mutex
1067 # Hardcoding to 0 for now. Other states not implemented yet.
1068 state = 0
1069
1070 if args.verbose:
1071 print cpu, timestamp, pid, tid, cookie
1072
1073 writeBinary(blob,\
1074 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1075
1076# Writes selected gem5 statistics as Streamline counters
1077def writeCounters(blob, stats):
1078 timestamp_list = []
1079 for tick in stats.tick_list:
1080 if tick > end_tick:
1081 break
1082 timestamp_list.append(ticksToNs(tick))
1083
1084 for stat in stats.stats_list:
1085 if stat.per_cpu:
1086 stat_length = len(stat.values[0])
1087 else:
1088 stat_length = len(stat.values)
1089
1090 for n in range(len(timestamp_list)):
1091 for stat in stats.stats_list:
1092 if stat.per_cpu:
1093 for i in range(num_cpus):
1094 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1095 stat.key, int(float(stat.values[i][n]))))
1096 else:
1097 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1098 stat.key, int(float(stat.values[n]))))
1099
1100# Streamline can display LCD frame buffer dumps (gzipped bmp)
1101# This function converts the frame buffer dumps to the Streamline format
1102def writeVisualAnnotations(blob, input_path, output_path):
1103 frame_path = input_path + "/frames_system.vncserver"
1104 if not os.path.exists(frame_path):
1105 return
1106
1107 frame_count = 0
1108 file_list = os.listdir(frame_path)
1109 file_list.sort()
1110 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1111
1112 # Use first non-negative pid to tag visual annotations
1113 annotate_pid = -1
1114 for e in unified_event_list:
1115 pid = e.task.pid
1116 if pid >= 0:
1117 annotate_pid = pid
1118 break
1119
1120 for fn in file_list:
1121 m = re_fb.match(fn)
1122 if m:
1123 seq = m.group(1)
1124 tick = int(m.group(2))
1125 if tick > end_tick:
1126 break
1127 frame_count += 1
1128
1129 userspace_body = []
1130 userspace_body += packed32(0x1C) # escape code
1131 userspace_body += packed32(0x04) # visual code
1132
1133 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1134 userspace_body += int16(len(text_annotation))
1135 userspace_body += utf8StringList(text_annotation)
1136
1137 if gzipped_bmp_supported:
1138 # copy gzipped bmp directly
1139 bytes_read = open(frame_path + "/" + fn, "rb").read()
1140 else:
1141 # copy uncompressed bmp
1142 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1143
1144 userspace_body += int32(len(bytes_read))
1145 userspace_body += bytes_read
1146
1147 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1148 len(userspace_body), userspace_body))
1149
1150 print "\nfound", frame_count, "frames for visual annotation.\n"
1151
1152
1153def createApcProject(input_path, output_path, stats):
1154 initOutput(output_path)
1155
1156 blob = open(output_path + "/0000000000", "wb")
1157
1158 # Summary frame takes current system time and system uptime.
1159 # Filling in with random values for now.
1160 writeBinary(blob, summaryFrame(1234, 5678))
1161
1162 writeCookiesThreads(blob)
1163
1164 print "writing Events"
1165 writeSchedEvents(blob)
1166
1167 print "writing Counters"
1168 writeCounters(blob, stats)
1169
1170 print "writing Visual Annotations"
1171 writeVisualAnnotations(blob, input_path, output_path)
1172
1173 doSessionXML(output_path)
1174 doCapturedXML(output_path, stats)
1175
1176 blob.close()
1177
1178
1179
1180#######################
1181# Main Routine
1182
1183input_path = args.input_path
1184output_path = args.output_path
1185
1186####
1187# Make sure input path exists
1188####
1189if not os.path.exists(input_path):
1190 print "ERROR: Input path %s does not exist!" % input_path
1191 sys.exit(1)
1192
1193####
1194# Parse gem5 configuration file to find # of CPUs and L2s
1195####
1196(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1197
1198####
1199# Parse task file to find process/thread info
1200####
1201parseProcessInfo(input_path + "/system.tasks.txt")
1202
1203####
1204# Parse stat config file and register stats
1205####
1206stat_config_file = args.stat_config_file
1207stats = registerStats(stat_config_file)
1208
1209####
1210# Parse gem5 stats
1211####
1212# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1213if os.path.exists(input_path + "/stats.txt") and \
1214 os.path.exists(input_path + "/stats.txt.gz"):
1215 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1216 Using stats.txt.gz by default."
1217
1218gem5_stats_file = input_path + "/stats.txt.gz"
1219if not os.path.exists(gem5_stats_file):
1220 gem5_stats_file = input_path + "/stats.txt"
1221if not os.path.exists(gem5_stats_file):
1222 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1223 sys.exit(1)
1224
1225readGem5Stats(stats, gem5_stats_file)
1226
1227####
1228# Create Streamline .apc project folder
1229####
1230createApcProject(input_path, output_path, stats)
1231
1232print "All done!"