2
3# Copyright (c) 2012, 2014 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v17 (DS-5 v5.17)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
81 APC project generation based on Gator v17 (DS-5 v5.17)
82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2_cache"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2_cache" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
201def packed32(x):
202 ret = []
203 more = True
204 while more:
205 b = x & 0x7f
206 x = x >> 7
207 if (((x == 0) and ((b & 0x40) == 0)) or \
208 ((x == -1) and ((b & 0x40) != 0))):
209 more = False
210 else:
211 b = b | 0x80
212 ret.append(b)
213 return ret
214
215# For historical reasons, 32/64-bit versions of functions are presevered
216def packed64(x):
217 return packed32(x)
218
219# variable length packed 4-byte signed value
220def unsigned_packed32(x):
221 ret = []
222 if ((x & 0xffffff80) == 0):
223 ret.append(x & 0x7f)
224 elif ((x & 0xffffc000) == 0):
225 ret.append((x | 0x80) & 0xff)
226 ret.append((x >> 7) & 0x7f)
227 elif ((x & 0xffe00000) == 0):
228 ret.append((x | 0x80) & 0xff)
229 ret.append(((x >> 7) | 0x80) & 0xff)
230 ret.append((x >> 14) & 0x7f)
231 elif ((x & 0xf0000000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append(((x >> 7) | 0x80) & 0xff)
234 ret.append(((x >> 14) | 0x80) & 0xff)
235 ret.append((x >> 21) & 0x7f)
236 else:
237 ret.append((x | 0x80) & 0xff)
238 ret.append(((x >> 7) | 0x80) & 0xff)
239 ret.append(((x >> 14) | 0x80) & 0xff)
240 ret.append(((x >> 21) | 0x80) & 0xff)
241 ret.append((x >> 28) & 0x0f)
242 return ret
243
244# variable length packed 8-byte signed value
245def unsigned_packed64(x):
246 ret = []
247 if ((x & 0xffffffffffffff80) == 0):
248 ret.append(x & 0x7f)
249 elif ((x & 0xffffffffffffc000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append((x >> 7) & 0x7f)
252 elif ((x & 0xffffffffffe00000) == 0):
253 ret.append((x | 0x80) & 0xff)
254 ret.append(((x >> 7) | 0x80) & 0xff)
255 ret.append((x >> 14) & 0x7f)
256 elif ((x & 0xfffffffff0000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append((x >> 21) & 0x7f)
261 elif ((x & 0xfffffff800000000) == 0):
262 ret.append((x | 0x80) & 0xff)
263 ret.append(((x >> 7) | 0x80) & 0xff)
264 ret.append(((x >> 14) | 0x80) & 0xff)
265 ret.append(((x >> 21) | 0x80) & 0xff)
266 ret.append((x >> 28) & 0x7f)
267 elif ((x & 0xfffffc0000000000) == 0):
268 ret.append((x | 0x80) & 0xff)
269 ret.append(((x >> 7) | 0x80) & 0xff)
270 ret.append(((x >> 14) | 0x80) & 0xff)
271 ret.append(((x >> 21) | 0x80) & 0xff)
272 ret.append(((x >> 28) | 0x80) & 0xff)
273 ret.append((x >> 35) & 0x7f)
274 elif ((x & 0xfffe000000000000) == 0):
275 ret.append((x | 0x80) & 0xff)
276 ret.append(((x >> 7) | 0x80) & 0xff)
277 ret.append(((x >> 14) | 0x80) & 0xff)
278 ret.append(((x >> 21) | 0x80) & 0xff)
279 ret.append(((x >> 28) | 0x80) & 0xff)
280 ret.append(((x >> 35) | 0x80) & 0xff)
281 ret.append((x >> 42) & 0x7f)
282 elif ((x & 0xff00000000000000) == 0):
283 ret.append((x | 0x80) & 0xff)
284 ret.append(((x >> 7) | 0x80) & 0xff)
285 ret.append(((x >> 14) | 0x80) & 0xff)
286 ret.append(((x >> 21) | 0x80) & 0xff)
287 ret.append(((x >> 28) | 0x80) & 0xff)
288 ret.append(((x >> 35) | 0x80) & 0xff)
289 ret.append(((x >> 42) | 0x80) & 0xff)
290 ret.append((x >> 49) & 0x7f)
291 elif ((x & 0x8000000000000000) == 0):
292 ret.append((x | 0x80) & 0xff)
293 ret.append(((x >> 7) | 0x80) & 0xff)
294 ret.append(((x >> 14) | 0x80) & 0xff)
295 ret.append(((x >> 21) | 0x80) & 0xff)
296 ret.append(((x >> 28) | 0x80) & 0xff)
297 ret.append(((x >> 35) | 0x80) & 0xff)
298 ret.append(((x >> 42) | 0x80) & 0xff)
299 ret.append(((x >> 49) | 0x80) & 0xff)
300 ret.append((x >> 56) & 0x7f)
301 else:
302 ret.append((x | 0x80) & 0xff)
303 ret.append(((x >> 7) | 0x80) & 0xff)
304 ret.append(((x >> 14) | 0x80) & 0xff)
305 ret.append(((x >> 21) | 0x80) & 0xff)
306 ret.append(((x >> 28) | 0x80) & 0xff)
307 ret.append(((x >> 35) | 0x80) & 0xff)
308 ret.append(((x >> 42) | 0x80) & 0xff)
309 ret.append(((x >> 49) | 0x80) & 0xff)
310 ret.append(((x >> 56) | 0x80) & 0xff)
311 ret.append((x >> 63) & 0x7f)
312 return ret
313
314# 4-byte signed little endian
315def int32(x):
316 ret = []
317 ret.append(x & 0xff)
318 ret.append((x >> 8) & 0xff)
319 ret.append((x >> 16) & 0xff)
320 ret.append((x >> 24) & 0xff)
321 return ret
322
323# 2-byte signed little endian
324def int16(x):
325 ret = []
326 ret.append(x & 0xff)
327 ret.append((x >> 8) & 0xff)
328 return ret
329
330# a packed32 length followed by the specified number of characters
331def stringList(x):
332 ret = []
333 ret += packed32(len(x))
334 for i in x:
335 ret.append(i)
336 return ret
337
338def utf8StringList(x):
339 ret = []
340 for i in x:
341 ret.append(ord(i))
342 return ret
343
344# packed64 time value in nanoseconds relative to the uptime from the
345# Summary message.
346def timestampList(x):
347 ret = packed64(x)
348 return ret
349
350
351############################################################
352# Write binary
353############################################################
354
355def writeBinary(outfile, binary_list):
356 for i in binary_list:
357 outfile.write("%c" % i)
358
359############################################################
360# APC Protocol Frame Types
361############################################################
362
363def addFrameHeader(frame_type, body, core):
364 ret = []
365
366 if frame_type == "Summary":
367 code = 1
368 elif frame_type == "Backtrace":
369 code = 2
370 elif frame_type == "Name":
371 code = 3
372 elif frame_type == "Counter":
373 code = 4
374 elif frame_type == "Block Counter":
375 code = 5
376 elif frame_type == "Annotate":
377 code = 6
378 elif frame_type == "Sched Trace":
379 code = 7
380 elif frame_type == "GPU Trace":
381 code = 8
382 elif frame_type == "Idle":
383 code = 9
384 else:
385 print "ERROR: Unknown frame type:", frame_type
386 sys.exit(1)
387
388 packed_code = packed32(code)
389
390 packed_core = packed32(core)
391
392 length = int32(len(packed_code) + len(packed_core) + len(body))
393
394 ret = length + packed_code + packed_core + body
395 return ret
396
397
398# Summary frame
399# - timestamp: packed64
400# - uptime: packed64
401def summaryFrame(timestamp, uptime):
402 frame_type = "Summary"
403 newline_canary = stringList("1\n2\r\n3\r4\n\r5")
404 monotonic_delta = packed64(0)
405 end_of_attr = stringList("")
406 body = newline_canary + packed64(timestamp) + packed64(uptime)
407 body += monotonic_delta + end_of_attr
408 ret = addFrameHeader(frame_type, body, 0)
409 return ret
410
411# Backtrace frame
412# - not implemented yet
413def backtraceFrame():
414 pass
415
416# Cookie name message
417# - cookie: packed32
418# - name: string
419def cookieNameFrame(cookie, name):
420 frame_type = "Name"
421 packed_code = packed32(1)
422 body = packed_code + packed32(cookie) + stringList(name)
423 ret = addFrameHeader(frame_type, body, 0)
424 return ret
425
426# Thread name message
427# - timestamp: timestamp
428# - thread id: packed32
429# - name: string
430def threadNameFrame(timestamp, thread_id, name):
431 frame_type = "Name"
432 packed_code = packed32(2)
433 body = packed_code + timestampList(timestamp) + \
434 packed32(thread_id) + stringList(name)
435 ret = addFrameHeader(frame_type, body, 0)
436 return ret
437
438# Core name message
439# - name: string
440# - core_id: packed32
441# - cpuid: packed32
442def coreNameFrame(name, core_id, cpuid):
443 frame_type = "Name"
444 packed_code = packed32(3)
445 body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
446 ret = addFrameHeader(frame_type, body, 0)
447 return ret
448
449# IRQ Cookie name message
450# - cookie: packed32
451# - name: string
452# - irq: packed32
453def irqCookieNameFrame(cookie, name, irq):
454 frame_type = "Name"
455 packed_code = packed32(5)
456 body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
457 ret = addFrameHeader(frame_type, body, 0)
458 return ret
459
460# Counter frame message
461# - timestamp: timestamp
462# - core: packed32
463# - key: packed32
464# - value: packed64
465def counterFrame(timestamp, core, key, value):
466 frame_type = "Counter"
467 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
468 packed64(value)
469 ret = addFrameHeader(frame_type, body, core)
470 return ret
471
472# Block Counter frame message
473# - key: packed32
474# - value: packed64
475def blockCounterFrame(core, key, value):
476 frame_type = "Block Counter"
477 body = packed32(key) + packed64(value)
478 ret = addFrameHeader(frame_type, body, core)
479 return ret
480
481# Annotate frame messages
482# - core: packed32
483# - tid: packed32
484# - timestamp: timestamp
485# - size: packed32
486# - body
487def annotateFrame(core, tid, timestamp, size, userspace_body):
488 frame_type = "Annotate"
489 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
490 packed32(size) + userspace_body
491 ret = addFrameHeader(frame_type, body, core)
492 return ret
493
494# Scheduler Trace frame messages
495# Sched Switch
496# - Code: 1
497# - timestamp: timestamp
498# - pid: packed32
499# - tid: packed32
500# - cookie: packed32
501# - state: packed32
502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
503 frame_type = "Sched Trace"
504 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
505 packed32(tid) + packed32(cookie) + packed32(state)
506 ret = addFrameHeader(frame_type, body, core)
507 return ret
508
509# Sched Thread Exit
510# - Code: 2
511# - timestamp: timestamp
512# - tid: packed32
513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
514 frame_type = "Sched Trace"
515 body = packed32(2) + timestampList(timestamp) + packed32(tid)
516 ret = addFrameHeader(frame_type, body, core)
517 return ret
518
519# GPU Trace frame messages
520# - Not implemented yet
521def gpuTraceFrame():
522 pass
523
524# Idle frame messages
525# Enter Idle
526# - code: 1
527# - timestamp: timestamp
528# - core: packed32
529def enterIdleFrame(timestamp, core):
530 frame_type = "Idle"
531 body = packed32(1) + timestampList(timestamp) + packed32(core)
532 ret = addFrameHeader(frame_type, body, core)
533 return ret
534
535# Exit Idle
536# - code: 2
537# - timestamp: timestamp
538# - core: packed32
539def exitIdleFrame(timestamp, core):
540 frame_type = "Idle"
541 body = packed32(2) + timestampList(timestamp) + packed32(core)
542 ret = addFrameHeader(frame_type, body, core)
543 return ret
544
545
546####################################################################
547def parseProcessInfo(task_file):
548 print "\n==============================="
549 print "Parsing Task file..."
550 print task_file
551 print "===============================\n"
552
553 global start_tick, end_tick, num_cpus
554 global process_dict, thread_dict, process_list
555 global event_list, unified_event_list
556 global idle_uid, kernel_uid
557
558 event_list = []
559 unified_event_list = []
560 for cpu in range(num_cpus):
561 event_list.append([])
562
563 uid = 1 # uid 0 is reserved for idle
564
565 # Dummy Tasks for frame buffers and system diagrams
566 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
567 process_list.append(process)
568 uid += 1
569 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
570 process.children.append(thread)
571 uid += 1
572 process = Task(uid, 9998, 9998, "System", True, 0)
573 process_list.append(process)
574 # if we don't find the real kernel, use this to keep things going
575 kernel_uid = uid
576 uid += 1
577 thread = Task(uid, 9998, 9998, "System", False, 0)
578 process.children.append(thread)
579 uid += 1
580
581 ext = os.path.splitext(task_file)[1]
582
583 try:
584 if ext == ".gz":
585 process_file = gzip.open(task_file, 'rb')
586 else:
587 process_file = open(task_file, 'rb')
588 except:
589 print "ERROR opening task file:", task_file
590 print "Make sure context switch task dumping is enabled in gem5."
591 sys.exit(1)
592
593 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
594 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
595
596 task_name_failure_warned = False
597
598 for line in process_file:
599 match = re.match(process_re, line)
600 if match:
601 tick = int(match.group(1))
602 if (start_tick < 0):
603 start_tick = tick
604 cpu_id = int(match.group(3))
605 pid = int(match.group(4))
606 tgid = int(match.group(5))
607 task_name = match.group(6)
608
609 if not task_name_failure_warned:
610 if task_name == "FailureIn_curTaskName":
611 print "-------------------------------------------------"
612 print "WARNING: Task name not set correctly!"
613 print "Process/Thread info will not be displayed correctly"
614 print "Perhaps forgot to apply m5struct.patch to kernel?"
615 print "-------------------------------------------------"
616 task_name_failure_warned = True
617
618 if not tgid in process_dict:
619 if tgid == pid:
620 # new task is parent as well
621 if args.verbose:
622 print "new process", uid, pid, tgid, task_name
623 if tgid == 0:
624 # new process is the "idle" task
625 process = Task(uid, pid, tgid, "idle", True, tick)
626 idle_uid = 0
627 else:
628 process = Task(uid, pid, tgid, task_name, True, tick)
629 else:
630 if tgid == 0:
631 process = Task(uid, tgid, tgid, "idle", True, tick)
632 idle_uid = 0
633 else:
634 # parent process name not known yet
635 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
636 if tgid == -1: # kernel
637 kernel_uid = 0
638 uid += 1
639 process_dict[tgid] = process
640 process_list.append(process)
641 else:
642 if tgid == pid:
643 if process_dict[tgid].task_name == "_Unknown_":
644 if args.verbose:
645 print "new process", \
646 process_dict[tgid].uid, pid, tgid, task_name
647 process_dict[tgid].task_name = task_name
648 if process_dict[tgid].task_name != task_name and tgid != 0:
649 process_dict[tgid].task_name = task_name
650
651 if not pid in thread_dict:
652 if args.verbose:
653 print "new thread", \
654 uid, process_dict[tgid].uid, pid, tgid, task_name
655 thread = Task(uid, pid, tgid, task_name, False, tick)
656 uid += 1
657 thread_dict[pid] = thread
658 process_dict[tgid].children.append(thread)
659 else:
660 if thread_dict[pid].task_name != task_name:
661 thread_dict[pid].task_name = task_name
662
663 if args.verbose:
664 print tick, uid, cpu_id, pid, tgid, task_name
665
666 task = thread_dict[pid]
667 event = Event(tick, task)
668 event_list[cpu_id].append(event)
669 unified_event_list.append(event)
670
671 if len(unified_event_list) == num_events:
672 print "Truncating at", num_events, "events!"
673 break
674 print "Found %d events." % len(unified_event_list)
675
676 for process in process_list:
677 if process.pid > 9990: # fix up framebuffer ticks
678 process.tick = start_tick
679 print process.uid, process.pid, process.tgid, \
680 process.task_name, str(process.tick)
681 for thread in process.children:
682 if thread.pid > 9990:
683 thread.tick = start_tick
684 print "\t", thread.uid, thread.pid, thread.tgid, \
685 thread.task_name, str(thread.tick)
686
687 end_tick = tick
688
689 print "Start tick:", start_tick
690 print "End tick: ", end_tick
691 print ""
692
693 return
694
695
696def initOutput(output_path):
697 if not os.path.exists(output_path):
698 os.mkdir(output_path)
699
700def ticksToNs(tick):
701 if ticks_in_ns < 0:
702 print "ticks_in_ns not set properly!"
703 sys.exit(1)
704
705 return tick / ticks_in_ns
706
707def writeXmlFile(xml, filename):
708 f = open(filename, "w")
709 txt = ET.tostring(xml)
710 f.write(minidom.parseString(txt).toprettyxml())
711 f.close()
712
713
714# StatsEntry that contains individual statistics
715class StatsEntry(object):
716 def __init__(self, name, group, group_index, per_cpu, key):
717
718 # Full name of statistics
719 self.name = name
720
721 # Streamline group name that statistic will belong to
722 self.group = group
723
724 # Index of statistics within group (used to change colors within groups)
725 self.group_index = group_index
726
727 # Shorter name with "system" stripped off
728 # and symbols converted to alphanumerics
729 self.short_name = re.sub("system\.", "", name)
730 self.short_name = re.sub(":", "_", name)
731
732 # Regex for this stat (string version used to construct union regex)
733 self.regex_string = "^" + name + "\s+([\d\.]+)"
734 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
735 self.description = ""
736
737 # Whether this stat is use per CPU or not
738 self.per_cpu = per_cpu
739
740 # Key used in .apc protocol (as described in captured.xml)
741 self.key = key
742
743 # List of values of stat per timestamp
744 self.values = []
745
746 # Whether this stat has been found for the current timestamp
747 self.found = False
748
749 # Whether this stat has been found at least once
750 # (to suppress too many warnings)
751 self.not_found_at_least_once = False
752
753 # Field used to hold ElementTree subelement for this stat
754 self.ET_element = None
755
756 # Create per-CPU stat name and regex, etc.
757 if self.per_cpu:
758 self.per_cpu_regex_string = []
759 self.per_cpu_regex = []
760 self.per_cpu_name = []
761 self.per_cpu_found = []
762 for i in range(num_cpus):
763 if num_cpus > 1:
764 per_cpu_name = re.sub("#", str(i), self.name)
765 else:
766 per_cpu_name = re.sub("#", "", self.name)
767
768 self.per_cpu_name.append(per_cpu_name)
769 print "\t", per_cpu_name
770
771 self.per_cpu_regex_string.\
772 append("^" + per_cpu_name + "\s+[\d\.]+")
773 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
774 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
775 self.values.append([])
776 self.per_cpu_found.append(False)
777
778 def append_value(self, val, per_cpu_index = None):
779 if self.per_cpu:
780 self.values[per_cpu_index].append(str(val))
781 else:
782 self.values.append(str(val))
783
784# Global stats object that contains the list of stats entries
785# and other utility functions
786class Stats(object):
787 def __init__(self):
788 self.stats_list = []
789 self.tick_list = []
790 self.next_key = 1
791
792 def register(self, name, group, group_index, per_cpu):
793 print "registering stat:", name, "group:", group, group_index
794 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
795 self.next_key))
796 self.next_key += 1
797
798 # Union of all stats to accelerate parsing speed
799 def createStatsRegex(self):
800 regex_strings = [];
801 print "\nnum entries in stats_list", len(self.stats_list)
802 for entry in self.stats_list:
803 if entry.per_cpu:
804 for i in range(num_cpus):
805 regex_strings.append(entry.per_cpu_regex_string[i])
806 else:
807 regex_strings.append(entry.regex_string)
808
809 self.regex = re.compile('|'.join(regex_strings))
810
811
812def registerStats(config_file):
813 print "==============================="
814 print "Parsing stats config.ini file..."
815 print config_file
816 print "==============================="
817
818 config = ConfigParser()
819 if not config.read(config_file):
820 print "ERROR: config file '", config_file, "' not found!"
821 sys.exit(1)
822
823 print "\nRegistering Stats..."
824
825 stats = Stats()
826
827 per_cpu_stat_groups = config.options('PER_CPU_STATS')
828 for group in per_cpu_stat_groups:
829 i = 0
830 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
831 for item in per_cpu_stats_list:
832 if item:
833 stats.register(item, group, i, True)
834 i += 1
835
836 per_l2_stat_groups = config.options('PER_L2_STATS')
837 for group in per_l2_stat_groups:
838 i = 0
839 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
840 for item in per_l2_stats_list:
841 if item:
842 for l2 in range(num_l2):
843 if num_l2 > 1:
844 name = re.sub("#", str(l2), item)
845 else:
846 name = re.sub("#", "", item)
847 stats.register(name, group, i, False)
848 i += 1
849
850 other_stat_groups = config.options('OTHER_STATS')
851 for group in other_stat_groups:
852 i = 0
853 other_stats_list = config.get('OTHER_STATS', group).split('\n')
854 for item in other_stats_list:
855 if item:
856 stats.register(item, group, i, False)
857 i += 1
858
859 stats.createStatsRegex()
860
861 return stats
862
863# Parse and read in gem5 stats file
864# Streamline counters are organized per CPU
865def readGem5Stats(stats, gem5_stats_file):
866 print "\n==============================="
867 print "Parsing gem5 stats file..."
868 print gem5_stats_file
869 print "===============================\n"
870 ext = os.path.splitext(gem5_stats_file)[1]
871
872 window_start_regex = \
873 re.compile("^---------- Begin Simulation Statistics ----------")
874 window_end_regex = \
875 re.compile("^---------- End Simulation Statistics ----------")
876 final_tick_regex = re.compile("^final_tick\s+(\d+)")
877
878 global ticks_in_ns
879 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
880 sim_freq = -1
881
882 try:
883 if ext == ".gz":
884 f = gzip.open(gem5_stats_file, "r")
885 else:
886 f = open(gem5_stats_file, "r")
887 except:
888 print "ERROR opening stats file", gem5_stats_file, "!"
889 sys.exit(1)
890
891 stats_not_found_list = stats.stats_list[:]
892 window_num = 0
893
894 while (True):
895 error = False
896 try:
897 line = f.readline()
898 except IOError:
899 print ""
900 print "WARNING: IO error in stats file"
901 print "(gzip stream not closed properly?)...continuing for now"
902 error = True
903 if not line:
904 break
905
906 # Find out how many gem5 ticks in 1ns
907 if sim_freq < 0:
908 m = sim_freq_regex.match(line)
909 if m:
910 sim_freq = int(m.group(1)) # ticks in 1 sec
911 ticks_in_ns = int(sim_freq / 1e9)
912 print "Simulation frequency found! 1 tick == %e sec\n" \
913 % (1.0 / sim_freq)
914
915 # Final tick in gem5 stats: current absolute timestamp
916 m = final_tick_regex.match(line)
917 if m:
918 tick = int(m.group(1))
919 if tick > end_tick:
920 break
921 stats.tick_list.append(tick)
922
923
924 if (window_end_regex.match(line) or error):
925 if args.verbose:
926 print "new window"
927 for stat in stats.stats_list:
928 if stat.per_cpu:
929 for i in range(num_cpus):
930 if not stat.per_cpu_found[i]:
931 if not stat.not_found_at_least_once:
932 print "WARNING: stat not found in window #", \
933 window_num, ":", stat.per_cpu_name[i]
934 print "suppressing further warnings for " + \
935 "this stat"
936 stat.not_found_at_least_once = True
937 stat.values[i].append(str(0))
938 stat.per_cpu_found[i] = False
939 else:
940 if not stat.found:
941 if not stat.not_found_at_least_once:
942 print "WARNING: stat not found in window #", \
943 window_num, ":", stat.name
944 print "suppressing further warnings for this stat"
945 stat.not_found_at_least_once = True
946 stat.values.append(str(0))
947 stat.found = False
948 stats_not_found_list = stats.stats_list[:]
949 window_num += 1
950 if error:
951 break
952
953 # Do a single regex of the union of all stats first for speed
954 if stats.regex.match(line):
955 # Then loop through only the stats we haven't seen in this window
956 for stat in stats_not_found_list[:]:
957 if stat.per_cpu:
958 for i in range(num_cpus):
959 m = stat.per_cpu_regex[i].match(line)
960 if m:
961 if stat.name == "ipc":
962 value = str(int(float(m.group(1)) * 1000))
963 else:
964 value = str(int(float(m.group(1))))
965 if args.verbose:
966 print stat.per_cpu_name[i], value
967 stat.values[i].append(value)
968 stat.per_cpu_found[i] = True
969 all_found = True
970 for j in range(num_cpus):
971 if not stat.per_cpu_found[j]:
972 all_found = False
973 if all_found:
974 stats_not_found_list.remove(stat)
975 if stat.description == "":
976 stat.description = m.group(2)
977 else:
978 m = stat.regex.match(line)
979 if m:
980 value = str(int(float(m.group(1))))
981 if args.verbose:
982 print stat.name, value
983 stat.values.append(value)
984 stat.found = True
985 stats_not_found_list.remove(stat)
986 if stat.description == "":
987 stat.description = m.group(2)
988 f.close()
989
990
991# Create session.xml file in .apc folder
992def doSessionXML(output_path):
993 session_file = output_path + "/session.xml"
994
995 xml = ET.Element("session")
996
997 xml.set("version", "1")
998 xml.set("call_stack_unwinding", "no")
999 xml.set("parse_debug_info", "no")
1000 xml.set("high_resolution", "yes")
1001 xml.set("buffer_mode", "streaming")
1002 xml.set("sample_rate", "low")
1003
1004 # Setting duration to zero for now. Doesn't affect visualization.
1005 xml.set("duration", "0")
1006
1007 xml.set("target_host", "")
1008 xml.set("target_port", "8080")
1009
1010 writeXmlFile(xml, session_file)
1011
1012
1013# Create captured.xml file in .apc folder
1014def doCapturedXML(output_path, stats):
1015 captured_file = output_path + "/captured.xml"
1016
1017 xml = ET.Element("captured")
1018 xml.set("version", "1")
1019 xml.set("protocol", "17")
1020 xml.set("backtrace_processing", "none")
1021
1022 target = ET.SubElement(xml, "target")
1023 target.set("name", "gem5")
1024 target.set("sample_rate", "1000")
1025 target.set("cores", str(num_cpus))
1026
1027 counters = ET.SubElement(xml, "counters")
1028 for stat in stats.stats_list:
1029 s = ET.SubElement(counters, "counter")
1030 stat_name = re.sub("\.", "_", stat.short_name)
1031 stat_name = re.sub("#", "", stat_name)
1032 s.set("title", stat.group)
1033 s.set("name", stat_name)
1034 s.set("color", "0x00000000")
1035 s.set("key", "0x%08x" % stat.key)
1036 s.set("type", stat_name)
1037 s.set("event", "0x00000000")
1038 if stat.per_cpu:
1039 s.set("per_cpu", "yes")
1040 else:
1041 s.set("per_cpu", "no")
1042 s.set("display", "")
1043 s.set("units", "")
1044 s.set("average_selection", "no")
1045 s.set("description", stat.description)
1046
1047 writeXmlFile(xml, captured_file)
1048
1049# Writes out Streamline cookies (unique IDs per process/thread)
1050def writeCookiesThreads(blob):
1051 thread_list = []
1052 for process in process_list:
1053 if process.uid > 0:
1054 print "cookie", process.task_name, process.uid
1055 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1056
1057 # pid and tgid need to be positive values -- no longer true?
1058 for thread in process.children:
1059 thread_list.append(thread)
1060
1061 # Threads need to be sorted in timestamp order
1062 thread_list.sort(key = lambda x: x.tick)
1063 for thread in thread_list:
1064 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1065 thread.tgid, thread.pid
1066 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1067 thread.pid, thread.task_name))
1068
1069# Writes context switch info as Streamline scheduling events
1070def writeSchedEvents(blob):
1071 for cpu in range(num_cpus):
1072 for event in event_list[cpu]:
1073 timestamp = ticksToNs(event.tick)
1074 pid = event.task.tgid
1075 tid = event.task.pid
1076 if process_dict.has_key(event.task.tgid):
1077 cookie = process_dict[event.task.tgid].uid
1078 else:
1079 cookie = 0
1080
1081 # State:
1082 # 0: waiting on other event besides I/O
1083 # 1: Contention/pre-emption
1084 # 2: Waiting on I/O
1085 # 3: Waiting on mutex
1086 # Hardcoding to 0 for now. Other states not implemented yet.
1087 state = 0
1088
1089 if args.verbose:
1090 print cpu, timestamp, pid, tid, cookie
1091
1092 writeBinary(blob,\
1093 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1094
1095# Writes selected gem5 statistics as Streamline counters
1096def writeCounters(blob, stats):
1097 timestamp_list = []
1098 for tick in stats.tick_list:
1099 if tick > end_tick:
1100 break
1101 timestamp_list.append(ticksToNs(tick))
1102
1103 for stat in stats.stats_list:
1104 if stat.per_cpu:
1105 stat_length = len(stat.values[0])
1106 else:
1107 stat_length = len(stat.values)
1108
1109 for n in range(len(timestamp_list)):
1110 for stat in stats.stats_list:
1111 if stat.per_cpu:
1112 for i in range(num_cpus):
1113 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1114 stat.key, int(float(stat.values[i][n]))))
1115 else:
1116 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1117 stat.key, int(float(stat.values[n]))))
1118
1119# Streamline can display LCD frame buffer dumps (gzipped bmp)
1120# This function converts the frame buffer dumps to the Streamline format
1121def writeVisualAnnotations(blob, input_path, output_path):
1122 frame_path = input_path + "/frames_system.vncserver"
1123 if not os.path.exists(frame_path):
1124 return
1125
1126 frame_count = 0
1127 file_list = os.listdir(frame_path)
1128 file_list.sort()
1129 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1130
1131 # Use first non-negative pid to tag visual annotations
1132 annotate_pid = -1
1133 for e in unified_event_list:
1134 pid = e.task.pid
1135 if pid >= 0:
1136 annotate_pid = pid
1137 break
1138
1139 for fn in file_list:
1140 m = re_fb.match(fn)
1141 if m:
1142 seq = m.group(1)
1143 tick = int(m.group(2))
1144 if tick > end_tick:
1145 break
1146 frame_count += 1
1147
1148 userspace_body = []
1149 userspace_body += packed32(0x1C) # escape code
1150 userspace_body += packed32(0x04) # visual code
1151
1152 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1153 userspace_body += int16(len(text_annotation))
1154 userspace_body += utf8StringList(text_annotation)
1155
1156 if gzipped_bmp_supported:
1157 # copy gzipped bmp directly
1158 bytes_read = open(frame_path + "/" + fn, "rb").read()
1159 else:
1160 # copy uncompressed bmp
1161 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1162
1163 userspace_body += int32(len(bytes_read))
1164 userspace_body += bytes_read
1165
1166 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1167 len(userspace_body), userspace_body))
1168
1169 print "\nfound", frame_count, "frames for visual annotation.\n"
1170
1171
1172def createApcProject(input_path, output_path, stats):
1173 initOutput(output_path)
1174
1175 blob = open(output_path + "/0000000000", "wb")
1176
1177 # Summary frame takes current system time and system uptime.
1178 # Filling in with random values for now.
1179 writeBinary(blob, summaryFrame(1234, 5678))
1180
1181 writeCookiesThreads(blob)
1182
1183 print "writing Events"
1184 writeSchedEvents(blob)
1185
1186 print "writing Counters"
1187 writeCounters(blob, stats)
1188
1189 print "writing Visual Annotations"
1190 writeVisualAnnotations(blob, input_path, output_path)
1191
1192 doSessionXML(output_path)
1193 doCapturedXML(output_path, stats)
1194
1195 blob.close()
1196
1197
1198
1199#######################
1200# Main Routine
1201
1202input_path = args.input_path
1203output_path = args.output_path
1204
1205####
1206# Make sure input path exists
1207####
1208if not os.path.exists(input_path):
1209 print "ERROR: Input path %s does not exist!" % input_path
1210 sys.exit(1)
1211
1212####
1213# Parse gem5 configuration file to find # of CPUs and L2s
1214####
1215(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1216
1217####
1218# Parse task file to find process/thread info
1219####
1220parseProcessInfo(input_path + "/system.tasks.txt")
1221
1222####
1223# Parse stat config file and register stats
1224####
1225stat_config_file = args.stat_config_file
1226stats = registerStats(stat_config_file)
1227
1228####
1229# Parse gem5 stats
1230####
1231# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1232if os.path.exists(input_path + "/stats.txt") and \
1233 os.path.exists(input_path + "/stats.txt.gz"):
1234 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1235 Using stats.txt.gz by default."
1236
1237gem5_stats_file = input_path + "/stats.txt.gz"
1238if not os.path.exists(gem5_stats_file):
1239 gem5_stats_file = input_path + "/stats.txt"
1240if not os.path.exists(gem5_stats_file):
1241 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1242 sys.exit(1)
1243
1244readGem5Stats(stats, gem5_stats_file)
1245
1246####
1247# Create Streamline .apc project folder
1248####
1249createApcProject(input_path, output_path, stats)
1250
1251print "All done!"
| 2
3# Copyright (c) 2012, 2014 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
59# APC project generation based on Gator v17 (DS-5 v5.17)
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
81 APC project generation based on Gator v17 (DS-5 v5.17)
82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2_cache"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2_cache" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
201def packed32(x):
202 ret = []
203 more = True
204 while more:
205 b = x & 0x7f
206 x = x >> 7
207 if (((x == 0) and ((b & 0x40) == 0)) or \
208 ((x == -1) and ((b & 0x40) != 0))):
209 more = False
210 else:
211 b = b | 0x80
212 ret.append(b)
213 return ret
214
215# For historical reasons, 32/64-bit versions of functions are presevered
216def packed64(x):
217 return packed32(x)
218
219# variable length packed 4-byte signed value
220def unsigned_packed32(x):
221 ret = []
222 if ((x & 0xffffff80) == 0):
223 ret.append(x & 0x7f)
224 elif ((x & 0xffffc000) == 0):
225 ret.append((x | 0x80) & 0xff)
226 ret.append((x >> 7) & 0x7f)
227 elif ((x & 0xffe00000) == 0):
228 ret.append((x | 0x80) & 0xff)
229 ret.append(((x >> 7) | 0x80) & 0xff)
230 ret.append((x >> 14) & 0x7f)
231 elif ((x & 0xf0000000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append(((x >> 7) | 0x80) & 0xff)
234 ret.append(((x >> 14) | 0x80) & 0xff)
235 ret.append((x >> 21) & 0x7f)
236 else:
237 ret.append((x | 0x80) & 0xff)
238 ret.append(((x >> 7) | 0x80) & 0xff)
239 ret.append(((x >> 14) | 0x80) & 0xff)
240 ret.append(((x >> 21) | 0x80) & 0xff)
241 ret.append((x >> 28) & 0x0f)
242 return ret
243
244# variable length packed 8-byte signed value
245def unsigned_packed64(x):
246 ret = []
247 if ((x & 0xffffffffffffff80) == 0):
248 ret.append(x & 0x7f)
249 elif ((x & 0xffffffffffffc000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append((x >> 7) & 0x7f)
252 elif ((x & 0xffffffffffe00000) == 0):
253 ret.append((x | 0x80) & 0xff)
254 ret.append(((x >> 7) | 0x80) & 0xff)
255 ret.append((x >> 14) & 0x7f)
256 elif ((x & 0xfffffffff0000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append((x >> 21) & 0x7f)
261 elif ((x & 0xfffffff800000000) == 0):
262 ret.append((x | 0x80) & 0xff)
263 ret.append(((x >> 7) | 0x80) & 0xff)
264 ret.append(((x >> 14) | 0x80) & 0xff)
265 ret.append(((x >> 21) | 0x80) & 0xff)
266 ret.append((x >> 28) & 0x7f)
267 elif ((x & 0xfffffc0000000000) == 0):
268 ret.append((x | 0x80) & 0xff)
269 ret.append(((x >> 7) | 0x80) & 0xff)
270 ret.append(((x >> 14) | 0x80) & 0xff)
271 ret.append(((x >> 21) | 0x80) & 0xff)
272 ret.append(((x >> 28) | 0x80) & 0xff)
273 ret.append((x >> 35) & 0x7f)
274 elif ((x & 0xfffe000000000000) == 0):
275 ret.append((x | 0x80) & 0xff)
276 ret.append(((x >> 7) | 0x80) & 0xff)
277 ret.append(((x >> 14) | 0x80) & 0xff)
278 ret.append(((x >> 21) | 0x80) & 0xff)
279 ret.append(((x >> 28) | 0x80) & 0xff)
280 ret.append(((x >> 35) | 0x80) & 0xff)
281 ret.append((x >> 42) & 0x7f)
282 elif ((x & 0xff00000000000000) == 0):
283 ret.append((x | 0x80) & 0xff)
284 ret.append(((x >> 7) | 0x80) & 0xff)
285 ret.append(((x >> 14) | 0x80) & 0xff)
286 ret.append(((x >> 21) | 0x80) & 0xff)
287 ret.append(((x >> 28) | 0x80) & 0xff)
288 ret.append(((x >> 35) | 0x80) & 0xff)
289 ret.append(((x >> 42) | 0x80) & 0xff)
290 ret.append((x >> 49) & 0x7f)
291 elif ((x & 0x8000000000000000) == 0):
292 ret.append((x | 0x80) & 0xff)
293 ret.append(((x >> 7) | 0x80) & 0xff)
294 ret.append(((x >> 14) | 0x80) & 0xff)
295 ret.append(((x >> 21) | 0x80) & 0xff)
296 ret.append(((x >> 28) | 0x80) & 0xff)
297 ret.append(((x >> 35) | 0x80) & 0xff)
298 ret.append(((x >> 42) | 0x80) & 0xff)
299 ret.append(((x >> 49) | 0x80) & 0xff)
300 ret.append((x >> 56) & 0x7f)
301 else:
302 ret.append((x | 0x80) & 0xff)
303 ret.append(((x >> 7) | 0x80) & 0xff)
304 ret.append(((x >> 14) | 0x80) & 0xff)
305 ret.append(((x >> 21) | 0x80) & 0xff)
306 ret.append(((x >> 28) | 0x80) & 0xff)
307 ret.append(((x >> 35) | 0x80) & 0xff)
308 ret.append(((x >> 42) | 0x80) & 0xff)
309 ret.append(((x >> 49) | 0x80) & 0xff)
310 ret.append(((x >> 56) | 0x80) & 0xff)
311 ret.append((x >> 63) & 0x7f)
312 return ret
313
314# 4-byte signed little endian
315def int32(x):
316 ret = []
317 ret.append(x & 0xff)
318 ret.append((x >> 8) & 0xff)
319 ret.append((x >> 16) & 0xff)
320 ret.append((x >> 24) & 0xff)
321 return ret
322
323# 2-byte signed little endian
324def int16(x):
325 ret = []
326 ret.append(x & 0xff)
327 ret.append((x >> 8) & 0xff)
328 return ret
329
330# a packed32 length followed by the specified number of characters
331def stringList(x):
332 ret = []
333 ret += packed32(len(x))
334 for i in x:
335 ret.append(i)
336 return ret
337
338def utf8StringList(x):
339 ret = []
340 for i in x:
341 ret.append(ord(i))
342 return ret
343
344# packed64 time value in nanoseconds relative to the uptime from the
345# Summary message.
346def timestampList(x):
347 ret = packed64(x)
348 return ret
349
350
351############################################################
352# Write binary
353############################################################
354
355def writeBinary(outfile, binary_list):
356 for i in binary_list:
357 outfile.write("%c" % i)
358
359############################################################
360# APC Protocol Frame Types
361############################################################
362
363def addFrameHeader(frame_type, body, core):
364 ret = []
365
366 if frame_type == "Summary":
367 code = 1
368 elif frame_type == "Backtrace":
369 code = 2
370 elif frame_type == "Name":
371 code = 3
372 elif frame_type == "Counter":
373 code = 4
374 elif frame_type == "Block Counter":
375 code = 5
376 elif frame_type == "Annotate":
377 code = 6
378 elif frame_type == "Sched Trace":
379 code = 7
380 elif frame_type == "GPU Trace":
381 code = 8
382 elif frame_type == "Idle":
383 code = 9
384 else:
385 print "ERROR: Unknown frame type:", frame_type
386 sys.exit(1)
387
388 packed_code = packed32(code)
389
390 packed_core = packed32(core)
391
392 length = int32(len(packed_code) + len(packed_core) + len(body))
393
394 ret = length + packed_code + packed_core + body
395 return ret
396
397
398# Summary frame
399# - timestamp: packed64
400# - uptime: packed64
401def summaryFrame(timestamp, uptime):
402 frame_type = "Summary"
403 newline_canary = stringList("1\n2\r\n3\r4\n\r5")
404 monotonic_delta = packed64(0)
405 end_of_attr = stringList("")
406 body = newline_canary + packed64(timestamp) + packed64(uptime)
407 body += monotonic_delta + end_of_attr
408 ret = addFrameHeader(frame_type, body, 0)
409 return ret
410
411# Backtrace frame
412# - not implemented yet
413def backtraceFrame():
414 pass
415
416# Cookie name message
417# - cookie: packed32
418# - name: string
419def cookieNameFrame(cookie, name):
420 frame_type = "Name"
421 packed_code = packed32(1)
422 body = packed_code + packed32(cookie) + stringList(name)
423 ret = addFrameHeader(frame_type, body, 0)
424 return ret
425
426# Thread name message
427# - timestamp: timestamp
428# - thread id: packed32
429# - name: string
430def threadNameFrame(timestamp, thread_id, name):
431 frame_type = "Name"
432 packed_code = packed32(2)
433 body = packed_code + timestampList(timestamp) + \
434 packed32(thread_id) + stringList(name)
435 ret = addFrameHeader(frame_type, body, 0)
436 return ret
437
438# Core name message
439# - name: string
440# - core_id: packed32
441# - cpuid: packed32
442def coreNameFrame(name, core_id, cpuid):
443 frame_type = "Name"
444 packed_code = packed32(3)
445 body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
446 ret = addFrameHeader(frame_type, body, 0)
447 return ret
448
449# IRQ Cookie name message
450# - cookie: packed32
451# - name: string
452# - irq: packed32
453def irqCookieNameFrame(cookie, name, irq):
454 frame_type = "Name"
455 packed_code = packed32(5)
456 body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
457 ret = addFrameHeader(frame_type, body, 0)
458 return ret
459
460# Counter frame message
461# - timestamp: timestamp
462# - core: packed32
463# - key: packed32
464# - value: packed64
465def counterFrame(timestamp, core, key, value):
466 frame_type = "Counter"
467 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
468 packed64(value)
469 ret = addFrameHeader(frame_type, body, core)
470 return ret
471
472# Block Counter frame message
473# - key: packed32
474# - value: packed64
475def blockCounterFrame(core, key, value):
476 frame_type = "Block Counter"
477 body = packed32(key) + packed64(value)
478 ret = addFrameHeader(frame_type, body, core)
479 return ret
480
481# Annotate frame messages
482# - core: packed32
483# - tid: packed32
484# - timestamp: timestamp
485# - size: packed32
486# - body
487def annotateFrame(core, tid, timestamp, size, userspace_body):
488 frame_type = "Annotate"
489 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
490 packed32(size) + userspace_body
491 ret = addFrameHeader(frame_type, body, core)
492 return ret
493
494# Scheduler Trace frame messages
495# Sched Switch
496# - Code: 1
497# - timestamp: timestamp
498# - pid: packed32
499# - tid: packed32
500# - cookie: packed32
501# - state: packed32
502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
503 frame_type = "Sched Trace"
504 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
505 packed32(tid) + packed32(cookie) + packed32(state)
506 ret = addFrameHeader(frame_type, body, core)
507 return ret
508
509# Sched Thread Exit
510# - Code: 2
511# - timestamp: timestamp
512# - tid: packed32
513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
514 frame_type = "Sched Trace"
515 body = packed32(2) + timestampList(timestamp) + packed32(tid)
516 ret = addFrameHeader(frame_type, body, core)
517 return ret
518
519# GPU Trace frame messages
520# - Not implemented yet
521def gpuTraceFrame():
522 pass
523
524# Idle frame messages
525# Enter Idle
526# - code: 1
527# - timestamp: timestamp
528# - core: packed32
529def enterIdleFrame(timestamp, core):
530 frame_type = "Idle"
531 body = packed32(1) + timestampList(timestamp) + packed32(core)
532 ret = addFrameHeader(frame_type, body, core)
533 return ret
534
535# Exit Idle
536# - code: 2
537# - timestamp: timestamp
538# - core: packed32
539def exitIdleFrame(timestamp, core):
540 frame_type = "Idle"
541 body = packed32(2) + timestampList(timestamp) + packed32(core)
542 ret = addFrameHeader(frame_type, body, core)
543 return ret
544
545
546####################################################################
547def parseProcessInfo(task_file):
548 print "\n==============================="
549 print "Parsing Task file..."
550 print task_file
551 print "===============================\n"
552
553 global start_tick, end_tick, num_cpus
554 global process_dict, thread_dict, process_list
555 global event_list, unified_event_list
556 global idle_uid, kernel_uid
557
558 event_list = []
559 unified_event_list = []
560 for cpu in range(num_cpus):
561 event_list.append([])
562
563 uid = 1 # uid 0 is reserved for idle
564
565 # Dummy Tasks for frame buffers and system diagrams
566 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
567 process_list.append(process)
568 uid += 1
569 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
570 process.children.append(thread)
571 uid += 1
572 process = Task(uid, 9998, 9998, "System", True, 0)
573 process_list.append(process)
574 # if we don't find the real kernel, use this to keep things going
575 kernel_uid = uid
576 uid += 1
577 thread = Task(uid, 9998, 9998, "System", False, 0)
578 process.children.append(thread)
579 uid += 1
580
581 ext = os.path.splitext(task_file)[1]
582
583 try:
584 if ext == ".gz":
585 process_file = gzip.open(task_file, 'rb')
586 else:
587 process_file = open(task_file, 'rb')
588 except:
589 print "ERROR opening task file:", task_file
590 print "Make sure context switch task dumping is enabled in gem5."
591 sys.exit(1)
592
593 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
594 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
595
596 task_name_failure_warned = False
597
598 for line in process_file:
599 match = re.match(process_re, line)
600 if match:
601 tick = int(match.group(1))
602 if (start_tick < 0):
603 start_tick = tick
604 cpu_id = int(match.group(3))
605 pid = int(match.group(4))
606 tgid = int(match.group(5))
607 task_name = match.group(6)
608
609 if not task_name_failure_warned:
610 if task_name == "FailureIn_curTaskName":
611 print "-------------------------------------------------"
612 print "WARNING: Task name not set correctly!"
613 print "Process/Thread info will not be displayed correctly"
614 print "Perhaps forgot to apply m5struct.patch to kernel?"
615 print "-------------------------------------------------"
616 task_name_failure_warned = True
617
618 if not tgid in process_dict:
619 if tgid == pid:
620 # new task is parent as well
621 if args.verbose:
622 print "new process", uid, pid, tgid, task_name
623 if tgid == 0:
624 # new process is the "idle" task
625 process = Task(uid, pid, tgid, "idle", True, tick)
626 idle_uid = 0
627 else:
628 process = Task(uid, pid, tgid, task_name, True, tick)
629 else:
630 if tgid == 0:
631 process = Task(uid, tgid, tgid, "idle", True, tick)
632 idle_uid = 0
633 else:
634 # parent process name not known yet
635 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
636 if tgid == -1: # kernel
637 kernel_uid = 0
638 uid += 1
639 process_dict[tgid] = process
640 process_list.append(process)
641 else:
642 if tgid == pid:
643 if process_dict[tgid].task_name == "_Unknown_":
644 if args.verbose:
645 print "new process", \
646 process_dict[tgid].uid, pid, tgid, task_name
647 process_dict[tgid].task_name = task_name
648 if process_dict[tgid].task_name != task_name and tgid != 0:
649 process_dict[tgid].task_name = task_name
650
651 if not pid in thread_dict:
652 if args.verbose:
653 print "new thread", \
654 uid, process_dict[tgid].uid, pid, tgid, task_name
655 thread = Task(uid, pid, tgid, task_name, False, tick)
656 uid += 1
657 thread_dict[pid] = thread
658 process_dict[tgid].children.append(thread)
659 else:
660 if thread_dict[pid].task_name != task_name:
661 thread_dict[pid].task_name = task_name
662
663 if args.verbose:
664 print tick, uid, cpu_id, pid, tgid, task_name
665
666 task = thread_dict[pid]
667 event = Event(tick, task)
668 event_list[cpu_id].append(event)
669 unified_event_list.append(event)
670
671 if len(unified_event_list) == num_events:
672 print "Truncating at", num_events, "events!"
673 break
674 print "Found %d events." % len(unified_event_list)
675
676 for process in process_list:
677 if process.pid > 9990: # fix up framebuffer ticks
678 process.tick = start_tick
679 print process.uid, process.pid, process.tgid, \
680 process.task_name, str(process.tick)
681 for thread in process.children:
682 if thread.pid > 9990:
683 thread.tick = start_tick
684 print "\t", thread.uid, thread.pid, thread.tgid, \
685 thread.task_name, str(thread.tick)
686
687 end_tick = tick
688
689 print "Start tick:", start_tick
690 print "End tick: ", end_tick
691 print ""
692
693 return
694
695
696def initOutput(output_path):
697 if not os.path.exists(output_path):
698 os.mkdir(output_path)
699
700def ticksToNs(tick):
701 if ticks_in_ns < 0:
702 print "ticks_in_ns not set properly!"
703 sys.exit(1)
704
705 return tick / ticks_in_ns
706
707def writeXmlFile(xml, filename):
708 f = open(filename, "w")
709 txt = ET.tostring(xml)
710 f.write(minidom.parseString(txt).toprettyxml())
711 f.close()
712
713
714# StatsEntry that contains individual statistics
715class StatsEntry(object):
716 def __init__(self, name, group, group_index, per_cpu, key):
717
718 # Full name of statistics
719 self.name = name
720
721 # Streamline group name that statistic will belong to
722 self.group = group
723
724 # Index of statistics within group (used to change colors within groups)
725 self.group_index = group_index
726
727 # Shorter name with "system" stripped off
728 # and symbols converted to alphanumerics
729 self.short_name = re.sub("system\.", "", name)
730 self.short_name = re.sub(":", "_", name)
731
732 # Regex for this stat (string version used to construct union regex)
733 self.regex_string = "^" + name + "\s+([\d\.]+)"
734 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
735 self.description = ""
736
737 # Whether this stat is use per CPU or not
738 self.per_cpu = per_cpu
739
740 # Key used in .apc protocol (as described in captured.xml)
741 self.key = key
742
743 # List of values of stat per timestamp
744 self.values = []
745
746 # Whether this stat has been found for the current timestamp
747 self.found = False
748
749 # Whether this stat has been found at least once
750 # (to suppress too many warnings)
751 self.not_found_at_least_once = False
752
753 # Field used to hold ElementTree subelement for this stat
754 self.ET_element = None
755
756 # Create per-CPU stat name and regex, etc.
757 if self.per_cpu:
758 self.per_cpu_regex_string = []
759 self.per_cpu_regex = []
760 self.per_cpu_name = []
761 self.per_cpu_found = []
762 for i in range(num_cpus):
763 if num_cpus > 1:
764 per_cpu_name = re.sub("#", str(i), self.name)
765 else:
766 per_cpu_name = re.sub("#", "", self.name)
767
768 self.per_cpu_name.append(per_cpu_name)
769 print "\t", per_cpu_name
770
771 self.per_cpu_regex_string.\
772 append("^" + per_cpu_name + "\s+[\d\.]+")
773 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
774 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
775 self.values.append([])
776 self.per_cpu_found.append(False)
777
778 def append_value(self, val, per_cpu_index = None):
779 if self.per_cpu:
780 self.values[per_cpu_index].append(str(val))
781 else:
782 self.values.append(str(val))
783
784# Global stats object that contains the list of stats entries
785# and other utility functions
786class Stats(object):
787 def __init__(self):
788 self.stats_list = []
789 self.tick_list = []
790 self.next_key = 1
791
792 def register(self, name, group, group_index, per_cpu):
793 print "registering stat:", name, "group:", group, group_index
794 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
795 self.next_key))
796 self.next_key += 1
797
798 # Union of all stats to accelerate parsing speed
799 def createStatsRegex(self):
800 regex_strings = [];
801 print "\nnum entries in stats_list", len(self.stats_list)
802 for entry in self.stats_list:
803 if entry.per_cpu:
804 for i in range(num_cpus):
805 regex_strings.append(entry.per_cpu_regex_string[i])
806 else:
807 regex_strings.append(entry.regex_string)
808
809 self.regex = re.compile('|'.join(regex_strings))
810
811
812def registerStats(config_file):
813 print "==============================="
814 print "Parsing stats config.ini file..."
815 print config_file
816 print "==============================="
817
818 config = ConfigParser()
819 if not config.read(config_file):
820 print "ERROR: config file '", config_file, "' not found!"
821 sys.exit(1)
822
823 print "\nRegistering Stats..."
824
825 stats = Stats()
826
827 per_cpu_stat_groups = config.options('PER_CPU_STATS')
828 for group in per_cpu_stat_groups:
829 i = 0
830 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
831 for item in per_cpu_stats_list:
832 if item:
833 stats.register(item, group, i, True)
834 i += 1
835
836 per_l2_stat_groups = config.options('PER_L2_STATS')
837 for group in per_l2_stat_groups:
838 i = 0
839 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
840 for item in per_l2_stats_list:
841 if item:
842 for l2 in range(num_l2):
843 if num_l2 > 1:
844 name = re.sub("#", str(l2), item)
845 else:
846 name = re.sub("#", "", item)
847 stats.register(name, group, i, False)
848 i += 1
849
850 other_stat_groups = config.options('OTHER_STATS')
851 for group in other_stat_groups:
852 i = 0
853 other_stats_list = config.get('OTHER_STATS', group).split('\n')
854 for item in other_stats_list:
855 if item:
856 stats.register(item, group, i, False)
857 i += 1
858
859 stats.createStatsRegex()
860
861 return stats
862
863# Parse and read in gem5 stats file
864# Streamline counters are organized per CPU
865def readGem5Stats(stats, gem5_stats_file):
866 print "\n==============================="
867 print "Parsing gem5 stats file..."
868 print gem5_stats_file
869 print "===============================\n"
870 ext = os.path.splitext(gem5_stats_file)[1]
871
872 window_start_regex = \
873 re.compile("^---------- Begin Simulation Statistics ----------")
874 window_end_regex = \
875 re.compile("^---------- End Simulation Statistics ----------")
876 final_tick_regex = re.compile("^final_tick\s+(\d+)")
877
878 global ticks_in_ns
879 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
880 sim_freq = -1
881
882 try:
883 if ext == ".gz":
884 f = gzip.open(gem5_stats_file, "r")
885 else:
886 f = open(gem5_stats_file, "r")
887 except:
888 print "ERROR opening stats file", gem5_stats_file, "!"
889 sys.exit(1)
890
891 stats_not_found_list = stats.stats_list[:]
892 window_num = 0
893
894 while (True):
895 error = False
896 try:
897 line = f.readline()
898 except IOError:
899 print ""
900 print "WARNING: IO error in stats file"
901 print "(gzip stream not closed properly?)...continuing for now"
902 error = True
903 if not line:
904 break
905
906 # Find out how many gem5 ticks in 1ns
907 if sim_freq < 0:
908 m = sim_freq_regex.match(line)
909 if m:
910 sim_freq = int(m.group(1)) # ticks in 1 sec
911 ticks_in_ns = int(sim_freq / 1e9)
912 print "Simulation frequency found! 1 tick == %e sec\n" \
913 % (1.0 / sim_freq)
914
915 # Final tick in gem5 stats: current absolute timestamp
916 m = final_tick_regex.match(line)
917 if m:
918 tick = int(m.group(1))
919 if tick > end_tick:
920 break
921 stats.tick_list.append(tick)
922
923
924 if (window_end_regex.match(line) or error):
925 if args.verbose:
926 print "new window"
927 for stat in stats.stats_list:
928 if stat.per_cpu:
929 for i in range(num_cpus):
930 if not stat.per_cpu_found[i]:
931 if not stat.not_found_at_least_once:
932 print "WARNING: stat not found in window #", \
933 window_num, ":", stat.per_cpu_name[i]
934 print "suppressing further warnings for " + \
935 "this stat"
936 stat.not_found_at_least_once = True
937 stat.values[i].append(str(0))
938 stat.per_cpu_found[i] = False
939 else:
940 if not stat.found:
941 if not stat.not_found_at_least_once:
942 print "WARNING: stat not found in window #", \
943 window_num, ":", stat.name
944 print "suppressing further warnings for this stat"
945 stat.not_found_at_least_once = True
946 stat.values.append(str(0))
947 stat.found = False
948 stats_not_found_list = stats.stats_list[:]
949 window_num += 1
950 if error:
951 break
952
953 # Do a single regex of the union of all stats first for speed
954 if stats.regex.match(line):
955 # Then loop through only the stats we haven't seen in this window
956 for stat in stats_not_found_list[:]:
957 if stat.per_cpu:
958 for i in range(num_cpus):
959 m = stat.per_cpu_regex[i].match(line)
960 if m:
961 if stat.name == "ipc":
962 value = str(int(float(m.group(1)) * 1000))
963 else:
964 value = str(int(float(m.group(1))))
965 if args.verbose:
966 print stat.per_cpu_name[i], value
967 stat.values[i].append(value)
968 stat.per_cpu_found[i] = True
969 all_found = True
970 for j in range(num_cpus):
971 if not stat.per_cpu_found[j]:
972 all_found = False
973 if all_found:
974 stats_not_found_list.remove(stat)
975 if stat.description == "":
976 stat.description = m.group(2)
977 else:
978 m = stat.regex.match(line)
979 if m:
980 value = str(int(float(m.group(1))))
981 if args.verbose:
982 print stat.name, value
983 stat.values.append(value)
984 stat.found = True
985 stats_not_found_list.remove(stat)
986 if stat.description == "":
987 stat.description = m.group(2)
988 f.close()
989
990
991# Create session.xml file in .apc folder
992def doSessionXML(output_path):
993 session_file = output_path + "/session.xml"
994
995 xml = ET.Element("session")
996
997 xml.set("version", "1")
998 xml.set("call_stack_unwinding", "no")
999 xml.set("parse_debug_info", "no")
1000 xml.set("high_resolution", "yes")
1001 xml.set("buffer_mode", "streaming")
1002 xml.set("sample_rate", "low")
1003
1004 # Setting duration to zero for now. Doesn't affect visualization.
1005 xml.set("duration", "0")
1006
1007 xml.set("target_host", "")
1008 xml.set("target_port", "8080")
1009
1010 writeXmlFile(xml, session_file)
1011
1012
1013# Create captured.xml file in .apc folder
1014def doCapturedXML(output_path, stats):
1015 captured_file = output_path + "/captured.xml"
1016
1017 xml = ET.Element("captured")
1018 xml.set("version", "1")
1019 xml.set("protocol", "17")
1020 xml.set("backtrace_processing", "none")
1021
1022 target = ET.SubElement(xml, "target")
1023 target.set("name", "gem5")
1024 target.set("sample_rate", "1000")
1025 target.set("cores", str(num_cpus))
1026
1027 counters = ET.SubElement(xml, "counters")
1028 for stat in stats.stats_list:
1029 s = ET.SubElement(counters, "counter")
1030 stat_name = re.sub("\.", "_", stat.short_name)
1031 stat_name = re.sub("#", "", stat_name)
1032 s.set("title", stat.group)
1033 s.set("name", stat_name)
1034 s.set("color", "0x00000000")
1035 s.set("key", "0x%08x" % stat.key)
1036 s.set("type", stat_name)
1037 s.set("event", "0x00000000")
1038 if stat.per_cpu:
1039 s.set("per_cpu", "yes")
1040 else:
1041 s.set("per_cpu", "no")
1042 s.set("display", "")
1043 s.set("units", "")
1044 s.set("average_selection", "no")
1045 s.set("description", stat.description)
1046
1047 writeXmlFile(xml, captured_file)
1048
1049# Writes out Streamline cookies (unique IDs per process/thread)
1050def writeCookiesThreads(blob):
1051 thread_list = []
1052 for process in process_list:
1053 if process.uid > 0:
1054 print "cookie", process.task_name, process.uid
1055 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1056
1057 # pid and tgid need to be positive values -- no longer true?
1058 for thread in process.children:
1059 thread_list.append(thread)
1060
1061 # Threads need to be sorted in timestamp order
1062 thread_list.sort(key = lambda x: x.tick)
1063 for thread in thread_list:
1064 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1065 thread.tgid, thread.pid
1066 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1067 thread.pid, thread.task_name))
1068
1069# Writes context switch info as Streamline scheduling events
1070def writeSchedEvents(blob):
1071 for cpu in range(num_cpus):
1072 for event in event_list[cpu]:
1073 timestamp = ticksToNs(event.tick)
1074 pid = event.task.tgid
1075 tid = event.task.pid
1076 if process_dict.has_key(event.task.tgid):
1077 cookie = process_dict[event.task.tgid].uid
1078 else:
1079 cookie = 0
1080
1081 # State:
1082 # 0: waiting on other event besides I/O
1083 # 1: Contention/pre-emption
1084 # 2: Waiting on I/O
1085 # 3: Waiting on mutex
1086 # Hardcoding to 0 for now. Other states not implemented yet.
1087 state = 0
1088
1089 if args.verbose:
1090 print cpu, timestamp, pid, tid, cookie
1091
1092 writeBinary(blob,\
1093 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1094
1095# Writes selected gem5 statistics as Streamline counters
1096def writeCounters(blob, stats):
1097 timestamp_list = []
1098 for tick in stats.tick_list:
1099 if tick > end_tick:
1100 break
1101 timestamp_list.append(ticksToNs(tick))
1102
1103 for stat in stats.stats_list:
1104 if stat.per_cpu:
1105 stat_length = len(stat.values[0])
1106 else:
1107 stat_length = len(stat.values)
1108
1109 for n in range(len(timestamp_list)):
1110 for stat in stats.stats_list:
1111 if stat.per_cpu:
1112 for i in range(num_cpus):
1113 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1114 stat.key, int(float(stat.values[i][n]))))
1115 else:
1116 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1117 stat.key, int(float(stat.values[n]))))
1118
1119# Streamline can display LCD frame buffer dumps (gzipped bmp)
1120# This function converts the frame buffer dumps to the Streamline format
1121def writeVisualAnnotations(blob, input_path, output_path):
1122 frame_path = input_path + "/frames_system.vncserver"
1123 if not os.path.exists(frame_path):
1124 return
1125
1126 frame_count = 0
1127 file_list = os.listdir(frame_path)
1128 file_list.sort()
1129 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1130
1131 # Use first non-negative pid to tag visual annotations
1132 annotate_pid = -1
1133 for e in unified_event_list:
1134 pid = e.task.pid
1135 if pid >= 0:
1136 annotate_pid = pid
1137 break
1138
1139 for fn in file_list:
1140 m = re_fb.match(fn)
1141 if m:
1142 seq = m.group(1)
1143 tick = int(m.group(2))
1144 if tick > end_tick:
1145 break
1146 frame_count += 1
1147
1148 userspace_body = []
1149 userspace_body += packed32(0x1C) # escape code
1150 userspace_body += packed32(0x04) # visual code
1151
1152 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1153 userspace_body += int16(len(text_annotation))
1154 userspace_body += utf8StringList(text_annotation)
1155
1156 if gzipped_bmp_supported:
1157 # copy gzipped bmp directly
1158 bytes_read = open(frame_path + "/" + fn, "rb").read()
1159 else:
1160 # copy uncompressed bmp
1161 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1162
1163 userspace_body += int32(len(bytes_read))
1164 userspace_body += bytes_read
1165
1166 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1167 len(userspace_body), userspace_body))
1168
1169 print "\nfound", frame_count, "frames for visual annotation.\n"
1170
1171
1172def createApcProject(input_path, output_path, stats):
1173 initOutput(output_path)
1174
1175 blob = open(output_path + "/0000000000", "wb")
1176
1177 # Summary frame takes current system time and system uptime.
1178 # Filling in with random values for now.
1179 writeBinary(blob, summaryFrame(1234, 5678))
1180
1181 writeCookiesThreads(blob)
1182
1183 print "writing Events"
1184 writeSchedEvents(blob)
1185
1186 print "writing Counters"
1187 writeCounters(blob, stats)
1188
1189 print "writing Visual Annotations"
1190 writeVisualAnnotations(blob, input_path, output_path)
1191
1192 doSessionXML(output_path)
1193 doCapturedXML(output_path, stats)
1194
1195 blob.close()
1196
1197
1198
1199#######################
1200# Main Routine
1201
1202input_path = args.input_path
1203output_path = args.output_path
1204
1205####
1206# Make sure input path exists
1207####
1208if not os.path.exists(input_path):
1209 print "ERROR: Input path %s does not exist!" % input_path
1210 sys.exit(1)
1211
1212####
1213# Parse gem5 configuration file to find # of CPUs and L2s
1214####
1215(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1216
1217####
1218# Parse task file to find process/thread info
1219####
1220parseProcessInfo(input_path + "/system.tasks.txt")
1221
1222####
1223# Parse stat config file and register stats
1224####
1225stat_config_file = args.stat_config_file
1226stats = registerStats(stat_config_file)
1227
1228####
1229# Parse gem5 stats
1230####
1231# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1232if os.path.exists(input_path + "/stats.txt") and \
1233 os.path.exists(input_path + "/stats.txt.gz"):
1234 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1235 Using stats.txt.gz by default."
1236
1237gem5_stats_file = input_path + "/stats.txt.gz"
1238if not os.path.exists(gem5_stats_file):
1239 gem5_stats_file = input_path + "/stats.txt"
1240if not os.path.exists(gem5_stats_file):
1241 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1242 sys.exit(1)
1243
1244readGem5Stats(stats, gem5_stats_file)
1245
1246####
1247# Create Streamline .apc project folder
1248####
1249createApcProject(input_path, output_path, stats)
1250
1251print "All done!"
|