m5stats2streamline.py revision 10016
1#!/usr/bin/env python 2 3# Copyright (c) 2012 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Author: Dam Sunwoo 39# 40 41# This script converts gem5 output to ARM DS-5 Streamline .apc project file 42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and 43# some patches applied to target Linux kernel.) 44# Visit http://www.gem5.org/Streamline for more details. 45# 46# Usage: 47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder> 48# 49# <stat_config.ini>: .ini file that describes which stats to be included 50# in conversion. Sample .ini files can be found in 51# util/streamline. 52# NOTE: this is NOT the gem5 config.ini file. 53# 54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini, 55# stats.txt[.gz], and system.tasks.txt.) 56# 57# <dest .apc folder>: Destination .apc folder path 58# 59# APC project generation based on Gator v17 (DS-5 v5.17) 60# Subsequent versions should be backward compatible 61 62import re, sys, os 63from ConfigParser import ConfigParser 64import gzip 65import xml.etree.ElementTree as ET 66import xml.dom.minidom as minidom 67import shutil 68import zlib 69 70import argparse 71 72parser = argparse.ArgumentParser( 73 formatter_class=argparse.RawDescriptionHelpFormatter, 74 description=""" 75 Converts gem5 runs to ARM DS-5 Streamline .apc project file. 76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump 77 enabled and some patches applied to the target Linux kernel.) 78 79 Visit http://www.gem5.org/Streamline for more details. 80 81 APC project generation based on Gator v17 (DS-5 v5.17) 82 Subsequent versions should be backward compatible 83 """) 84 85parser.add_argument("stat_config_file", metavar="<stat_config.ini>", 86 help=".ini file that describes which stats to be included \ 87 in conversion. Sample .ini files can be found in \ 88 util/streamline. NOTE: this is NOT the gem5 config.ini \ 89 file.") 90 91parser.add_argument("input_path", metavar="<gem5 run folder>", 92 help="Path to gem5 run folder (must contain config.ini, \ 93 stats.txt[.gz], and system.tasks.txt.)") 94 95parser.add_argument("output_path", metavar="<dest .apc folder>", 96 help="Destination .apc folder path") 97 98parser.add_argument("--num-events", action="store", type=int, 99 default=1000000, 100 help="Maximum number of scheduling (context switch) \ 101 events to be processed. Set to truncate early. \ 102 Default=1000000") 103 104parser.add_argument("--gzipped-bmp-not-supported", action="store_true", 105 help="Do not use gzipped .bmp files for visual annotations. \ 106 This option is only required when using Streamline versions \ 107 older than 5.14") 108 109parser.add_argument("--verbose", action="store_true", 110 help="Enable verbose output") 111 112args = parser.parse_args() 113 114if not re.match("(.*)\.apc", args.output_path): 115 print "ERROR: <dest .apc folder> should end with '.apc'!" 116 sys.exit(1) 117 118# gzipped BMP files for visual annotation is supported in Streamline 5.14. 119# Setting this to True will significantly compress the .apc binary file that 120# includes frame buffer snapshots. 121gzipped_bmp_supported = not args.gzipped_bmp_not_supported 122 123ticks_in_ns = -1 124 125# Default max # of events. Increase this for longer runs. 126num_events = args.num_events 127 128start_tick = -1 129end_tick = -1 130 131# Parse gem5 config.ini file to determine some system configurations. 132# Number of CPUs, L2s, etc. 133def parseConfig(config_file): 134 global num_cpus, num_l2 135 136 print "\n===============================" 137 print "Parsing gem5 config.ini file..." 138 print config_file 139 print "===============================\n" 140 config = ConfigParser() 141 if not config.read(config_file): 142 print "ERROR: config file '", config_file, "' not found" 143 sys.exit(1) 144 145 if config.has_section("system.cpu"): 146 num_cpus = 1 147 else: 148 num_cpus = 0 149 while config.has_section("system.cpu" + str(num_cpus)): 150 num_cpus += 1 151 152 if config.has_section("system.l2"): 153 num_l2 = 1 154 else: 155 num_l2 = 0 156 while config.has_section("system.l2" + str(num_l2)): 157 num_l2 += 1 158 159 print "Num CPUs:", num_cpus 160 print "Num L2s:", num_l2 161 print "" 162 163 return (num_cpus, num_l2) 164 165 166process_dict = {} 167thread_dict = {} 168 169process_list = [] 170 171idle_uid = -1 172kernel_uid = -1 173 174class Task(object): 175 def __init__(self, uid, pid, tgid, task_name, is_process, tick): 176 if pid == 0: # Idle 177 self.uid = 0 178 elif pid == -1: # Kernel 179 self.uid = 0 180 else: 181 self.uid = uid 182 self.pid = pid 183 self.tgid = tgid 184 self.is_process = is_process 185 self.task_name = task_name 186 self.children = [] 187 self.tick = tick # time this task first appeared 188 189class Event(object): 190 def __init__(self, tick, task): 191 self.tick = tick 192 self.task = task 193 194############################################################ 195# Types used in APC Protocol 196# - packed32, packed64 197# - int32 198# - string 199############################################################ 200 201def packed32(x): 202 ret = [] 203 more = True 204 while more: 205 b = x & 0x7f 206 x = x >> 7 207 if (((x == 0) and ((b & 0x40) == 0)) or \ 208 ((x == -1) and ((b & 0x40) != 0))): 209 more = False 210 else: 211 b = b | 0x80 212 ret.append(b) 213 return ret 214 215# For historical reasons, 32/64-bit versions of functions are presevered 216def packed64(x): 217 return packed32(x) 218 219# variable length packed 4-byte signed value 220def unsigned_packed32(x): 221 ret = [] 222 if ((x & 0xffffff80) == 0): 223 ret.append(x & 0x7f) 224 elif ((x & 0xffffc000) == 0): 225 ret.append((x | 0x80) & 0xff) 226 ret.append((x >> 7) & 0x7f) 227 elif ((x & 0xffe00000) == 0): 228 ret.append((x | 0x80) & 0xff) 229 ret.append(((x >> 7) | 0x80) & 0xff) 230 ret.append((x >> 14) & 0x7f) 231 elif ((x & 0xf0000000) == 0): 232 ret.append((x | 0x80) & 0xff) 233 ret.append(((x >> 7) | 0x80) & 0xff) 234 ret.append(((x >> 14) | 0x80) & 0xff) 235 ret.append((x >> 21) & 0x7f) 236 else: 237 ret.append((x | 0x80) & 0xff) 238 ret.append(((x >> 7) | 0x80) & 0xff) 239 ret.append(((x >> 14) | 0x80) & 0xff) 240 ret.append(((x >> 21) | 0x80) & 0xff) 241 ret.append((x >> 28) & 0x0f) 242 return ret 243 244# variable length packed 8-byte signed value 245def unsigned_packed64(x): 246 ret = [] 247 if ((x & 0xffffffffffffff80) == 0): 248 ret.append(x & 0x7f) 249 elif ((x & 0xffffffffffffc000) == 0): 250 ret.append((x | 0x80) & 0xff) 251 ret.append((x >> 7) & 0x7f) 252 elif ((x & 0xffffffffffe00000) == 0): 253 ret.append((x | 0x80) & 0xff) 254 ret.append(((x >> 7) | 0x80) & 0xff) 255 ret.append((x >> 14) & 0x7f) 256 elif ((x & 0xfffffffff0000000) == 0): 257 ret.append((x | 0x80) & 0xff) 258 ret.append(((x >> 7) | 0x80) & 0xff) 259 ret.append(((x >> 14) | 0x80) & 0xff) 260 ret.append((x >> 21) & 0x7f) 261 elif ((x & 0xfffffff800000000) == 0): 262 ret.append((x | 0x80) & 0xff) 263 ret.append(((x >> 7) | 0x80) & 0xff) 264 ret.append(((x >> 14) | 0x80) & 0xff) 265 ret.append(((x >> 21) | 0x80) & 0xff) 266 ret.append((x >> 28) & 0x7f) 267 elif ((x & 0xfffffc0000000000) == 0): 268 ret.append((x | 0x80) & 0xff) 269 ret.append(((x >> 7) | 0x80) & 0xff) 270 ret.append(((x >> 14) | 0x80) & 0xff) 271 ret.append(((x >> 21) | 0x80) & 0xff) 272 ret.append(((x >> 28) | 0x80) & 0xff) 273 ret.append((x >> 35) & 0x7f) 274 elif ((x & 0xfffe000000000000) == 0): 275 ret.append((x | 0x80) & 0xff) 276 ret.append(((x >> 7) | 0x80) & 0xff) 277 ret.append(((x >> 14) | 0x80) & 0xff) 278 ret.append(((x >> 21) | 0x80) & 0xff) 279 ret.append(((x >> 28) | 0x80) & 0xff) 280 ret.append(((x >> 35) | 0x80) & 0xff) 281 ret.append((x >> 42) & 0x7f) 282 elif ((x & 0xff00000000000000) == 0): 283 ret.append((x | 0x80) & 0xff) 284 ret.append(((x >> 7) | 0x80) & 0xff) 285 ret.append(((x >> 14) | 0x80) & 0xff) 286 ret.append(((x >> 21) | 0x80) & 0xff) 287 ret.append(((x >> 28) | 0x80) & 0xff) 288 ret.append(((x >> 35) | 0x80) & 0xff) 289 ret.append(((x >> 42) | 0x80) & 0xff) 290 ret.append((x >> 49) & 0x7f) 291 elif ((x & 0x8000000000000000) == 0): 292 ret.append((x | 0x80) & 0xff) 293 ret.append(((x >> 7) | 0x80) & 0xff) 294 ret.append(((x >> 14) | 0x80) & 0xff) 295 ret.append(((x >> 21) | 0x80) & 0xff) 296 ret.append(((x >> 28) | 0x80) & 0xff) 297 ret.append(((x >> 35) | 0x80) & 0xff) 298 ret.append(((x >> 42) | 0x80) & 0xff) 299 ret.append(((x >> 49) | 0x80) & 0xff) 300 ret.append((x >> 56) & 0x7f) 301 else: 302 ret.append((x | 0x80) & 0xff) 303 ret.append(((x >> 7) | 0x80) & 0xff) 304 ret.append(((x >> 14) | 0x80) & 0xff) 305 ret.append(((x >> 21) | 0x80) & 0xff) 306 ret.append(((x >> 28) | 0x80) & 0xff) 307 ret.append(((x >> 35) | 0x80) & 0xff) 308 ret.append(((x >> 42) | 0x80) & 0xff) 309 ret.append(((x >> 49) | 0x80) & 0xff) 310 ret.append(((x >> 56) | 0x80) & 0xff) 311 ret.append((x >> 63) & 0x7f) 312 return ret 313 314# 4-byte signed little endian 315def int32(x): 316 ret = [] 317 ret.append(x & 0xff) 318 ret.append((x >> 8) & 0xff) 319 ret.append((x >> 16) & 0xff) 320 ret.append((x >> 24) & 0xff) 321 return ret 322 323# 2-byte signed little endian 324def int16(x): 325 ret = [] 326 ret.append(x & 0xff) 327 ret.append((x >> 8) & 0xff) 328 return ret 329 330# a packed32 length followed by the specified number of characters 331def stringList(x): 332 ret = [] 333 ret += packed32(len(x)) 334 for i in x: 335 ret.append(i) 336 return ret 337 338def utf8StringList(x): 339 ret = [] 340 for i in x: 341 ret.append(ord(i)) 342 return ret 343 344# packed64 time value in nanoseconds relative to the uptime from the 345# Summary message. 346def timestampList(x): 347 ret = packed64(x) 348 return ret 349 350 351############################################################ 352# Write binary 353############################################################ 354 355def writeBinary(outfile, binary_list): 356 for i in binary_list: 357 outfile.write("%c" % i) 358 359############################################################ 360# APC Protocol Frame Types 361############################################################ 362 363def addFrameHeader(frame_type, body, core): 364 ret = [] 365 366 if frame_type == "Summary": 367 code = 1 368 elif frame_type == "Backtrace": 369 code = 2 370 elif frame_type == "Name": 371 code = 3 372 elif frame_type == "Counter": 373 code = 4 374 elif frame_type == "Block Counter": 375 code = 5 376 elif frame_type == "Annotate": 377 code = 6 378 elif frame_type == "Sched Trace": 379 code = 7 380 elif frame_type == "GPU Trace": 381 code = 8 382 elif frame_type == "Idle": 383 code = 9 384 else: 385 print "ERROR: Unknown frame type:", frame_type 386 sys.exit(1) 387 388 packed_code = packed32(code) 389 390 packed_core = packed32(core) 391 392 length = int32(len(packed_code) + len(packed_core) + len(body)) 393 394 ret = length + packed_code + packed_core + body 395 return ret 396 397 398# Summary frame 399# - timestamp: packed64 400# - uptime: packed64 401def summaryFrame(timestamp, uptime): 402 frame_type = "Summary" 403 newline_canary = stringList("1\n2\r\n3\r4\n\r5") 404 monotonic_delta = packed64(0) 405 end_of_attr = stringList("") 406 body = newline_canary + packed64(timestamp) + packed64(uptime) 407 body += monotonic_delta + end_of_attr 408 ret = addFrameHeader(frame_type, body, 0) 409 return ret 410 411# Backtrace frame 412# - not implemented yet 413def backtraceFrame(): 414 pass 415 416# Cookie name message 417# - cookie: packed32 418# - name: string 419def cookieNameFrame(cookie, name): 420 frame_type = "Name" 421 packed_code = packed32(1) 422 body = packed_code + packed32(cookie) + stringList(name) 423 ret = addFrameHeader(frame_type, body, 0) 424 return ret 425 426# Thread name message 427# - timestamp: timestamp 428# - thread id: packed32 429# - name: string 430def threadNameFrame(timestamp, thread_id, name): 431 frame_type = "Name" 432 packed_code = packed32(2) 433 body = packed_code + timestampList(timestamp) + \ 434 packed32(thread_id) + stringList(name) 435 ret = addFrameHeader(frame_type, body, 0) 436 return ret 437 438# Core name message 439# - name: string 440# - core_id: packed32 441# - cpuid: packed32 442def coreNameFrame(name, core_id, cpuid): 443 frame_type = "Name" 444 packed_code = packed32(3) 445 body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name) 446 ret = addFrameHeader(frame_type, body, 0) 447 return ret 448 449# IRQ Cookie name message 450# - cookie: packed32 451# - name: string 452# - irq: packed32 453def irqCookieNameFrame(cookie, name, irq): 454 frame_type = "Name" 455 packed_code = packed32(5) 456 body = packed_code + packed32(cookie) + stringList(name) + packed32(irq) 457 ret = addFrameHeader(frame_type, body, 0) 458 return ret 459 460# Counter frame message 461# - timestamp: timestamp 462# - core: packed32 463# - key: packed32 464# - value: packed64 465def counterFrame(timestamp, core, key, value): 466 frame_type = "Counter" 467 body = timestampList(timestamp) + packed32(core) + packed32(key) + \ 468 packed64(value) 469 ret = addFrameHeader(frame_type, body, core) 470 return ret 471 472# Block Counter frame message 473# - key: packed32 474# - value: packed64 475def blockCounterFrame(core, key, value): 476 frame_type = "Block Counter" 477 body = packed32(key) + packed64(value) 478 ret = addFrameHeader(frame_type, body, core) 479 return ret 480 481# Annotate frame messages 482# - core: packed32 483# - tid: packed32 484# - timestamp: timestamp 485# - size: packed32 486# - body 487def annotateFrame(core, tid, timestamp, size, userspace_body): 488 frame_type = "Annotate" 489 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \ 490 packed32(size) + userspace_body 491 ret = addFrameHeader(frame_type, body, core) 492 return ret 493 494# Scheduler Trace frame messages 495# Sched Switch 496# - Code: 1 497# - timestamp: timestamp 498# - pid: packed32 499# - tid: packed32 500# - cookie: packed32 501# - state: packed32 502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state): 503 frame_type = "Sched Trace" 504 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \ 505 packed32(tid) + packed32(cookie) + packed32(state) 506 ret = addFrameHeader(frame_type, body, core) 507 return ret 508 509# Sched Thread Exit 510# - Code: 2 511# - timestamp: timestamp 512# - tid: packed32 513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state): 514 frame_type = "Sched Trace" 515 body = packed32(2) + timestampList(timestamp) + packed32(tid) 516 ret = addFrameHeader(frame_type, body, core) 517 return ret 518 519# GPU Trace frame messages 520# - Not implemented yet 521def gpuTraceFrame(): 522 pass 523 524# Idle frame messages 525# Enter Idle 526# - code: 1 527# - timestamp: timestamp 528# - core: packed32 529def enterIdleFrame(timestamp, core): 530 frame_type = "Idle" 531 body = packed32(1) + timestampList(timestamp) + packed32(core) 532 ret = addFrameHeader(frame_type, body, core) 533 return ret 534 535# Exit Idle 536# - code: 2 537# - timestamp: timestamp 538# - core: packed32 539def exitIdleFrame(timestamp, core): 540 frame_type = "Idle" 541 body = packed32(2) + timestampList(timestamp) + packed32(core) 542 ret = addFrameHeader(frame_type, body, core) 543 return ret 544 545 546#################################################################### 547def parseProcessInfo(task_file): 548 print "\n===============================" 549 print "Parsing Task file..." 550 print task_file 551 print "===============================\n" 552 553 global start_tick, end_tick, num_cpus 554 global process_dict, thread_dict, process_list 555 global event_list, unified_event_list 556 global idle_uid, kernel_uid 557 558 event_list = [] 559 unified_event_list = [] 560 for cpu in range(num_cpus): 561 event_list.append([]) 562 563 uid = 1 # uid 0 is reserved for idle 564 565 # Dummy Tasks for frame buffers and system diagrams 566 process = Task(uid, 9999, 9999, "framebuffer", True, 0) 567 process_list.append(process) 568 uid += 1 569 thread = Task(uid, 9999, 9999, "framebuffer", False, 0) 570 process.children.append(thread) 571 uid += 1 572 process = Task(uid, 9998, 9998, "System", True, 0) 573 process_list.append(process) 574 # if we don't find the real kernel, use this to keep things going 575 kernel_uid = uid 576 uid += 1 577 thread = Task(uid, 9998, 9998, "System", False, 0) 578 process.children.append(thread) 579 uid += 1 580 581 ext = os.path.splitext(task_file)[1] 582 583 try: 584 if ext == ".gz": 585 process_file = gzip.open(task_file, 'rb') 586 else: 587 process_file = open(task_file, 'rb') 588 except: 589 print "ERROR opening task file:", task_file 590 print "Make sure context switch task dumping is enabled in gem5." 591 sys.exit(1) 592 593 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" + 594 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)") 595 596 task_name_failure_warned = False 597 598 for line in process_file: 599 match = re.match(process_re, line) 600 if match: 601 tick = int(match.group(1)) 602 if (start_tick < 0): 603 start_tick = tick 604 cpu_id = int(match.group(3)) 605 pid = int(match.group(4)) 606 tgid = int(match.group(5)) 607 task_name = match.group(6) 608 609 if not task_name_failure_warned: 610 if task_name == "FailureIn_curTaskName": 611 print "-------------------------------------------------" 612 print "WARNING: Task name not set correctly!" 613 print "Process/Thread info will not be displayed correctly" 614 print "Perhaps forgot to apply m5struct.patch to kernel?" 615 print "-------------------------------------------------" 616 task_name_failure_warned = True 617 618 if not tgid in process_dict: 619 if tgid == pid: 620 # new task is parent as well 621 if args.verbose: 622 print "new process", uid, pid, tgid, task_name 623 if tgid == 0: 624 # new process is the "idle" task 625 process = Task(uid, pid, tgid, "idle", True, tick) 626 idle_uid = 0 627 else: 628 process = Task(uid, pid, tgid, task_name, True, tick) 629 else: 630 if tgid == 0: 631 process = Task(uid, tgid, tgid, "idle", True, tick) 632 idle_uid = 0 633 else: 634 # parent process name not known yet 635 process = Task(uid, tgid, tgid, "_Unknown_", True, tick) 636 if tgid == -1: # kernel 637 kernel_uid = 0 638 uid += 1 639 process_dict[tgid] = process 640 process_list.append(process) 641 else: 642 if tgid == pid: 643 if process_dict[tgid].task_name == "_Unknown_": 644 if args.verbose: 645 print "new process", \ 646 process_dict[tgid].uid, pid, tgid, task_name 647 process_dict[tgid].task_name = task_name 648 if process_dict[tgid].task_name != task_name and tgid != 0: 649 process_dict[tgid].task_name = task_name 650 651 if not pid in thread_dict: 652 if args.verbose: 653 print "new thread", \ 654 uid, process_dict[tgid].uid, pid, tgid, task_name 655 thread = Task(uid, pid, tgid, task_name, False, tick) 656 uid += 1 657 thread_dict[pid] = thread 658 process_dict[tgid].children.append(thread) 659 else: 660 if thread_dict[pid].task_name != task_name: 661 thread_dict[pid].task_name = task_name 662 663 if args.verbose: 664 print tick, uid, cpu_id, pid, tgid, task_name 665 666 task = thread_dict[pid] 667 event = Event(tick, task) 668 event_list[cpu_id].append(event) 669 unified_event_list.append(event) 670 671 if len(unified_event_list) == num_events: 672 print "Truncating at", num_events, "events!" 673 break 674 print "Found %d events." % len(unified_event_list) 675 676 for process in process_list: 677 if process.pid > 9990: # fix up framebuffer ticks 678 process.tick = start_tick 679 print process.uid, process.pid, process.tgid, \ 680 process.task_name, str(process.tick) 681 for thread in process.children: 682 if thread.pid > 9990: 683 thread.tick = start_tick 684 print "\t", thread.uid, thread.pid, thread.tgid, \ 685 thread.task_name, str(thread.tick) 686 687 end_tick = tick 688 689 print "Start tick:", start_tick 690 print "End tick: ", end_tick 691 print "" 692 693 return 694 695 696def initOutput(output_path): 697 if not os.path.exists(output_path): 698 os.mkdir(output_path) 699 700def ticksToNs(tick): 701 if ticks_in_ns < 0: 702 print "ticks_in_ns not set properly!" 703 sys.exit(1) 704 705 return tick / ticks_in_ns 706 707def writeXmlFile(xml, filename): 708 f = open(filename, "w") 709 txt = ET.tostring(xml) 710 f.write(minidom.parseString(txt).toprettyxml()) 711 f.close() 712 713 714# StatsEntry that contains individual statistics 715class StatsEntry(object): 716 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key): 717 718 # Full name of statistics 719 self.name = name 720 721 # Streamline group name that statistic will belong to 722 self.group = group 723 724 # Index of statistics within group (used to change colors within groups) 725 self.group_index = group_index 726 727 # Shorter name with "system" stripped off 728 # and symbols converted to alphanumerics 729 self.short_name = re.sub("system\.", "", name) 730 self.short_name = re.sub(":", "_", name) 731 732 # Regex for this stat (string version used to construct union regex) 733 self.regex_string = "^" + name + "\s+([\d\.]+)" 734 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M) 735 self.description = "" 736 737 # Whether this stat is use per CPU or not 738 self.per_cpu = per_cpu 739 self.per_switchcpu = per_switchcpu 740 741 # Key used in .apc protocol (as described in captured.xml) 742 self.key = key 743 744 # List of values of stat per timestamp 745 self.values = [] 746 747 # Whether this stat has been found for the current timestamp 748 self.found = False 749 750 # Whether this stat has been found at least once 751 # (to suppress too many warnings) 752 self.not_found_at_least_once = False 753 754 # Field used to hold ElementTree subelement for this stat 755 self.ET_element = None 756 757 # Create per-CPU stat name and regex, etc. 758 if self.per_cpu: 759 self.per_cpu_regex_string = [] 760 self.per_cpu_regex = [] 761 self.per_cpu_name = [] 762 self.per_cpu_found = [] 763 for i in range(num_cpus): 764 # Resuming from checkpoints results in using "switch_cpus" 765 if per_switchcpu: 766 per_cpu_name = "system.switch_cpus" 767 else: 768 per_cpu_name = "system.cpu" 769 770 # No CPU number appends if num_cpus == 1 771 if num_cpus > 1: 772 per_cpu_name += str(i) 773 per_cpu_name += "." + self.name 774 self.per_cpu_name.append(per_cpu_name) 775 print "\t", per_cpu_name 776 777 self.per_cpu_regex_string.\ 778 append("^" + per_cpu_name + "\s+[\d\.]+") 779 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \ 780 "\s+([\d\.e\-]+)\s+# (.*)$", re.M)) 781 self.values.append([]) 782 self.per_cpu_found.append(False) 783 784 def append_value(self, val, per_cpu_index = None): 785 if self.per_cpu: 786 self.values[per_cpu_index].append(str(val)) 787 else: 788 self.values.append(str(val)) 789 790# Global stats object that contains the list of stats entries 791# and other utility functions 792class Stats(object): 793 def __init__(self): 794 self.stats_list = [] 795 self.tick_list = [] 796 self.next_key = 1 797 798 def register(self, name, group, group_index, per_cpu, per_switchcpu): 799 print "registering stat:", name, "group:", group, group_index 800 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \ 801 per_switchcpu, self.next_key)) 802 self.next_key += 1 803 804 # Union of all stats to accelerate parsing speed 805 def createStatsRegex(self): 806 regex_strings = []; 807 print "\nnum entries in stats_list", len(self.stats_list) 808 for entry in self.stats_list: 809 if entry.per_cpu: 810 for i in range(num_cpus): 811 regex_strings.append(entry.per_cpu_regex_string[i]) 812 else: 813 regex_strings.append(entry.regex_string) 814 815 self.regex = re.compile('|'.join(regex_strings)) 816 817 818def registerStats(config_file): 819 print "===============================" 820 print "Parsing stats config.ini file..." 821 print config_file 822 print "===============================" 823 824 config = ConfigParser() 825 if not config.read(config_file): 826 print "ERROR: config file '", config_file, "' not found!" 827 sys.exit(1) 828 829 print "\nRegistering Stats..." 830 831 stats = Stats() 832 833 per_cpu_stat_groups = config.options('PER_CPU_STATS') 834 for group in per_cpu_stat_groups: 835 i = 0 836 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n') 837 for item in per_cpu_stats_list: 838 if item: 839 stats.register(item, group, i, True, False) 840 i += 1 841 842 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS') 843 for group in per_cpu_stat_groups: 844 i = 0 845 per_cpu_stats_list = \ 846 config.get('PER_SWITCHCPU_STATS', group).split('\n') 847 for item in per_cpu_stats_list: 848 if item: 849 stats.register(item, group, i, True, True) 850 i += 1 851 852 per_l2_stat_groups = config.options('PER_L2_STATS') 853 for group in per_l2_stat_groups: 854 i = 0 855 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n') 856 for item in per_l2_stats_list: 857 if item: 858 for l2 in range(num_l2): 859 name = item 860 prefix = "system.l2" 861 if num_l2 > 1: 862 prefix += str(l2) 863 prefix += "." 864 name = prefix + name 865 stats.register(name, group, i, False, False) 866 i += 1 867 868 other_stat_groups = config.options('OTHER_STATS') 869 for group in other_stat_groups: 870 i = 0 871 other_stats_list = config.get('OTHER_STATS', group).split('\n') 872 for item in other_stats_list: 873 if item: 874 stats.register(item, group, i, False, False) 875 i += 1 876 877 stats.createStatsRegex() 878 879 return stats 880 881# Parse and read in gem5 stats file 882# Streamline counters are organized per CPU 883def readGem5Stats(stats, gem5_stats_file): 884 print "\n===============================" 885 print "Parsing gem5 stats file..." 886 print gem5_stats_file 887 print "===============================\n" 888 ext = os.path.splitext(gem5_stats_file)[1] 889 890 window_start_regex = \ 891 re.compile("^---------- Begin Simulation Statistics ----------") 892 window_end_regex = \ 893 re.compile("^---------- End Simulation Statistics ----------") 894 final_tick_regex = re.compile("^final_tick\s+(\d+)") 895 896 global ticks_in_ns 897 sim_freq_regex = re.compile("^sim_freq\s+(\d+)") 898 sim_freq = -1 899 900 try: 901 if ext == ".gz": 902 f = gzip.open(gem5_stats_file, "r") 903 else: 904 f = open(gem5_stats_file, "r") 905 except: 906 print "ERROR opening stats file", gem5_stats_file, "!" 907 sys.exit(1) 908 909 stats_not_found_list = stats.stats_list[:] 910 window_num = 0 911 912 while (True): 913 error = False 914 try: 915 line = f.readline() 916 except IOError: 917 print "" 918 print "WARNING: IO error in stats file" 919 print "(gzip stream not closed properly?)...continuing for now" 920 error = True 921 if not line: 922 break 923 924 # Find out how many gem5 ticks in 1ns 925 if sim_freq < 0: 926 m = sim_freq_regex.match(line) 927 if m: 928 sim_freq = int(m.group(1)) # ticks in 1 sec 929 ticks_in_ns = int(sim_freq / 1e9) 930 print "Simulation frequency found! 1 tick == %e sec\n" \ 931 % (1.0 / sim_freq) 932 933 # Final tick in gem5 stats: current absolute timestamp 934 m = final_tick_regex.match(line) 935 if m: 936 tick = int(m.group(1)) 937 if tick > end_tick: 938 break 939 stats.tick_list.append(tick) 940 941 942 if (window_end_regex.match(line) or error): 943 if args.verbose: 944 print "new window" 945 for stat in stats.stats_list: 946 if stat.per_cpu: 947 for i in range(num_cpus): 948 if not stat.per_cpu_found[i]: 949 if not stat.not_found_at_least_once: 950 print "WARNING: stat not found in window #", \ 951 window_num, ":", stat.per_cpu_name[i] 952 print "suppressing further warnings for " + \ 953 "this stat" 954 stat.not_found_at_least_once = True 955 stat.values[i].append(str(0)) 956 stat.per_cpu_found[i] = False 957 else: 958 if not stat.found: 959 if not stat.not_found_at_least_once: 960 print "WARNING: stat not found in window #", \ 961 window_num, ":", stat.name 962 print "suppressing further warnings for this stat" 963 stat.not_found_at_least_once = True 964 stat.values.append(str(0)) 965 stat.found = False 966 stats_not_found_list = stats.stats_list[:] 967 window_num += 1 968 if error: 969 break 970 971 # Do a single regex of the union of all stats first for speed 972 if stats.regex.match(line): 973 # Then loop through only the stats we haven't seen in this window 974 for stat in stats_not_found_list[:]: 975 if stat.per_cpu: 976 for i in range(num_cpus): 977 m = stat.per_cpu_regex[i].match(line) 978 if m: 979 if stat.name == "ipc": 980 value = str(int(float(m.group(1)) * 1000)) 981 else: 982 value = str(int(float(m.group(1)))) 983 if args.verbose: 984 print stat.per_cpu_name[i], value 985 stat.values[i].append(value) 986 stat.per_cpu_found[i] = True 987 all_found = True 988 for j in range(num_cpus): 989 if not stat.per_cpu_found[j]: 990 all_found = False 991 if all_found: 992 stats_not_found_list.remove(stat) 993 if stat.description == "": 994 stat.description = m.group(2) 995 else: 996 m = stat.regex.match(line) 997 if m: 998 value = str(int(float(m.group(1)))) 999 if args.verbose: 1000 print stat.name, value 1001 stat.values.append(value) 1002 stat.found = True 1003 stats_not_found_list.remove(stat) 1004 if stat.description == "": 1005 stat.description = m.group(2) 1006 f.close() 1007 1008 1009# Create session.xml file in .apc folder 1010def doSessionXML(output_path): 1011 session_file = output_path + "/session.xml" 1012 1013 xml = ET.Element("session") 1014 1015 xml.set("version", "1") 1016 xml.set("call_stack_unwinding", "no") 1017 xml.set("parse_debug_info", "no") 1018 xml.set("high_resolution", "yes") 1019 xml.set("buffer_mode", "streaming") 1020 xml.set("sample_rate", "low") 1021 1022 # Setting duration to zero for now. Doesn't affect visualization. 1023 xml.set("duration", "0") 1024 1025 xml.set("target_host", "") 1026 xml.set("target_port", "8080") 1027 1028 writeXmlFile(xml, session_file) 1029 1030 1031# Create captured.xml file in .apc folder 1032def doCapturedXML(output_path, stats): 1033 captured_file = output_path + "/captured.xml" 1034 1035 xml = ET.Element("captured") 1036 xml.set("version", "1") 1037 xml.set("protocol", "17") 1038 xml.set("backtrace_processing", "none") 1039 1040 target = ET.SubElement(xml, "target") 1041 target.set("name", "gem5") 1042 target.set("sample_rate", "1000") 1043 target.set("cores", str(num_cpus)) 1044 1045 counters = ET.SubElement(xml, "counters") 1046 for stat in stats.stats_list: 1047 s = ET.SubElement(counters, "counter") 1048 stat_name = re.sub("\.", "_", stat.short_name) 1049 s.set("title", stat.group) 1050 s.set("name", stat_name) 1051 s.set("color", "0x00000000") 1052 s.set("key", "0x%08x" % stat.key) 1053 s.set("type", stat_name) 1054 s.set("event", "0x00000000") 1055 if stat.per_cpu: 1056 s.set("per_cpu", "yes") 1057 else: 1058 s.set("per_cpu", "no") 1059 s.set("display", "") 1060 s.set("units", "") 1061 s.set("average_selection", "no") 1062 s.set("description", stat.description) 1063 1064 writeXmlFile(xml, captured_file) 1065 1066# Writes out Streamline cookies (unique IDs per process/thread) 1067def writeCookiesThreads(blob): 1068 thread_list = [] 1069 for process in process_list: 1070 if process.uid > 0: 1071 print "cookie", process.task_name, process.uid 1072 writeBinary(blob, cookieNameFrame(process.uid, process.task_name)) 1073 1074 # pid and tgid need to be positive values -- no longer true? 1075 for thread in process.children: 1076 thread_list.append(thread) 1077 1078 # Threads need to be sorted in timestamp order 1079 thread_list.sort(key = lambda x: x.tick) 1080 for thread in thread_list: 1081 print "thread", thread.task_name, (ticksToNs(thread.tick)),\ 1082 thread.tgid, thread.pid 1083 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\ 1084 thread.pid, thread.task_name)) 1085 1086# Writes context switch info as Streamline scheduling events 1087def writeSchedEvents(blob): 1088 for cpu in range(num_cpus): 1089 for event in event_list[cpu]: 1090 timestamp = ticksToNs(event.tick) 1091 pid = event.task.tgid 1092 tid = event.task.pid 1093 if process_dict.has_key(event.task.tgid): 1094 cookie = process_dict[event.task.tgid].uid 1095 else: 1096 cookie = 0 1097 1098 # State: 1099 # 0: waiting on other event besides I/O 1100 # 1: Contention/pre-emption 1101 # 2: Waiting on I/O 1102 # 3: Waiting on mutex 1103 # Hardcoding to 0 for now. Other states not implemented yet. 1104 state = 0 1105 1106 if args.verbose: 1107 print cpu, timestamp, pid, tid, cookie 1108 1109 writeBinary(blob,\ 1110 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state)) 1111 1112# Writes selected gem5 statistics as Streamline counters 1113def writeCounters(blob, stats): 1114 timestamp_list = [] 1115 for tick in stats.tick_list: 1116 if tick > end_tick: 1117 break 1118 timestamp_list.append(ticksToNs(tick)) 1119 1120 for stat in stats.stats_list: 1121 if stat.per_cpu: 1122 stat_length = len(stat.values[0]) 1123 else: 1124 stat_length = len(stat.values) 1125 1126 for n in range(len(timestamp_list)): 1127 for stat in stats.stats_list: 1128 if stat.per_cpu: 1129 for i in range(num_cpus): 1130 writeBinary(blob, counterFrame(timestamp_list[n], i, \ 1131 stat.key, int(float(stat.values[i][n])))) 1132 else: 1133 writeBinary(blob, counterFrame(timestamp_list[n], 0, \ 1134 stat.key, int(float(stat.values[n])))) 1135 1136# Streamline can display LCD frame buffer dumps (gzipped bmp) 1137# This function converts the frame buffer dumps to the Streamline format 1138def writeVisualAnnotations(blob, input_path, output_path): 1139 frame_path = input_path + "/frames_system.vncserver" 1140 if not os.path.exists(frame_path): 1141 return 1142 1143 frame_count = 0 1144 file_list = os.listdir(frame_path) 1145 file_list.sort() 1146 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz") 1147 1148 # Use first non-negative pid to tag visual annotations 1149 annotate_pid = -1 1150 for e in unified_event_list: 1151 pid = e.task.pid 1152 if pid >= 0: 1153 annotate_pid = pid 1154 break 1155 1156 for fn in file_list: 1157 m = re_fb.match(fn) 1158 if m: 1159 seq = m.group(1) 1160 tick = int(m.group(2)) 1161 if tick > end_tick: 1162 break 1163 frame_count += 1 1164 1165 userspace_body = [] 1166 userspace_body += packed32(0x1C) # escape code 1167 userspace_body += packed32(0x04) # visual code 1168 1169 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz" 1170 userspace_body += int16(len(text_annotation)) 1171 userspace_body += utf8StringList(text_annotation) 1172 1173 if gzipped_bmp_supported: 1174 # copy gzipped bmp directly 1175 bytes_read = open(frame_path + "/" + fn, "rb").read() 1176 else: 1177 # copy uncompressed bmp 1178 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read() 1179 1180 userspace_body += int32(len(bytes_read)) 1181 userspace_body += bytes_read 1182 1183 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \ 1184 len(userspace_body), userspace_body)) 1185 1186 print "\nfound", frame_count, "frames for visual annotation.\n" 1187 1188 1189def createApcProject(input_path, output_path, stats): 1190 initOutput(output_path) 1191 1192 blob = open(output_path + "/0000000000", "wb") 1193 1194 # Summary frame takes current system time and system uptime. 1195 # Filling in with random values for now. 1196 writeBinary(blob, summaryFrame(1234, 5678)) 1197 1198 writeCookiesThreads(blob) 1199 1200 print "writing Events" 1201 writeSchedEvents(blob) 1202 1203 print "writing Counters" 1204 writeCounters(blob, stats) 1205 1206 print "writing Visual Annotations" 1207 writeVisualAnnotations(blob, input_path, output_path) 1208 1209 doSessionXML(output_path) 1210 doCapturedXML(output_path, stats) 1211 1212 blob.close() 1213 1214 1215 1216####################### 1217# Main Routine 1218 1219input_path = args.input_path 1220output_path = args.output_path 1221 1222#### 1223# Make sure input path exists 1224#### 1225if not os.path.exists(input_path): 1226 print "ERROR: Input path %s does not exist!" % input_path 1227 sys.exit(1) 1228 1229#### 1230# Parse gem5 configuration file to find # of CPUs and L2s 1231#### 1232(num_cpus, num_l2) = parseConfig(input_path + "/config.ini") 1233 1234#### 1235# Parse task file to find process/thread info 1236#### 1237parseProcessInfo(input_path + "/system.tasks.txt") 1238 1239#### 1240# Parse stat config file and register stats 1241#### 1242stat_config_file = args.stat_config_file 1243stats = registerStats(stat_config_file) 1244 1245#### 1246# Parse gem5 stats 1247#### 1248# Check if both stats.txt and stats.txt.gz exist and warn if both exist 1249if os.path.exists(input_path + "/stats.txt") and \ 1250 os.path.exists(input_path + "/stats.txt.gz"): 1251 print "WARNING: Both stats.txt.gz and stats.txt exist. \ 1252 Using stats.txt.gz by default." 1253 1254gem5_stats_file = input_path + "/stats.txt.gz" 1255if not os.path.exists(gem5_stats_file): 1256 gem5_stats_file = input_path + "/stats.txt" 1257if not os.path.exists(gem5_stats_file): 1258 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path 1259 sys.exit(1) 1260 1261readGem5Stats(stats, gem5_stats_file) 1262 1263#### 1264# Create Streamline .apc project folder 1265#### 1266createApcProject(input_path, output_path, stats) 1267 1268print "All done!" 1269