2 3# Copyright (c) 2012, 2014 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Author: Dam Sunwoo 39# 40 41# This script converts gem5 output to ARM DS-5 Streamline .apc project file 42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and 43# some patches applied to target Linux kernel.) 44# Visit http://www.gem5.org/Streamline for more details. 45# 46# Usage: 47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder> 48# 49# <stat_config.ini>: .ini file that describes which stats to be included 50# in conversion. Sample .ini files can be found in 51# util/streamline. 52# NOTE: this is NOT the gem5 config.ini file. 53# 54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini, 55# stats.txt[.gz], and system.tasks.txt.) 56# 57# <dest .apc folder>: Destination .apc folder path 58# 59# APC project generation based on Gator v17 (DS-5 v5.17) 60# Subsequent versions should be backward compatible 61 62import re, sys, os 63from ConfigParser import ConfigParser 64import gzip 65import xml.etree.ElementTree as ET 66import xml.dom.minidom as minidom 67import shutil 68import zlib 69 70import argparse 71 72parser = argparse.ArgumentParser( 73 formatter_class=argparse.RawDescriptionHelpFormatter, 74 description=""" 75 Converts gem5 runs to ARM DS-5 Streamline .apc project file. 76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump 77 enabled and some patches applied to the target Linux kernel.) 78 79 Visit http://www.gem5.org/Streamline for more details. 80 81 APC project generation based on Gator v17 (DS-5 v5.17) 82 Subsequent versions should be backward compatible 83 """) 84 85parser.add_argument("stat_config_file", metavar="<stat_config.ini>", 86 help=".ini file that describes which stats to be included \ 87 in conversion. Sample .ini files can be found in \ 88 util/streamline. NOTE: this is NOT the gem5 config.ini \ 89 file.") 90 91parser.add_argument("input_path", metavar="<gem5 run folder>", 92 help="Path to gem5 run folder (must contain config.ini, \ 93 stats.txt[.gz], and system.tasks.txt.)") 94 95parser.add_argument("output_path", metavar="<dest .apc folder>", 96 help="Destination .apc folder path") 97 98parser.add_argument("--num-events", action="store", type=int, 99 default=1000000, 100 help="Maximum number of scheduling (context switch) \ 101 events to be processed. Set to truncate early. \ 102 Default=1000000") 103 104parser.add_argument("--gzipped-bmp-not-supported", action="store_true", 105 help="Do not use gzipped .bmp files for visual annotations. \ 106 This option is only required when using Streamline versions \ 107 older than 5.14") 108 109parser.add_argument("--verbose", action="store_true", 110 help="Enable verbose output") 111 112args = parser.parse_args() 113 114if not re.match("(.*)\.apc", args.output_path): 115 print "ERROR: <dest .apc folder> should end with '.apc'!" 116 sys.exit(1) 117 118# gzipped BMP files for visual annotation is supported in Streamline 5.14. 119# Setting this to True will significantly compress the .apc binary file that 120# includes frame buffer snapshots. 121gzipped_bmp_supported = not args.gzipped_bmp_not_supported 122 123ticks_in_ns = -1 124 125# Default max # of events. Increase this for longer runs. 126num_events = args.num_events 127 128start_tick = -1 129end_tick = -1 130 131# Parse gem5 config.ini file to determine some system configurations. 132# Number of CPUs, L2s, etc. 133def parseConfig(config_file): 134 global num_cpus, num_l2 135 136 print "\n===============================" 137 print "Parsing gem5 config.ini file..." 138 print config_file 139 print "===============================\n" 140 config = ConfigParser() 141 if not config.read(config_file): 142 print "ERROR: config file '", config_file, "' not found" 143 sys.exit(1) 144 145 if config.has_section("system.cpu"): 146 num_cpus = 1 147 else: 148 num_cpus = 0 149 while config.has_section("system.cpu" + str(num_cpus)): 150 num_cpus += 1 151 152 if config.has_section("system.l2_cache"): 153 num_l2 = 1 154 else: 155 num_l2 = 0 156 while config.has_section("system.l2_cache" + str(num_l2)): 157 num_l2 += 1 158 159 print "Num CPUs:", num_cpus 160 print "Num L2s:", num_l2 161 print "" 162 163 return (num_cpus, num_l2) 164 165 166process_dict = {} 167thread_dict = {} 168 169process_list = [] 170 171idle_uid = -1 172kernel_uid = -1 173 174class Task(object): 175 def __init__(self, uid, pid, tgid, task_name, is_process, tick): 176 if pid == 0: # Idle 177 self.uid = 0 178 elif pid == -1: # Kernel 179 self.uid = 0 180 else: 181 self.uid = uid 182 self.pid = pid 183 self.tgid = tgid 184 self.is_process = is_process 185 self.task_name = task_name 186 self.children = [] 187 self.tick = tick # time this task first appeared 188 189class Event(object): 190 def __init__(self, tick, task): 191 self.tick = tick 192 self.task = task 193 194############################################################ 195# Types used in APC Protocol 196# - packed32, packed64 197# - int32 198# - string 199############################################################ 200 201def packed32(x): 202 ret = [] 203 more = True 204 while more: 205 b = x & 0x7f 206 x = x >> 7 207 if (((x == 0) and ((b & 0x40) == 0)) or \ 208 ((x == -1) and ((b & 0x40) != 0))): 209 more = False 210 else: 211 b = b | 0x80 212 ret.append(b) 213 return ret 214 215# For historical reasons, 32/64-bit versions of functions are presevered 216def packed64(x): 217 return packed32(x) 218 219# variable length packed 4-byte signed value 220def unsigned_packed32(x): 221 ret = [] 222 if ((x & 0xffffff80) == 0): 223 ret.append(x & 0x7f) 224 elif ((x & 0xffffc000) == 0): 225 ret.append((x | 0x80) & 0xff) 226 ret.append((x >> 7) & 0x7f) 227 elif ((x & 0xffe00000) == 0): 228 ret.append((x | 0x80) & 0xff) 229 ret.append(((x >> 7) | 0x80) & 0xff) 230 ret.append((x >> 14) & 0x7f) 231 elif ((x & 0xf0000000) == 0): 232 ret.append((x | 0x80) & 0xff) 233 ret.append(((x >> 7) | 0x80) & 0xff) 234 ret.append(((x >> 14) | 0x80) & 0xff) 235 ret.append((x >> 21) & 0x7f) 236 else: 237 ret.append((x | 0x80) & 0xff) 238 ret.append(((x >> 7) | 0x80) & 0xff) 239 ret.append(((x >> 14) | 0x80) & 0xff) 240 ret.append(((x >> 21) | 0x80) & 0xff) 241 ret.append((x >> 28) & 0x0f) 242 return ret 243 244# variable length packed 8-byte signed value 245def unsigned_packed64(x): 246 ret = [] 247 if ((x & 0xffffffffffffff80) == 0): 248 ret.append(x & 0x7f) 249 elif ((x & 0xffffffffffffc000) == 0): 250 ret.append((x | 0x80) & 0xff) 251 ret.append((x >> 7) & 0x7f) 252 elif ((x & 0xffffffffffe00000) == 0): 253 ret.append((x | 0x80) & 0xff) 254 ret.append(((x >> 7) | 0x80) & 0xff) 255 ret.append((x >> 14) & 0x7f) 256 elif ((x & 0xfffffffff0000000) == 0): 257 ret.append((x | 0x80) & 0xff) 258 ret.append(((x >> 7) | 0x80) & 0xff) 259 ret.append(((x >> 14) | 0x80) & 0xff) 260 ret.append((x >> 21) & 0x7f) 261 elif ((x & 0xfffffff800000000) == 0): 262 ret.append((x | 0x80) & 0xff) 263 ret.append(((x >> 7) | 0x80) & 0xff) 264 ret.append(((x >> 14) | 0x80) & 0xff) 265 ret.append(((x >> 21) | 0x80) & 0xff) 266 ret.append((x >> 28) & 0x7f) 267 elif ((x & 0xfffffc0000000000) == 0): 268 ret.append((x | 0x80) & 0xff) 269 ret.append(((x >> 7) | 0x80) & 0xff) 270 ret.append(((x >> 14) | 0x80) & 0xff) 271 ret.append(((x >> 21) | 0x80) & 0xff) 272 ret.append(((x >> 28) | 0x80) & 0xff) 273 ret.append((x >> 35) & 0x7f) 274 elif ((x & 0xfffe000000000000) == 0): 275 ret.append((x | 0x80) & 0xff) 276 ret.append(((x >> 7) | 0x80) & 0xff) 277 ret.append(((x >> 14) | 0x80) & 0xff) 278 ret.append(((x >> 21) | 0x80) & 0xff) 279 ret.append(((x >> 28) | 0x80) & 0xff) 280 ret.append(((x >> 35) | 0x80) & 0xff) 281 ret.append((x >> 42) & 0x7f) 282 elif ((x & 0xff00000000000000) == 0): 283 ret.append((x | 0x80) & 0xff) 284 ret.append(((x >> 7) | 0x80) & 0xff) 285 ret.append(((x >> 14) | 0x80) & 0xff) 286 ret.append(((x >> 21) | 0x80) & 0xff) 287 ret.append(((x >> 28) | 0x80) & 0xff) 288 ret.append(((x >> 35) | 0x80) & 0xff) 289 ret.append(((x >> 42) | 0x80) & 0xff) 290 ret.append((x >> 49) & 0x7f) 291 elif ((x & 0x8000000000000000) == 0): 292 ret.append((x | 0x80) & 0xff) 293 ret.append(((x >> 7) | 0x80) & 0xff) 294 ret.append(((x >> 14) | 0x80) & 0xff) 295 ret.append(((x >> 21) | 0x80) & 0xff) 296 ret.append(((x >> 28) | 0x80) & 0xff) 297 ret.append(((x >> 35) | 0x80) & 0xff) 298 ret.append(((x >> 42) | 0x80) & 0xff) 299 ret.append(((x >> 49) | 0x80) & 0xff) 300 ret.append((x >> 56) & 0x7f) 301 else: 302 ret.append((x | 0x80) & 0xff) 303 ret.append(((x >> 7) | 0x80) & 0xff) 304 ret.append(((x >> 14) | 0x80) & 0xff) 305 ret.append(((x >> 21) | 0x80) & 0xff) 306 ret.append(((x >> 28) | 0x80) & 0xff) 307 ret.append(((x >> 35) | 0x80) & 0xff) 308 ret.append(((x >> 42) | 0x80) & 0xff) 309 ret.append(((x >> 49) | 0x80) & 0xff) 310 ret.append(((x >> 56) | 0x80) & 0xff) 311 ret.append((x >> 63) & 0x7f) 312 return ret 313 314# 4-byte signed little endian 315def int32(x): 316 ret = [] 317 ret.append(x & 0xff) 318 ret.append((x >> 8) & 0xff) 319 ret.append((x >> 16) & 0xff) 320 ret.append((x >> 24) & 0xff) 321 return ret 322 323# 2-byte signed little endian 324def int16(x): 325 ret = [] 326 ret.append(x & 0xff) 327 ret.append((x >> 8) & 0xff) 328 return ret 329 330# a packed32 length followed by the specified number of characters 331def stringList(x): 332 ret = [] 333 ret += packed32(len(x)) 334 for i in x: 335 ret.append(i) 336 return ret 337 338def utf8StringList(x): 339 ret = [] 340 for i in x: 341 ret.append(ord(i)) 342 return ret 343 344# packed64 time value in nanoseconds relative to the uptime from the 345# Summary message. 346def timestampList(x): 347 ret = packed64(x) 348 return ret 349 350 351############################################################ 352# Write binary 353############################################################ 354 355def writeBinary(outfile, binary_list): 356 for i in binary_list: 357 outfile.write("%c" % i) 358 359############################################################ 360# APC Protocol Frame Types 361############################################################ 362 363def addFrameHeader(frame_type, body, core): 364 ret = [] 365 366 if frame_type == "Summary": 367 code = 1 368 elif frame_type == "Backtrace": 369 code = 2 370 elif frame_type == "Name": 371 code = 3 372 elif frame_type == "Counter": 373 code = 4 374 elif frame_type == "Block Counter": 375 code = 5 376 elif frame_type == "Annotate": 377 code = 6 378 elif frame_type == "Sched Trace": 379 code = 7 380 elif frame_type == "GPU Trace": 381 code = 8 382 elif frame_type == "Idle": 383 code = 9 384 else: 385 print "ERROR: Unknown frame type:", frame_type 386 sys.exit(1) 387 388 packed_code = packed32(code) 389 390 packed_core = packed32(core) 391 392 length = int32(len(packed_code) + len(packed_core) + len(body)) 393 394 ret = length + packed_code + packed_core + body 395 return ret 396 397 398# Summary frame 399# - timestamp: packed64 400# - uptime: packed64 401def summaryFrame(timestamp, uptime): 402 frame_type = "Summary" 403 newline_canary = stringList("1\n2\r\n3\r4\n\r5") 404 monotonic_delta = packed64(0) 405 end_of_attr = stringList("") 406 body = newline_canary + packed64(timestamp) + packed64(uptime) 407 body += monotonic_delta + end_of_attr 408 ret = addFrameHeader(frame_type, body, 0) 409 return ret 410 411# Backtrace frame 412# - not implemented yet 413def backtraceFrame(): 414 pass 415 416# Cookie name message 417# - cookie: packed32 418# - name: string 419def cookieNameFrame(cookie, name): 420 frame_type = "Name" 421 packed_code = packed32(1) 422 body = packed_code + packed32(cookie) + stringList(name) 423 ret = addFrameHeader(frame_type, body, 0) 424 return ret 425 426# Thread name message 427# - timestamp: timestamp 428# - thread id: packed32 429# - name: string 430def threadNameFrame(timestamp, thread_id, name): 431 frame_type = "Name" 432 packed_code = packed32(2) 433 body = packed_code + timestampList(timestamp) + \ 434 packed32(thread_id) + stringList(name) 435 ret = addFrameHeader(frame_type, body, 0) 436 return ret 437 438# Core name message 439# - name: string 440# - core_id: packed32 441# - cpuid: packed32 442def coreNameFrame(name, core_id, cpuid): 443 frame_type = "Name" 444 packed_code = packed32(3) 445 body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name) 446 ret = addFrameHeader(frame_type, body, 0) 447 return ret 448 449# IRQ Cookie name message 450# - cookie: packed32 451# - name: string 452# - irq: packed32 453def irqCookieNameFrame(cookie, name, irq): 454 frame_type = "Name" 455 packed_code = packed32(5) 456 body = packed_code + packed32(cookie) + stringList(name) + packed32(irq) 457 ret = addFrameHeader(frame_type, body, 0) 458 return ret 459 460# Counter frame message 461# - timestamp: timestamp 462# - core: packed32 463# - key: packed32 464# - value: packed64 465def counterFrame(timestamp, core, key, value): 466 frame_type = "Counter" 467 body = timestampList(timestamp) + packed32(core) + packed32(key) + \ 468 packed64(value) 469 ret = addFrameHeader(frame_type, body, core) 470 return ret 471 472# Block Counter frame message 473# - key: packed32 474# - value: packed64 475def blockCounterFrame(core, key, value): 476 frame_type = "Block Counter" 477 body = packed32(key) + packed64(value) 478 ret = addFrameHeader(frame_type, body, core) 479 return ret 480 481# Annotate frame messages 482# - core: packed32 483# - tid: packed32 484# - timestamp: timestamp 485# - size: packed32 486# - body 487def annotateFrame(core, tid, timestamp, size, userspace_body): 488 frame_type = "Annotate" 489 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \ 490 packed32(size) + userspace_body 491 ret = addFrameHeader(frame_type, body, core) 492 return ret 493 494# Scheduler Trace frame messages 495# Sched Switch 496# - Code: 1 497# - timestamp: timestamp 498# - pid: packed32 499# - tid: packed32 500# - cookie: packed32 501# - state: packed32 502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state): 503 frame_type = "Sched Trace" 504 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \ 505 packed32(tid) + packed32(cookie) + packed32(state) 506 ret = addFrameHeader(frame_type, body, core) 507 return ret 508 509# Sched Thread Exit 510# - Code: 2 511# - timestamp: timestamp 512# - tid: packed32 513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state): 514 frame_type = "Sched Trace" 515 body = packed32(2) + timestampList(timestamp) + packed32(tid) 516 ret = addFrameHeader(frame_type, body, core) 517 return ret 518 519# GPU Trace frame messages 520# - Not implemented yet 521def gpuTraceFrame(): 522 pass 523 524# Idle frame messages 525# Enter Idle 526# - code: 1 527# - timestamp: timestamp 528# - core: packed32 529def enterIdleFrame(timestamp, core): 530 frame_type = "Idle" 531 body = packed32(1) + timestampList(timestamp) + packed32(core) 532 ret = addFrameHeader(frame_type, body, core) 533 return ret 534 535# Exit Idle 536# - code: 2 537# - timestamp: timestamp 538# - core: packed32 539def exitIdleFrame(timestamp, core): 540 frame_type = "Idle" 541 body = packed32(2) + timestampList(timestamp) + packed32(core) 542 ret = addFrameHeader(frame_type, body, core) 543 return ret 544 545 546#################################################################### 547def parseProcessInfo(task_file): 548 print "\n===============================" 549 print "Parsing Task file..." 550 print task_file 551 print "===============================\n" 552 553 global start_tick, end_tick, num_cpus 554 global process_dict, thread_dict, process_list 555 global event_list, unified_event_list 556 global idle_uid, kernel_uid 557 558 event_list = [] 559 unified_event_list = [] 560 for cpu in range(num_cpus): 561 event_list.append([]) 562 563 uid = 1 # uid 0 is reserved for idle 564 565 # Dummy Tasks for frame buffers and system diagrams 566 process = Task(uid, 9999, 9999, "framebuffer", True, 0) 567 process_list.append(process) 568 uid += 1 569 thread = Task(uid, 9999, 9999, "framebuffer", False, 0) 570 process.children.append(thread) 571 uid += 1 572 process = Task(uid, 9998, 9998, "System", True, 0) 573 process_list.append(process) 574 # if we don't find the real kernel, use this to keep things going 575 kernel_uid = uid 576 uid += 1 577 thread = Task(uid, 9998, 9998, "System", False, 0) 578 process.children.append(thread) 579 uid += 1 580 581 ext = os.path.splitext(task_file)[1] 582 583 try: 584 if ext == ".gz": 585 process_file = gzip.open(task_file, 'rb') 586 else: 587 process_file = open(task_file, 'rb') 588 except: 589 print "ERROR opening task file:", task_file 590 print "Make sure context switch task dumping is enabled in gem5." 591 sys.exit(1) 592 593 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" + 594 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)") 595 596 task_name_failure_warned = False 597 598 for line in process_file: 599 match = re.match(process_re, line) 600 if match: 601 tick = int(match.group(1)) 602 if (start_tick < 0): 603 start_tick = tick 604 cpu_id = int(match.group(3)) 605 pid = int(match.group(4)) 606 tgid = int(match.group(5)) 607 task_name = match.group(6) 608 609 if not task_name_failure_warned: 610 if task_name == "FailureIn_curTaskName": 611 print "-------------------------------------------------" 612 print "WARNING: Task name not set correctly!" 613 print "Process/Thread info will not be displayed correctly" 614 print "Perhaps forgot to apply m5struct.patch to kernel?" 615 print "-------------------------------------------------" 616 task_name_failure_warned = True 617 618 if not tgid in process_dict: 619 if tgid == pid: 620 # new task is parent as well 621 if args.verbose: 622 print "new process", uid, pid, tgid, task_name 623 if tgid == 0: 624 # new process is the "idle" task 625 process = Task(uid, pid, tgid, "idle", True, tick) 626 idle_uid = 0 627 else: 628 process = Task(uid, pid, tgid, task_name, True, tick) 629 else: 630 if tgid == 0: 631 process = Task(uid, tgid, tgid, "idle", True, tick) 632 idle_uid = 0 633 else: 634 # parent process name not known yet 635 process = Task(uid, tgid, tgid, "_Unknown_", True, tick) 636 if tgid == -1: # kernel 637 kernel_uid = 0 638 uid += 1 639 process_dict[tgid] = process 640 process_list.append(process) 641 else: 642 if tgid == pid: 643 if process_dict[tgid].task_name == "_Unknown_": 644 if args.verbose: 645 print "new process", \ 646 process_dict[tgid].uid, pid, tgid, task_name 647 process_dict[tgid].task_name = task_name 648 if process_dict[tgid].task_name != task_name and tgid != 0: 649 process_dict[tgid].task_name = task_name 650 651 if not pid in thread_dict: 652 if args.verbose: 653 print "new thread", \ 654 uid, process_dict[tgid].uid, pid, tgid, task_name 655 thread = Task(uid, pid, tgid, task_name, False, tick) 656 uid += 1 657 thread_dict[pid] = thread 658 process_dict[tgid].children.append(thread) 659 else: 660 if thread_dict[pid].task_name != task_name: 661 thread_dict[pid].task_name = task_name 662 663 if args.verbose: 664 print tick, uid, cpu_id, pid, tgid, task_name 665 666 task = thread_dict[pid] 667 event = Event(tick, task) 668 event_list[cpu_id].append(event) 669 unified_event_list.append(event) 670 671 if len(unified_event_list) == num_events: 672 print "Truncating at", num_events, "events!" 673 break 674 print "Found %d events." % len(unified_event_list) 675 676 for process in process_list: 677 if process.pid > 9990: # fix up framebuffer ticks 678 process.tick = start_tick 679 print process.uid, process.pid, process.tgid, \ 680 process.task_name, str(process.tick) 681 for thread in process.children: 682 if thread.pid > 9990: 683 thread.tick = start_tick 684 print "\t", thread.uid, thread.pid, thread.tgid, \ 685 thread.task_name, str(thread.tick) 686 687 end_tick = tick 688 689 print "Start tick:", start_tick 690 print "End tick: ", end_tick 691 print "" 692 693 return 694 695 696def initOutput(output_path): 697 if not os.path.exists(output_path): 698 os.mkdir(output_path) 699 700def ticksToNs(tick): 701 if ticks_in_ns < 0: 702 print "ticks_in_ns not set properly!" 703 sys.exit(1) 704 705 return tick / ticks_in_ns 706 707def writeXmlFile(xml, filename): 708 f = open(filename, "w") 709 txt = ET.tostring(xml) 710 f.write(minidom.parseString(txt).toprettyxml()) 711 f.close() 712 713 714# StatsEntry that contains individual statistics 715class StatsEntry(object): 716 def __init__(self, name, group, group_index, per_cpu, key): 717 718 # Full name of statistics 719 self.name = name 720 721 # Streamline group name that statistic will belong to 722 self.group = group 723 724 # Index of statistics within group (used to change colors within groups) 725 self.group_index = group_index 726 727 # Shorter name with "system" stripped off 728 # and symbols converted to alphanumerics 729 self.short_name = re.sub("system\.", "", name) 730 self.short_name = re.sub(":", "_", name) 731 732 # Regex for this stat (string version used to construct union regex) 733 self.regex_string = "^" + name + "\s+([\d\.]+)" 734 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M) 735 self.description = "" 736 737 # Whether this stat is use per CPU or not 738 self.per_cpu = per_cpu 739 740 # Key used in .apc protocol (as described in captured.xml) 741 self.key = key 742 743 # List of values of stat per timestamp 744 self.values = [] 745 746 # Whether this stat has been found for the current timestamp 747 self.found = False 748 749 # Whether this stat has been found at least once 750 # (to suppress too many warnings) 751 self.not_found_at_least_once = False 752 753 # Field used to hold ElementTree subelement for this stat 754 self.ET_element = None 755 756 # Create per-CPU stat name and regex, etc. 757 if self.per_cpu: 758 self.per_cpu_regex_string = [] 759 self.per_cpu_regex = [] 760 self.per_cpu_name = [] 761 self.per_cpu_found = [] 762 for i in range(num_cpus): 763 if num_cpus > 1: 764 per_cpu_name = re.sub("#", str(i), self.name) 765 else: 766 per_cpu_name = re.sub("#", "", self.name) 767 768 self.per_cpu_name.append(per_cpu_name) 769 print "\t", per_cpu_name 770 771 self.per_cpu_regex_string.\ 772 append("^" + per_cpu_name + "\s+[\d\.]+") 773 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \ 774 "\s+([\d\.e\-]+)\s+# (.*)$", re.M)) 775 self.values.append([]) 776 self.per_cpu_found.append(False) 777 778 def append_value(self, val, per_cpu_index = None): 779 if self.per_cpu: 780 self.values[per_cpu_index].append(str(val)) 781 else: 782 self.values.append(str(val)) 783 784# Global stats object that contains the list of stats entries 785# and other utility functions 786class Stats(object): 787 def __init__(self): 788 self.stats_list = [] 789 self.tick_list = [] 790 self.next_key = 1 791 792 def register(self, name, group, group_index, per_cpu): 793 print "registering stat:", name, "group:", group, group_index 794 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \ 795 self.next_key)) 796 self.next_key += 1 797 798 # Union of all stats to accelerate parsing speed 799 def createStatsRegex(self): 800 regex_strings = []; 801 print "\nnum entries in stats_list", len(self.stats_list) 802 for entry in self.stats_list: 803 if entry.per_cpu: 804 for i in range(num_cpus): 805 regex_strings.append(entry.per_cpu_regex_string[i]) 806 else: 807 regex_strings.append(entry.regex_string) 808 809 self.regex = re.compile('|'.join(regex_strings)) 810 811 812def registerStats(config_file): 813 print "===============================" 814 print "Parsing stats config.ini file..." 815 print config_file 816 print "===============================" 817 818 config = ConfigParser() 819 if not config.read(config_file): 820 print "ERROR: config file '", config_file, "' not found!" 821 sys.exit(1) 822 823 print "\nRegistering Stats..." 824 825 stats = Stats() 826 827 per_cpu_stat_groups = config.options('PER_CPU_STATS') 828 for group in per_cpu_stat_groups: 829 i = 0 830 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n') 831 for item in per_cpu_stats_list: 832 if item: 833 stats.register(item, group, i, True) 834 i += 1 835 836 per_l2_stat_groups = config.options('PER_L2_STATS') 837 for group in per_l2_stat_groups: 838 i = 0 839 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n') 840 for item in per_l2_stats_list: 841 if item: 842 for l2 in range(num_l2): 843 if num_l2 > 1: 844 name = re.sub("#", str(l2), item) 845 else: 846 name = re.sub("#", "", item) 847 stats.register(name, group, i, False) 848 i += 1 849 850 other_stat_groups = config.options('OTHER_STATS') 851 for group in other_stat_groups: 852 i = 0 853 other_stats_list = config.get('OTHER_STATS', group).split('\n') 854 for item in other_stats_list: 855 if item: 856 stats.register(item, group, i, False) 857 i += 1 858 859 stats.createStatsRegex() 860 861 return stats 862 863# Parse and read in gem5 stats file 864# Streamline counters are organized per CPU 865def readGem5Stats(stats, gem5_stats_file): 866 print "\n===============================" 867 print "Parsing gem5 stats file..." 868 print gem5_stats_file 869 print "===============================\n" 870 ext = os.path.splitext(gem5_stats_file)[1] 871 872 window_start_regex = \ 873 re.compile("^---------- Begin Simulation Statistics ----------") 874 window_end_regex = \ 875 re.compile("^---------- End Simulation Statistics ----------") 876 final_tick_regex = re.compile("^final_tick\s+(\d+)") 877 878 global ticks_in_ns 879 sim_freq_regex = re.compile("^sim_freq\s+(\d+)") 880 sim_freq = -1 881 882 try: 883 if ext == ".gz": 884 f = gzip.open(gem5_stats_file, "r") 885 else: 886 f = open(gem5_stats_file, "r") 887 except: 888 print "ERROR opening stats file", gem5_stats_file, "!" 889 sys.exit(1) 890 891 stats_not_found_list = stats.stats_list[:] 892 window_num = 0 893 894 while (True): 895 error = False 896 try: 897 line = f.readline() 898 except IOError: 899 print "" 900 print "WARNING: IO error in stats file" 901 print "(gzip stream not closed properly?)...continuing for now" 902 error = True 903 if not line: 904 break 905 906 # Find out how many gem5 ticks in 1ns 907 if sim_freq < 0: 908 m = sim_freq_regex.match(line) 909 if m: 910 sim_freq = int(m.group(1)) # ticks in 1 sec 911 ticks_in_ns = int(sim_freq / 1e9) 912 print "Simulation frequency found! 1 tick == %e sec\n" \ 913 % (1.0 / sim_freq) 914 915 # Final tick in gem5 stats: current absolute timestamp 916 m = final_tick_regex.match(line) 917 if m: 918 tick = int(m.group(1)) 919 if tick > end_tick: 920 break 921 stats.tick_list.append(tick) 922 923 924 if (window_end_regex.match(line) or error): 925 if args.verbose: 926 print "new window" 927 for stat in stats.stats_list: 928 if stat.per_cpu: 929 for i in range(num_cpus): 930 if not stat.per_cpu_found[i]: 931 if not stat.not_found_at_least_once: 932 print "WARNING: stat not found in window #", \ 933 window_num, ":", stat.per_cpu_name[i] 934 print "suppressing further warnings for " + \ 935 "this stat" 936 stat.not_found_at_least_once = True 937 stat.values[i].append(str(0)) 938 stat.per_cpu_found[i] = False 939 else: 940 if not stat.found: 941 if not stat.not_found_at_least_once: 942 print "WARNING: stat not found in window #", \ 943 window_num, ":", stat.name 944 print "suppressing further warnings for this stat" 945 stat.not_found_at_least_once = True 946 stat.values.append(str(0)) 947 stat.found = False 948 stats_not_found_list = stats.stats_list[:] 949 window_num += 1 950 if error: 951 break 952 953 # Do a single regex of the union of all stats first for speed 954 if stats.regex.match(line): 955 # Then loop through only the stats we haven't seen in this window 956 for stat in stats_not_found_list[:]: 957 if stat.per_cpu: 958 for i in range(num_cpus): 959 m = stat.per_cpu_regex[i].match(line) 960 if m: 961 if stat.name == "ipc": 962 value = str(int(float(m.group(1)) * 1000)) 963 else: 964 value = str(int(float(m.group(1)))) 965 if args.verbose: 966 print stat.per_cpu_name[i], value 967 stat.values[i].append(value) 968 stat.per_cpu_found[i] = True 969 all_found = True 970 for j in range(num_cpus): 971 if not stat.per_cpu_found[j]: 972 all_found = False 973 if all_found: 974 stats_not_found_list.remove(stat) 975 if stat.description == "": 976 stat.description = m.group(2) 977 else: 978 m = stat.regex.match(line) 979 if m: 980 value = str(int(float(m.group(1)))) 981 if args.verbose: 982 print stat.name, value 983 stat.values.append(value) 984 stat.found = True 985 stats_not_found_list.remove(stat) 986 if stat.description == "": 987 stat.description = m.group(2) 988 f.close() 989 990 991# Create session.xml file in .apc folder 992def doSessionXML(output_path): 993 session_file = output_path + "/session.xml" 994 995 xml = ET.Element("session") 996 997 xml.set("version", "1") 998 xml.set("call_stack_unwinding", "no") 999 xml.set("parse_debug_info", "no") 1000 xml.set("high_resolution", "yes") 1001 xml.set("buffer_mode", "streaming") 1002 xml.set("sample_rate", "low") 1003 1004 # Setting duration to zero for now. Doesn't affect visualization. 1005 xml.set("duration", "0") 1006 1007 xml.set("target_host", "") 1008 xml.set("target_port", "8080") 1009 1010 writeXmlFile(xml, session_file) 1011 1012 1013# Create captured.xml file in .apc folder 1014def doCapturedXML(output_path, stats): 1015 captured_file = output_path + "/captured.xml" 1016 1017 xml = ET.Element("captured") 1018 xml.set("version", "1") 1019 xml.set("protocol", "17") 1020 xml.set("backtrace_processing", "none") 1021 1022 target = ET.SubElement(xml, "target") 1023 target.set("name", "gem5") 1024 target.set("sample_rate", "1000") 1025 target.set("cores", str(num_cpus)) 1026 1027 counters = ET.SubElement(xml, "counters") 1028 for stat in stats.stats_list: 1029 s = ET.SubElement(counters, "counter") 1030 stat_name = re.sub("\.", "_", stat.short_name) 1031 stat_name = re.sub("#", "", stat_name) 1032 s.set("title", stat.group) 1033 s.set("name", stat_name) 1034 s.set("color", "0x00000000") 1035 s.set("key", "0x%08x" % stat.key) 1036 s.set("type", stat_name) 1037 s.set("event", "0x00000000") 1038 if stat.per_cpu: 1039 s.set("per_cpu", "yes") 1040 else: 1041 s.set("per_cpu", "no") 1042 s.set("display", "") 1043 s.set("units", "") 1044 s.set("average_selection", "no") 1045 s.set("description", stat.description) 1046 1047 writeXmlFile(xml, captured_file) 1048 1049# Writes out Streamline cookies (unique IDs per process/thread) 1050def writeCookiesThreads(blob): 1051 thread_list = [] 1052 for process in process_list: 1053 if process.uid > 0: 1054 print "cookie", process.task_name, process.uid 1055 writeBinary(blob, cookieNameFrame(process.uid, process.task_name)) 1056 1057 # pid and tgid need to be positive values -- no longer true? 1058 for thread in process.children: 1059 thread_list.append(thread) 1060 1061 # Threads need to be sorted in timestamp order 1062 thread_list.sort(key = lambda x: x.tick) 1063 for thread in thread_list: 1064 print "thread", thread.task_name, (ticksToNs(thread.tick)),\ 1065 thread.tgid, thread.pid 1066 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\ 1067 thread.pid, thread.task_name)) 1068 1069# Writes context switch info as Streamline scheduling events 1070def writeSchedEvents(blob): 1071 for cpu in range(num_cpus): 1072 for event in event_list[cpu]: 1073 timestamp = ticksToNs(event.tick) 1074 pid = event.task.tgid 1075 tid = event.task.pid 1076 if process_dict.has_key(event.task.tgid): 1077 cookie = process_dict[event.task.tgid].uid 1078 else: 1079 cookie = 0 1080 1081 # State: 1082 # 0: waiting on other event besides I/O 1083 # 1: Contention/pre-emption 1084 # 2: Waiting on I/O 1085 # 3: Waiting on mutex 1086 # Hardcoding to 0 for now. Other states not implemented yet. 1087 state = 0 1088 1089 if args.verbose: 1090 print cpu, timestamp, pid, tid, cookie 1091 1092 writeBinary(blob,\ 1093 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state)) 1094 1095# Writes selected gem5 statistics as Streamline counters 1096def writeCounters(blob, stats): 1097 timestamp_list = [] 1098 for tick in stats.tick_list: 1099 if tick > end_tick: 1100 break 1101 timestamp_list.append(ticksToNs(tick)) 1102 1103 for stat in stats.stats_list: 1104 if stat.per_cpu: 1105 stat_length = len(stat.values[0]) 1106 else: 1107 stat_length = len(stat.values) 1108 1109 for n in range(len(timestamp_list)): 1110 for stat in stats.stats_list: 1111 if stat.per_cpu: 1112 for i in range(num_cpus): 1113 writeBinary(blob, counterFrame(timestamp_list[n], i, \ 1114 stat.key, int(float(stat.values[i][n])))) 1115 else: 1116 writeBinary(blob, counterFrame(timestamp_list[n], 0, \ 1117 stat.key, int(float(stat.values[n])))) 1118 1119# Streamline can display LCD frame buffer dumps (gzipped bmp) 1120# This function converts the frame buffer dumps to the Streamline format 1121def writeVisualAnnotations(blob, input_path, output_path): 1122 frame_path = input_path + "/frames_system.vncserver" 1123 if not os.path.exists(frame_path): 1124 return 1125 1126 frame_count = 0 1127 file_list = os.listdir(frame_path) 1128 file_list.sort() 1129 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz") 1130 1131 # Use first non-negative pid to tag visual annotations 1132 annotate_pid = -1 1133 for e in unified_event_list: 1134 pid = e.task.pid 1135 if pid >= 0: 1136 annotate_pid = pid 1137 break 1138 1139 for fn in file_list: 1140 m = re_fb.match(fn) 1141 if m: 1142 seq = m.group(1) 1143 tick = int(m.group(2)) 1144 if tick > end_tick: 1145 break 1146 frame_count += 1 1147 1148 userspace_body = [] 1149 userspace_body += packed32(0x1C) # escape code 1150 userspace_body += packed32(0x04) # visual code 1151 1152 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz" 1153 userspace_body += int16(len(text_annotation)) 1154 userspace_body += utf8StringList(text_annotation) 1155 1156 if gzipped_bmp_supported: 1157 # copy gzipped bmp directly 1158 bytes_read = open(frame_path + "/" + fn, "rb").read() 1159 else: 1160 # copy uncompressed bmp 1161 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read() 1162 1163 userspace_body += int32(len(bytes_read)) 1164 userspace_body += bytes_read 1165 1166 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \ 1167 len(userspace_body), userspace_body)) 1168 1169 print "\nfound", frame_count, "frames for visual annotation.\n" 1170 1171 1172def createApcProject(input_path, output_path, stats): 1173 initOutput(output_path) 1174 1175 blob = open(output_path + "/0000000000", "wb") 1176 1177 # Summary frame takes current system time and system uptime. 1178 # Filling in with random values for now. 1179 writeBinary(blob, summaryFrame(1234, 5678)) 1180 1181 writeCookiesThreads(blob) 1182 1183 print "writing Events" 1184 writeSchedEvents(blob) 1185 1186 print "writing Counters" 1187 writeCounters(blob, stats) 1188 1189 print "writing Visual Annotations" 1190 writeVisualAnnotations(blob, input_path, output_path) 1191 1192 doSessionXML(output_path) 1193 doCapturedXML(output_path, stats) 1194 1195 blob.close() 1196 1197 1198 1199####################### 1200# Main Routine 1201 1202input_path = args.input_path 1203output_path = args.output_path 1204 1205#### 1206# Make sure input path exists 1207#### 1208if not os.path.exists(input_path): 1209 print "ERROR: Input path %s does not exist!" % input_path 1210 sys.exit(1) 1211 1212#### 1213# Parse gem5 configuration file to find # of CPUs and L2s 1214#### 1215(num_cpus, num_l2) = parseConfig(input_path + "/config.ini") 1216 1217#### 1218# Parse task file to find process/thread info 1219#### 1220parseProcessInfo(input_path + "/system.tasks.txt") 1221 1222#### 1223# Parse stat config file and register stats 1224#### 1225stat_config_file = args.stat_config_file 1226stats = registerStats(stat_config_file) 1227 1228#### 1229# Parse gem5 stats 1230#### 1231# Check if both stats.txt and stats.txt.gz exist and warn if both exist 1232if os.path.exists(input_path + "/stats.txt") and \ 1233 os.path.exists(input_path + "/stats.txt.gz"): 1234 print "WARNING: Both stats.txt.gz and stats.txt exist. \ 1235 Using stats.txt.gz by default." 1236 1237gem5_stats_file = input_path + "/stats.txt.gz" 1238if not os.path.exists(gem5_stats_file): 1239 gem5_stats_file = input_path + "/stats.txt" 1240if not os.path.exists(gem5_stats_file): 1241 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path 1242 sys.exit(1) 1243 1244readGem5Stats(stats, gem5_stats_file) 1245 1246#### 1247# Create Streamline .apc project folder 1248#### 1249createApcProject(input_path, output_path, stats) 1250 1251print "All done!"
|