m5stats2streamline.py revision 9935
1#!/usr/bin/env python 2 3# Copyright (c) 2012 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Author: Dam Sunwoo 39# 40 41# This script converts gem5 output to ARM DS-5 Streamline .apc project file 42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and 43# some patches applied to target Linux kernel.) 44# Visit http://www.gem5.org/Streamline for more details. 45# 46# Usage: 47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder> 48# 49# <stat_config.ini>: .ini file that describes which stats to be included 50# in conversion. Sample .ini files can be found in 51# util/streamline. 52# NOTE: this is NOT the gem5 config.ini file. 53# 54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini, 55# stats.txt[.gz], and system.tasks.txt.) 56# 57# <dest .apc folder>: Destination .apc folder path 58# 59# APC project generation based on Gator v12 (DS-5 v5.13) 60# Subsequent versions should be backward compatible 61 62import re, sys, os 63from ConfigParser import ConfigParser 64import gzip 65import xml.etree.ElementTree as ET 66import xml.dom.minidom as minidom 67import shutil 68import zlib 69 70import argparse 71 72parser = argparse.ArgumentParser( 73 formatter_class=argparse.RawDescriptionHelpFormatter, 74 description=""" 75 Converts gem5 runs to ARM DS-5 Streamline .apc project file. 76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump 77 enabled and some patches applied to the target Linux kernel.) 78 79 Visit http://www.gem5.org/Streamline for more details. 80 81 APC project generation based on Gator v12 (DS-5 v5.13) 82 Subsequent versions should be backward compatible 83 """) 84 85parser.add_argument("stat_config_file", metavar="<stat_config.ini>", 86 help=".ini file that describes which stats to be included \ 87 in conversion. Sample .ini files can be found in \ 88 util/streamline. NOTE: this is NOT the gem5 config.ini \ 89 file.") 90 91parser.add_argument("input_path", metavar="<gem5 run folder>", 92 help="Path to gem5 run folder (must contain config.ini, \ 93 stats.txt[.gz], and system.tasks.txt.)") 94 95parser.add_argument("output_path", metavar="<dest .apc folder>", 96 help="Destination .apc folder path") 97 98parser.add_argument("--num-events", action="store", type=int, 99 default=1000000, 100 help="Maximum number of scheduling (context switch) \ 101 events to be processed. Set to truncate early. \ 102 Default=1000000") 103 104parser.add_argument("--gzipped-bmp-not-supported", action="store_true", 105 help="Do not use gzipped .bmp files for visual annotations. \ 106 This option is only required when using Streamline versions \ 107 older than 5.14") 108 109parser.add_argument("--verbose", action="store_true", 110 help="Enable verbose output") 111 112args = parser.parse_args() 113 114if not re.match("(.*)\.apc", args.output_path): 115 print "ERROR: <dest .apc folder> should end with '.apc'!" 116 sys.exit(1) 117 118# gzipped BMP files for visual annotation is supported in Streamline 5.14. 119# Setting this to True will significantly compress the .apc binary file that 120# includes frame buffer snapshots. 121gzipped_bmp_supported = not args.gzipped_bmp_not_supported 122 123ticks_in_ns = -1 124 125# Default max # of events. Increase this for longer runs. 126num_events = args.num_events 127 128start_tick = -1 129end_tick = -1 130 131# Parse gem5 config.ini file to determine some system configurations. 132# Number of CPUs, L2s, etc. 133def parseConfig(config_file): 134 global num_cpus, num_l2 135 136 print "\n===============================" 137 print "Parsing gem5 config.ini file..." 138 print config_file 139 print "===============================\n" 140 config = ConfigParser() 141 if not config.read(config_file): 142 print "ERROR: config file '", config_file, "' not found" 143 sys.exit(1) 144 145 if config.has_section("system.cpu"): 146 num_cpus = 1 147 else: 148 num_cpus = 0 149 while config.has_section("system.cpu" + str(num_cpus)): 150 num_cpus += 1 151 152 if config.has_section("system.l2"): 153 num_l2 = 1 154 else: 155 num_l2 = 0 156 while config.has_section("system.l2" + str(num_l2)): 157 num_l2 += 1 158 159 print "Num CPUs:", num_cpus 160 print "Num L2s:", num_l2 161 print "" 162 163 return (num_cpus, num_l2) 164 165 166process_dict = {} 167thread_dict = {} 168 169process_list = [] 170 171idle_uid = -1 172kernel_uid = -1 173 174class Task(object): 175 def __init__(self, uid, pid, tgid, task_name, is_process, tick): 176 if pid == 0: # Idle 177 self.uid = 0 178 elif pid == -1: # Kernel 179 self.uid = 0 180 else: 181 self.uid = uid 182 self.pid = pid 183 self.tgid = tgid 184 self.is_process = is_process 185 self.task_name = task_name 186 self.children = [] 187 self.tick = tick # time this task first appeared 188 189class Event(object): 190 def __init__(self, tick, task): 191 self.tick = tick 192 self.task = task 193 194############################################################ 195# Types used in APC Protocol 196# - packed32, packed64 197# - int32 198# - string 199############################################################ 200 201# variable length packed 4-byte signed value 202def packed32(x): 203 ret = [] 204 if ((x & 0xffffff80) == 0): 205 ret.append(x & 0x7f) 206 elif ((x & 0xffffc000) == 0): 207 ret.append((x | 0x80) & 0xff) 208 ret.append((x >> 7) & 0x7f) 209 elif ((x & 0xffe00000) == 0): 210 ret.append((x | 0x80) & 0xff) 211 ret.append(((x >> 7) | 0x80) & 0xff) 212 ret.append((x >> 14) & 0x7f) 213 elif ((x & 0xf0000000) == 0): 214 ret.append((x | 0x80) & 0xff) 215 ret.append(((x >> 7) | 0x80) & 0xff) 216 ret.append(((x >> 14) | 0x80) & 0xff) 217 ret.append((x >> 21) & 0x7f) 218 else: 219 ret.append((x | 0x80) & 0xff) 220 ret.append(((x >> 7) | 0x80) & 0xff) 221 ret.append(((x >> 14) | 0x80) & 0xff) 222 ret.append(((x >> 21) | 0x80) & 0xff) 223 ret.append((x >> 28) & 0x0f) 224 return ret 225 226# variable length packed 8-byte signed value 227def packed64(x): 228 ret = [] 229 if ((x & 0xffffffffffffff80) == 0): 230 ret.append(x & 0x7f) 231 elif ((x & 0xffffffffffffc000) == 0): 232 ret.append((x | 0x80) & 0xff) 233 ret.append((x >> 7) & 0x7f) 234 elif ((x & 0xffffffffffe00000) == 0): 235 ret.append((x | 0x80) & 0xff) 236 ret.append(((x >> 7) | 0x80) & 0xff) 237 ret.append((x >> 14) & 0x7f) 238 elif ((x & 0xfffffffff0000000) == 0): 239 ret.append((x | 0x80) & 0xff) 240 ret.append(((x >> 7) | 0x80) & 0xff) 241 ret.append(((x >> 14) | 0x80) & 0xff) 242 ret.append((x >> 21) & 0x7f) 243 elif ((x & 0xfffffff800000000) == 0): 244 ret.append((x | 0x80) & 0xff) 245 ret.append(((x >> 7) | 0x80) & 0xff) 246 ret.append(((x >> 14) | 0x80) & 0xff) 247 ret.append(((x >> 21) | 0x80) & 0xff) 248 ret.append((x >> 28) & 0x7f) 249 elif ((x & 0xfffffc0000000000) == 0): 250 ret.append((x | 0x80) & 0xff) 251 ret.append(((x >> 7) | 0x80) & 0xff) 252 ret.append(((x >> 14) | 0x80) & 0xff) 253 ret.append(((x >> 21) | 0x80) & 0xff) 254 ret.append(((x >> 28) | 0x80) & 0xff) 255 ret.append((x >> 35) & 0x7f) 256 elif ((x & 0xfffe000000000000) == 0): 257 ret.append((x | 0x80) & 0xff) 258 ret.append(((x >> 7) | 0x80) & 0xff) 259 ret.append(((x >> 14) | 0x80) & 0xff) 260 ret.append(((x >> 21) | 0x80) & 0xff) 261 ret.append(((x >> 28) | 0x80) & 0xff) 262 ret.append(((x >> 35) | 0x80) & 0xff) 263 ret.append((x >> 42) & 0x7f) 264 elif ((x & 0xff00000000000000) == 0): 265 ret.append((x | 0x80) & 0xff) 266 ret.append(((x >> 7) | 0x80) & 0xff) 267 ret.append(((x >> 14) | 0x80) & 0xff) 268 ret.append(((x >> 21) | 0x80) & 0xff) 269 ret.append(((x >> 28) | 0x80) & 0xff) 270 ret.append(((x >> 35) | 0x80) & 0xff) 271 ret.append(((x >> 42) | 0x80) & 0xff) 272 ret.append((x >> 49) & 0x7f) 273 elif ((x & 0x8000000000000000) == 0): 274 ret.append((x | 0x80) & 0xff) 275 ret.append(((x >> 7) | 0x80) & 0xff) 276 ret.append(((x >> 14) | 0x80) & 0xff) 277 ret.append(((x >> 21) | 0x80) & 0xff) 278 ret.append(((x >> 28) | 0x80) & 0xff) 279 ret.append(((x >> 35) | 0x80) & 0xff) 280 ret.append(((x >> 42) | 0x80) & 0xff) 281 ret.append(((x >> 49) | 0x80) & 0xff) 282 ret.append((x >> 56) & 0x7f) 283 else: 284 ret.append((x | 0x80) & 0xff) 285 ret.append(((x >> 7) | 0x80) & 0xff) 286 ret.append(((x >> 14) | 0x80) & 0xff) 287 ret.append(((x >> 21) | 0x80) & 0xff) 288 ret.append(((x >> 28) | 0x80) & 0xff) 289 ret.append(((x >> 35) | 0x80) & 0xff) 290 ret.append(((x >> 42) | 0x80) & 0xff) 291 ret.append(((x >> 49) | 0x80) & 0xff) 292 ret.append(((x >> 56) | 0x80) & 0xff) 293 ret.append((x >> 63) & 0x7f) 294 return ret 295 296# 4-byte signed little endian 297def int32(x): 298 ret = [] 299 ret.append(x & 0xff) 300 ret.append((x >> 8) & 0xff) 301 ret.append((x >> 16) & 0xff) 302 ret.append((x >> 24) & 0xff) 303 return ret 304 305# 2-byte signed little endian 306def int16(x): 307 ret = [] 308 ret.append(x & 0xff) 309 ret.append((x >> 8) & 0xff) 310 return ret 311 312# a packed32 length followed by the specified number of characters 313def stringList(x): 314 ret = [] 315 ret += packed32(len(x)) 316 for i in x: 317 ret.append(i) 318 return ret 319 320def utf8StringList(x): 321 ret = [] 322 for i in x: 323 ret.append(ord(i)) 324 return ret 325 326# packed64 time value in nanoseconds relative to the uptime from the 327# Summary message. 328def timestampList(x): 329 ret = packed64(x) 330 return ret 331 332 333############################################################ 334# Write binary 335############################################################ 336 337def writeBinary(outfile, binary_list): 338 for i in binary_list: 339 outfile.write("%c" % i) 340 341############################################################ 342# APC Protocol Frame Types 343############################################################ 344 345def addFrameHeader(frame_type, body, core): 346 ret = [] 347 348 if frame_type == "Summary": 349 code = 1 350 elif frame_type == "Backtrace": 351 code = 2 352 elif frame_type == "Name": 353 code = 3 354 elif frame_type == "Counter": 355 code = 4 356 elif frame_type == "Block Counter": 357 code = 5 358 elif frame_type == "Annotate": 359 code = 6 360 elif frame_type == "Sched Trace": 361 code = 7 362 elif frame_type == "GPU Trace": 363 code = 8 364 elif frame_type == "Idle": 365 code = 9 366 else: 367 print "ERROR: Unknown frame type:", frame_type 368 sys.exit(1) 369 370 packed_code = packed32(code) 371 372 packed_core = packed32(core) 373 374 length = int32(len(packed_code) + len(packed_core) + len(body)) 375 376 ret = length + packed_code + packed_core + body 377 return ret 378 379 380# Summary frame 381# - timestamp: packed64 382# - uptime: packed64 383def summaryFrame(timestamp, uptime): 384 frame_type = "Summary" 385 body = packed64(timestamp) + packed64(uptime) 386 ret = addFrameHeader(frame_type, body, 0) 387 return ret 388 389# Backtrace frame 390# - not implemented yet 391def backtraceFrame(): 392 pass 393 394# Cookie name message 395# - cookie: packed32 396# - name: string 397def cookieNameFrame(cookie, name): 398 frame_type = "Name" 399 packed_code = packed32(1) 400 body = packed_code + packed32(cookie) + stringList(name) 401 ret = addFrameHeader(frame_type, body, 0) 402 return ret 403 404# Thread name message 405# - timestamp: timestamp 406# - thread id: packed32 407# - name: string 408def threadNameFrame(timestamp, thread_id, name): 409 frame_type = "Name" 410 packed_code = packed32(2) 411 body = packed_code + timestampList(timestamp) + \ 412 packed32(thread_id) + stringList(name) 413 ret = addFrameHeader(frame_type, body, 0) 414 return ret 415 416# Core name message 417# - name: string 418def coreNameFrame(name): 419 frame_type = "Name" 420 packed_code = packed32(3) 421 body = packed_code + stringList(name) 422 ret = addFrameHeader(frame_type, body, 0) 423 return ret 424 425# Counter frame message 426# - timestamp: timestamp 427# - core: packed32 428# - key: packed32 429# - value: packed64 430def counterFrame(timestamp, core, key, value): 431 frame_type = "Counter" 432 body = timestampList(timestamp) + packed32(core) + packed32(key) + \ 433 packed64(value) 434 ret = addFrameHeader(frame_type, body, core) 435 return ret 436 437# Block Counter frame message 438# - key: packed32 439# - value: packed64 440def blockCounterFrame(core, key, value): 441 frame_type = "Block Counter" 442 body = packed32(key) + packed64(value) 443 ret = addFrameHeader(frame_type, body, core) 444 return ret 445 446# Annotate frame messages 447# - core: packed32 448# - tid: packed32 449# - timestamp: timestamp 450# - size: packed32 451# - body 452def annotateFrame(core, tid, timestamp, size, userspace_body): 453 frame_type = "Annotate" 454 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \ 455 packed32(size) + userspace_body 456 ret = addFrameHeader(frame_type, body, core) 457 return ret 458 459# Scheduler Trace frame messages 460# Sched Switch 461# - Code: 1 462# - timestamp: timestamp 463# - pid: packed32 464# - tid: packed32 465# - cookie: packed32 466# - state: packed32 467def schedSwitchFrame(core, timestamp, pid, tid, cookie, state): 468 frame_type = "Sched Trace" 469 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \ 470 packed32(tid) + packed32(cookie) + packed32(state) 471 ret = addFrameHeader(frame_type, body, core) 472 return ret 473 474# Sched Thread Exit 475# - Code: 2 476# - timestamp: timestamp 477# - tid: packed32 478def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state): 479 frame_type = "Sched Trace" 480 body = packed32(2) + timestampList(timestamp) + packed32(tid) 481 ret = addFrameHeader(frame_type, body, core) 482 return ret 483 484# GPU Trace frame messages 485# - Not implemented yet 486def gpuTraceFrame(): 487 pass 488 489# Idle frame messages 490# Enter Idle 491# - code: 1 492# - timestamp: timestamp 493# - core: packed32 494def enterIdleFrame(timestamp, core): 495 frame_type = "Idle" 496 body = packed32(1) + timestampList(timestamp) + packed32(core) 497 ret = addFrameHeader(frame_type, body, core) 498 return ret 499 500# Exit Idle 501# - code: 2 502# - timestamp: timestamp 503# - core: packed32 504def exitIdleFrame(timestamp, core): 505 frame_type = "Idle" 506 body = packed32(2) + timestampList(timestamp) + packed32(core) 507 ret = addFrameHeader(frame_type, body, core) 508 return ret 509 510 511#################################################################### 512def parseProcessInfo(task_file): 513 print "\n===============================" 514 print "Parsing Task file..." 515 print task_file 516 print "===============================\n" 517 518 global start_tick, end_tick, num_cpus 519 global process_dict, thread_dict, process_list 520 global event_list, unified_event_list 521 global idle_uid, kernel_uid 522 523 event_list = [] 524 unified_event_list = [] 525 for cpu in range(num_cpus): 526 event_list.append([]) 527 528 uid = 1 # uid 0 is reserved for idle 529 530 # Dummy Tasks for frame buffers and system diagrams 531 process = Task(uid, 9999, 9999, "framebuffer", True, 0) 532 process_list.append(process) 533 uid += 1 534 thread = Task(uid, 9999, 9999, "framebuffer", False, 0) 535 process.children.append(thread) 536 uid += 1 537 process = Task(uid, 9998, 9998, "System", True, 0) 538 process_list.append(process) 539 # if we don't find the real kernel, use this to keep things going 540 kernel_uid = uid 541 uid += 1 542 thread = Task(uid, 9998, 9998, "System", False, 0) 543 process.children.append(thread) 544 uid += 1 545 546 ext = os.path.splitext(task_file)[1] 547 548 try: 549 if ext == ".gz": 550 process_file = gzip.open(task_file, 'rb') 551 else: 552 process_file = open(task_file, 'rb') 553 except: 554 print "ERROR opening task file:", task_file 555 print "Make sure context switch task dumping is enabled in gem5." 556 sys.exit(1) 557 558 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" + 559 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)") 560 561 task_name_failure_warned = False 562 563 for line in process_file: 564 match = re.match(process_re, line) 565 if match: 566 tick = int(match.group(1)) 567 if (start_tick < 0): 568 start_tick = tick 569 cpu_id = int(match.group(3)) 570 pid = int(match.group(4)) 571 tgid = int(match.group(5)) 572 task_name = match.group(6) 573 574 if not task_name_failure_warned: 575 if task_name == "FailureIn_curTaskName": 576 print "-------------------------------------------------" 577 print "WARNING: Task name not set correctly!" 578 print "Process/Thread info will not be displayed correctly" 579 print "Perhaps forgot to apply m5struct.patch to kernel?" 580 print "-------------------------------------------------" 581 task_name_failure_warned = True 582 583 if not tgid in process_dict: 584 if tgid == pid: 585 # new task is parent as well 586 if args.verbose: 587 print "new process", uid, pid, tgid, task_name 588 if tgid == 0: 589 # new process is the "idle" task 590 process = Task(uid, pid, tgid, "idle", True, tick) 591 idle_uid = 0 592 else: 593 process = Task(uid, pid, tgid, task_name, True, tick) 594 else: 595 if tgid == 0: 596 process = Task(uid, tgid, tgid, "idle", True, tick) 597 idle_uid = 0 598 else: 599 # parent process name not known yet 600 process = Task(uid, tgid, tgid, "_Unknown_", True, tick) 601 if tgid == -1: # kernel 602 kernel_uid = 0 603 uid += 1 604 process_dict[tgid] = process 605 process_list.append(process) 606 else: 607 if tgid == pid: 608 if process_dict[tgid].task_name == "_Unknown_": 609 if args.verbose: 610 print "new process", \ 611 process_dict[tgid].uid, pid, tgid, task_name 612 process_dict[tgid].task_name = task_name 613 if process_dict[tgid].task_name != task_name and tgid != 0: 614 process_dict[tgid].task_name = task_name 615 616 if not pid in thread_dict: 617 if args.verbose: 618 print "new thread", \ 619 uid, process_dict[tgid].uid, pid, tgid, task_name 620 thread = Task(uid, pid, tgid, task_name, False, tick) 621 uid += 1 622 thread_dict[pid] = thread 623 process_dict[tgid].children.append(thread) 624 else: 625 if thread_dict[pid].task_name != task_name: 626 thread_dict[pid].task_name = task_name 627 628 if args.verbose: 629 print tick, uid, cpu_id, pid, tgid, task_name 630 631 task = thread_dict[pid] 632 event = Event(tick, task) 633 event_list[cpu_id].append(event) 634 unified_event_list.append(event) 635 636 if len(unified_event_list) == num_events: 637 print "Truncating at", num_events, "events!" 638 break 639 print "Found %d events." % len(unified_event_list) 640 641 for process in process_list: 642 if process.pid > 9990: # fix up framebuffer ticks 643 process.tick = start_tick 644 print process.uid, process.pid, process.tgid, \ 645 process.task_name, str(process.tick) 646 for thread in process.children: 647 if thread.pid > 9990: 648 thread.tick = start_tick 649 print "\t", thread.uid, thread.pid, thread.tgid, \ 650 thread.task_name, str(thread.tick) 651 652 end_tick = tick 653 654 print "Start tick:", start_tick 655 print "End tick: ", end_tick 656 print "" 657 658 return 659 660 661def initOutput(output_path): 662 if not os.path.exists(output_path): 663 os.mkdir(output_path) 664 665def ticksToNs(tick): 666 if ticks_in_ns < 0: 667 print "ticks_in_ns not set properly!" 668 sys.exit(1) 669 670 return tick / ticks_in_ns 671 672def writeXmlFile(xml, filename): 673 f = open(filename, "w") 674 txt = ET.tostring(xml) 675 f.write(minidom.parseString(txt).toprettyxml()) 676 f.close() 677 678 679# StatsEntry that contains individual statistics 680class StatsEntry(object): 681 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key): 682 683 # Full name of statistics 684 self.name = name 685 686 # Streamline group name that statistic will belong to 687 self.group = group 688 689 # Index of statistics within group (used to change colors within groups) 690 self.group_index = group_index 691 692 # Shorter name with "system" stripped off 693 # and symbols converted to alphanumerics 694 self.short_name = re.sub("system\.", "", name) 695 self.short_name = re.sub(":", "_", name) 696 697 # Regex for this stat (string version used to construct union regex) 698 self.regex_string = "^" + name + "\s+([\d\.]+)" 699 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M) 700 self.description = "" 701 702 # Whether this stat is use per CPU or not 703 self.per_cpu = per_cpu 704 self.per_switchcpu = per_switchcpu 705 706 # Key used in .apc protocol (as described in captured.xml) 707 self.key = key 708 709 # List of values of stat per timestamp 710 self.values = [] 711 712 # Whether this stat has been found for the current timestamp 713 self.found = False 714 715 # Whether this stat has been found at least once 716 # (to suppress too many warnings) 717 self.not_found_at_least_once = False 718 719 # Field used to hold ElementTree subelement for this stat 720 self.ET_element = None 721 722 # Create per-CPU stat name and regex, etc. 723 if self.per_cpu: 724 self.per_cpu_regex_string = [] 725 self.per_cpu_regex = [] 726 self.per_cpu_name = [] 727 self.per_cpu_found = [] 728 for i in range(num_cpus): 729 # Resuming from checkpoints results in using "switch_cpus" 730 if per_switchcpu: 731 per_cpu_name = "system.switch_cpus" 732 else: 733 per_cpu_name = "system.cpu" 734 735 # No CPU number appends if num_cpus == 1 736 if num_cpus > 1: 737 per_cpu_name += str(i) 738 per_cpu_name += "." + self.name 739 self.per_cpu_name.append(per_cpu_name) 740 print "\t", per_cpu_name 741 742 self.per_cpu_regex_string.\ 743 append("^" + per_cpu_name + "\s+[\d\.]+") 744 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \ 745 "\s+([\d\.e\-]+)\s+# (.*)$", re.M)) 746 self.values.append([]) 747 self.per_cpu_found.append(False) 748 749 def append_value(self, val, per_cpu_index = None): 750 if self.per_cpu: 751 self.values[per_cpu_index].append(str(val)) 752 else: 753 self.values.append(str(val)) 754 755# Global stats object that contains the list of stats entries 756# and other utility functions 757class Stats(object): 758 def __init__(self): 759 self.stats_list = [] 760 self.tick_list = [] 761 self.next_key = 1 762 763 def register(self, name, group, group_index, per_cpu, per_switchcpu): 764 print "registering stat:", name, "group:", group, group_index 765 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \ 766 per_switchcpu, self.next_key)) 767 self.next_key += 1 768 769 # Union of all stats to accelerate parsing speed 770 def createStatsRegex(self): 771 regex_strings = []; 772 print "\nnum entries in stats_list", len(self.stats_list) 773 for entry in self.stats_list: 774 if entry.per_cpu: 775 for i in range(num_cpus): 776 regex_strings.append(entry.per_cpu_regex_string[i]) 777 else: 778 regex_strings.append(entry.regex_string) 779 780 self.regex = re.compile('|'.join(regex_strings)) 781 782 783def registerStats(config_file): 784 print "===============================" 785 print "Parsing stats config.ini file..." 786 print config_file 787 print "===============================" 788 789 config = ConfigParser() 790 if not config.read(config_file): 791 print "ERROR: config file '", config_file, "' not found!" 792 sys.exit(1) 793 794 print "\nRegistering Stats..." 795 796 stats = Stats() 797 798 per_cpu_stat_groups = config.options('PER_CPU_STATS') 799 for group in per_cpu_stat_groups: 800 i = 0 801 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n') 802 for item in per_cpu_stats_list: 803 if item: 804 stats.register(item, group, i, True, False) 805 i += 1 806 807 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS') 808 for group in per_cpu_stat_groups: 809 i = 0 810 per_cpu_stats_list = \ 811 config.get('PER_SWITCHCPU_STATS', group).split('\n') 812 for item in per_cpu_stats_list: 813 if item: 814 stats.register(item, group, i, True, True) 815 i += 1 816 817 per_l2_stat_groups = config.options('PER_L2_STATS') 818 for group in per_l2_stat_groups: 819 i = 0 820 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n') 821 for item in per_l2_stats_list: 822 if item: 823 for l2 in range(num_l2): 824 name = item 825 prefix = "system.l2" 826 if num_l2 > 1: 827 prefix += str(l2) 828 prefix += "." 829 name = prefix + name 830 stats.register(name, group, i, False, False) 831 i += 1 832 833 other_stat_groups = config.options('OTHER_STATS') 834 for group in other_stat_groups: 835 i = 0 836 other_stats_list = config.get('OTHER_STATS', group).split('\n') 837 for item in other_stats_list: 838 if item: 839 stats.register(item, group, i, False, False) 840 i += 1 841 842 stats.createStatsRegex() 843 844 return stats 845 846# Parse and read in gem5 stats file 847# Streamline counters are organized per CPU 848def readGem5Stats(stats, gem5_stats_file): 849 print "\n===============================" 850 print "Parsing gem5 stats file..." 851 print gem5_stats_file 852 print "===============================\n" 853 ext = os.path.splitext(gem5_stats_file)[1] 854 855 window_start_regex = \ 856 re.compile("^---------- Begin Simulation Statistics ----------") 857 window_end_regex = \ 858 re.compile("^---------- End Simulation Statistics ----------") 859 final_tick_regex = re.compile("^final_tick\s+(\d+)") 860 861 global ticks_in_ns 862 sim_freq_regex = re.compile("^sim_freq\s+(\d+)") 863 sim_freq = -1 864 865 try: 866 if ext == ".gz": 867 f = gzip.open(gem5_stats_file, "r") 868 else: 869 f = open(gem5_stats_file, "r") 870 except: 871 print "ERROR opening stats file", gem5_stats_file, "!" 872 sys.exit(1) 873 874 stats_not_found_list = stats.stats_list[:] 875 window_num = 0 876 877 while (True): 878 error = False 879 try: 880 line = f.readline() 881 except IOError: 882 print "" 883 print "WARNING: IO error in stats file" 884 print "(gzip stream not closed properly?)...continuing for now" 885 error = True 886 if not line: 887 break 888 889 # Find out how many gem5 ticks in 1ns 890 if sim_freq < 0: 891 m = sim_freq_regex.match(line) 892 if m: 893 sim_freq = int(m.group(1)) # ticks in 1 sec 894 ticks_in_ns = int(sim_freq / 1e9) 895 print "Simulation frequency found! 1 tick == %e sec\n" \ 896 % (1.0 / sim_freq) 897 898 # Final tick in gem5 stats: current absolute timestamp 899 m = final_tick_regex.match(line) 900 if m: 901 tick = int(m.group(1)) 902 if tick > end_tick: 903 break 904 stats.tick_list.append(tick) 905 906 907 if (window_end_regex.match(line) or error): 908 if args.verbose: 909 print "new window" 910 for stat in stats.stats_list: 911 if stat.per_cpu: 912 for i in range(num_cpus): 913 if not stat.per_cpu_found[i]: 914 if not stat.not_found_at_least_once: 915 print "WARNING: stat not found in window #", \ 916 window_num, ":", stat.per_cpu_name[i] 917 print "suppressing further warnings for " + \ 918 "this stat" 919 stat.not_found_at_least_once = True 920 stat.values[i].append(str(0)) 921 stat.per_cpu_found[i] = False 922 else: 923 if not stat.found: 924 if not stat.not_found_at_least_once: 925 print "WARNING: stat not found in window #", \ 926 window_num, ":", stat.name 927 print "suppressing further warnings for this stat" 928 stat.not_found_at_least_once = True 929 stat.values.append(str(0)) 930 stat.found = False 931 stats_not_found_list = stats.stats_list[:] 932 window_num += 1 933 if error: 934 break 935 936 # Do a single regex of the union of all stats first for speed 937 if stats.regex.match(line): 938 # Then loop through only the stats we haven't seen in this window 939 for stat in stats_not_found_list[:]: 940 if stat.per_cpu: 941 for i in range(num_cpus): 942 m = stat.per_cpu_regex[i].match(line) 943 if m: 944 if stat.name == "ipc": 945 value = str(int(float(m.group(1)) * 1000)) 946 else: 947 value = str(int(float(m.group(1)))) 948 if args.verbose: 949 print stat.per_cpu_name[i], value 950 stat.values[i].append(value) 951 stat.per_cpu_found[i] = True 952 all_found = True 953 for j in range(num_cpus): 954 if not stat.per_cpu_found[j]: 955 all_found = False 956 if all_found: 957 stats_not_found_list.remove(stat) 958 if stat.description == "": 959 stat.description = m.group(2) 960 else: 961 m = stat.regex.match(line) 962 if m: 963 value = str(int(float(m.group(1)))) 964 if args.verbose: 965 print stat.name, value 966 stat.values.append(value) 967 stat.found = True 968 stats_not_found_list.remove(stat) 969 if stat.description == "": 970 stat.description = m.group(2) 971 f.close() 972 973 974# Create session.xml file in .apc folder 975def doSessionXML(output_path): 976 session_file = output_path + "/session.xml" 977 978 xml = ET.Element("session") 979 980 xml.set("version", "1") 981 xml.set("call_stack_unwinding", "no") 982 xml.set("parse_debug_info", "no") 983 xml.set("high_resolution", "yes") 984 xml.set("buffer_mode", "streaming") 985 xml.set("sample_rate", "low") 986 987 # Setting duration to zero for now. Doesn't affect visualization. 988 xml.set("duration", "0") 989 990 xml.set("target_host", "") 991 xml.set("target_port", "8080") 992 993 writeXmlFile(xml, session_file) 994 995 996# Create captured.xml file in .apc folder 997def doCapturedXML(output_path, stats): 998 captured_file = output_path + "/captured.xml" 999 1000 xml = ET.Element("captured") 1001 xml.set("version", "1") 1002 xml.set("protocol", "12") 1003 1004 target = ET.SubElement(xml, "target") 1005 target.set("name", "gem5") 1006 target.set("sample_rate", "1000") 1007 target.set("cores", str(num_cpus)) 1008 1009 counters = ET.SubElement(xml, "counters") 1010 for stat in stats.stats_list: 1011 s = ET.SubElement(counters, "counter") 1012 stat_name = re.sub("\.", "_", stat.short_name) 1013 s.set("title", stat.group) 1014 s.set("name", stat_name) 1015 s.set("color", "0x00000000") 1016 s.set("key", "0x%08x" % stat.key) 1017 s.set("type", stat_name) 1018 s.set("event", "0x00000000") 1019 if stat.per_cpu: 1020 s.set("per_cpu", "yes") 1021 else: 1022 s.set("per_cpu", "no") 1023 s.set("display", "") 1024 s.set("units", "") 1025 s.set("average_selection", "no") 1026 s.set("description", stat.description) 1027 1028 writeXmlFile(xml, captured_file) 1029 1030# Writes out Streamline cookies (unique IDs per process/thread) 1031def writeCookiesThreads(blob): 1032 thread_list = [] 1033 for process in process_list: 1034 if process.uid > 0: 1035 print "cookie", process.task_name, process.uid 1036 writeBinary(blob, cookieNameFrame(process.uid, process.task_name)) 1037 1038 # pid and tgid need to be positive values -- no longer true? 1039 for thread in process.children: 1040 thread_list.append(thread) 1041 1042 # Threads need to be sorted in timestamp order 1043 thread_list.sort(key = lambda x: x.tick) 1044 for thread in thread_list: 1045 print "thread", thread.task_name, (ticksToNs(thread.tick)),\ 1046 thread.tgid, thread.pid 1047 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\ 1048 thread.pid, thread.task_name)) 1049 1050# Writes context switch info as Streamline scheduling events 1051def writeSchedEvents(blob): 1052 for cpu in range(num_cpus): 1053 for event in event_list[cpu]: 1054 timestamp = ticksToNs(event.tick) 1055 pid = event.task.tgid 1056 tid = event.task.pid 1057 if process_dict.has_key(event.task.tgid): 1058 cookie = process_dict[event.task.tgid].uid 1059 else: 1060 cookie = 0 1061 1062 # State: 1063 # 0: waiting on other event besides I/O 1064 # 1: Contention/pre-emption 1065 # 2: Waiting on I/O 1066 # 3: Waiting on mutex 1067 # Hardcoding to 0 for now. Other states not implemented yet. 1068 state = 0 1069 1070 if args.verbose: 1071 print cpu, timestamp, pid, tid, cookie 1072 1073 writeBinary(blob,\ 1074 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state)) 1075 1076# Writes selected gem5 statistics as Streamline counters 1077def writeCounters(blob, stats): 1078 timestamp_list = [] 1079 for tick in stats.tick_list: 1080 if tick > end_tick: 1081 break 1082 timestamp_list.append(ticksToNs(tick)) 1083 1084 for stat in stats.stats_list: 1085 if stat.per_cpu: 1086 stat_length = len(stat.values[0]) 1087 else: 1088 stat_length = len(stat.values) 1089 1090 for n in range(len(timestamp_list)): 1091 for stat in stats.stats_list: 1092 if stat.per_cpu: 1093 for i in range(num_cpus): 1094 writeBinary(blob, counterFrame(timestamp_list[n], i, \ 1095 stat.key, int(float(stat.values[i][n])))) 1096 else: 1097 writeBinary(blob, counterFrame(timestamp_list[n], 0, \ 1098 stat.key, int(float(stat.values[n])))) 1099 1100# Streamline can display LCD frame buffer dumps (gzipped bmp) 1101# This function converts the frame buffer dumps to the Streamline format 1102def writeVisualAnnotations(blob, input_path, output_path): 1103 frame_path = input_path + "/frames_system.vncserver" 1104 if not os.path.exists(frame_path): 1105 return 1106 1107 frame_count = 0 1108 file_list = os.listdir(frame_path) 1109 file_list.sort() 1110 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz") 1111 1112 # Use first non-negative pid to tag visual annotations 1113 annotate_pid = -1 1114 for e in unified_event_list: 1115 pid = e.task.pid 1116 if pid >= 0: 1117 annotate_pid = pid 1118 break 1119 1120 for fn in file_list: 1121 m = re_fb.match(fn) 1122 if m: 1123 seq = m.group(1) 1124 tick = int(m.group(2)) 1125 if tick > end_tick: 1126 break 1127 frame_count += 1 1128 1129 userspace_body = [] 1130 userspace_body += packed32(0x1C) # escape code 1131 userspace_body += packed32(0x04) # visual code 1132 1133 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz" 1134 userspace_body += int16(len(text_annotation)) 1135 userspace_body += utf8StringList(text_annotation) 1136 1137 if gzipped_bmp_supported: 1138 # copy gzipped bmp directly 1139 bytes_read = open(frame_path + "/" + fn, "rb").read() 1140 else: 1141 # copy uncompressed bmp 1142 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read() 1143 1144 userspace_body += int32(len(bytes_read)) 1145 userspace_body += bytes_read 1146 1147 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \ 1148 len(userspace_body), userspace_body)) 1149 1150 print "\nfound", frame_count, "frames for visual annotation.\n" 1151 1152 1153def createApcProject(input_path, output_path, stats): 1154 initOutput(output_path) 1155 1156 blob = open(output_path + "/0000000000", "wb") 1157 1158 # Summary frame takes current system time and system uptime. 1159 # Filling in with random values for now. 1160 writeBinary(blob, summaryFrame(1234, 5678)) 1161 1162 writeCookiesThreads(blob) 1163 1164 print "writing Events" 1165 writeSchedEvents(blob) 1166 1167 print "writing Counters" 1168 writeCounters(blob, stats) 1169 1170 print "writing Visual Annotations" 1171 writeVisualAnnotations(blob, input_path, output_path) 1172 1173 doSessionXML(output_path) 1174 doCapturedXML(output_path, stats) 1175 1176 blob.close() 1177 1178 1179 1180####################### 1181# Main Routine 1182 1183input_path = args.input_path 1184output_path = args.output_path 1185 1186#### 1187# Make sure input path exists 1188#### 1189if not os.path.exists(input_path): 1190 print "ERROR: Input path %s does not exist!" % input_path 1191 sys.exit(1) 1192 1193#### 1194# Parse gem5 configuration file to find # of CPUs and L2s 1195#### 1196(num_cpus, num_l2) = parseConfig(input_path + "/config.ini") 1197 1198#### 1199# Parse task file to find process/thread info 1200#### 1201parseProcessInfo(input_path + "/system.tasks.txt") 1202 1203#### 1204# Parse stat config file and register stats 1205#### 1206stat_config_file = args.stat_config_file 1207stats = registerStats(stat_config_file) 1208 1209#### 1210# Parse gem5 stats 1211#### 1212# Check if both stats.txt and stats.txt.gz exist and warn if both exist 1213if os.path.exists(input_path + "/stats.txt") and \ 1214 os.path.exists(input_path + "/stats.txt.gz"): 1215 print "WARNING: Both stats.txt.gz and stats.txt exist. \ 1216 Using stats.txt.gz by default." 1217 1218gem5_stats_file = input_path + "/stats.txt.gz" 1219if not os.path.exists(gem5_stats_file): 1220 gem5_stats_file = input_path + "/stats.txt" 1221if not os.path.exists(gem5_stats_file): 1222 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path 1223 sys.exit(1) 1224 1225readGem5Stats(stats, gem5_stats_file) 1226 1227#### 1228# Create Streamline .apc project folder 1229#### 1230createApcProject(input_path, output_path, stats) 1231 1232print "All done!" 1233