1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
| 1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Author: Dam Sunwoo
39#
40
41# This script converts gem5 output to ARM DS-5 Streamline .apc project file
42# (Requires the gem5 runs to be run with ContextSwitchStatsDump enabled and
43# some patches applied to target Linux kernel.)
44# Visit http://www.gem5.org/Streamline for more details.
45#
46# Usage:
47# m5stats2streamline.py <stat_config.ini> <gem5 run folder> <dest .apc folder>
48#
49# <stat_config.ini>: .ini file that describes which stats to be included
50# in conversion. Sample .ini files can be found in
51# util/streamline.
52# NOTE: this is NOT the gem5 config.ini file.
53#
54# <gem5 run folder>: Path to gem5 run folder (must contain config.ini,
55# stats.txt[.gz], and system.tasks.txt.)
56#
57# <dest .apc folder>: Destination .apc folder path
58#
|
59# APC project generation based on Gator v12 (DS-5 v5.13)
| 59# APC project generation based on Gator v17 (DS-5 v5.17)
|
60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
| 60# Subsequent versions should be backward compatible
61
62import re, sys, os
63from ConfigParser import ConfigParser
64import gzip
65import xml.etree.ElementTree as ET
66import xml.dom.minidom as minidom
67import shutil
68import zlib
69
70import argparse
71
72parser = argparse.ArgumentParser(
73 formatter_class=argparse.RawDescriptionHelpFormatter,
74 description="""
75 Converts gem5 runs to ARM DS-5 Streamline .apc project file.
76 (NOTE: Requires gem5 runs to be run with ContextSwitchStatsDump
77 enabled and some patches applied to the target Linux kernel.)
78
79 Visit http://www.gem5.org/Streamline for more details.
80
|
81 APC project generation based on Gator v12 (DS-5 v5.13)
| 81 APC project generation based on Gator v17 (DS-5 v5.17)
|
82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
| 82 Subsequent versions should be backward compatible
83 """)
84
85parser.add_argument("stat_config_file", metavar="<stat_config.ini>",
86 help=".ini file that describes which stats to be included \
87 in conversion. Sample .ini files can be found in \
88 util/streamline. NOTE: this is NOT the gem5 config.ini \
89 file.")
90
91parser.add_argument("input_path", metavar="<gem5 run folder>",
92 help="Path to gem5 run folder (must contain config.ini, \
93 stats.txt[.gz], and system.tasks.txt.)")
94
95parser.add_argument("output_path", metavar="<dest .apc folder>",
96 help="Destination .apc folder path")
97
98parser.add_argument("--num-events", action="store", type=int,
99 default=1000000,
100 help="Maximum number of scheduling (context switch) \
101 events to be processed. Set to truncate early. \
102 Default=1000000")
103
104parser.add_argument("--gzipped-bmp-not-supported", action="store_true",
105 help="Do not use gzipped .bmp files for visual annotations. \
106 This option is only required when using Streamline versions \
107 older than 5.14")
108
109parser.add_argument("--verbose", action="store_true",
110 help="Enable verbose output")
111
112args = parser.parse_args()
113
114if not re.match("(.*)\.apc", args.output_path):
115 print "ERROR: <dest .apc folder> should end with '.apc'!"
116 sys.exit(1)
117
118# gzipped BMP files for visual annotation is supported in Streamline 5.14.
119# Setting this to True will significantly compress the .apc binary file that
120# includes frame buffer snapshots.
121gzipped_bmp_supported = not args.gzipped_bmp_not_supported
122
123ticks_in_ns = -1
124
125# Default max # of events. Increase this for longer runs.
126num_events = args.num_events
127
128start_tick = -1
129end_tick = -1
130
131# Parse gem5 config.ini file to determine some system configurations.
132# Number of CPUs, L2s, etc.
133def parseConfig(config_file):
134 global num_cpus, num_l2
135
136 print "\n==============================="
137 print "Parsing gem5 config.ini file..."
138 print config_file
139 print "===============================\n"
140 config = ConfigParser()
141 if not config.read(config_file):
142 print "ERROR: config file '", config_file, "' not found"
143 sys.exit(1)
144
145 if config.has_section("system.cpu"):
146 num_cpus = 1
147 else:
148 num_cpus = 0
149 while config.has_section("system.cpu" + str(num_cpus)):
150 num_cpus += 1
151
152 if config.has_section("system.l2"):
153 num_l2 = 1
154 else:
155 num_l2 = 0
156 while config.has_section("system.l2" + str(num_l2)):
157 num_l2 += 1
158
159 print "Num CPUs:", num_cpus
160 print "Num L2s:", num_l2
161 print ""
162
163 return (num_cpus, num_l2)
164
165
166process_dict = {}
167thread_dict = {}
168
169process_list = []
170
171idle_uid = -1
172kernel_uid = -1
173
174class Task(object):
175 def __init__(self, uid, pid, tgid, task_name, is_process, tick):
176 if pid == 0: # Idle
177 self.uid = 0
178 elif pid == -1: # Kernel
179 self.uid = 0
180 else:
181 self.uid = uid
182 self.pid = pid
183 self.tgid = tgid
184 self.is_process = is_process
185 self.task_name = task_name
186 self.children = []
187 self.tick = tick # time this task first appeared
188
189class Event(object):
190 def __init__(self, tick, task):
191 self.tick = tick
192 self.task = task
193
194############################################################
195# Types used in APC Protocol
196# - packed32, packed64
197# - int32
198# - string
199############################################################
200
|
201# variable length packed 4-byte signed value
| |
202def packed32(x):
203 ret = []
| 201def packed32(x):
202 ret = []
|
| 203 more = True
204 while more:
205 b = x & 0x7f
206 x = x >> 7
207 if (((x == 0) and ((b & 0x40) == 0)) or \
208 ((x == -1) and ((b & 0x40) != 0))):
209 more = False
210 else:
211 b = b | 0x80
212 ret.append(b)
213 return ret
214
215# For historical reasons, 32/64-bit versions of functions are presevered
216def packed64(x):
217 return packed32(x)
218
219# variable length packed 4-byte signed value
220def unsigned_packed32(x):
221 ret = []
|
204 if ((x & 0xffffff80) == 0):
205 ret.append(x & 0x7f)
206 elif ((x & 0xffffc000) == 0):
207 ret.append((x | 0x80) & 0xff)
208 ret.append((x >> 7) & 0x7f)
209 elif ((x & 0xffe00000) == 0):
210 ret.append((x | 0x80) & 0xff)
211 ret.append(((x >> 7) | 0x80) & 0xff)
212 ret.append((x >> 14) & 0x7f)
213 elif ((x & 0xf0000000) == 0):
214 ret.append((x | 0x80) & 0xff)
215 ret.append(((x >> 7) | 0x80) & 0xff)
216 ret.append(((x >> 14) | 0x80) & 0xff)
217 ret.append((x >> 21) & 0x7f)
218 else:
219 ret.append((x | 0x80) & 0xff)
220 ret.append(((x >> 7) | 0x80) & 0xff)
221 ret.append(((x >> 14) | 0x80) & 0xff)
222 ret.append(((x >> 21) | 0x80) & 0xff)
223 ret.append((x >> 28) & 0x0f)
224 return ret
225
226# variable length packed 8-byte signed value
| 222 if ((x & 0xffffff80) == 0):
223 ret.append(x & 0x7f)
224 elif ((x & 0xffffc000) == 0):
225 ret.append((x | 0x80) & 0xff)
226 ret.append((x >> 7) & 0x7f)
227 elif ((x & 0xffe00000) == 0):
228 ret.append((x | 0x80) & 0xff)
229 ret.append(((x >> 7) | 0x80) & 0xff)
230 ret.append((x >> 14) & 0x7f)
231 elif ((x & 0xf0000000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append(((x >> 7) | 0x80) & 0xff)
234 ret.append(((x >> 14) | 0x80) & 0xff)
235 ret.append((x >> 21) & 0x7f)
236 else:
237 ret.append((x | 0x80) & 0xff)
238 ret.append(((x >> 7) | 0x80) & 0xff)
239 ret.append(((x >> 14) | 0x80) & 0xff)
240 ret.append(((x >> 21) | 0x80) & 0xff)
241 ret.append((x >> 28) & 0x0f)
242 return ret
243
244# variable length packed 8-byte signed value
|
227def packed64(x):
| 245def unsigned_packed64(x):
|
228 ret = []
229 if ((x & 0xffffffffffffff80) == 0):
230 ret.append(x & 0x7f)
231 elif ((x & 0xffffffffffffc000) == 0):
232 ret.append((x | 0x80) & 0xff)
233 ret.append((x >> 7) & 0x7f)
234 elif ((x & 0xffffffffffe00000) == 0):
235 ret.append((x | 0x80) & 0xff)
236 ret.append(((x >> 7) | 0x80) & 0xff)
237 ret.append((x >> 14) & 0x7f)
238 elif ((x & 0xfffffffff0000000) == 0):
239 ret.append((x | 0x80) & 0xff)
240 ret.append(((x >> 7) | 0x80) & 0xff)
241 ret.append(((x >> 14) | 0x80) & 0xff)
242 ret.append((x >> 21) & 0x7f)
243 elif ((x & 0xfffffff800000000) == 0):
244 ret.append((x | 0x80) & 0xff)
245 ret.append(((x >> 7) | 0x80) & 0xff)
246 ret.append(((x >> 14) | 0x80) & 0xff)
247 ret.append(((x >> 21) | 0x80) & 0xff)
248 ret.append((x >> 28) & 0x7f)
249 elif ((x & 0xfffffc0000000000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append(((x >> 7) | 0x80) & 0xff)
252 ret.append(((x >> 14) | 0x80) & 0xff)
253 ret.append(((x >> 21) | 0x80) & 0xff)
254 ret.append(((x >> 28) | 0x80) & 0xff)
255 ret.append((x >> 35) & 0x7f)
256 elif ((x & 0xfffe000000000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append(((x >> 21) | 0x80) & 0xff)
261 ret.append(((x >> 28) | 0x80) & 0xff)
262 ret.append(((x >> 35) | 0x80) & 0xff)
263 ret.append((x >> 42) & 0x7f)
264 elif ((x & 0xff00000000000000) == 0):
265 ret.append((x | 0x80) & 0xff)
266 ret.append(((x >> 7) | 0x80) & 0xff)
267 ret.append(((x >> 14) | 0x80) & 0xff)
268 ret.append(((x >> 21) | 0x80) & 0xff)
269 ret.append(((x >> 28) | 0x80) & 0xff)
270 ret.append(((x >> 35) | 0x80) & 0xff)
271 ret.append(((x >> 42) | 0x80) & 0xff)
272 ret.append((x >> 49) & 0x7f)
273 elif ((x & 0x8000000000000000) == 0):
274 ret.append((x | 0x80) & 0xff)
275 ret.append(((x >> 7) | 0x80) & 0xff)
276 ret.append(((x >> 14) | 0x80) & 0xff)
277 ret.append(((x >> 21) | 0x80) & 0xff)
278 ret.append(((x >> 28) | 0x80) & 0xff)
279 ret.append(((x >> 35) | 0x80) & 0xff)
280 ret.append(((x >> 42) | 0x80) & 0xff)
281 ret.append(((x >> 49) | 0x80) & 0xff)
282 ret.append((x >> 56) & 0x7f)
283 else:
284 ret.append((x | 0x80) & 0xff)
285 ret.append(((x >> 7) | 0x80) & 0xff)
286 ret.append(((x >> 14) | 0x80) & 0xff)
287 ret.append(((x >> 21) | 0x80) & 0xff)
288 ret.append(((x >> 28) | 0x80) & 0xff)
289 ret.append(((x >> 35) | 0x80) & 0xff)
290 ret.append(((x >> 42) | 0x80) & 0xff)
291 ret.append(((x >> 49) | 0x80) & 0xff)
292 ret.append(((x >> 56) | 0x80) & 0xff)
293 ret.append((x >> 63) & 0x7f)
294 return ret
295
296# 4-byte signed little endian
297def int32(x):
298 ret = []
299 ret.append(x & 0xff)
300 ret.append((x >> 8) & 0xff)
301 ret.append((x >> 16) & 0xff)
302 ret.append((x >> 24) & 0xff)
303 return ret
304
305# 2-byte signed little endian
306def int16(x):
307 ret = []
308 ret.append(x & 0xff)
309 ret.append((x >> 8) & 0xff)
310 return ret
311
312# a packed32 length followed by the specified number of characters
313def stringList(x):
314 ret = []
315 ret += packed32(len(x))
316 for i in x:
317 ret.append(i)
318 return ret
319
320def utf8StringList(x):
321 ret = []
322 for i in x:
323 ret.append(ord(i))
324 return ret
325
326# packed64 time value in nanoseconds relative to the uptime from the
327# Summary message.
328def timestampList(x):
329 ret = packed64(x)
330 return ret
331
332
333############################################################
334# Write binary
335############################################################
336
337def writeBinary(outfile, binary_list):
338 for i in binary_list:
339 outfile.write("%c" % i)
340
341############################################################
342# APC Protocol Frame Types
343############################################################
344
345def addFrameHeader(frame_type, body, core):
346 ret = []
347
348 if frame_type == "Summary":
349 code = 1
350 elif frame_type == "Backtrace":
351 code = 2
352 elif frame_type == "Name":
353 code = 3
354 elif frame_type == "Counter":
355 code = 4
356 elif frame_type == "Block Counter":
357 code = 5
358 elif frame_type == "Annotate":
359 code = 6
360 elif frame_type == "Sched Trace":
361 code = 7
362 elif frame_type == "GPU Trace":
363 code = 8
364 elif frame_type == "Idle":
365 code = 9
366 else:
367 print "ERROR: Unknown frame type:", frame_type
368 sys.exit(1)
369
370 packed_code = packed32(code)
371
372 packed_core = packed32(core)
373
374 length = int32(len(packed_code) + len(packed_core) + len(body))
375
376 ret = length + packed_code + packed_core + body
377 return ret
378
379
380# Summary frame
381# - timestamp: packed64
382# - uptime: packed64
383def summaryFrame(timestamp, uptime):
384 frame_type = "Summary"
| 246 ret = []
247 if ((x & 0xffffffffffffff80) == 0):
248 ret.append(x & 0x7f)
249 elif ((x & 0xffffffffffffc000) == 0):
250 ret.append((x | 0x80) & 0xff)
251 ret.append((x >> 7) & 0x7f)
252 elif ((x & 0xffffffffffe00000) == 0):
253 ret.append((x | 0x80) & 0xff)
254 ret.append(((x >> 7) | 0x80) & 0xff)
255 ret.append((x >> 14) & 0x7f)
256 elif ((x & 0xfffffffff0000000) == 0):
257 ret.append((x | 0x80) & 0xff)
258 ret.append(((x >> 7) | 0x80) & 0xff)
259 ret.append(((x >> 14) | 0x80) & 0xff)
260 ret.append((x >> 21) & 0x7f)
261 elif ((x & 0xfffffff800000000) == 0):
262 ret.append((x | 0x80) & 0xff)
263 ret.append(((x >> 7) | 0x80) & 0xff)
264 ret.append(((x >> 14) | 0x80) & 0xff)
265 ret.append(((x >> 21) | 0x80) & 0xff)
266 ret.append((x >> 28) & 0x7f)
267 elif ((x & 0xfffffc0000000000) == 0):
268 ret.append((x | 0x80) & 0xff)
269 ret.append(((x >> 7) | 0x80) & 0xff)
270 ret.append(((x >> 14) | 0x80) & 0xff)
271 ret.append(((x >> 21) | 0x80) & 0xff)
272 ret.append(((x >> 28) | 0x80) & 0xff)
273 ret.append((x >> 35) & 0x7f)
274 elif ((x & 0xfffe000000000000) == 0):
275 ret.append((x | 0x80) & 0xff)
276 ret.append(((x >> 7) | 0x80) & 0xff)
277 ret.append(((x >> 14) | 0x80) & 0xff)
278 ret.append(((x >> 21) | 0x80) & 0xff)
279 ret.append(((x >> 28) | 0x80) & 0xff)
280 ret.append(((x >> 35) | 0x80) & 0xff)
281 ret.append((x >> 42) & 0x7f)
282 elif ((x & 0xff00000000000000) == 0):
283 ret.append((x | 0x80) & 0xff)
284 ret.append(((x >> 7) | 0x80) & 0xff)
285 ret.append(((x >> 14) | 0x80) & 0xff)
286 ret.append(((x >> 21) | 0x80) & 0xff)
287 ret.append(((x >> 28) | 0x80) & 0xff)
288 ret.append(((x >> 35) | 0x80) & 0xff)
289 ret.append(((x >> 42) | 0x80) & 0xff)
290 ret.append((x >> 49) & 0x7f)
291 elif ((x & 0x8000000000000000) == 0):
292 ret.append((x | 0x80) & 0xff)
293 ret.append(((x >> 7) | 0x80) & 0xff)
294 ret.append(((x >> 14) | 0x80) & 0xff)
295 ret.append(((x >> 21) | 0x80) & 0xff)
296 ret.append(((x >> 28) | 0x80) & 0xff)
297 ret.append(((x >> 35) | 0x80) & 0xff)
298 ret.append(((x >> 42) | 0x80) & 0xff)
299 ret.append(((x >> 49) | 0x80) & 0xff)
300 ret.append((x >> 56) & 0x7f)
301 else:
302 ret.append((x | 0x80) & 0xff)
303 ret.append(((x >> 7) | 0x80) & 0xff)
304 ret.append(((x >> 14) | 0x80) & 0xff)
305 ret.append(((x >> 21) | 0x80) & 0xff)
306 ret.append(((x >> 28) | 0x80) & 0xff)
307 ret.append(((x >> 35) | 0x80) & 0xff)
308 ret.append(((x >> 42) | 0x80) & 0xff)
309 ret.append(((x >> 49) | 0x80) & 0xff)
310 ret.append(((x >> 56) | 0x80) & 0xff)
311 ret.append((x >> 63) & 0x7f)
312 return ret
313
314# 4-byte signed little endian
315def int32(x):
316 ret = []
317 ret.append(x & 0xff)
318 ret.append((x >> 8) & 0xff)
319 ret.append((x >> 16) & 0xff)
320 ret.append((x >> 24) & 0xff)
321 return ret
322
323# 2-byte signed little endian
324def int16(x):
325 ret = []
326 ret.append(x & 0xff)
327 ret.append((x >> 8) & 0xff)
328 return ret
329
330# a packed32 length followed by the specified number of characters
331def stringList(x):
332 ret = []
333 ret += packed32(len(x))
334 for i in x:
335 ret.append(i)
336 return ret
337
338def utf8StringList(x):
339 ret = []
340 for i in x:
341 ret.append(ord(i))
342 return ret
343
344# packed64 time value in nanoseconds relative to the uptime from the
345# Summary message.
346def timestampList(x):
347 ret = packed64(x)
348 return ret
349
350
351############################################################
352# Write binary
353############################################################
354
355def writeBinary(outfile, binary_list):
356 for i in binary_list:
357 outfile.write("%c" % i)
358
359############################################################
360# APC Protocol Frame Types
361############################################################
362
363def addFrameHeader(frame_type, body, core):
364 ret = []
365
366 if frame_type == "Summary":
367 code = 1
368 elif frame_type == "Backtrace":
369 code = 2
370 elif frame_type == "Name":
371 code = 3
372 elif frame_type == "Counter":
373 code = 4
374 elif frame_type == "Block Counter":
375 code = 5
376 elif frame_type == "Annotate":
377 code = 6
378 elif frame_type == "Sched Trace":
379 code = 7
380 elif frame_type == "GPU Trace":
381 code = 8
382 elif frame_type == "Idle":
383 code = 9
384 else:
385 print "ERROR: Unknown frame type:", frame_type
386 sys.exit(1)
387
388 packed_code = packed32(code)
389
390 packed_core = packed32(core)
391
392 length = int32(len(packed_code) + len(packed_core) + len(body))
393
394 ret = length + packed_code + packed_core + body
395 return ret
396
397
398# Summary frame
399# - timestamp: packed64
400# - uptime: packed64
401def summaryFrame(timestamp, uptime):
402 frame_type = "Summary"
|
385 body = packed64(timestamp) + packed64(uptime)
| 403 newline_canary = stringList("1\n2\r\n3\r4\n\r5")
404 monotonic_delta = packed64(0)
405 end_of_attr = stringList("")
406 body = newline_canary + packed64(timestamp) + packed64(uptime)
407 body += monotonic_delta + end_of_attr
|
386 ret = addFrameHeader(frame_type, body, 0)
387 return ret
388
389# Backtrace frame
390# - not implemented yet
391def backtraceFrame():
392 pass
393
394# Cookie name message
395# - cookie: packed32
396# - name: string
397def cookieNameFrame(cookie, name):
398 frame_type = "Name"
399 packed_code = packed32(1)
400 body = packed_code + packed32(cookie) + stringList(name)
401 ret = addFrameHeader(frame_type, body, 0)
402 return ret
403
404# Thread name message
405# - timestamp: timestamp
406# - thread id: packed32
407# - name: string
408def threadNameFrame(timestamp, thread_id, name):
409 frame_type = "Name"
410 packed_code = packed32(2)
411 body = packed_code + timestampList(timestamp) + \
412 packed32(thread_id) + stringList(name)
413 ret = addFrameHeader(frame_type, body, 0)
414 return ret
415
416# Core name message
417# - name: string
| 408 ret = addFrameHeader(frame_type, body, 0)
409 return ret
410
411# Backtrace frame
412# - not implemented yet
413def backtraceFrame():
414 pass
415
416# Cookie name message
417# - cookie: packed32
418# - name: string
419def cookieNameFrame(cookie, name):
420 frame_type = "Name"
421 packed_code = packed32(1)
422 body = packed_code + packed32(cookie) + stringList(name)
423 ret = addFrameHeader(frame_type, body, 0)
424 return ret
425
426# Thread name message
427# - timestamp: timestamp
428# - thread id: packed32
429# - name: string
430def threadNameFrame(timestamp, thread_id, name):
431 frame_type = "Name"
432 packed_code = packed32(2)
433 body = packed_code + timestampList(timestamp) + \
434 packed32(thread_id) + stringList(name)
435 ret = addFrameHeader(frame_type, body, 0)
436 return ret
437
438# Core name message
439# - name: string
|
418def coreNameFrame(name):
| 440# - core_id: packed32
441# - cpuid: packed32
442def coreNameFrame(name, core_id, cpuid):
|
419 frame_type = "Name"
420 packed_code = packed32(3)
| 443 frame_type = "Name"
444 packed_code = packed32(3)
|
421 body = packed_code + stringList(name)
| 445 body = packed_code + packed32(core_id) + packed32(cpuid) + stringList(name)
|
422 ret = addFrameHeader(frame_type, body, 0)
423 return ret
424
| 446 ret = addFrameHeader(frame_type, body, 0)
447 return ret
448
|
| 449# IRQ Cookie name message
450# - cookie: packed32
451# - name: string
452# - irq: packed32
453def irqCookieNameFrame(cookie, name, irq):
454 frame_type = "Name"
455 packed_code = packed32(5)
456 body = packed_code + packed32(cookie) + stringList(name) + packed32(irq)
457 ret = addFrameHeader(frame_type, body, 0)
458 return ret
459
|
425# Counter frame message
426# - timestamp: timestamp
427# - core: packed32
428# - key: packed32
429# - value: packed64
430def counterFrame(timestamp, core, key, value):
431 frame_type = "Counter"
432 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
433 packed64(value)
434 ret = addFrameHeader(frame_type, body, core)
435 return ret
436
437# Block Counter frame message
438# - key: packed32
439# - value: packed64
440def blockCounterFrame(core, key, value):
441 frame_type = "Block Counter"
442 body = packed32(key) + packed64(value)
443 ret = addFrameHeader(frame_type, body, core)
444 return ret
445
446# Annotate frame messages
447# - core: packed32
448# - tid: packed32
449# - timestamp: timestamp
450# - size: packed32
451# - body
452def annotateFrame(core, tid, timestamp, size, userspace_body):
453 frame_type = "Annotate"
454 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
455 packed32(size) + userspace_body
456 ret = addFrameHeader(frame_type, body, core)
457 return ret
458
459# Scheduler Trace frame messages
460# Sched Switch
461# - Code: 1
462# - timestamp: timestamp
463# - pid: packed32
464# - tid: packed32
465# - cookie: packed32
466# - state: packed32
467def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
468 frame_type = "Sched Trace"
469 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
470 packed32(tid) + packed32(cookie) + packed32(state)
471 ret = addFrameHeader(frame_type, body, core)
472 return ret
473
474# Sched Thread Exit
475# - Code: 2
476# - timestamp: timestamp
477# - tid: packed32
478def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
479 frame_type = "Sched Trace"
480 body = packed32(2) + timestampList(timestamp) + packed32(tid)
481 ret = addFrameHeader(frame_type, body, core)
482 return ret
483
484# GPU Trace frame messages
485# - Not implemented yet
486def gpuTraceFrame():
487 pass
488
489# Idle frame messages
490# Enter Idle
491# - code: 1
492# - timestamp: timestamp
493# - core: packed32
494def enterIdleFrame(timestamp, core):
495 frame_type = "Idle"
496 body = packed32(1) + timestampList(timestamp) + packed32(core)
497 ret = addFrameHeader(frame_type, body, core)
498 return ret
499
500# Exit Idle
501# - code: 2
502# - timestamp: timestamp
503# - core: packed32
504def exitIdleFrame(timestamp, core):
505 frame_type = "Idle"
506 body = packed32(2) + timestampList(timestamp) + packed32(core)
507 ret = addFrameHeader(frame_type, body, core)
508 return ret
509
510
511####################################################################
512def parseProcessInfo(task_file):
513 print "\n==============================="
514 print "Parsing Task file..."
515 print task_file
516 print "===============================\n"
517
518 global start_tick, end_tick, num_cpus
519 global process_dict, thread_dict, process_list
520 global event_list, unified_event_list
521 global idle_uid, kernel_uid
522
523 event_list = []
524 unified_event_list = []
525 for cpu in range(num_cpus):
526 event_list.append([])
527
528 uid = 1 # uid 0 is reserved for idle
529
530 # Dummy Tasks for frame buffers and system diagrams
531 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
532 process_list.append(process)
533 uid += 1
534 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
535 process.children.append(thread)
536 uid += 1
537 process = Task(uid, 9998, 9998, "System", True, 0)
538 process_list.append(process)
539 # if we don't find the real kernel, use this to keep things going
540 kernel_uid = uid
541 uid += 1
542 thread = Task(uid, 9998, 9998, "System", False, 0)
543 process.children.append(thread)
544 uid += 1
545
546 ext = os.path.splitext(task_file)[1]
547
548 try:
549 if ext == ".gz":
550 process_file = gzip.open(task_file, 'rb')
551 else:
552 process_file = open(task_file, 'rb')
553 except:
554 print "ERROR opening task file:", task_file
555 print "Make sure context switch task dumping is enabled in gem5."
556 sys.exit(1)
557
558 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
559 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
560
561 task_name_failure_warned = False
562
563 for line in process_file:
564 match = re.match(process_re, line)
565 if match:
566 tick = int(match.group(1))
567 if (start_tick < 0):
568 start_tick = tick
569 cpu_id = int(match.group(3))
570 pid = int(match.group(4))
571 tgid = int(match.group(5))
572 task_name = match.group(6)
573
574 if not task_name_failure_warned:
575 if task_name == "FailureIn_curTaskName":
576 print "-------------------------------------------------"
577 print "WARNING: Task name not set correctly!"
578 print "Process/Thread info will not be displayed correctly"
579 print "Perhaps forgot to apply m5struct.patch to kernel?"
580 print "-------------------------------------------------"
581 task_name_failure_warned = True
582
583 if not tgid in process_dict:
584 if tgid == pid:
585 # new task is parent as well
586 if args.verbose:
587 print "new process", uid, pid, tgid, task_name
588 if tgid == 0:
589 # new process is the "idle" task
590 process = Task(uid, pid, tgid, "idle", True, tick)
591 idle_uid = 0
592 else:
593 process = Task(uid, pid, tgid, task_name, True, tick)
594 else:
595 if tgid == 0:
596 process = Task(uid, tgid, tgid, "idle", True, tick)
597 idle_uid = 0
598 else:
599 # parent process name not known yet
600 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
601 if tgid == -1: # kernel
602 kernel_uid = 0
603 uid += 1
604 process_dict[tgid] = process
605 process_list.append(process)
606 else:
607 if tgid == pid:
608 if process_dict[tgid].task_name == "_Unknown_":
609 if args.verbose:
610 print "new process", \
611 process_dict[tgid].uid, pid, tgid, task_name
612 process_dict[tgid].task_name = task_name
613 if process_dict[tgid].task_name != task_name and tgid != 0:
614 process_dict[tgid].task_name = task_name
615
616 if not pid in thread_dict:
617 if args.verbose:
618 print "new thread", \
619 uid, process_dict[tgid].uid, pid, tgid, task_name
620 thread = Task(uid, pid, tgid, task_name, False, tick)
621 uid += 1
622 thread_dict[pid] = thread
623 process_dict[tgid].children.append(thread)
624 else:
625 if thread_dict[pid].task_name != task_name:
626 thread_dict[pid].task_name = task_name
627
628 if args.verbose:
629 print tick, uid, cpu_id, pid, tgid, task_name
630
631 task = thread_dict[pid]
632 event = Event(tick, task)
633 event_list[cpu_id].append(event)
634 unified_event_list.append(event)
635
636 if len(unified_event_list) == num_events:
637 print "Truncating at", num_events, "events!"
638 break
639 print "Found %d events." % len(unified_event_list)
640
641 for process in process_list:
642 if process.pid > 9990: # fix up framebuffer ticks
643 process.tick = start_tick
644 print process.uid, process.pid, process.tgid, \
645 process.task_name, str(process.tick)
646 for thread in process.children:
647 if thread.pid > 9990:
648 thread.tick = start_tick
649 print "\t", thread.uid, thread.pid, thread.tgid, \
650 thread.task_name, str(thread.tick)
651
652 end_tick = tick
653
654 print "Start tick:", start_tick
655 print "End tick: ", end_tick
656 print ""
657
658 return
659
660
661def initOutput(output_path):
662 if not os.path.exists(output_path):
663 os.mkdir(output_path)
664
665def ticksToNs(tick):
666 if ticks_in_ns < 0:
667 print "ticks_in_ns not set properly!"
668 sys.exit(1)
669
670 return tick / ticks_in_ns
671
672def writeXmlFile(xml, filename):
673 f = open(filename, "w")
674 txt = ET.tostring(xml)
675 f.write(minidom.parseString(txt).toprettyxml())
676 f.close()
677
678
679# StatsEntry that contains individual statistics
680class StatsEntry(object):
681 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
682
683 # Full name of statistics
684 self.name = name
685
686 # Streamline group name that statistic will belong to
687 self.group = group
688
689 # Index of statistics within group (used to change colors within groups)
690 self.group_index = group_index
691
692 # Shorter name with "system" stripped off
693 # and symbols converted to alphanumerics
694 self.short_name = re.sub("system\.", "", name)
695 self.short_name = re.sub(":", "_", name)
696
697 # Regex for this stat (string version used to construct union regex)
698 self.regex_string = "^" + name + "\s+([\d\.]+)"
699 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
700 self.description = ""
701
702 # Whether this stat is use per CPU or not
703 self.per_cpu = per_cpu
704 self.per_switchcpu = per_switchcpu
705
706 # Key used in .apc protocol (as described in captured.xml)
707 self.key = key
708
709 # List of values of stat per timestamp
710 self.values = []
711
712 # Whether this stat has been found for the current timestamp
713 self.found = False
714
715 # Whether this stat has been found at least once
716 # (to suppress too many warnings)
717 self.not_found_at_least_once = False
718
719 # Field used to hold ElementTree subelement for this stat
720 self.ET_element = None
721
722 # Create per-CPU stat name and regex, etc.
723 if self.per_cpu:
724 self.per_cpu_regex_string = []
725 self.per_cpu_regex = []
726 self.per_cpu_name = []
727 self.per_cpu_found = []
728 for i in range(num_cpus):
729 # Resuming from checkpoints results in using "switch_cpus"
730 if per_switchcpu:
731 per_cpu_name = "system.switch_cpus"
732 else:
733 per_cpu_name = "system.cpu"
734
735 # No CPU number appends if num_cpus == 1
736 if num_cpus > 1:
737 per_cpu_name += str(i)
738 per_cpu_name += "." + self.name
739 self.per_cpu_name.append(per_cpu_name)
740 print "\t", per_cpu_name
741
742 self.per_cpu_regex_string.\
743 append("^" + per_cpu_name + "\s+[\d\.]+")
744 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
745 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
746 self.values.append([])
747 self.per_cpu_found.append(False)
748
749 def append_value(self, val, per_cpu_index = None):
750 if self.per_cpu:
751 self.values[per_cpu_index].append(str(val))
752 else:
753 self.values.append(str(val))
754
755# Global stats object that contains the list of stats entries
756# and other utility functions
757class Stats(object):
758 def __init__(self):
759 self.stats_list = []
760 self.tick_list = []
761 self.next_key = 1
762
763 def register(self, name, group, group_index, per_cpu, per_switchcpu):
764 print "registering stat:", name, "group:", group, group_index
765 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
766 per_switchcpu, self.next_key))
767 self.next_key += 1
768
769 # Union of all stats to accelerate parsing speed
770 def createStatsRegex(self):
771 regex_strings = [];
772 print "\nnum entries in stats_list", len(self.stats_list)
773 for entry in self.stats_list:
774 if entry.per_cpu:
775 for i in range(num_cpus):
776 regex_strings.append(entry.per_cpu_regex_string[i])
777 else:
778 regex_strings.append(entry.regex_string)
779
780 self.regex = re.compile('|'.join(regex_strings))
781
782
783def registerStats(config_file):
784 print "==============================="
785 print "Parsing stats config.ini file..."
786 print config_file
787 print "==============================="
788
789 config = ConfigParser()
790 if not config.read(config_file):
791 print "ERROR: config file '", config_file, "' not found!"
792 sys.exit(1)
793
794 print "\nRegistering Stats..."
795
796 stats = Stats()
797
798 per_cpu_stat_groups = config.options('PER_CPU_STATS')
799 for group in per_cpu_stat_groups:
800 i = 0
801 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
802 for item in per_cpu_stats_list:
803 if item:
804 stats.register(item, group, i, True, False)
805 i += 1
806
807 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
808 for group in per_cpu_stat_groups:
809 i = 0
810 per_cpu_stats_list = \
811 config.get('PER_SWITCHCPU_STATS', group).split('\n')
812 for item in per_cpu_stats_list:
813 if item:
814 stats.register(item, group, i, True, True)
815 i += 1
816
817 per_l2_stat_groups = config.options('PER_L2_STATS')
818 for group in per_l2_stat_groups:
819 i = 0
820 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
821 for item in per_l2_stats_list:
822 if item:
823 for l2 in range(num_l2):
824 name = item
825 prefix = "system.l2"
826 if num_l2 > 1:
827 prefix += str(l2)
828 prefix += "."
829 name = prefix + name
830 stats.register(name, group, i, False, False)
831 i += 1
832
833 other_stat_groups = config.options('OTHER_STATS')
834 for group in other_stat_groups:
835 i = 0
836 other_stats_list = config.get('OTHER_STATS', group).split('\n')
837 for item in other_stats_list:
838 if item:
839 stats.register(item, group, i, False, False)
840 i += 1
841
842 stats.createStatsRegex()
843
844 return stats
845
846# Parse and read in gem5 stats file
847# Streamline counters are organized per CPU
848def readGem5Stats(stats, gem5_stats_file):
849 print "\n==============================="
850 print "Parsing gem5 stats file..."
851 print gem5_stats_file
852 print "===============================\n"
853 ext = os.path.splitext(gem5_stats_file)[1]
854
855 window_start_regex = \
856 re.compile("^---------- Begin Simulation Statistics ----------")
857 window_end_regex = \
858 re.compile("^---------- End Simulation Statistics ----------")
859 final_tick_regex = re.compile("^final_tick\s+(\d+)")
860
861 global ticks_in_ns
862 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
863 sim_freq = -1
864
865 try:
866 if ext == ".gz":
867 f = gzip.open(gem5_stats_file, "r")
868 else:
869 f = open(gem5_stats_file, "r")
870 except:
871 print "ERROR opening stats file", gem5_stats_file, "!"
872 sys.exit(1)
873
874 stats_not_found_list = stats.stats_list[:]
875 window_num = 0
876
877 while (True):
878 error = False
879 try:
880 line = f.readline()
881 except IOError:
882 print ""
883 print "WARNING: IO error in stats file"
884 print "(gzip stream not closed properly?)...continuing for now"
885 error = True
886 if not line:
887 break
888
889 # Find out how many gem5 ticks in 1ns
890 if sim_freq < 0:
891 m = sim_freq_regex.match(line)
892 if m:
893 sim_freq = int(m.group(1)) # ticks in 1 sec
894 ticks_in_ns = int(sim_freq / 1e9)
895 print "Simulation frequency found! 1 tick == %e sec\n" \
896 % (1.0 / sim_freq)
897
898 # Final tick in gem5 stats: current absolute timestamp
899 m = final_tick_regex.match(line)
900 if m:
901 tick = int(m.group(1))
902 if tick > end_tick:
903 break
904 stats.tick_list.append(tick)
905
906
907 if (window_end_regex.match(line) or error):
908 if args.verbose:
909 print "new window"
910 for stat in stats.stats_list:
911 if stat.per_cpu:
912 for i in range(num_cpus):
913 if not stat.per_cpu_found[i]:
914 if not stat.not_found_at_least_once:
915 print "WARNING: stat not found in window #", \
916 window_num, ":", stat.per_cpu_name[i]
917 print "suppressing further warnings for " + \
918 "this stat"
919 stat.not_found_at_least_once = True
920 stat.values[i].append(str(0))
921 stat.per_cpu_found[i] = False
922 else:
923 if not stat.found:
924 if not stat.not_found_at_least_once:
925 print "WARNING: stat not found in window #", \
926 window_num, ":", stat.name
927 print "suppressing further warnings for this stat"
928 stat.not_found_at_least_once = True
929 stat.values.append(str(0))
930 stat.found = False
931 stats_not_found_list = stats.stats_list[:]
932 window_num += 1
933 if error:
934 break
935
936 # Do a single regex of the union of all stats first for speed
937 if stats.regex.match(line):
938 # Then loop through only the stats we haven't seen in this window
939 for stat in stats_not_found_list[:]:
940 if stat.per_cpu:
941 for i in range(num_cpus):
942 m = stat.per_cpu_regex[i].match(line)
943 if m:
944 if stat.name == "ipc":
945 value = str(int(float(m.group(1)) * 1000))
946 else:
947 value = str(int(float(m.group(1))))
948 if args.verbose:
949 print stat.per_cpu_name[i], value
950 stat.values[i].append(value)
951 stat.per_cpu_found[i] = True
952 all_found = True
953 for j in range(num_cpus):
954 if not stat.per_cpu_found[j]:
955 all_found = False
956 if all_found:
957 stats_not_found_list.remove(stat)
958 if stat.description == "":
959 stat.description = m.group(2)
960 else:
961 m = stat.regex.match(line)
962 if m:
963 value = str(int(float(m.group(1))))
964 if args.verbose:
965 print stat.name, value
966 stat.values.append(value)
967 stat.found = True
968 stats_not_found_list.remove(stat)
969 if stat.description == "":
970 stat.description = m.group(2)
971 f.close()
972
973
974# Create session.xml file in .apc folder
975def doSessionXML(output_path):
976 session_file = output_path + "/session.xml"
977
978 xml = ET.Element("session")
979
980 xml.set("version", "1")
981 xml.set("call_stack_unwinding", "no")
982 xml.set("parse_debug_info", "no")
983 xml.set("high_resolution", "yes")
984 xml.set("buffer_mode", "streaming")
985 xml.set("sample_rate", "low")
986
987 # Setting duration to zero for now. Doesn't affect visualization.
988 xml.set("duration", "0")
989
990 xml.set("target_host", "")
991 xml.set("target_port", "8080")
992
993 writeXmlFile(xml, session_file)
994
995
996# Create captured.xml file in .apc folder
997def doCapturedXML(output_path, stats):
998 captured_file = output_path + "/captured.xml"
999
1000 xml = ET.Element("captured")
1001 xml.set("version", "1")
| 460# Counter frame message
461# - timestamp: timestamp
462# - core: packed32
463# - key: packed32
464# - value: packed64
465def counterFrame(timestamp, core, key, value):
466 frame_type = "Counter"
467 body = timestampList(timestamp) + packed32(core) + packed32(key) + \
468 packed64(value)
469 ret = addFrameHeader(frame_type, body, core)
470 return ret
471
472# Block Counter frame message
473# - key: packed32
474# - value: packed64
475def blockCounterFrame(core, key, value):
476 frame_type = "Block Counter"
477 body = packed32(key) + packed64(value)
478 ret = addFrameHeader(frame_type, body, core)
479 return ret
480
481# Annotate frame messages
482# - core: packed32
483# - tid: packed32
484# - timestamp: timestamp
485# - size: packed32
486# - body
487def annotateFrame(core, tid, timestamp, size, userspace_body):
488 frame_type = "Annotate"
489 body = packed32(core) + packed32(tid) + timestampList(timestamp) + \
490 packed32(size) + userspace_body
491 ret = addFrameHeader(frame_type, body, core)
492 return ret
493
494# Scheduler Trace frame messages
495# Sched Switch
496# - Code: 1
497# - timestamp: timestamp
498# - pid: packed32
499# - tid: packed32
500# - cookie: packed32
501# - state: packed32
502def schedSwitchFrame(core, timestamp, pid, tid, cookie, state):
503 frame_type = "Sched Trace"
504 body = packed32(1) + timestampList(timestamp) + packed32(pid) + \
505 packed32(tid) + packed32(cookie) + packed32(state)
506 ret = addFrameHeader(frame_type, body, core)
507 return ret
508
509# Sched Thread Exit
510# - Code: 2
511# - timestamp: timestamp
512# - tid: packed32
513def schedThreadExitFrame(core, timestamp, pid, tid, cookie, state):
514 frame_type = "Sched Trace"
515 body = packed32(2) + timestampList(timestamp) + packed32(tid)
516 ret = addFrameHeader(frame_type, body, core)
517 return ret
518
519# GPU Trace frame messages
520# - Not implemented yet
521def gpuTraceFrame():
522 pass
523
524# Idle frame messages
525# Enter Idle
526# - code: 1
527# - timestamp: timestamp
528# - core: packed32
529def enterIdleFrame(timestamp, core):
530 frame_type = "Idle"
531 body = packed32(1) + timestampList(timestamp) + packed32(core)
532 ret = addFrameHeader(frame_type, body, core)
533 return ret
534
535# Exit Idle
536# - code: 2
537# - timestamp: timestamp
538# - core: packed32
539def exitIdleFrame(timestamp, core):
540 frame_type = "Idle"
541 body = packed32(2) + timestampList(timestamp) + packed32(core)
542 ret = addFrameHeader(frame_type, body, core)
543 return ret
544
545
546####################################################################
547def parseProcessInfo(task_file):
548 print "\n==============================="
549 print "Parsing Task file..."
550 print task_file
551 print "===============================\n"
552
553 global start_tick, end_tick, num_cpus
554 global process_dict, thread_dict, process_list
555 global event_list, unified_event_list
556 global idle_uid, kernel_uid
557
558 event_list = []
559 unified_event_list = []
560 for cpu in range(num_cpus):
561 event_list.append([])
562
563 uid = 1 # uid 0 is reserved for idle
564
565 # Dummy Tasks for frame buffers and system diagrams
566 process = Task(uid, 9999, 9999, "framebuffer", True, 0)
567 process_list.append(process)
568 uid += 1
569 thread = Task(uid, 9999, 9999, "framebuffer", False, 0)
570 process.children.append(thread)
571 uid += 1
572 process = Task(uid, 9998, 9998, "System", True, 0)
573 process_list.append(process)
574 # if we don't find the real kernel, use this to keep things going
575 kernel_uid = uid
576 uid += 1
577 thread = Task(uid, 9998, 9998, "System", False, 0)
578 process.children.append(thread)
579 uid += 1
580
581 ext = os.path.splitext(task_file)[1]
582
583 try:
584 if ext == ".gz":
585 process_file = gzip.open(task_file, 'rb')
586 else:
587 process_file = open(task_file, 'rb')
588 except:
589 print "ERROR opening task file:", task_file
590 print "Make sure context switch task dumping is enabled in gem5."
591 sys.exit(1)
592
593 process_re = re.compile("tick=(\d+)\s+(\d+)\s+cpu_id=(\d+)\s+" +
594 "next_pid=([-\d]+)\s+next_tgid=([-\d]+)\s+next_task=(.*)")
595
596 task_name_failure_warned = False
597
598 for line in process_file:
599 match = re.match(process_re, line)
600 if match:
601 tick = int(match.group(1))
602 if (start_tick < 0):
603 start_tick = tick
604 cpu_id = int(match.group(3))
605 pid = int(match.group(4))
606 tgid = int(match.group(5))
607 task_name = match.group(6)
608
609 if not task_name_failure_warned:
610 if task_name == "FailureIn_curTaskName":
611 print "-------------------------------------------------"
612 print "WARNING: Task name not set correctly!"
613 print "Process/Thread info will not be displayed correctly"
614 print "Perhaps forgot to apply m5struct.patch to kernel?"
615 print "-------------------------------------------------"
616 task_name_failure_warned = True
617
618 if not tgid in process_dict:
619 if tgid == pid:
620 # new task is parent as well
621 if args.verbose:
622 print "new process", uid, pid, tgid, task_name
623 if tgid == 0:
624 # new process is the "idle" task
625 process = Task(uid, pid, tgid, "idle", True, tick)
626 idle_uid = 0
627 else:
628 process = Task(uid, pid, tgid, task_name, True, tick)
629 else:
630 if tgid == 0:
631 process = Task(uid, tgid, tgid, "idle", True, tick)
632 idle_uid = 0
633 else:
634 # parent process name not known yet
635 process = Task(uid, tgid, tgid, "_Unknown_", True, tick)
636 if tgid == -1: # kernel
637 kernel_uid = 0
638 uid += 1
639 process_dict[tgid] = process
640 process_list.append(process)
641 else:
642 if tgid == pid:
643 if process_dict[tgid].task_name == "_Unknown_":
644 if args.verbose:
645 print "new process", \
646 process_dict[tgid].uid, pid, tgid, task_name
647 process_dict[tgid].task_name = task_name
648 if process_dict[tgid].task_name != task_name and tgid != 0:
649 process_dict[tgid].task_name = task_name
650
651 if not pid in thread_dict:
652 if args.verbose:
653 print "new thread", \
654 uid, process_dict[tgid].uid, pid, tgid, task_name
655 thread = Task(uid, pid, tgid, task_name, False, tick)
656 uid += 1
657 thread_dict[pid] = thread
658 process_dict[tgid].children.append(thread)
659 else:
660 if thread_dict[pid].task_name != task_name:
661 thread_dict[pid].task_name = task_name
662
663 if args.verbose:
664 print tick, uid, cpu_id, pid, tgid, task_name
665
666 task = thread_dict[pid]
667 event = Event(tick, task)
668 event_list[cpu_id].append(event)
669 unified_event_list.append(event)
670
671 if len(unified_event_list) == num_events:
672 print "Truncating at", num_events, "events!"
673 break
674 print "Found %d events." % len(unified_event_list)
675
676 for process in process_list:
677 if process.pid > 9990: # fix up framebuffer ticks
678 process.tick = start_tick
679 print process.uid, process.pid, process.tgid, \
680 process.task_name, str(process.tick)
681 for thread in process.children:
682 if thread.pid > 9990:
683 thread.tick = start_tick
684 print "\t", thread.uid, thread.pid, thread.tgid, \
685 thread.task_name, str(thread.tick)
686
687 end_tick = tick
688
689 print "Start tick:", start_tick
690 print "End tick: ", end_tick
691 print ""
692
693 return
694
695
696def initOutput(output_path):
697 if not os.path.exists(output_path):
698 os.mkdir(output_path)
699
700def ticksToNs(tick):
701 if ticks_in_ns < 0:
702 print "ticks_in_ns not set properly!"
703 sys.exit(1)
704
705 return tick / ticks_in_ns
706
707def writeXmlFile(xml, filename):
708 f = open(filename, "w")
709 txt = ET.tostring(xml)
710 f.write(minidom.parseString(txt).toprettyxml())
711 f.close()
712
713
714# StatsEntry that contains individual statistics
715class StatsEntry(object):
716 def __init__(self, name, group, group_index, per_cpu, per_switchcpu, key):
717
718 # Full name of statistics
719 self.name = name
720
721 # Streamline group name that statistic will belong to
722 self.group = group
723
724 # Index of statistics within group (used to change colors within groups)
725 self.group_index = group_index
726
727 # Shorter name with "system" stripped off
728 # and symbols converted to alphanumerics
729 self.short_name = re.sub("system\.", "", name)
730 self.short_name = re.sub(":", "_", name)
731
732 # Regex for this stat (string version used to construct union regex)
733 self.regex_string = "^" + name + "\s+([\d\.]+)"
734 self.regex = re.compile("^" + name + "\s+([\d\.e\-]+)\s+# (.*)$", re.M)
735 self.description = ""
736
737 # Whether this stat is use per CPU or not
738 self.per_cpu = per_cpu
739 self.per_switchcpu = per_switchcpu
740
741 # Key used in .apc protocol (as described in captured.xml)
742 self.key = key
743
744 # List of values of stat per timestamp
745 self.values = []
746
747 # Whether this stat has been found for the current timestamp
748 self.found = False
749
750 # Whether this stat has been found at least once
751 # (to suppress too many warnings)
752 self.not_found_at_least_once = False
753
754 # Field used to hold ElementTree subelement for this stat
755 self.ET_element = None
756
757 # Create per-CPU stat name and regex, etc.
758 if self.per_cpu:
759 self.per_cpu_regex_string = []
760 self.per_cpu_regex = []
761 self.per_cpu_name = []
762 self.per_cpu_found = []
763 for i in range(num_cpus):
764 # Resuming from checkpoints results in using "switch_cpus"
765 if per_switchcpu:
766 per_cpu_name = "system.switch_cpus"
767 else:
768 per_cpu_name = "system.cpu"
769
770 # No CPU number appends if num_cpus == 1
771 if num_cpus > 1:
772 per_cpu_name += str(i)
773 per_cpu_name += "." + self.name
774 self.per_cpu_name.append(per_cpu_name)
775 print "\t", per_cpu_name
776
777 self.per_cpu_regex_string.\
778 append("^" + per_cpu_name + "\s+[\d\.]+")
779 self.per_cpu_regex.append(re.compile("^" + per_cpu_name + \
780 "\s+([\d\.e\-]+)\s+# (.*)$", re.M))
781 self.values.append([])
782 self.per_cpu_found.append(False)
783
784 def append_value(self, val, per_cpu_index = None):
785 if self.per_cpu:
786 self.values[per_cpu_index].append(str(val))
787 else:
788 self.values.append(str(val))
789
790# Global stats object that contains the list of stats entries
791# and other utility functions
792class Stats(object):
793 def __init__(self):
794 self.stats_list = []
795 self.tick_list = []
796 self.next_key = 1
797
798 def register(self, name, group, group_index, per_cpu, per_switchcpu):
799 print "registering stat:", name, "group:", group, group_index
800 self.stats_list.append(StatsEntry(name, group, group_index, per_cpu, \
801 per_switchcpu, self.next_key))
802 self.next_key += 1
803
804 # Union of all stats to accelerate parsing speed
805 def createStatsRegex(self):
806 regex_strings = [];
807 print "\nnum entries in stats_list", len(self.stats_list)
808 for entry in self.stats_list:
809 if entry.per_cpu:
810 for i in range(num_cpus):
811 regex_strings.append(entry.per_cpu_regex_string[i])
812 else:
813 regex_strings.append(entry.regex_string)
814
815 self.regex = re.compile('|'.join(regex_strings))
816
817
818def registerStats(config_file):
819 print "==============================="
820 print "Parsing stats config.ini file..."
821 print config_file
822 print "==============================="
823
824 config = ConfigParser()
825 if not config.read(config_file):
826 print "ERROR: config file '", config_file, "' not found!"
827 sys.exit(1)
828
829 print "\nRegistering Stats..."
830
831 stats = Stats()
832
833 per_cpu_stat_groups = config.options('PER_CPU_STATS')
834 for group in per_cpu_stat_groups:
835 i = 0
836 per_cpu_stats_list = config.get('PER_CPU_STATS', group).split('\n')
837 for item in per_cpu_stats_list:
838 if item:
839 stats.register(item, group, i, True, False)
840 i += 1
841
842 per_cpu_stat_groups = config.options('PER_SWITCHCPU_STATS')
843 for group in per_cpu_stat_groups:
844 i = 0
845 per_cpu_stats_list = \
846 config.get('PER_SWITCHCPU_STATS', group).split('\n')
847 for item in per_cpu_stats_list:
848 if item:
849 stats.register(item, group, i, True, True)
850 i += 1
851
852 per_l2_stat_groups = config.options('PER_L2_STATS')
853 for group in per_l2_stat_groups:
854 i = 0
855 per_l2_stats_list = config.get('PER_L2_STATS', group).split('\n')
856 for item in per_l2_stats_list:
857 if item:
858 for l2 in range(num_l2):
859 name = item
860 prefix = "system.l2"
861 if num_l2 > 1:
862 prefix += str(l2)
863 prefix += "."
864 name = prefix + name
865 stats.register(name, group, i, False, False)
866 i += 1
867
868 other_stat_groups = config.options('OTHER_STATS')
869 for group in other_stat_groups:
870 i = 0
871 other_stats_list = config.get('OTHER_STATS', group).split('\n')
872 for item in other_stats_list:
873 if item:
874 stats.register(item, group, i, False, False)
875 i += 1
876
877 stats.createStatsRegex()
878
879 return stats
880
881# Parse and read in gem5 stats file
882# Streamline counters are organized per CPU
883def readGem5Stats(stats, gem5_stats_file):
884 print "\n==============================="
885 print "Parsing gem5 stats file..."
886 print gem5_stats_file
887 print "===============================\n"
888 ext = os.path.splitext(gem5_stats_file)[1]
889
890 window_start_regex = \
891 re.compile("^---------- Begin Simulation Statistics ----------")
892 window_end_regex = \
893 re.compile("^---------- End Simulation Statistics ----------")
894 final_tick_regex = re.compile("^final_tick\s+(\d+)")
895
896 global ticks_in_ns
897 sim_freq_regex = re.compile("^sim_freq\s+(\d+)")
898 sim_freq = -1
899
900 try:
901 if ext == ".gz":
902 f = gzip.open(gem5_stats_file, "r")
903 else:
904 f = open(gem5_stats_file, "r")
905 except:
906 print "ERROR opening stats file", gem5_stats_file, "!"
907 sys.exit(1)
908
909 stats_not_found_list = stats.stats_list[:]
910 window_num = 0
911
912 while (True):
913 error = False
914 try:
915 line = f.readline()
916 except IOError:
917 print ""
918 print "WARNING: IO error in stats file"
919 print "(gzip stream not closed properly?)...continuing for now"
920 error = True
921 if not line:
922 break
923
924 # Find out how many gem5 ticks in 1ns
925 if sim_freq < 0:
926 m = sim_freq_regex.match(line)
927 if m:
928 sim_freq = int(m.group(1)) # ticks in 1 sec
929 ticks_in_ns = int(sim_freq / 1e9)
930 print "Simulation frequency found! 1 tick == %e sec\n" \
931 % (1.0 / sim_freq)
932
933 # Final tick in gem5 stats: current absolute timestamp
934 m = final_tick_regex.match(line)
935 if m:
936 tick = int(m.group(1))
937 if tick > end_tick:
938 break
939 stats.tick_list.append(tick)
940
941
942 if (window_end_regex.match(line) or error):
943 if args.verbose:
944 print "new window"
945 for stat in stats.stats_list:
946 if stat.per_cpu:
947 for i in range(num_cpus):
948 if not stat.per_cpu_found[i]:
949 if not stat.not_found_at_least_once:
950 print "WARNING: stat not found in window #", \
951 window_num, ":", stat.per_cpu_name[i]
952 print "suppressing further warnings for " + \
953 "this stat"
954 stat.not_found_at_least_once = True
955 stat.values[i].append(str(0))
956 stat.per_cpu_found[i] = False
957 else:
958 if not stat.found:
959 if not stat.not_found_at_least_once:
960 print "WARNING: stat not found in window #", \
961 window_num, ":", stat.name
962 print "suppressing further warnings for this stat"
963 stat.not_found_at_least_once = True
964 stat.values.append(str(0))
965 stat.found = False
966 stats_not_found_list = stats.stats_list[:]
967 window_num += 1
968 if error:
969 break
970
971 # Do a single regex of the union of all stats first for speed
972 if stats.regex.match(line):
973 # Then loop through only the stats we haven't seen in this window
974 for stat in stats_not_found_list[:]:
975 if stat.per_cpu:
976 for i in range(num_cpus):
977 m = stat.per_cpu_regex[i].match(line)
978 if m:
979 if stat.name == "ipc":
980 value = str(int(float(m.group(1)) * 1000))
981 else:
982 value = str(int(float(m.group(1))))
983 if args.verbose:
984 print stat.per_cpu_name[i], value
985 stat.values[i].append(value)
986 stat.per_cpu_found[i] = True
987 all_found = True
988 for j in range(num_cpus):
989 if not stat.per_cpu_found[j]:
990 all_found = False
991 if all_found:
992 stats_not_found_list.remove(stat)
993 if stat.description == "":
994 stat.description = m.group(2)
995 else:
996 m = stat.regex.match(line)
997 if m:
998 value = str(int(float(m.group(1))))
999 if args.verbose:
1000 print stat.name, value
1001 stat.values.append(value)
1002 stat.found = True
1003 stats_not_found_list.remove(stat)
1004 if stat.description == "":
1005 stat.description = m.group(2)
1006 f.close()
1007
1008
1009# Create session.xml file in .apc folder
1010def doSessionXML(output_path):
1011 session_file = output_path + "/session.xml"
1012
1013 xml = ET.Element("session")
1014
1015 xml.set("version", "1")
1016 xml.set("call_stack_unwinding", "no")
1017 xml.set("parse_debug_info", "no")
1018 xml.set("high_resolution", "yes")
1019 xml.set("buffer_mode", "streaming")
1020 xml.set("sample_rate", "low")
1021
1022 # Setting duration to zero for now. Doesn't affect visualization.
1023 xml.set("duration", "0")
1024
1025 xml.set("target_host", "")
1026 xml.set("target_port", "8080")
1027
1028 writeXmlFile(xml, session_file)
1029
1030
1031# Create captured.xml file in .apc folder
1032def doCapturedXML(output_path, stats):
1033 captured_file = output_path + "/captured.xml"
1034
1035 xml = ET.Element("captured")
1036 xml.set("version", "1")
|
1002 xml.set("protocol", "12")
| 1037 xml.set("protocol", "17")
1038 xml.set("backtrace_processing", "none")
|
1003
1004 target = ET.SubElement(xml, "target")
1005 target.set("name", "gem5")
1006 target.set("sample_rate", "1000")
1007 target.set("cores", str(num_cpus))
1008
1009 counters = ET.SubElement(xml, "counters")
1010 for stat in stats.stats_list:
1011 s = ET.SubElement(counters, "counter")
1012 stat_name = re.sub("\.", "_", stat.short_name)
1013 s.set("title", stat.group)
1014 s.set("name", stat_name)
1015 s.set("color", "0x00000000")
1016 s.set("key", "0x%08x" % stat.key)
1017 s.set("type", stat_name)
1018 s.set("event", "0x00000000")
1019 if stat.per_cpu:
1020 s.set("per_cpu", "yes")
1021 else:
1022 s.set("per_cpu", "no")
1023 s.set("display", "")
1024 s.set("units", "")
1025 s.set("average_selection", "no")
1026 s.set("description", stat.description)
1027
1028 writeXmlFile(xml, captured_file)
1029
1030# Writes out Streamline cookies (unique IDs per process/thread)
1031def writeCookiesThreads(blob):
1032 thread_list = []
1033 for process in process_list:
1034 if process.uid > 0:
1035 print "cookie", process.task_name, process.uid
1036 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1037
1038 # pid and tgid need to be positive values -- no longer true?
1039 for thread in process.children:
1040 thread_list.append(thread)
1041
1042 # Threads need to be sorted in timestamp order
1043 thread_list.sort(key = lambda x: x.tick)
1044 for thread in thread_list:
1045 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1046 thread.tgid, thread.pid
1047 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1048 thread.pid, thread.task_name))
1049
1050# Writes context switch info as Streamline scheduling events
1051def writeSchedEvents(blob):
1052 for cpu in range(num_cpus):
1053 for event in event_list[cpu]:
1054 timestamp = ticksToNs(event.tick)
1055 pid = event.task.tgid
1056 tid = event.task.pid
1057 if process_dict.has_key(event.task.tgid):
1058 cookie = process_dict[event.task.tgid].uid
1059 else:
1060 cookie = 0
1061
1062 # State:
1063 # 0: waiting on other event besides I/O
1064 # 1: Contention/pre-emption
1065 # 2: Waiting on I/O
1066 # 3: Waiting on mutex
1067 # Hardcoding to 0 for now. Other states not implemented yet.
1068 state = 0
1069
1070 if args.verbose:
1071 print cpu, timestamp, pid, tid, cookie
1072
1073 writeBinary(blob,\
1074 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1075
1076# Writes selected gem5 statistics as Streamline counters
1077def writeCounters(blob, stats):
1078 timestamp_list = []
1079 for tick in stats.tick_list:
1080 if tick > end_tick:
1081 break
1082 timestamp_list.append(ticksToNs(tick))
1083
1084 for stat in stats.stats_list:
1085 if stat.per_cpu:
1086 stat_length = len(stat.values[0])
1087 else:
1088 stat_length = len(stat.values)
1089
1090 for n in range(len(timestamp_list)):
1091 for stat in stats.stats_list:
1092 if stat.per_cpu:
1093 for i in range(num_cpus):
1094 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1095 stat.key, int(float(stat.values[i][n]))))
1096 else:
1097 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1098 stat.key, int(float(stat.values[n]))))
1099
1100# Streamline can display LCD frame buffer dumps (gzipped bmp)
1101# This function converts the frame buffer dumps to the Streamline format
1102def writeVisualAnnotations(blob, input_path, output_path):
1103 frame_path = input_path + "/frames_system.vncserver"
1104 if not os.path.exists(frame_path):
1105 return
1106
1107 frame_count = 0
1108 file_list = os.listdir(frame_path)
1109 file_list.sort()
1110 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1111
1112 # Use first non-negative pid to tag visual annotations
1113 annotate_pid = -1
1114 for e in unified_event_list:
1115 pid = e.task.pid
1116 if pid >= 0:
1117 annotate_pid = pid
1118 break
1119
1120 for fn in file_list:
1121 m = re_fb.match(fn)
1122 if m:
1123 seq = m.group(1)
1124 tick = int(m.group(2))
1125 if tick > end_tick:
1126 break
1127 frame_count += 1
1128
1129 userspace_body = []
1130 userspace_body += packed32(0x1C) # escape code
1131 userspace_body += packed32(0x04) # visual code
1132
1133 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1134 userspace_body += int16(len(text_annotation))
1135 userspace_body += utf8StringList(text_annotation)
1136
1137 if gzipped_bmp_supported:
1138 # copy gzipped bmp directly
1139 bytes_read = open(frame_path + "/" + fn, "rb").read()
1140 else:
1141 # copy uncompressed bmp
1142 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1143
1144 userspace_body += int32(len(bytes_read))
1145 userspace_body += bytes_read
1146
1147 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1148 len(userspace_body), userspace_body))
1149
1150 print "\nfound", frame_count, "frames for visual annotation.\n"
1151
1152
1153def createApcProject(input_path, output_path, stats):
1154 initOutput(output_path)
1155
1156 blob = open(output_path + "/0000000000", "wb")
1157
1158 # Summary frame takes current system time and system uptime.
1159 # Filling in with random values for now.
1160 writeBinary(blob, summaryFrame(1234, 5678))
1161
1162 writeCookiesThreads(blob)
1163
1164 print "writing Events"
1165 writeSchedEvents(blob)
1166
1167 print "writing Counters"
1168 writeCounters(blob, stats)
1169
1170 print "writing Visual Annotations"
1171 writeVisualAnnotations(blob, input_path, output_path)
1172
1173 doSessionXML(output_path)
1174 doCapturedXML(output_path, stats)
1175
1176 blob.close()
1177
1178
1179
1180#######################
1181# Main Routine
1182
1183input_path = args.input_path
1184output_path = args.output_path
1185
1186####
1187# Make sure input path exists
1188####
1189if not os.path.exists(input_path):
1190 print "ERROR: Input path %s does not exist!" % input_path
1191 sys.exit(1)
1192
1193####
1194# Parse gem5 configuration file to find # of CPUs and L2s
1195####
1196(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1197
1198####
1199# Parse task file to find process/thread info
1200####
1201parseProcessInfo(input_path + "/system.tasks.txt")
1202
1203####
1204# Parse stat config file and register stats
1205####
1206stat_config_file = args.stat_config_file
1207stats = registerStats(stat_config_file)
1208
1209####
1210# Parse gem5 stats
1211####
1212# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1213if os.path.exists(input_path + "/stats.txt") and \
1214 os.path.exists(input_path + "/stats.txt.gz"):
1215 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1216 Using stats.txt.gz by default."
1217
1218gem5_stats_file = input_path + "/stats.txt.gz"
1219if not os.path.exists(gem5_stats_file):
1220 gem5_stats_file = input_path + "/stats.txt"
1221if not os.path.exists(gem5_stats_file):
1222 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1223 sys.exit(1)
1224
1225readGem5Stats(stats, gem5_stats_file)
1226
1227####
1228# Create Streamline .apc project folder
1229####
1230createApcProject(input_path, output_path, stats)
1231
1232print "All done!"
| 1039
1040 target = ET.SubElement(xml, "target")
1041 target.set("name", "gem5")
1042 target.set("sample_rate", "1000")
1043 target.set("cores", str(num_cpus))
1044
1045 counters = ET.SubElement(xml, "counters")
1046 for stat in stats.stats_list:
1047 s = ET.SubElement(counters, "counter")
1048 stat_name = re.sub("\.", "_", stat.short_name)
1049 s.set("title", stat.group)
1050 s.set("name", stat_name)
1051 s.set("color", "0x00000000")
1052 s.set("key", "0x%08x" % stat.key)
1053 s.set("type", stat_name)
1054 s.set("event", "0x00000000")
1055 if stat.per_cpu:
1056 s.set("per_cpu", "yes")
1057 else:
1058 s.set("per_cpu", "no")
1059 s.set("display", "")
1060 s.set("units", "")
1061 s.set("average_selection", "no")
1062 s.set("description", stat.description)
1063
1064 writeXmlFile(xml, captured_file)
1065
1066# Writes out Streamline cookies (unique IDs per process/thread)
1067def writeCookiesThreads(blob):
1068 thread_list = []
1069 for process in process_list:
1070 if process.uid > 0:
1071 print "cookie", process.task_name, process.uid
1072 writeBinary(blob, cookieNameFrame(process.uid, process.task_name))
1073
1074 # pid and tgid need to be positive values -- no longer true?
1075 for thread in process.children:
1076 thread_list.append(thread)
1077
1078 # Threads need to be sorted in timestamp order
1079 thread_list.sort(key = lambda x: x.tick)
1080 for thread in thread_list:
1081 print "thread", thread.task_name, (ticksToNs(thread.tick)),\
1082 thread.tgid, thread.pid
1083 writeBinary(blob, threadNameFrame(ticksToNs(thread.tick),\
1084 thread.pid, thread.task_name))
1085
1086# Writes context switch info as Streamline scheduling events
1087def writeSchedEvents(blob):
1088 for cpu in range(num_cpus):
1089 for event in event_list[cpu]:
1090 timestamp = ticksToNs(event.tick)
1091 pid = event.task.tgid
1092 tid = event.task.pid
1093 if process_dict.has_key(event.task.tgid):
1094 cookie = process_dict[event.task.tgid].uid
1095 else:
1096 cookie = 0
1097
1098 # State:
1099 # 0: waiting on other event besides I/O
1100 # 1: Contention/pre-emption
1101 # 2: Waiting on I/O
1102 # 3: Waiting on mutex
1103 # Hardcoding to 0 for now. Other states not implemented yet.
1104 state = 0
1105
1106 if args.verbose:
1107 print cpu, timestamp, pid, tid, cookie
1108
1109 writeBinary(blob,\
1110 schedSwitchFrame(cpu, timestamp, pid, tid, cookie, state))
1111
1112# Writes selected gem5 statistics as Streamline counters
1113def writeCounters(blob, stats):
1114 timestamp_list = []
1115 for tick in stats.tick_list:
1116 if tick > end_tick:
1117 break
1118 timestamp_list.append(ticksToNs(tick))
1119
1120 for stat in stats.stats_list:
1121 if stat.per_cpu:
1122 stat_length = len(stat.values[0])
1123 else:
1124 stat_length = len(stat.values)
1125
1126 for n in range(len(timestamp_list)):
1127 for stat in stats.stats_list:
1128 if stat.per_cpu:
1129 for i in range(num_cpus):
1130 writeBinary(blob, counterFrame(timestamp_list[n], i, \
1131 stat.key, int(float(stat.values[i][n]))))
1132 else:
1133 writeBinary(blob, counterFrame(timestamp_list[n], 0, \
1134 stat.key, int(float(stat.values[n]))))
1135
1136# Streamline can display LCD frame buffer dumps (gzipped bmp)
1137# This function converts the frame buffer dumps to the Streamline format
1138def writeVisualAnnotations(blob, input_path, output_path):
1139 frame_path = input_path + "/frames_system.vncserver"
1140 if not os.path.exists(frame_path):
1141 return
1142
1143 frame_count = 0
1144 file_list = os.listdir(frame_path)
1145 file_list.sort()
1146 re_fb = re.compile("fb\.(\d+)\.(\d+)\.bmp.gz")
1147
1148 # Use first non-negative pid to tag visual annotations
1149 annotate_pid = -1
1150 for e in unified_event_list:
1151 pid = e.task.pid
1152 if pid >= 0:
1153 annotate_pid = pid
1154 break
1155
1156 for fn in file_list:
1157 m = re_fb.match(fn)
1158 if m:
1159 seq = m.group(1)
1160 tick = int(m.group(2))
1161 if tick > end_tick:
1162 break
1163 frame_count += 1
1164
1165 userspace_body = []
1166 userspace_body += packed32(0x1C) # escape code
1167 userspace_body += packed32(0x04) # visual code
1168
1169 text_annotation = "image_" + str(ticksToNs(tick)) + ".bmp.gz"
1170 userspace_body += int16(len(text_annotation))
1171 userspace_body += utf8StringList(text_annotation)
1172
1173 if gzipped_bmp_supported:
1174 # copy gzipped bmp directly
1175 bytes_read = open(frame_path + "/" + fn, "rb").read()
1176 else:
1177 # copy uncompressed bmp
1178 bytes_read = gzip.open(frame_path + "/" + fn, "rb").read()
1179
1180 userspace_body += int32(len(bytes_read))
1181 userspace_body += bytes_read
1182
1183 writeBinary(blob, annotateFrame(0, annotate_pid, ticksToNs(tick), \
1184 len(userspace_body), userspace_body))
1185
1186 print "\nfound", frame_count, "frames for visual annotation.\n"
1187
1188
1189def createApcProject(input_path, output_path, stats):
1190 initOutput(output_path)
1191
1192 blob = open(output_path + "/0000000000", "wb")
1193
1194 # Summary frame takes current system time and system uptime.
1195 # Filling in with random values for now.
1196 writeBinary(blob, summaryFrame(1234, 5678))
1197
1198 writeCookiesThreads(blob)
1199
1200 print "writing Events"
1201 writeSchedEvents(blob)
1202
1203 print "writing Counters"
1204 writeCounters(blob, stats)
1205
1206 print "writing Visual Annotations"
1207 writeVisualAnnotations(blob, input_path, output_path)
1208
1209 doSessionXML(output_path)
1210 doCapturedXML(output_path, stats)
1211
1212 blob.close()
1213
1214
1215
1216#######################
1217# Main Routine
1218
1219input_path = args.input_path
1220output_path = args.output_path
1221
1222####
1223# Make sure input path exists
1224####
1225if not os.path.exists(input_path):
1226 print "ERROR: Input path %s does not exist!" % input_path
1227 sys.exit(1)
1228
1229####
1230# Parse gem5 configuration file to find # of CPUs and L2s
1231####
1232(num_cpus, num_l2) = parseConfig(input_path + "/config.ini")
1233
1234####
1235# Parse task file to find process/thread info
1236####
1237parseProcessInfo(input_path + "/system.tasks.txt")
1238
1239####
1240# Parse stat config file and register stats
1241####
1242stat_config_file = args.stat_config_file
1243stats = registerStats(stat_config_file)
1244
1245####
1246# Parse gem5 stats
1247####
1248# Check if both stats.txt and stats.txt.gz exist and warn if both exist
1249if os.path.exists(input_path + "/stats.txt") and \
1250 os.path.exists(input_path + "/stats.txt.gz"):
1251 print "WARNING: Both stats.txt.gz and stats.txt exist. \
1252 Using stats.txt.gz by default."
1253
1254gem5_stats_file = input_path + "/stats.txt.gz"
1255if not os.path.exists(gem5_stats_file):
1256 gem5_stats_file = input_path + "/stats.txt"
1257if not os.path.exists(gem5_stats_file):
1258 print "ERROR: stats.txt[.gz] file does not exist in %s!" % input_path
1259 sys.exit(1)
1260
1261readGem5Stats(stats, gem5_stats_file)
1262
1263####
1264# Create Streamline .apc project folder
1265####
1266createApcProject(input_path, output_path, stats)
1267
1268print "All done!"
|