1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Ali Saidi
39#
40
41# This python code is used to migrate checkpoints that were created in one
42# version of the simulator to newer version. As features are added or bugs are
43# fixed some of the state that needs to be checkpointed can change. If you have
44# many historic checkpoints that you use, manually editing them to fix them is
45# both time consuming and error-prone.
46
47# This script provides a way to migrate checkpoints to the newer repository in
48# a programatic way. It can be imported into another script or used on the
49# command line. From the command line the script will either migrate every
50# checkpoint it finds recursively (-r option) or a single checkpoint. When a
51# change is made to the gem5 repository that breaks previous checkpoints a
52# from_N() method should be implemented here and the gem5CheckpointVersion
53# variable in src/sim/serialize.hh should be incremented. For each version
54# between the checkpoints current version and the new version the from_N()
55# method will be run, passing in a ConfigParser object which contains the open
56# file. As these operations can be isa specific the method can verify the isa
57# and use regexes to find the correct sections that need to be updated.
58
59
60import ConfigParser
61import sys, os
62import os.path as osp
63
64# An example of a translator
65def from_0(cpt):
66 if cpt.get('root','isa') == 'arm':
67 for sec in cpt.sections():
68 import re
69 # Search for all the execution contexts
70 if re.search('.*sys.*\.cpu.*\.x.\..*', sec):
71 # Update each one
72 mr = cpt.get(sec, 'miscRegs').split()
73 #mr.insert(21,0)
74 #mr.insert(26,0)
75 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
76
77# The backing store supporting the memories in the system has changed
78# in that it is now stored globally per address range. As a result the
79# actual storage is separate from the memory controllers themselves.
80def from_1(cpt):
81 for sec in cpt.sections():
82 import re
83 # Search for a physical memory
84 if re.search('.*sys.*\.physmem$', sec):
85 # Add the number of stores attribute to the global physmem
86 cpt.set(sec, 'nbr_of_stores', '1')
87
88 # Get the filename and size as this is moving to the
89 # specific backing store
90 mem_filename = cpt.get(sec, 'filename')
91 mem_size = cpt.get(sec, '_size')
92 cpt.remove_option(sec, 'filename')
93 cpt.remove_option(sec, '_size')
94
95 # Get the name so that we can create the new section
96 system_name = str(sec).split('.')[0]
97 section_name = system_name + '.physmem.store0'
98 cpt.add_section(section_name)
99 cpt.set(section_name, 'store_id', '0')
100 cpt.set(section_name, 'range_size', mem_size)
101 cpt.set(section_name, 'filename', mem_filename)
102 elif re.search('.*sys.*\.\w*mem$', sec):
103 # Due to the lack of information about a start address,
104 # this migration only works if there is a single memory in
105 # the system, thus starting at 0
106 raise ValueError("more than one memory detected (" + sec + ")")
107
108def from_2(cpt):
109 for sec in cpt.sections():
110 import re
111 # Search for a CPUs
112 if re.search('.*sys.*cpu', sec):
113 try:
114 junk = cpt.get(sec, 'instCnt')
115 cpt.set(sec, '_pid', '0')
116 except ConfigParser.NoOptionError:
117 pass
118
119# The ISA is now a separate SimObject, which means that we serialize
120# it in a separate section instead of as a part of the ThreadContext.
121def from_3(cpt):
122 isa = cpt.get('root','isa')
123 isa_fields = {
124 "alpha" : ( "fpcr", "uniq", "lock_flag", "lock_addr", "ipr" ),
125 "arm" : ( "miscRegs" ),
126 "sparc" : ( "asi", "tick", "fprs", "gsr", "softint", "tick_cmpr",
127 "stick", "stick_cmpr", "tpc", "tnpc", "tstate", "tt",
128 "tba", "pstate", "tl", "pil", "cwp", "gl", "hpstate",
129 "htstate", "hintp", "htba", "hstick_cmpr",
130 "strandStatusReg", "fsr", "priContext", "secContext",
131 "partId", "lsuCtrlReg", "scratchPad",
132 "cpu_mondo_head", "cpu_mondo_tail",
133 "dev_mondo_head", "dev_mondo_tail",
134 "res_error_head", "res_error_tail",
135 "nres_error_head", "nres_error_tail",
136 "tick_intr_sched",
137 "cpu", "tc_num", "tick_cmp", "stick_cmp", "hstick_cmp"),
138 "x86" : ( "regVal" ),
139 }
140
141 isa_fields = isa_fields.get(isa, [])
142 isa_sections = []
143 for sec in cpt.sections():
144 import re
145
146 re_cpu_match = re.match('^(.*sys.*\.cpu[^.]*)\.xc\.(.+)$', sec)
147 # Search for all the execution contexts
148 if not re_cpu_match:
149 continue
150
151 if re_cpu_match.group(2) != "0":
152 # This shouldn't happen as we didn't support checkpointing
153 # of in-order and O3 CPUs.
154 raise ValueError("Don't know how to migrate multi-threaded CPUs "
155 "from version 1")
156
157 isa_section = []
158 for fspec in isa_fields:
159 for (key, value) in cpt.items(sec, raw=True):
160 if key in isa_fields:
161 isa_section.append((key, value))
162
163 name = "%s.isa" % re_cpu_match.group(1)
164 isa_sections.append((name, isa_section))
165
166 for (key, value) in isa_section:
167 cpt.remove_option(sec, key)
168
169 for (sec, options) in isa_sections:
170 # Some intermediate versions of gem5 have empty ISA sections
171 # (after we made the ISA a SimObject, but before we started to
172 # serialize into a separate ISA section).
173 if not cpt.has_section(sec):
174 cpt.add_section(sec)
175 else:
176 if cpt.items(sec):
177 raise ValueError("Unexpected populated ISA section in old "
178 "checkpoint")
179
180 for (key, value) in options:
181 cpt.set(sec, key, value)
182
183# Version 5 of the checkpoint format removes the MISCREG_CPSR_MODE
184# register from the ARM register file.
185def from_4(cpt):
186 if cpt.get('root','isa') == 'arm':
187 for sec in cpt.sections():
188 import re
189 # Search for all ISA sections
190 if re.search('.*sys.*\.cpu.*\.isa', sec):
191 mr = cpt.get(sec, 'miscRegs').split()
192 # Remove MISCREG_CPSR_MODE
193 del mr[137]
194 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
195
| 1#!/usr/bin/env python
2
3# Copyright (c) 2012 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Ali Saidi
39#
40
41# This python code is used to migrate checkpoints that were created in one
42# version of the simulator to newer version. As features are added or bugs are
43# fixed some of the state that needs to be checkpointed can change. If you have
44# many historic checkpoints that you use, manually editing them to fix them is
45# both time consuming and error-prone.
46
47# This script provides a way to migrate checkpoints to the newer repository in
48# a programatic way. It can be imported into another script or used on the
49# command line. From the command line the script will either migrate every
50# checkpoint it finds recursively (-r option) or a single checkpoint. When a
51# change is made to the gem5 repository that breaks previous checkpoints a
52# from_N() method should be implemented here and the gem5CheckpointVersion
53# variable in src/sim/serialize.hh should be incremented. For each version
54# between the checkpoints current version and the new version the from_N()
55# method will be run, passing in a ConfigParser object which contains the open
56# file. As these operations can be isa specific the method can verify the isa
57# and use regexes to find the correct sections that need to be updated.
58
59
60import ConfigParser
61import sys, os
62import os.path as osp
63
64# An example of a translator
65def from_0(cpt):
66 if cpt.get('root','isa') == 'arm':
67 for sec in cpt.sections():
68 import re
69 # Search for all the execution contexts
70 if re.search('.*sys.*\.cpu.*\.x.\..*', sec):
71 # Update each one
72 mr = cpt.get(sec, 'miscRegs').split()
73 #mr.insert(21,0)
74 #mr.insert(26,0)
75 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
76
77# The backing store supporting the memories in the system has changed
78# in that it is now stored globally per address range. As a result the
79# actual storage is separate from the memory controllers themselves.
80def from_1(cpt):
81 for sec in cpt.sections():
82 import re
83 # Search for a physical memory
84 if re.search('.*sys.*\.physmem$', sec):
85 # Add the number of stores attribute to the global physmem
86 cpt.set(sec, 'nbr_of_stores', '1')
87
88 # Get the filename and size as this is moving to the
89 # specific backing store
90 mem_filename = cpt.get(sec, 'filename')
91 mem_size = cpt.get(sec, '_size')
92 cpt.remove_option(sec, 'filename')
93 cpt.remove_option(sec, '_size')
94
95 # Get the name so that we can create the new section
96 system_name = str(sec).split('.')[0]
97 section_name = system_name + '.physmem.store0'
98 cpt.add_section(section_name)
99 cpt.set(section_name, 'store_id', '0')
100 cpt.set(section_name, 'range_size', mem_size)
101 cpt.set(section_name, 'filename', mem_filename)
102 elif re.search('.*sys.*\.\w*mem$', sec):
103 # Due to the lack of information about a start address,
104 # this migration only works if there is a single memory in
105 # the system, thus starting at 0
106 raise ValueError("more than one memory detected (" + sec + ")")
107
108def from_2(cpt):
109 for sec in cpt.sections():
110 import re
111 # Search for a CPUs
112 if re.search('.*sys.*cpu', sec):
113 try:
114 junk = cpt.get(sec, 'instCnt')
115 cpt.set(sec, '_pid', '0')
116 except ConfigParser.NoOptionError:
117 pass
118
119# The ISA is now a separate SimObject, which means that we serialize
120# it in a separate section instead of as a part of the ThreadContext.
121def from_3(cpt):
122 isa = cpt.get('root','isa')
123 isa_fields = {
124 "alpha" : ( "fpcr", "uniq", "lock_flag", "lock_addr", "ipr" ),
125 "arm" : ( "miscRegs" ),
126 "sparc" : ( "asi", "tick", "fprs", "gsr", "softint", "tick_cmpr",
127 "stick", "stick_cmpr", "tpc", "tnpc", "tstate", "tt",
128 "tba", "pstate", "tl", "pil", "cwp", "gl", "hpstate",
129 "htstate", "hintp", "htba", "hstick_cmpr",
130 "strandStatusReg", "fsr", "priContext", "secContext",
131 "partId", "lsuCtrlReg", "scratchPad",
132 "cpu_mondo_head", "cpu_mondo_tail",
133 "dev_mondo_head", "dev_mondo_tail",
134 "res_error_head", "res_error_tail",
135 "nres_error_head", "nres_error_tail",
136 "tick_intr_sched",
137 "cpu", "tc_num", "tick_cmp", "stick_cmp", "hstick_cmp"),
138 "x86" : ( "regVal" ),
139 }
140
141 isa_fields = isa_fields.get(isa, [])
142 isa_sections = []
143 for sec in cpt.sections():
144 import re
145
146 re_cpu_match = re.match('^(.*sys.*\.cpu[^.]*)\.xc\.(.+)$', sec)
147 # Search for all the execution contexts
148 if not re_cpu_match:
149 continue
150
151 if re_cpu_match.group(2) != "0":
152 # This shouldn't happen as we didn't support checkpointing
153 # of in-order and O3 CPUs.
154 raise ValueError("Don't know how to migrate multi-threaded CPUs "
155 "from version 1")
156
157 isa_section = []
158 for fspec in isa_fields:
159 for (key, value) in cpt.items(sec, raw=True):
160 if key in isa_fields:
161 isa_section.append((key, value))
162
163 name = "%s.isa" % re_cpu_match.group(1)
164 isa_sections.append((name, isa_section))
165
166 for (key, value) in isa_section:
167 cpt.remove_option(sec, key)
168
169 for (sec, options) in isa_sections:
170 # Some intermediate versions of gem5 have empty ISA sections
171 # (after we made the ISA a SimObject, but before we started to
172 # serialize into a separate ISA section).
173 if not cpt.has_section(sec):
174 cpt.add_section(sec)
175 else:
176 if cpt.items(sec):
177 raise ValueError("Unexpected populated ISA section in old "
178 "checkpoint")
179
180 for (key, value) in options:
181 cpt.set(sec, key, value)
182
183# Version 5 of the checkpoint format removes the MISCREG_CPSR_MODE
184# register from the ARM register file.
185def from_4(cpt):
186 if cpt.get('root','isa') == 'arm':
187 for sec in cpt.sections():
188 import re
189 # Search for all ISA sections
190 if re.search('.*sys.*\.cpu.*\.isa', sec):
191 mr = cpt.get(sec, 'miscRegs').split()
192 # Remove MISCREG_CPSR_MODE
193 del mr[137]
194 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
195
|
204
205verbose_print = False
206
207def verboseprint(*args):
208 if not verbose_print:
209 return
210 for arg in args:
211 print arg,
212 print
213
214def process_file(path, **kwargs):
215 if not osp.isfile(path):
216 import errno
217 raise IOError(ennro.ENOENT, "No such file", path)
218
219 verboseprint("Processing file %s...." % path)
220
221 if kwargs.get('backup', True):
222 import shutil
223 shutil.copyfile(path, path + '.bak')
224
225 cpt = ConfigParser.SafeConfigParser()
226
227 # gem5 is case sensitive with paramaters
228 cpt.optionxform = str
229
230 # Read the current data
231 cpt_file = file(path, 'r')
232 cpt.readfp(cpt_file)
233 cpt_file.close()
234
235 # Make sure we know what we're starting from
236 if not cpt.has_option('root','cpt_ver'):
237 raise LookupError("cannot determine version of checkpoint")
238
239 cpt_ver = cpt.getint('root','cpt_ver')
240
241 # If the current checkpoint is longer than the migrations list, we have a problem
242 # and someone didn't update this file
243 if cpt_ver > len(migrations):
244 raise ValueError("upgrade script is too old and needs updating")
245
246 verboseprint("\t...file is at version %#x" % cpt_ver)
247
248 if cpt_ver == len(migrations):
249 verboseprint("\t...nothing to do")
250 return
251
252 # Walk through every function from now until the end fixing the checkpoint
253 for v in xrange(cpt_ver,len(migrations)):
254 verboseprint("\t...migrating to version %#x" % (v + 1))
255 migrations[v](cpt)
256 cpt.set('root','cpt_ver', str(v + 1))
257
258 # Write the old data back
259 verboseprint("\t...completed")
260 cpt.write(file(path, 'w'))
261
262
263if __name__ == '__main__':
264 from optparse import OptionParser
265 parser = OptionParser("usage: %prog [options] <filename or directory>")
266 parser.add_option("-r", "--recurse", action="store_true",
267 help="Recurse through all subdirectories modifying "\
268 "each checkpoint that is found")
269 parser.add_option("-N", "--no-backup", action="store_false",
270 dest="backup", default=True,
271 help="Do no backup each checkpoint before modifying it")
272 parser.add_option("-v", "--verbose", action="store_true",
273 help="Print out debugging information as")
274
275 (options, args) = parser.parse_args()
276 if len(args) != 1:
277 parser.error("You must specify a checkpoint file to modify or a "\
278 "directory of checkpoints to recursively update")
279
280 verbose_print = options.verbose
281
282 # Deal with shell variables and ~
283 path = osp.expandvars(osp.expanduser(args[0]))
284
285 # Process a single file if we have it
286 if osp.isfile(path):
287 process_file(path, **vars(options))
288 # Process an entire directory
289 elif osp.isdir(path):
290 cpt_file = osp.join(path, 'm5.cpt')
291 if options.recurse:
292 # Visit very file and see if it matches
293 for root,dirs,files in os.walk(path):
294 for name in files:
295 if name == 'm5.cpt':
296 process_file(osp.join(root,name), **vars(options))
297 for dir in dirs:
298 pass
299 # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
300 elif osp.isfile(cpt_file):
301 process_file(cpt_file, **vars(options))
302 else:
303 print "Error: checkpoint file not found at in %s " % path,
304 print "and recurse not specified"
305 sys.exit(1)
306 sys.exit(0)
307
| 219
220verbose_print = False
221
222def verboseprint(*args):
223 if not verbose_print:
224 return
225 for arg in args:
226 print arg,
227 print
228
229def process_file(path, **kwargs):
230 if not osp.isfile(path):
231 import errno
232 raise IOError(ennro.ENOENT, "No such file", path)
233
234 verboseprint("Processing file %s...." % path)
235
236 if kwargs.get('backup', True):
237 import shutil
238 shutil.copyfile(path, path + '.bak')
239
240 cpt = ConfigParser.SafeConfigParser()
241
242 # gem5 is case sensitive with paramaters
243 cpt.optionxform = str
244
245 # Read the current data
246 cpt_file = file(path, 'r')
247 cpt.readfp(cpt_file)
248 cpt_file.close()
249
250 # Make sure we know what we're starting from
251 if not cpt.has_option('root','cpt_ver'):
252 raise LookupError("cannot determine version of checkpoint")
253
254 cpt_ver = cpt.getint('root','cpt_ver')
255
256 # If the current checkpoint is longer than the migrations list, we have a problem
257 # and someone didn't update this file
258 if cpt_ver > len(migrations):
259 raise ValueError("upgrade script is too old and needs updating")
260
261 verboseprint("\t...file is at version %#x" % cpt_ver)
262
263 if cpt_ver == len(migrations):
264 verboseprint("\t...nothing to do")
265 return
266
267 # Walk through every function from now until the end fixing the checkpoint
268 for v in xrange(cpt_ver,len(migrations)):
269 verboseprint("\t...migrating to version %#x" % (v + 1))
270 migrations[v](cpt)
271 cpt.set('root','cpt_ver', str(v + 1))
272
273 # Write the old data back
274 verboseprint("\t...completed")
275 cpt.write(file(path, 'w'))
276
277
278if __name__ == '__main__':
279 from optparse import OptionParser
280 parser = OptionParser("usage: %prog [options] <filename or directory>")
281 parser.add_option("-r", "--recurse", action="store_true",
282 help="Recurse through all subdirectories modifying "\
283 "each checkpoint that is found")
284 parser.add_option("-N", "--no-backup", action="store_false",
285 dest="backup", default=True,
286 help="Do no backup each checkpoint before modifying it")
287 parser.add_option("-v", "--verbose", action="store_true",
288 help="Print out debugging information as")
289
290 (options, args) = parser.parse_args()
291 if len(args) != 1:
292 parser.error("You must specify a checkpoint file to modify or a "\
293 "directory of checkpoints to recursively update")
294
295 verbose_print = options.verbose
296
297 # Deal with shell variables and ~
298 path = osp.expandvars(osp.expanduser(args[0]))
299
300 # Process a single file if we have it
301 if osp.isfile(path):
302 process_file(path, **vars(options))
303 # Process an entire directory
304 elif osp.isdir(path):
305 cpt_file = osp.join(path, 'm5.cpt')
306 if options.recurse:
307 # Visit very file and see if it matches
308 for root,dirs,files in os.walk(path):
309 for name in files:
310 if name == 'm5.cpt':
311 process_file(osp.join(root,name), **vars(options))
312 for dir in dirs:
313 pass
314 # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
315 elif osp.isfile(cpt_file):
316 process_file(cpt_file, **vars(options))
317 else:
318 print "Error: checkpoint file not found at in %s " % path,
319 print "and recurse not specified"
320 sys.exit(1)
321 sys.exit(0)
322
|