cpt_upgrader.py revision 9431:8bb372a49e1b
1#!/usr/bin/env python 2 3# Copyright (c) 2012 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Ali Saidi 39# 40 41# This python code is used to migrate checkpoints that were created in one 42# version of the simulator to newer version. As features are added or bugs are 43# fixed some of the state that needs to be checkpointed can change. If you have 44# many historic checkpoints that you use, manually editing them to fix them is 45# both time consuming and error-prone. 46 47# This script provides a way to migrate checkpoints to the newer repository in 48# a programatic way. It can be imported into another script or used on the 49# command line. From the command line the script will either migrate every 50# checkpoint it finds recursively (-r option) or a single checkpoint. When a 51# change is made to the gem5 repository that breaks previous checkpoints a 52# from_N() method should be implemented here and the gem5CheckpointVersion 53# variable in src/sim/serialize.hh should be incremented. For each version 54# between the checkpoints current version and the new version the from_N() 55# method will be run, passing in a ConfigParser object which contains the open 56# file. As these operations can be isa specific the method can verify the isa 57# and use regexes to find the correct sections that need to be updated. 58 59 60import ConfigParser 61import sys, os 62import os.path as osp 63 64# An example of a translator 65def from_0(cpt): 66 if cpt.get('root','isa') == 'arm': 67 for sec in cpt.sections(): 68 import re 69 # Search for all the execution contexts 70 if re.search('.*sys.*\.cpu.*\.x.\..*', sec): 71 # Update each one 72 mr = cpt.get(sec, 'miscRegs').split() 73 #mr.insert(21,0) 74 #mr.insert(26,0) 75 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr)) 76 77# The backing store supporting the memories in the system has changed 78# in that it is now stored globally per address range. As a result the 79# actual storage is separate from the memory controllers themselves. 80def from_1(cpt): 81 for sec in cpt.sections(): 82 import re 83 # Search for a physical memory 84 if re.search('.*sys.*\.physmem$', sec): 85 # Add the number of stores attribute to the global physmem 86 cpt.set(sec, 'nbr_of_stores', '1') 87 88 # Get the filename and size as this is moving to the 89 # specific backing store 90 mem_filename = cpt.get(sec, 'filename') 91 mem_size = cpt.get(sec, '_size') 92 cpt.remove_option(sec, 'filename') 93 cpt.remove_option(sec, '_size') 94 95 # Get the name so that we can create the new section 96 system_name = str(sec).split('.')[0] 97 section_name = system_name + '.physmem.store0' 98 cpt.add_section(section_name) 99 cpt.set(section_name, 'store_id', '0') 100 cpt.set(section_name, 'range_size', mem_size) 101 cpt.set(section_name, 'filename', mem_filename) 102 elif re.search('.*sys.*\.\w*mem$', sec): 103 # Due to the lack of information about a start address, 104 # this migration only works if there is a single memory in 105 # the system, thus starting at 0 106 raise ValueError("more than one memory detected (" + sec + ")") 107 108def from_2(cpt): 109 for sec in cpt.sections(): 110 import re 111 # Search for a CPUs 112 if re.search('.*sys.*cpu', sec): 113 try: 114 junk = cpt.get(sec, 'instCnt') 115 cpt.set(sec, '_pid', '0') 116 except ConfigParser.NoOptionError: 117 pass 118 119# The ISA is now a separate SimObject, which means that we serialize 120# it in a separate section instead of as a part of the ThreadContext. 121def from_3(cpt): 122 isa = cpt.get('root','isa') 123 isa_fields = { 124 "alpha" : ( "fpcr", "uniq", "lock_flag", "lock_addr", "ipr" ), 125 "arm" : ( "miscRegs" ), 126 "sparc" : ( "asi", "tick", "fprs", "gsr", "softint", "tick_cmpr", 127 "stick", "stick_cmpr", "tpc", "tnpc", "tstate", "tt", 128 "tba", "pstate", "tl", "pil", "cwp", "gl", "hpstate", 129 "htstate", "hintp", "htba", "hstick_cmpr", 130 "strandStatusReg", "fsr", "priContext", "secContext", 131 "partId", "lsuCtrlReg", "scratchPad", 132 "cpu_mondo_head", "cpu_mondo_tail", 133 "dev_mondo_head", "dev_mondo_tail", 134 "res_error_head", "res_error_tail", 135 "nres_error_head", "nres_error_tail", 136 "tick_intr_sched", 137 "cpu", "tc_num", "tick_cmp", "stick_cmp", "hstick_cmp"), 138 "x86" : ( "regVal" ), 139 } 140 141 isa_fields = isa_fields.get(isa, []) 142 isa_sections = [] 143 for sec in cpt.sections(): 144 import re 145 146 re_cpu_match = re.match('^(.*sys.*\.cpu[^.]*)\.xc\.(.+)$', sec) 147 # Search for all the execution contexts 148 if not re_cpu_match: 149 continue 150 151 if re_cpu_match.group(2) != "0": 152 # This shouldn't happen as we didn't support checkpointing 153 # of in-order and O3 CPUs. 154 raise ValueError("Don't know how to migrate multi-threaded CPUs " 155 "from version 1") 156 157 isa_section = [] 158 for fspec in isa_fields: 159 for (key, value) in cpt.items(sec, raw=True): 160 if key in isa_fields: 161 isa_section.append((key, value)) 162 163 name = "%s.isa" % re_cpu_match.group(1) 164 isa_sections.append((name, isa_section)) 165 166 for (key, value) in isa_section: 167 cpt.remove_option(sec, key) 168 169 for (sec, options) in isa_sections: 170 # Some intermediate versions of gem5 have empty ISA sections 171 # (after we made the ISA a SimObject, but before we started to 172 # serialize into a separate ISA section). 173 if not cpt.has_section(sec): 174 cpt.add_section(sec) 175 else: 176 if cpt.items(sec): 177 raise ValueError("Unexpected populated ISA section in old " 178 "checkpoint") 179 180 for (key, value) in options: 181 cpt.set(sec, key, value) 182 183# Version 5 of the checkpoint format removes the MISCREG_CPSR_MODE 184# register from the ARM register file. 185def from_4(cpt): 186 if cpt.get('root','isa') == 'arm': 187 for sec in cpt.sections(): 188 import re 189 # Search for all ISA sections 190 if re.search('.*sys.*\.cpu.*\.isa', sec): 191 mr = cpt.get(sec, 'miscRegs').split() 192 # Remove MISCREG_CPSR_MODE 193 del mr[137] 194 cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr)) 195 196 197 198migrations = [] 199migrations.append(from_0) 200migrations.append(from_1) 201migrations.append(from_2) 202migrations.append(from_3) 203migrations.append(from_4) 204 205verbose_print = False 206 207def verboseprint(*args): 208 if not verbose_print: 209 return 210 for arg in args: 211 print arg, 212 print 213 214def process_file(path, **kwargs): 215 if not osp.isfile(path): 216 import errno 217 raise IOError(ennro.ENOENT, "No such file", path) 218 219 verboseprint("Processing file %s...." % path) 220 221 if kwargs.get('backup', True): 222 import shutil 223 shutil.copyfile(path, path + '.bak') 224 225 cpt = ConfigParser.SafeConfigParser() 226 227 # gem5 is case sensitive with paramaters 228 cpt.optionxform = str 229 230 # Read the current data 231 cpt_file = file(path, 'r') 232 cpt.readfp(cpt_file) 233 cpt_file.close() 234 235 # Make sure we know what we're starting from 236 if not cpt.has_option('root','cpt_ver'): 237 raise LookupError("cannot determine version of checkpoint") 238 239 cpt_ver = cpt.getint('root','cpt_ver') 240 241 # If the current checkpoint is longer than the migrations list, we have a problem 242 # and someone didn't update this file 243 if cpt_ver > len(migrations): 244 raise ValueError("upgrade script is too old and needs updating") 245 246 verboseprint("\t...file is at version %#x" % cpt_ver) 247 248 if cpt_ver == len(migrations): 249 verboseprint("\t...nothing to do") 250 return 251 252 # Walk through every function from now until the end fixing the checkpoint 253 for v in xrange(cpt_ver,len(migrations)): 254 verboseprint("\t...migrating to version %#x" % (v + 1)) 255 migrations[v](cpt) 256 cpt.set('root','cpt_ver', str(v + 1)) 257 258 # Write the old data back 259 verboseprint("\t...completed") 260 cpt.write(file(path, 'w')) 261 262 263if __name__ == '__main__': 264 from optparse import OptionParser 265 parser = OptionParser("usage: %prog [options] <filename or directory>") 266 parser.add_option("-r", "--recurse", action="store_true", 267 help="Recurse through all subdirectories modifying "\ 268 "each checkpoint that is found") 269 parser.add_option("-N", "--no-backup", action="store_false", 270 dest="backup", default=True, 271 help="Do no backup each checkpoint before modifying it") 272 parser.add_option("-v", "--verbose", action="store_true", 273 help="Print out debugging information as") 274 275 (options, args) = parser.parse_args() 276 if len(args) != 1: 277 parser.error("You must specify a checkpoint file to modify or a "\ 278 "directory of checkpoints to recursively update") 279 280 verbose_print = options.verbose 281 282 # Deal with shell variables and ~ 283 path = osp.expandvars(osp.expanduser(args[0])) 284 285 # Process a single file if we have it 286 if osp.isfile(path): 287 process_file(path, **vars(options)) 288 # Process an entire directory 289 elif osp.isdir(path): 290 cpt_file = osp.join(path, 'm5.cpt') 291 if options.recurse: 292 # Visit very file and see if it matches 293 for root,dirs,files in os.walk(path): 294 for name in files: 295 if name == 'm5.cpt': 296 process_file(osp.join(root,name), **vars(options)) 297 for dir in dirs: 298 pass 299 # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt 300 elif osp.isfile(cpt_file): 301 process_file(cpt_file, **vars(options)) 302 else: 303 print "Error: checkpoint file not found at in %s " % path, 304 print "and recurse not specified" 305 sys.exit(1) 306 sys.exit(0) 307 308