cpt_upgrader.py revision 11834:29f0d1d70282
15643Sgblack@eecs.umich.edu#!/usr/bin/env python2 25643Sgblack@eecs.umich.edu 35643Sgblack@eecs.umich.edu# Copyright (c) 2012-2013,2015-2016 ARM Limited 45643Sgblack@eecs.umich.edu# All rights reserved 55643Sgblack@eecs.umich.edu# 65643Sgblack@eecs.umich.edu# The license below extends only to copyright in the software and shall 75643Sgblack@eecs.umich.edu# not be construed as granting a license to any other intellectual 85643Sgblack@eecs.umich.edu# property including but not limited to intellectual property relating 95643Sgblack@eecs.umich.edu# to a hardware implementation of the functionality of the software 105643Sgblack@eecs.umich.edu# licensed hereunder. You may use the software subject to the license 115643Sgblack@eecs.umich.edu# terms below provided that you ensure that this notice is replicated 125643Sgblack@eecs.umich.edu# unmodified and in its entirety in all distributions of the software, 135643Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form. 145643Sgblack@eecs.umich.edu# 155643Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 165643Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are 175643Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright 185643Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer; 195643Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 205643Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 215643Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution; 225643Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its 235643Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 245643Sgblack@eecs.umich.edu# this software without specific prior written permission. 255643Sgblack@eecs.umich.edu# 265643Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 275643Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 285643Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 295643Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 305643Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3111793Sbrandon.potter@amd.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211793Sbrandon.potter@amd.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 336138Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 345651Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 358746Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 368232Snate@binkert.org# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 375657Sgblack@eecs.umich.edu# 385643Sgblack@eecs.umich.edu# Authors: Ali Saidi 395643Sgblack@eecs.umich.edu# Curtis Dunham 405643Sgblack@eecs.umich.edu# 415643Sgblack@eecs.umich.edu 429805Sstever@gmail.com# This python code is used to migrate checkpoints that were created in one 439808Sstever@gmail.com# version of the simulator to newer version. As features are added or bugs are 449805Sstever@gmail.com# fixed some of the state that needs to be checkpointed can change. If you have 455643Sgblack@eecs.umich.edu# many historic checkpoints that you use, manually editing them to fix them is 467913SBrad.Beckmann@amd.com# both time consuming and error-prone. 477913SBrad.Beckmann@amd.com 487913SBrad.Beckmann@amd.com# This script provides a way to migrate checkpoints to the newer repository in 497913SBrad.Beckmann@amd.com# a programmatic way. It can be imported into another script or used on the 507913SBrad.Beckmann@amd.com# command line. From the command line the script will either migrate every 516136Sgblack@eecs.umich.edu# checkpoint it finds recursively (-r option) or a single checkpoint. When a 525643Sgblack@eecs.umich.edu# change is made to the gem5 repository that breaks previous checkpoints an 535643Sgblack@eecs.umich.edu# upgrade() method should be implemented in its own .py file and placed in 545653Sgblack@eecs.umich.edu# src/util/cpt_upgraders/. For each upgrader whose tag is not present in 555653Sgblack@eecs.umich.edu# the checkpoint tag list, the upgrade() method will be run, passing in a 565653Sgblack@eecs.umich.edu# ConfigParser object which contains the open file. As these operations can 575653Sgblack@eecs.umich.edu# be isa specific the method can verify the isa and use regexes to find the 585827Sgblack@eecs.umich.edu# correct sections that need to be updated. 595653Sgblack@eecs.umich.edu 605643Sgblack@eecs.umich.edu# It is also possible to use this mechanism to revert prior tags. In this 615643Sgblack@eecs.umich.edu# case, implement a downgrade() method instead. Dependencies should still 627913SBrad.Beckmann@amd.com# work naturally - a tag depending on a tag with a downgrader means that it 637913SBrad.Beckmann@amd.com# insists on the other tag being removed and its downgrader executed before 647913SBrad.Beckmann@amd.com# its upgrader (or downgrader) can run. It is still the case that a tag 657913SBrad.Beckmann@amd.com# can only be used once. 667913SBrad.Beckmann@amd.com 679807Sstever@gmail.com 687913SBrad.Beckmann@amd.comimport ConfigParser 699805Sstever@gmail.comimport glob, types, sys, os 709807Sstever@gmail.comimport os.path as osp 717913SBrad.Beckmann@amd.com 727913SBrad.Beckmann@amd.comverbose_print = False 7313784Sgabeblack@google.com 7413784Sgabeblack@google.comdef verboseprint(*args): 759805Sstever@gmail.com if not verbose_print: 769805Sstever@gmail.com return 779805Sstever@gmail.com for arg in args: 7813784Sgabeblack@google.com print arg, 799805Sstever@gmail.com print 809805Sstever@gmail.com 819805Sstever@gmail.comclass Upgrader: 829805Sstever@gmail.com tag_set = set() 839805Sstever@gmail.com untag_set = set() # tags to remove by downgrading 849805Sstever@gmail.com by_tag = {} 859805Sstever@gmail.com legacy = {} 869805Sstever@gmail.com def __init__(self, filename): 879805Sstever@gmail.com self.filename = filename 889805Sstever@gmail.com execfile(filename, {}, self.__dict__) 899805Sstever@gmail.com 909805Sstever@gmail.com if not hasattr(self, 'tag'): 915643Sgblack@eecs.umich.edu self.tag = osp.basename(filename)[:-3] 9211144Sjthestness@gmail.com if not hasattr(self, 'depends'): 9311144Sjthestness@gmail.com self.depends = [] 9411144Sjthestness@gmail.com elif isinstance(self.depends, str): 9511144Sjthestness@gmail.com self.depends = [self.depends] 9611144Sjthestness@gmail.com 9711144Sjthestness@gmail.com if hasattr(self, 'upgrader'): 9811144Sjthestness@gmail.com if not isinstance(self.upgrader, types.FunctionType): 9911144Sjthestness@gmail.com print "Error: 'upgrader' for %s is %s, not function" \ 1005643Sgblack@eecs.umich.edu % (self.tag, type(self)) 1015643Sgblack@eecs.umich.edu sys.exit(1) 1025643Sgblack@eecs.umich.edu Upgrader.tag_set.add(self.tag) 1035643Sgblack@eecs.umich.edu elif hasattr(self, 'downgrader'): 1045643Sgblack@eecs.umich.edu if not isinstance(self.downgrader, types.FunctionType): 1055643Sgblack@eecs.umich.edu print "Error: 'downgrader' for %s is %s, not function" \ 10613229Sgabeblack@google.com % (self.tag, type(self)) 1075643Sgblack@eecs.umich.edu sys.exit(1) 1085643Sgblack@eecs.umich.edu Upgrader.untag_set.add(self.tag) 10913229Sgabeblack@google.com else: 1105643Sgblack@eecs.umich.edu print "Error: no upgrader or downgrader method for", self.tag 1115643Sgblack@eecs.umich.edu sys.exit(1) 1125643Sgblack@eecs.umich.edu 1135643Sgblack@eecs.umich.edu if hasattr(self, 'legacy_version'): 1145898Sgblack@eecs.umich.edu Upgrader.legacy[self.legacy_version] = self 1159805Sstever@gmail.com 1165643Sgblack@eecs.umich.edu Upgrader.by_tag[self.tag] = self 1175643Sgblack@eecs.umich.edu 1185643Sgblack@eecs.umich.edu def ready(self, tags): 1195643Sgblack@eecs.umich.edu for dep in self.depends: 1205643Sgblack@eecs.umich.edu if dep not in tags: 1215643Sgblack@eecs.umich.edu return False 1225643Sgblack@eecs.umich.edu return True 1235643Sgblack@eecs.umich.edu 1245643Sgblack@eecs.umich.edu def update(self, cpt, tags): 12513229Sgabeblack@google.com if hasattr(self, 'upgrader'): 1265643Sgblack@eecs.umich.edu self.upgrader(cpt) 1275643Sgblack@eecs.umich.edu tags.add(self.tag) 12813229Sgabeblack@google.com verboseprint("applied upgrade for", self.tag) 1295643Sgblack@eecs.umich.edu else: 1305643Sgblack@eecs.umich.edu self.downgrader(cpt) 1315643Sgblack@eecs.umich.edu tags.remove(self.tag) 1325643Sgblack@eecs.umich.edu verboseprint("applied downgrade for", self.tag) 1335898Sgblack@eecs.umich.edu 1349805Sstever@gmail.com @staticmethod 1355643Sgblack@eecs.umich.edu def get(tag): 1365643Sgblack@eecs.umich.edu return Upgrader.by_tag[tag] 1375643Sgblack@eecs.umich.edu 1385643Sgblack@eecs.umich.edu @staticmethod 1395643Sgblack@eecs.umich.edu def load_all(): 1405643Sgblack@eecs.umich.edu util_dir = osp.dirname(osp.abspath(__file__)) 1417913SBrad.Beckmann@amd.com 1425643Sgblack@eecs.umich.edu for py in glob.glob(util_dir + '/cpt_upgraders/*.py'): 1435643Sgblack@eecs.umich.edu Upgrader(py) 1445643Sgblack@eecs.umich.edu 1457913SBrad.Beckmann@amd.com # make linear dependences for legacy versions 1465643Sgblack@eecs.umich.edu i = 3 1475643Sgblack@eecs.umich.edu while i in Upgrader.legacy: 1485643Sgblack@eecs.umich.edu Upgrader.legacy[i].depends = [Upgrader.legacy[i-1].tag] 1495643Sgblack@eecs.umich.edu i = i + 1 1505643Sgblack@eecs.umich.edu 1515643Sgblack@eecs.umich.edudef process_file(path, **kwargs): 1525643Sgblack@eecs.umich.edu if not osp.isfile(path): 1535643Sgblack@eecs.umich.edu import errno 1545643Sgblack@eecs.umich.edu raise IOError(ennro.ENOENT, "No such file", path) 1555643Sgblack@eecs.umich.edu 1565643Sgblack@eecs.umich.edu verboseprint("Processing file %s...." % path) 1575643Sgblack@eecs.umich.edu 1585643Sgblack@eecs.umich.edu if kwargs.get('backup', True): 1595643Sgblack@eecs.umich.edu import shutil 1605643Sgblack@eecs.umich.edu shutil.copyfile(path, path + '.bak') 1615643Sgblack@eecs.umich.edu 1625643Sgblack@eecs.umich.edu cpt = ConfigParser.SafeConfigParser() 1635643Sgblack@eecs.umich.edu 1645643Sgblack@eecs.umich.edu # gem5 is case sensitive with paramaters 1655643Sgblack@eecs.umich.edu cpt.optionxform = str 1665643Sgblack@eecs.umich.edu 1675643Sgblack@eecs.umich.edu # Read the current data 1685643Sgblack@eecs.umich.edu cpt_file = file(path, 'r') 1695643Sgblack@eecs.umich.edu cpt.readfp(cpt_file) 1705643Sgblack@eecs.umich.edu cpt_file.close() 1715643Sgblack@eecs.umich.edu 1725643Sgblack@eecs.umich.edu change = False 1735643Sgblack@eecs.umich.edu 1745643Sgblack@eecs.umich.edu # Make sure we know what we're starting from 1755643Sgblack@eecs.umich.edu if cpt.has_option('root','cpt_ver'): 1765643Sgblack@eecs.umich.edu cpt_ver = cpt.getint('root','cpt_ver') 1775643Sgblack@eecs.umich.edu 1785643Sgblack@eecs.umich.edu # Legacy linear checkpoint version 1795643Sgblack@eecs.umich.edu # convert to list of tags before proceeding 1805643Sgblack@eecs.umich.edu tags = set([]) 1815643Sgblack@eecs.umich.edu for i in xrange(2, cpt_ver+1): 1825643Sgblack@eecs.umich.edu tags.add(Upgrader.legacy[i].tag) 1835643Sgblack@eecs.umich.edu verboseprint("performed legacy version -> tags conversion") 1845643Sgblack@eecs.umich.edu change = True 1855643Sgblack@eecs.umich.edu 1865643Sgblack@eecs.umich.edu cpt.remove_option('root', 'cpt_ver') 1875643Sgblack@eecs.umich.edu elif cpt.has_option('Globals','version_tags'): 1885643Sgblack@eecs.umich.edu tags = set((''.join(cpt.get('Globals','version_tags'))).split()) 1895643Sgblack@eecs.umich.edu else: 1905643Sgblack@eecs.umich.edu print "fatal: no version information in checkpoint" 1915643Sgblack@eecs.umich.edu exit(1) 1925643Sgblack@eecs.umich.edu 1935643Sgblack@eecs.umich.edu verboseprint("has tags", ' '.join(tags)) 1945643Sgblack@eecs.umich.edu # If the current checkpoint has a tag we don't know about, we have 1955643Sgblack@eecs.umich.edu # a divergence that (in general) must be addressed by (e.g.) merging 1965643Sgblack@eecs.umich.edu # simulator support for its changes. 1976712Snate@binkert.org unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set) 1985651Sgblack@eecs.umich.edu if unknown_tags: 1995657Sgblack@eecs.umich.edu print "warning: upgrade script does not recognize the following "\ 2005657Sgblack@eecs.umich.edu "tags in this checkpoint:", ' '.join(unknown_tags) 2015657Sgblack@eecs.umich.edu 2025657Sgblack@eecs.umich.edu # Apply migrations for tags not in checkpoint and tags present for which 2035657Sgblack@eecs.umich.edu # downgraders are present, respecting dependences 2045657Sgblack@eecs.umich.edu to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags) 2055651Sgblack@eecs.umich.edu while to_apply: 2065651Sgblack@eecs.umich.edu ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ]) 2075654Sgblack@eecs.umich.edu if not ready: 2085654Sgblack@eecs.umich.edu print "could not apply these upgrades:", ' '.join(to_apply) 2096138Sgblack@eecs.umich.edu print "update dependences impossible to resolve; aborting" 2106138Sgblack@eecs.umich.edu exit(1) 2116138Sgblack@eecs.umich.edu 2126138Sgblack@eecs.umich.edu for tag in ready: 2136138Sgblack@eecs.umich.edu Upgrader.get(tag).update(cpt, tags) 2146138Sgblack@eecs.umich.edu change = True 2156138Sgblack@eecs.umich.edu 2166138Sgblack@eecs.umich.edu to_apply -= ready 2176138Sgblack@eecs.umich.edu 2186138Sgblack@eecs.umich.edu if not change: 2196138Sgblack@eecs.umich.edu verboseprint("...nothing to do") 2206138Sgblack@eecs.umich.edu return 2216138Sgblack@eecs.umich.edu 2226138Sgblack@eecs.umich.edu cpt.set('Globals', 'version_tags', ' '.join(tags)) 2236138Sgblack@eecs.umich.edu 2246138Sgblack@eecs.umich.edu # Write the old data back 2256138Sgblack@eecs.umich.edu verboseprint("...completed") 2268746Sgblack@eecs.umich.edu cpt.write(file(path, 'w')) 22711150Smitch.hayenga@arm.com 2286138Sgblack@eecs.umich.eduif __name__ == '__main__': 2296138Sgblack@eecs.umich.edu from optparse import OptionParser, SUPPRESS_HELP 2308746Sgblack@eecs.umich.edu parser = OptionParser("usage: %prog [options] <filename or directory>") 2316138Sgblack@eecs.umich.edu parser.add_option("-r", "--recurse", action="store_true", 2326138Sgblack@eecs.umich.edu help="Recurse through all subdirectories modifying "\ 2336139Sgblack@eecs.umich.edu "each checkpoint that is found") 2346139Sgblack@eecs.umich.edu parser.add_option("-N", "--no-backup", action="store_false", 2356139Sgblack@eecs.umich.edu dest="backup", default=True, 2366139Sgblack@eecs.umich.edu help="Do no backup each checkpoint before modifying it") 2376139Sgblack@eecs.umich.edu parser.add_option("-v", "--verbose", action="store_true", 2386139Sgblack@eecs.umich.edu help="Print out debugging information as") 2396139Sgblack@eecs.umich.edu parser.add_option("--get-cc-file", action="store_true", 2406139Sgblack@eecs.umich.edu # used during build; generate src/sim/tags.cc and exit 2416139Sgblack@eecs.umich.edu help=SUPPRESS_HELP) 2426139Sgblack@eecs.umich.edu 2436139Sgblack@eecs.umich.edu (options, args) = parser.parse_args() 2446139Sgblack@eecs.umich.edu verbose_print = options.verbose 2456139Sgblack@eecs.umich.edu 2466139Sgblack@eecs.umich.edu Upgrader.load_all() 2476139Sgblack@eecs.umich.edu 2486139Sgblack@eecs.umich.edu if options.get_cc_file: 2496138Sgblack@eecs.umich.edu print "// this file is auto-generated by util/cpt_upgrader.py" 2506138Sgblack@eecs.umich.edu print "#include <string>" 2519524SAndreas.Sandberg@ARM.com print "#include <set>" 2525643Sgblack@eecs.umich.edu print 2535643Sgblack@eecs.umich.edu print "std::set<std::string> version_tags = {" 2545643Sgblack@eecs.umich.edu for tag in Upgrader.tag_set: 2555827Sgblack@eecs.umich.edu print " \"%s\"," % tag 2565827Sgblack@eecs.umich.edu print "};" 2575827Sgblack@eecs.umich.edu exit(0) 2585827Sgblack@eecs.umich.edu elif len(args) != 1: 2595827Sgblack@eecs.umich.edu parser.error("You must specify a checkpoint file to modify or a "\ 2605827Sgblack@eecs.umich.edu "directory of checkpoints to recursively update") 2615827Sgblack@eecs.umich.edu 2625827Sgblack@eecs.umich.edu # Deal with shell variables and ~ 2635827Sgblack@eecs.umich.edu path = osp.expandvars(osp.expanduser(args[0])) 2645827Sgblack@eecs.umich.edu 2655827Sgblack@eecs.umich.edu # Process a single file if we have it 2665827Sgblack@eecs.umich.edu if osp.isfile(path): 2675827Sgblack@eecs.umich.edu process_file(path, **vars(options)) 2685827Sgblack@eecs.umich.edu # Process an entire directory 2695827Sgblack@eecs.umich.edu elif osp.isdir(path): 2705827Sgblack@eecs.umich.edu cpt_file = osp.join(path, 'm5.cpt') 2716137Sgblack@eecs.umich.edu if options.recurse: 27210905Sandreas.sandberg@arm.com # Visit very file and see if it matches 2737903Shestness@cs.utexas.edu for root,dirs,files in os.walk(path): 2747903Shestness@cs.utexas.edu for name in files: 2757903Shestness@cs.utexas.edu if name == 'm5.cpt': 2767903Shestness@cs.utexas.edu process_file(osp.join(root,name), **vars(options)) 2777903Shestness@cs.utexas.edu for dir in dirs: 2787903Shestness@cs.utexas.edu pass 2797903Shestness@cs.utexas.edu # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt 2807903Shestness@cs.utexas.edu elif osp.isfile(cpt_file): 2817903Shestness@cs.utexas.edu process_file(cpt_file, **vars(options)) 2827903Shestness@cs.utexas.edu else: 2837903Shestness@cs.utexas.edu print "Error: checkpoint file not found at in %s " % path, 2847903Shestness@cs.utexas.edu print "and recurse not specified" 28510905Sandreas.sandberg@arm.com sys.exit(1) 2867903Shestness@cs.utexas.edu sys.exit(0) 2877903Shestness@cs.utexas.edu 2887903Shestness@cs.utexas.edu