cpt_upgrader.py revision 11834:29f0d1d70282
15643Sgblack@eecs.umich.edu#!/usr/bin/env python2
25643Sgblack@eecs.umich.edu
35643Sgblack@eecs.umich.edu# Copyright (c) 2012-2013,2015-2016 ARM Limited
45643Sgblack@eecs.umich.edu# All rights reserved
55643Sgblack@eecs.umich.edu#
65643Sgblack@eecs.umich.edu# The license below extends only to copyright in the software and shall
75643Sgblack@eecs.umich.edu# not be construed as granting a license to any other intellectual
85643Sgblack@eecs.umich.edu# property including but not limited to intellectual property relating
95643Sgblack@eecs.umich.edu# to a hardware implementation of the functionality of the software
105643Sgblack@eecs.umich.edu# licensed hereunder.  You may use the software subject to the license
115643Sgblack@eecs.umich.edu# terms below provided that you ensure that this notice is replicated
125643Sgblack@eecs.umich.edu# unmodified and in its entirety in all distributions of the software,
135643Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form.
145643Sgblack@eecs.umich.edu#
155643Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without
165643Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are
175643Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright
185643Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer;
195643Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright
205643Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the
215643Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution;
225643Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its
235643Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from
245643Sgblack@eecs.umich.edu# this software without specific prior written permission.
255643Sgblack@eecs.umich.edu#
265643Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
275643Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
285643Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
295643Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
305643Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
3111793Sbrandon.potter@amd.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
3211793Sbrandon.potter@amd.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
336138Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
345651Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
358746Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
368232Snate@binkert.org# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
375657Sgblack@eecs.umich.edu#
385643Sgblack@eecs.umich.edu# Authors: Ali Saidi
395643Sgblack@eecs.umich.edu#          Curtis Dunham
405643Sgblack@eecs.umich.edu#
415643Sgblack@eecs.umich.edu
429805Sstever@gmail.com# This python code is used to migrate checkpoints that were created in one
439808Sstever@gmail.com# version of the simulator to newer version. As features are added or bugs are
449805Sstever@gmail.com# fixed some of the state that needs to be checkpointed can change. If you have
455643Sgblack@eecs.umich.edu# many historic checkpoints that you use, manually editing them to fix them is
467913SBrad.Beckmann@amd.com# both time consuming and error-prone.
477913SBrad.Beckmann@amd.com
487913SBrad.Beckmann@amd.com# This script provides a way to migrate checkpoints to the newer repository in
497913SBrad.Beckmann@amd.com# a programmatic way. It can be imported into another script or used on the
507913SBrad.Beckmann@amd.com# command line. From the command line the script will either migrate every
516136Sgblack@eecs.umich.edu# checkpoint it finds recursively (-r option) or a single checkpoint. When a
525643Sgblack@eecs.umich.edu# change is made to the gem5 repository that breaks previous checkpoints an
535643Sgblack@eecs.umich.edu# upgrade() method should be implemented in its own .py file and placed in
545653Sgblack@eecs.umich.edu# src/util/cpt_upgraders/.  For each upgrader whose tag is not present in
555653Sgblack@eecs.umich.edu# the checkpoint tag list, the upgrade() method will be run, passing in a
565653Sgblack@eecs.umich.edu# ConfigParser object which contains the open file. As these operations can
575653Sgblack@eecs.umich.edu# be isa specific the method can verify the isa and use regexes to find the
585827Sgblack@eecs.umich.edu# correct sections that need to be updated.
595653Sgblack@eecs.umich.edu
605643Sgblack@eecs.umich.edu# It is also possible to use this mechanism to revert prior tags.  In this
615643Sgblack@eecs.umich.edu# case, implement a downgrade() method instead.  Dependencies should still
627913SBrad.Beckmann@amd.com# work naturally - a tag depending on a tag with a downgrader means that it
637913SBrad.Beckmann@amd.com# insists on the other tag being removed and its downgrader executed before
647913SBrad.Beckmann@amd.com# its upgrader (or downgrader) can run.  It is still the case that a tag
657913SBrad.Beckmann@amd.com# can only be used once.
667913SBrad.Beckmann@amd.com
679807Sstever@gmail.com
687913SBrad.Beckmann@amd.comimport ConfigParser
699805Sstever@gmail.comimport glob, types, sys, os
709807Sstever@gmail.comimport os.path as osp
717913SBrad.Beckmann@amd.com
727913SBrad.Beckmann@amd.comverbose_print = False
7313784Sgabeblack@google.com
7413784Sgabeblack@google.comdef verboseprint(*args):
759805Sstever@gmail.com    if not verbose_print:
769805Sstever@gmail.com        return
779805Sstever@gmail.com    for arg in args:
7813784Sgabeblack@google.com        print arg,
799805Sstever@gmail.com    print
809805Sstever@gmail.com
819805Sstever@gmail.comclass Upgrader:
829805Sstever@gmail.com    tag_set = set()
839805Sstever@gmail.com    untag_set = set() # tags to remove by downgrading
849805Sstever@gmail.com    by_tag = {}
859805Sstever@gmail.com    legacy = {}
869805Sstever@gmail.com    def __init__(self, filename):
879805Sstever@gmail.com        self.filename = filename
889805Sstever@gmail.com        execfile(filename, {}, self.__dict__)
899805Sstever@gmail.com
909805Sstever@gmail.com        if not hasattr(self, 'tag'):
915643Sgblack@eecs.umich.edu            self.tag = osp.basename(filename)[:-3]
9211144Sjthestness@gmail.com        if not hasattr(self, 'depends'):
9311144Sjthestness@gmail.com            self.depends = []
9411144Sjthestness@gmail.com        elif isinstance(self.depends, str):
9511144Sjthestness@gmail.com            self.depends = [self.depends]
9611144Sjthestness@gmail.com
9711144Sjthestness@gmail.com        if hasattr(self, 'upgrader'):
9811144Sjthestness@gmail.com            if not isinstance(self.upgrader, types.FunctionType):
9911144Sjthestness@gmail.com                print "Error: 'upgrader' for %s is %s, not function" \
1005643Sgblack@eecs.umich.edu                    % (self.tag, type(self))
1015643Sgblack@eecs.umich.edu                sys.exit(1)
1025643Sgblack@eecs.umich.edu            Upgrader.tag_set.add(self.tag)
1035643Sgblack@eecs.umich.edu        elif hasattr(self, 'downgrader'):
1045643Sgblack@eecs.umich.edu            if not isinstance(self.downgrader, types.FunctionType):
1055643Sgblack@eecs.umich.edu                print "Error: 'downgrader' for %s is %s, not function" \
10613229Sgabeblack@google.com                    % (self.tag, type(self))
1075643Sgblack@eecs.umich.edu                sys.exit(1)
1085643Sgblack@eecs.umich.edu            Upgrader.untag_set.add(self.tag)
10913229Sgabeblack@google.com        else:
1105643Sgblack@eecs.umich.edu            print "Error: no upgrader or downgrader method for", self.tag
1115643Sgblack@eecs.umich.edu            sys.exit(1)
1125643Sgblack@eecs.umich.edu
1135643Sgblack@eecs.umich.edu        if hasattr(self, 'legacy_version'):
1145898Sgblack@eecs.umich.edu            Upgrader.legacy[self.legacy_version] = self
1159805Sstever@gmail.com
1165643Sgblack@eecs.umich.edu        Upgrader.by_tag[self.tag] = self
1175643Sgblack@eecs.umich.edu
1185643Sgblack@eecs.umich.edu    def ready(self, tags):
1195643Sgblack@eecs.umich.edu        for dep in self.depends:
1205643Sgblack@eecs.umich.edu            if dep not in tags:
1215643Sgblack@eecs.umich.edu                return False
1225643Sgblack@eecs.umich.edu        return True
1235643Sgblack@eecs.umich.edu
1245643Sgblack@eecs.umich.edu    def update(self, cpt, tags):
12513229Sgabeblack@google.com        if hasattr(self, 'upgrader'):
1265643Sgblack@eecs.umich.edu            self.upgrader(cpt)
1275643Sgblack@eecs.umich.edu            tags.add(self.tag)
12813229Sgabeblack@google.com            verboseprint("applied upgrade for", self.tag)
1295643Sgblack@eecs.umich.edu        else:
1305643Sgblack@eecs.umich.edu            self.downgrader(cpt)
1315643Sgblack@eecs.umich.edu            tags.remove(self.tag)
1325643Sgblack@eecs.umich.edu            verboseprint("applied downgrade for", self.tag)
1335898Sgblack@eecs.umich.edu
1349805Sstever@gmail.com    @staticmethod
1355643Sgblack@eecs.umich.edu    def get(tag):
1365643Sgblack@eecs.umich.edu        return Upgrader.by_tag[tag]
1375643Sgblack@eecs.umich.edu
1385643Sgblack@eecs.umich.edu    @staticmethod
1395643Sgblack@eecs.umich.edu    def load_all():
1405643Sgblack@eecs.umich.edu        util_dir = osp.dirname(osp.abspath(__file__))
1417913SBrad.Beckmann@amd.com
1425643Sgblack@eecs.umich.edu        for py in glob.glob(util_dir + '/cpt_upgraders/*.py'):
1435643Sgblack@eecs.umich.edu            Upgrader(py)
1445643Sgblack@eecs.umich.edu
1457913SBrad.Beckmann@amd.com        # make linear dependences for legacy versions
1465643Sgblack@eecs.umich.edu        i = 3
1475643Sgblack@eecs.umich.edu        while i in Upgrader.legacy:
1485643Sgblack@eecs.umich.edu            Upgrader.legacy[i].depends = [Upgrader.legacy[i-1].tag]
1495643Sgblack@eecs.umich.edu            i = i + 1
1505643Sgblack@eecs.umich.edu
1515643Sgblack@eecs.umich.edudef process_file(path, **kwargs):
1525643Sgblack@eecs.umich.edu    if not osp.isfile(path):
1535643Sgblack@eecs.umich.edu        import errno
1545643Sgblack@eecs.umich.edu        raise IOError(ennro.ENOENT, "No such file", path)
1555643Sgblack@eecs.umich.edu
1565643Sgblack@eecs.umich.edu    verboseprint("Processing file %s...." % path)
1575643Sgblack@eecs.umich.edu
1585643Sgblack@eecs.umich.edu    if kwargs.get('backup', True):
1595643Sgblack@eecs.umich.edu        import shutil
1605643Sgblack@eecs.umich.edu        shutil.copyfile(path, path + '.bak')
1615643Sgblack@eecs.umich.edu
1625643Sgblack@eecs.umich.edu    cpt = ConfigParser.SafeConfigParser()
1635643Sgblack@eecs.umich.edu
1645643Sgblack@eecs.umich.edu    # gem5 is case sensitive with paramaters
1655643Sgblack@eecs.umich.edu    cpt.optionxform = str
1665643Sgblack@eecs.umich.edu
1675643Sgblack@eecs.umich.edu    # Read the current data
1685643Sgblack@eecs.umich.edu    cpt_file = file(path, 'r')
1695643Sgblack@eecs.umich.edu    cpt.readfp(cpt_file)
1705643Sgblack@eecs.umich.edu    cpt_file.close()
1715643Sgblack@eecs.umich.edu
1725643Sgblack@eecs.umich.edu    change = False
1735643Sgblack@eecs.umich.edu
1745643Sgblack@eecs.umich.edu    # Make sure we know what we're starting from
1755643Sgblack@eecs.umich.edu    if cpt.has_option('root','cpt_ver'):
1765643Sgblack@eecs.umich.edu        cpt_ver = cpt.getint('root','cpt_ver')
1775643Sgblack@eecs.umich.edu
1785643Sgblack@eecs.umich.edu        # Legacy linear checkpoint version
1795643Sgblack@eecs.umich.edu        # convert to list of tags before proceeding
1805643Sgblack@eecs.umich.edu        tags = set([])
1815643Sgblack@eecs.umich.edu        for i in xrange(2, cpt_ver+1):
1825643Sgblack@eecs.umich.edu            tags.add(Upgrader.legacy[i].tag)
1835643Sgblack@eecs.umich.edu        verboseprint("performed legacy version -> tags conversion")
1845643Sgblack@eecs.umich.edu        change = True
1855643Sgblack@eecs.umich.edu
1865643Sgblack@eecs.umich.edu        cpt.remove_option('root', 'cpt_ver')
1875643Sgblack@eecs.umich.edu    elif cpt.has_option('Globals','version_tags'):
1885643Sgblack@eecs.umich.edu        tags = set((''.join(cpt.get('Globals','version_tags'))).split())
1895643Sgblack@eecs.umich.edu    else:
1905643Sgblack@eecs.umich.edu        print "fatal: no version information in checkpoint"
1915643Sgblack@eecs.umich.edu        exit(1)
1925643Sgblack@eecs.umich.edu
1935643Sgblack@eecs.umich.edu    verboseprint("has tags", ' '.join(tags))
1945643Sgblack@eecs.umich.edu    # If the current checkpoint has a tag we don't know about, we have
1955643Sgblack@eecs.umich.edu    # a divergence that (in general) must be addressed by (e.g.) merging
1965643Sgblack@eecs.umich.edu    # simulator support for its changes.
1976712Snate@binkert.org    unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set)
1985651Sgblack@eecs.umich.edu    if unknown_tags:
1995657Sgblack@eecs.umich.edu        print "warning: upgrade script does not recognize the following "\
2005657Sgblack@eecs.umich.edu              "tags in this checkpoint:", ' '.join(unknown_tags)
2015657Sgblack@eecs.umich.edu
2025657Sgblack@eecs.umich.edu    # Apply migrations for tags not in checkpoint and tags present for which
2035657Sgblack@eecs.umich.edu    # downgraders are present, respecting dependences
2045657Sgblack@eecs.umich.edu    to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags)
2055651Sgblack@eecs.umich.edu    while to_apply:
2065651Sgblack@eecs.umich.edu        ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ])
2075654Sgblack@eecs.umich.edu        if not ready:
2085654Sgblack@eecs.umich.edu            print "could not apply these upgrades:", ' '.join(to_apply)
2096138Sgblack@eecs.umich.edu            print "update dependences impossible to resolve; aborting"
2106138Sgblack@eecs.umich.edu            exit(1)
2116138Sgblack@eecs.umich.edu
2126138Sgblack@eecs.umich.edu        for tag in ready:
2136138Sgblack@eecs.umich.edu            Upgrader.get(tag).update(cpt, tags)
2146138Sgblack@eecs.umich.edu            change = True
2156138Sgblack@eecs.umich.edu
2166138Sgblack@eecs.umich.edu        to_apply -= ready
2176138Sgblack@eecs.umich.edu
2186138Sgblack@eecs.umich.edu    if not change:
2196138Sgblack@eecs.umich.edu        verboseprint("...nothing to do")
2206138Sgblack@eecs.umich.edu        return
2216138Sgblack@eecs.umich.edu
2226138Sgblack@eecs.umich.edu    cpt.set('Globals', 'version_tags', ' '.join(tags))
2236138Sgblack@eecs.umich.edu
2246138Sgblack@eecs.umich.edu    # Write the old data back
2256138Sgblack@eecs.umich.edu    verboseprint("...completed")
2268746Sgblack@eecs.umich.edu    cpt.write(file(path, 'w'))
22711150Smitch.hayenga@arm.com
2286138Sgblack@eecs.umich.eduif __name__ == '__main__':
2296138Sgblack@eecs.umich.edu    from optparse import OptionParser, SUPPRESS_HELP
2308746Sgblack@eecs.umich.edu    parser = OptionParser("usage: %prog [options] <filename or directory>")
2316138Sgblack@eecs.umich.edu    parser.add_option("-r", "--recurse", action="store_true",
2326138Sgblack@eecs.umich.edu                      help="Recurse through all subdirectories modifying "\
2336139Sgblack@eecs.umich.edu                           "each checkpoint that is found")
2346139Sgblack@eecs.umich.edu    parser.add_option("-N", "--no-backup", action="store_false",
2356139Sgblack@eecs.umich.edu                      dest="backup", default=True,
2366139Sgblack@eecs.umich.edu                      help="Do no backup each checkpoint before modifying it")
2376139Sgblack@eecs.umich.edu    parser.add_option("-v", "--verbose", action="store_true",
2386139Sgblack@eecs.umich.edu                      help="Print out debugging information as")
2396139Sgblack@eecs.umich.edu    parser.add_option("--get-cc-file", action="store_true",
2406139Sgblack@eecs.umich.edu                      # used during build; generate src/sim/tags.cc and exit
2416139Sgblack@eecs.umich.edu                      help=SUPPRESS_HELP)
2426139Sgblack@eecs.umich.edu
2436139Sgblack@eecs.umich.edu    (options, args) = parser.parse_args()
2446139Sgblack@eecs.umich.edu    verbose_print = options.verbose
2456139Sgblack@eecs.umich.edu
2466139Sgblack@eecs.umich.edu    Upgrader.load_all()
2476139Sgblack@eecs.umich.edu
2486139Sgblack@eecs.umich.edu    if options.get_cc_file:
2496138Sgblack@eecs.umich.edu        print "// this file is auto-generated by util/cpt_upgrader.py"
2506138Sgblack@eecs.umich.edu        print "#include <string>"
2519524SAndreas.Sandberg@ARM.com        print "#include <set>"
2525643Sgblack@eecs.umich.edu        print
2535643Sgblack@eecs.umich.edu        print "std::set<std::string> version_tags = {"
2545643Sgblack@eecs.umich.edu        for tag in Upgrader.tag_set:
2555827Sgblack@eecs.umich.edu            print "  \"%s\"," % tag
2565827Sgblack@eecs.umich.edu        print "};"
2575827Sgblack@eecs.umich.edu        exit(0)
2585827Sgblack@eecs.umich.edu    elif len(args) != 1:
2595827Sgblack@eecs.umich.edu        parser.error("You must specify a checkpoint file to modify or a "\
2605827Sgblack@eecs.umich.edu                     "directory of checkpoints to recursively update")
2615827Sgblack@eecs.umich.edu
2625827Sgblack@eecs.umich.edu    # Deal with shell variables and ~
2635827Sgblack@eecs.umich.edu    path = osp.expandvars(osp.expanduser(args[0]))
2645827Sgblack@eecs.umich.edu
2655827Sgblack@eecs.umich.edu    # Process a single file if we have it
2665827Sgblack@eecs.umich.edu    if osp.isfile(path):
2675827Sgblack@eecs.umich.edu        process_file(path, **vars(options))
2685827Sgblack@eecs.umich.edu    # Process an entire directory
2695827Sgblack@eecs.umich.edu    elif osp.isdir(path):
2705827Sgblack@eecs.umich.edu        cpt_file = osp.join(path, 'm5.cpt')
2716137Sgblack@eecs.umich.edu        if options.recurse:
27210905Sandreas.sandberg@arm.com            # Visit very file and see if it matches
2737903Shestness@cs.utexas.edu            for root,dirs,files in os.walk(path):
2747903Shestness@cs.utexas.edu                for name in files:
2757903Shestness@cs.utexas.edu                    if name == 'm5.cpt':
2767903Shestness@cs.utexas.edu                        process_file(osp.join(root,name), **vars(options))
2777903Shestness@cs.utexas.edu                for dir in dirs:
2787903Shestness@cs.utexas.edu                    pass
2797903Shestness@cs.utexas.edu        # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
2807903Shestness@cs.utexas.edu        elif osp.isfile(cpt_file):
2817903Shestness@cs.utexas.edu            process_file(cpt_file, **vars(options))
2827903Shestness@cs.utexas.edu        else:
2837903Shestness@cs.utexas.edu            print "Error: checkpoint file not found at in %s " % path,
2847903Shestness@cs.utexas.edu            print "and recurse not specified"
28510905Sandreas.sandberg@arm.com            sys.exit(1)
2867903Shestness@cs.utexas.edu    sys.exit(0)
2877903Shestness@cs.utexas.edu
2887903Shestness@cs.utexas.edu