cpt_upgrader.py revision 11834:29f0d1d70282 
1#!/usr/bin/env python2
2
3# Copyright (c) 2012-2013,2015-2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Ali Saidi
39#          Curtis Dunham
40#
41
42# This python code is used to migrate checkpoints that were created in one
43# version of the simulator to newer version. As features are added or bugs are
44# fixed some of the state that needs to be checkpointed can change. If you have
45# many historic checkpoints that you use, manually editing them to fix them is
46# both time consuming and error-prone.
47
48# This script provides a way to migrate checkpoints to the newer repository in
49# a programmatic way. It can be imported into another script or used on the
50# command line. From the command line the script will either migrate every
51# checkpoint it finds recursively (-r option) or a single checkpoint. When a
52# change is made to the gem5 repository that breaks previous checkpoints an
53# upgrade() method should be implemented in its own .py file and placed in
54# src/util/cpt_upgraders/.  For each upgrader whose tag is not present in
55# the checkpoint tag list, the upgrade() method will be run, passing in a
56# ConfigParser object which contains the open file. As these operations can
57# be isa specific the method can verify the isa and use regexes to find the
58# correct sections that need to be updated.
59
60# It is also possible to use this mechanism to revert prior tags.  In this
61# case, implement a downgrade() method instead.  Dependencies should still
62# work naturally - a tag depending on a tag with a downgrader means that it
63# insists on the other tag being removed and its downgrader executed before
64# its upgrader (or downgrader) can run.  It is still the case that a tag
65# can only be used once.
66
67
68import ConfigParser
69import glob, types, sys, os
70import os.path as osp
71
72verbose_print = False
73
74def verboseprint(*args):
75    if not verbose_print:
76        return
77    for arg in args:
78        print arg,
79    print
80
81class Upgrader:
82    tag_set = set()
83    untag_set = set() # tags to remove by downgrading
84    by_tag = {}
85    legacy = {}
86    def __init__(self, filename):
87        self.filename = filename
88        execfile(filename, {}, self.__dict__)
89
90        if not hasattr(self, 'tag'):
91            self.tag = osp.basename(filename)[:-3]
92        if not hasattr(self, 'depends'):
93            self.depends = []
94        elif isinstance(self.depends, str):
95            self.depends = [self.depends]
96
97        if hasattr(self, 'upgrader'):
98            if not isinstance(self.upgrader, types.FunctionType):
99                print "Error: 'upgrader' for %s is %s, not function" \
100                    % (self.tag, type(self))
101                sys.exit(1)
102            Upgrader.tag_set.add(self.tag)
103        elif hasattr(self, 'downgrader'):
104            if not isinstance(self.downgrader, types.FunctionType):
105                print "Error: 'downgrader' for %s is %s, not function" \
106                    % (self.tag, type(self))
107                sys.exit(1)
108            Upgrader.untag_set.add(self.tag)
109        else:
110            print "Error: no upgrader or downgrader method for", self.tag
111            sys.exit(1)
112
113        if hasattr(self, 'legacy_version'):
114            Upgrader.legacy[self.legacy_version] = self
115
116        Upgrader.by_tag[self.tag] = self
117
118    def ready(self, tags):
119        for dep in self.depends:
120            if dep not in tags:
121                return False
122        return True
123
124    def update(self, cpt, tags):
125        if hasattr(self, 'upgrader'):
126            self.upgrader(cpt)
127            tags.add(self.tag)
128            verboseprint("applied upgrade for", self.tag)
129        else:
130            self.downgrader(cpt)
131            tags.remove(self.tag)
132            verboseprint("applied downgrade for", self.tag)
133
134    @staticmethod
135    def get(tag):
136        return Upgrader.by_tag[tag]
137
138    @staticmethod
139    def load_all():
140        util_dir = osp.dirname(osp.abspath(__file__))
141
142        for py in glob.glob(util_dir + '/cpt_upgraders/*.py'):
143            Upgrader(py)
144
145        # make linear dependences for legacy versions
146        i = 3
147        while i in Upgrader.legacy:
148            Upgrader.legacy[i].depends = [Upgrader.legacy[i-1].tag]
149            i = i + 1
150
151def process_file(path, **kwargs):
152    if not osp.isfile(path):
153        import errno
154        raise IOError(ennro.ENOENT, "No such file", path)
155
156    verboseprint("Processing file %s...." % path)
157
158    if kwargs.get('backup', True):
159        import shutil
160        shutil.copyfile(path, path + '.bak')
161
162    cpt = ConfigParser.SafeConfigParser()
163
164    # gem5 is case sensitive with paramaters
165    cpt.optionxform = str
166
167    # Read the current data
168    cpt_file = file(path, 'r')
169    cpt.readfp(cpt_file)
170    cpt_file.close()
171
172    change = False
173
174    # Make sure we know what we're starting from
175    if cpt.has_option('root','cpt_ver'):
176        cpt_ver = cpt.getint('root','cpt_ver')
177
178        # Legacy linear checkpoint version
179        # convert to list of tags before proceeding
180        tags = set([])
181        for i in xrange(2, cpt_ver+1):
182            tags.add(Upgrader.legacy[i].tag)
183        verboseprint("performed legacy version -> tags conversion")
184        change = True
185
186        cpt.remove_option('root', 'cpt_ver')
187    elif cpt.has_option('Globals','version_tags'):
188        tags = set((''.join(cpt.get('Globals','version_tags'))).split())
189    else:
190        print "fatal: no version information in checkpoint"
191        exit(1)
192
193    verboseprint("has tags", ' '.join(tags))
194    # If the current checkpoint has a tag we don't know about, we have
195    # a divergence that (in general) must be addressed by (e.g.) merging
196    # simulator support for its changes.
197    unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set)
198    if unknown_tags:
199        print "warning: upgrade script does not recognize the following "\
200              "tags in this checkpoint:", ' '.join(unknown_tags)
201
202    # Apply migrations for tags not in checkpoint and tags present for which
203    # downgraders are present, respecting dependences
204    to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags)
205    while to_apply:
206        ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ])
207        if not ready:
208            print "could not apply these upgrades:", ' '.join(to_apply)
209            print "update dependences impossible to resolve; aborting"
210            exit(1)
211
212        for tag in ready:
213            Upgrader.get(tag).update(cpt, tags)
214            change = True
215
216        to_apply -= ready
217
218    if not change:
219        verboseprint("...nothing to do")
220        return
221
222    cpt.set('Globals', 'version_tags', ' '.join(tags))
223
224    # Write the old data back
225    verboseprint("...completed")
226    cpt.write(file(path, 'w'))
227
228if __name__ == '__main__':
229    from optparse import OptionParser, SUPPRESS_HELP
230    parser = OptionParser("usage: %prog [options] <filename or directory>")
231    parser.add_option("-r", "--recurse", action="store_true",
232                      help="Recurse through all subdirectories modifying "\
233                           "each checkpoint that is found")
234    parser.add_option("-N", "--no-backup", action="store_false",
235                      dest="backup", default=True,
236                      help="Do no backup each checkpoint before modifying it")
237    parser.add_option("-v", "--verbose", action="store_true",
238                      help="Print out debugging information as")
239    parser.add_option("--get-cc-file", action="store_true",
240                      # used during build; generate src/sim/tags.cc and exit
241                      help=SUPPRESS_HELP)
242
243    (options, args) = parser.parse_args()
244    verbose_print = options.verbose
245
246    Upgrader.load_all()
247
248    if options.get_cc_file:
249        print "// this file is auto-generated by util/cpt_upgrader.py"
250        print "#include <string>"
251        print "#include <set>"
252        print
253        print "std::set<std::string> version_tags = {"
254        for tag in Upgrader.tag_set:
255            print "  \"%s\"," % tag
256        print "};"
257        exit(0)
258    elif len(args) != 1:
259        parser.error("You must specify a checkpoint file to modify or a "\
260                     "directory of checkpoints to recursively update")
261
262    # Deal with shell variables and ~
263    path = osp.expandvars(osp.expanduser(args[0]))
264
265    # Process a single file if we have it
266    if osp.isfile(path):
267        process_file(path, **vars(options))
268    # Process an entire directory
269    elif osp.isdir(path):
270        cpt_file = osp.join(path, 'm5.cpt')
271        if options.recurse:
272            # Visit very file and see if it matches
273            for root,dirs,files in os.walk(path):
274                for name in files:
275                    if name == 'm5.cpt':
276                        process_file(osp.join(root,name), **vars(options))
277                for dir in dirs:
278                    pass
279        # Maybe someone passed a cpt.XXXXXXX directory and not m5.cpt
280        elif osp.isfile(cpt_file):
281            process_file(cpt_file, **vars(options))
282        else:
283            print "Error: checkpoint file not found at in %s " % path,
284            print "and recurse not specified"
285            sys.exit(1)
286    sys.exit(0)
287
288