checkpoint_aggregator.py revision 7443 
1# Copyright (c) 2009 The Regents of The University of Michigan
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Lisa Hsu
28
29from ConfigParser import ConfigParser
30import gzip
31
32import sys, re, optparse, os
33
34class myCP(ConfigParser):
35    def __init__(self):
36        ConfigParser.__init__(self)
37
38    def optionxform(self, optionstr):
39        return optionstr
40
41def aggregate(options, args):
42    merged = myCP()
43    page_ptr = 0
44
45    allfiles = os.listdir(os.getcwd())
46    cpts = []
47    for arg in args:
48        found = False
49        for f in allfiles:
50            if re.compile("cpt." + arg + ".\d+").search(f):
51                found = True
52                cpts.append(f)
53                break
54        if not found:
55            print "missing checkpoint: ", arg
56            sys.exit(1)
57
58    dirname = "-".join([options.prefix, "cpt"])
59    agg_name = "-".join(args)
60    print agg_name
61    fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000")
62    if not os.path.isdir(fullpath):
63        os.system("mkdir -p " + fullpath)
64    elif os.path.isfile(fullpath + "/system.physmem.physmem"):
65        if os.path.isfile(fullpath + "/m5.cpt"):
66            print fullpath, " already done"
67            return
68
69    myfile = open(fullpath + "/system.physmem.physmem", "wb+")
70    merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb")
71
72    max_curtick = 0
73    when = 0
74    for (i, arg) in enumerate(args):
75        print arg
76        config = myCP()
77        config.readfp(open(cpts[i] + "/m5.cpt"))
78
79        for sec in config.sections():
80            if re.compile("cpu").search(sec):
81                newsec = re.sub("cpu", "cpu" + str(i), sec)
82                merged.add_section(newsec)
83                if re.compile("workload$").search(sec):
84                    merged.set(newsec, "M5_pid", i)
85
86                items = config.items(sec)
87                for item in items:
88                    if item[0] == "ppn":
89                        if config.getint(sec, "tag") != 0:
90                            merged.set(newsec, item[0], int(item[1]) + page_ptr)
91                            continue
92                    elif item[0] == "asn":
93                        tmp = re.compile("(.*).Entry(\d+)").search(sec).groups()
94                        if config.has_option(tmp[0], "nlu"):
95                            size = config.getint(tmp[0], "nlu")
96                            if int(tmp[1]) < size:
97                                merged.set(newsec, item[0], i)
98                                continue
99                        else:
100                            merged.set(newsec, item[0], i)
101                            continue
102                    merged.set(newsec, item[0], item[1])
103            elif sec == "system":
104                pass
105            elif sec == "Globals":
106                tick = config.getint(sec, "curTick")
107                if tick > max_curtick:
108                    max_curtick = tick
109                    when = config.getint("system.cpu.tickEvent", "_when")
110            else:
111                if i == 0:
112                    merged.add_section(sec)
113                    for item in config.items(sec):
114                        merged.set(sec, item[0], item[1])
115                        if item[0] == "curtick":
116                            merged.optionxform(str("curTick"))
117                        elif item[0] == "numevents":
118                            merged.optionxform(str("numEvents"))
119
120        page_ptr = page_ptr + int(config.get("system", "page_ptr"))
121
122        ### memory stuff
123        f = open(cpts[i] + "/system.physmem.physmem", "rb")
124        gf = gzip.GzipFile(fileobj=f, mode="rb")
125        pages = int(config.get("system", "page_ptr"))
126        print "pages to be read: ", pages
127
128        x = 0
129        while x < pages:
130            bytesRead = gf.read(1 << 13)
131            merged_mem.write(bytesRead)
132            x += 1
133
134        gf.close()
135        f.close()
136
137    merged.add_section("system")
138    merged.set("system", "page_ptr", page_ptr)
139    print "WARNING: "
140    print "Make sure the simulation using this checkpoint has at least "
141    if page_ptr > (1<<20):
142        print "8G ",
143    elif page_ptr > (1<<19):
144        print "4G ",
145    elif page_ptr > (1<<18):
146        print "2G ",
147    elif page_ptr > (1<<17):
148        print "1G ",
149    elif page_ptr > (1<<16):
150        print "512KB ",
151    else:
152        print "this is a small sim, you're probably fine",
153    print "of memory."
154
155    merged.add_section("Globals")
156    merged.set("Globals", "curTick", max_curtick)
157
158    for i in xrange(len(args)):
159        merged.set("system.cpu" + str(i) + ".tickEvent", "_when", when)
160
161    merged.write(file(fullpath + "/m5.cpt", "wb"))
162    merged_mem.close()
163    myfile.close()
164
165if __name__ == "__main__":
166
167    parser = optparse.OptionParser()
168    parser.add_option("--prefix", type="string", default="agg")
169
170    (options, args) = parser.parse_args()
171
172    aggregate(options, args)
173
174