checkpoint_aggregator.py revision 9991 
1# Copyright (c) 2009 The Regents of The University of Michigan
2# Copyright (c) 2011 Advanced Micro Devices, Inc.
3# Copyright (c) 2013 Mark D. Hill and David A. Wood
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# Authors: Lisa Hsu
30#          Nilay Vaish
31
32from ConfigParser import ConfigParser
33import gzip
34
35import sys, re, os
36
37class myCP(ConfigParser):
38    def __init__(self):
39        ConfigParser.__init__(self)
40
41    def optionxform(self, optionstr):
42        return optionstr
43
44def aggregate(output_dir, cpts, no_compress, memory_size):
45    merged_config = None
46    page_ptr = 0
47
48    output_path = output_dir
49    if not os.path.isdir(output_path):
50        os.system("mkdir -p " + output_path)
51
52    agg_mem_file = open(output_path + "/system.physmem.store0.pmem", "wb+")
53    agg_config_file = open(output_path + "/m5.cpt", "wb+")
54
55    if not no_compress:
56        merged_mem = gzip.GzipFile(fileobj= agg_mem_file, mode="wb")
57
58    max_curtick = 0
59    num_digits = len(str(len(cpts)-1))
60
61    for (i, arg) in enumerate(cpts):
62        print arg
63        merged_config = myCP()
64        config = myCP()
65        config.readfp(open(cpts[i] + "/m5.cpt"))
66
67        for sec in config.sections():
68            if re.compile("cpu").search(sec):
69                newsec = re.sub("cpu", "cpu" + str(i).zfill(num_digits), sec)
70                merged_config.add_section(newsec)
71
72                items = config.items(sec)
73                for item in items:
74                    if item[0] == "paddr":
75                        merged_config.set(newsec, item[0], int(item[1]) + (page_ptr << 12))
76                        continue
77                    merged_config.set(newsec, item[0], item[1])
78
79                if re.compile("workload.FdMap256$").search(sec):
80                    merged_config.set(newsec, "M5_pid", i)
81
82            elif sec == "system":
83                pass
84            elif sec == "Globals":
85                tick = config.getint(sec, "curTick")
86                if tick > max_curtick:
87                    max_curtick = tick
88            else:
89                if i == len(cpts)-1:
90                    merged_config.add_section(sec)
91                    for item in config.items(sec):
92                        merged_config.set(sec, item[0], item[1])
93
94        if i != len(cpts)-1:
95            merged_config.write(agg_config_file)
96
97        ### memory stuff
98        pages = int(config.get("system", "pagePtr"))
99        page_ptr = page_ptr + pages
100        print "pages to be read: ", pages
101
102        f = open(cpts[i] + "/system.physmem.store0.pmem", "rb")
103        gf = gzip.GzipFile(fileobj=f, mode="rb")
104
105        x = 0
106        while x < pages:
107            bytesRead = gf.read(1 << 12)
108            if not no_compress:
109                merged_mem.write(bytesRead)
110            else:
111                agg_mem_file.write(bytesRead)
112            x += 1
113
114        gf.close()
115        f.close()
116
117    merged_config.add_section("system")
118    merged_config.set("system", "pagePtr", page_ptr)
119    merged_config.set("system", "nextPID", len(cpts))
120
121    file_size = page_ptr * 4 * 1024
122    dummy_data = "".zfill(4096)
123    while file_size < memory_size:
124        if not no_compress:
125            merged_mem.write(dummy_data)
126        else:
127            agg_mem_file.write(dummy_data)
128        file_size += 4 * 1024
129        page_ptr += 1
130
131    print "WARNING: "
132    print "Make sure the simulation using this checkpoint has at least ",
133    print page_ptr, "x 4K of memory"
134    merged_config.set("system.physmem.store0", "range_size", page_ptr * 4 * 1024)
135
136    merged_config.add_section("Globals")
137    merged_config.set("Globals", "curTick", max_curtick)
138
139    merged_config.write(agg_config_file)
140
141    if not no_compress:
142        merged_mem.close()
143        agg_mem_file.close()
144    else:
145        agg_mem_file.close()
146
147if __name__ == "__main__":
148    from argparse import ArgumentParser
149    parser = ArgumentParser("usage: %prog [options] <directory names which "\
150                            "hold the checkpoints to be combined>")
151    parser.add_argument("-o", "--output-dir", action="store",
152                        help="Output directory")
153    parser.add_argument("-c", "--no-compress", action="store_true")
154    parser.add_argument("--cpts", nargs='+')
155    parser.add_argument("--memory-size", action="store", type=int)
156
157    # Assume x86 ISA.  Any other ISAs would need extra stuff in this script
158    # to appropriately parse their page tables and understand page sizes.
159    options = parser.parse_args()
160    print options.cpts, len(options.cpts)
161    if len(options.cpts) <= 1:
162        parser.error("You must specify atleast two checkpoint files that "\
163                     "need to be combined.")
164
165    aggregate(options.output_dir, options.cpts, options.no_compress,
166              options.memory_size)
167