RubySystem.cc revision 8688 
1/*
2 * Copyright (c) 1999-2011 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <fcntl.h>
30#include <zlib.h>
31
32#include <cstdio>
33
34#include "base/intmath.hh"
35#include "base/output.hh"
36#include "debug/RubySystem.hh"
37#include "mem/ruby/common/Address.hh"
38#include "mem/ruby/network/Network.hh"
39#include "mem/ruby/profiler/Profiler.hh"
40#include "mem/ruby/system/System.hh"
41#include "sim/simulate.hh"
42
43using namespace std;
44
45int RubySystem::m_random_seed;
46bool RubySystem::m_randomization;
47Tick RubySystem::m_clock;
48int RubySystem::m_block_size_bytes;
49int RubySystem::m_block_size_bits;
50uint64 RubySystem::m_memory_size_bytes;
51int RubySystem::m_memory_size_bits;
52
53Network* RubySystem::m_network_ptr;
54Profiler* RubySystem::m_profiler_ptr;
55MemoryVector* RubySystem::m_mem_vec_ptr;
56
57RubySystem::RubySystem(const Params *p)
58    : SimObject(p)
59{
60    if (g_system_ptr != NULL)
61        fatal("Only one RubySystem object currently allowed.\n");
62
63    m_random_seed = p->random_seed;
64    srandom(m_random_seed);
65    m_randomization = p->randomization;
66    m_clock = p->clock;
67
68    m_block_size_bytes = p->block_size_bytes;
69    assert(isPowerOf2(m_block_size_bytes));
70    m_block_size_bits = floorLog2(m_block_size_bytes);
71
72    m_memory_size_bytes = p->mem_size;
73    if (m_memory_size_bytes == 0) {
74        m_memory_size_bits = 0;
75    } else {
76        m_memory_size_bits = floorLog2(m_memory_size_bytes);
77    }
78
79    g_eventQueue_ptr = new RubyEventQueue(p->eventq, m_clock);
80    g_system_ptr = this;
81    if (p->no_mem_vec) {
82        m_mem_vec_ptr = NULL;
83    } else {
84        m_mem_vec_ptr = new MemoryVector;
85        m_mem_vec_ptr->resize(m_memory_size_bytes);
86    }
87
88    //
89    // Print ruby configuration and stats at exit
90    //
91    RubyExitCallback* rubyExitCB = new RubyExitCallback(p->stats_filename);
92    registerExitCallback(rubyExitCB);
93    m_warmup_enabled = false;
94    m_cooldown_enabled = false;
95}
96
97void
98RubySystem::init()
99{
100    m_profiler_ptr->clearStats();
101}
102
103void
104RubySystem::registerNetwork(Network* network_ptr)
105{
106  m_network_ptr = network_ptr;
107}
108
109void
110RubySystem::registerProfiler(Profiler* profiler_ptr)
111{
112  m_profiler_ptr = profiler_ptr;
113}
114
115void
116RubySystem::registerAbstractController(AbstractController* cntrl)
117{
118  m_abs_cntrl_vec.push_back(cntrl);
119}
120
121void
122RubySystem::registerSparseMemory(SparseMemory* s)
123{
124    m_sparse_memory_vector.push_back(s);
125}
126
127RubySystem::~RubySystem()
128{
129    delete m_network_ptr;
130    delete m_profiler_ptr;
131    if (m_mem_vec_ptr)
132        delete m_mem_vec_ptr;
133}
134
135void
136RubySystem::printSystemConfig(ostream & out)
137{
138    out << "RubySystem config:" << endl
139        << "  random_seed: " << m_random_seed << endl
140        << "  randomization: " << m_randomization << endl
141        << "  cycle_period: " << m_clock << endl
142        << "  block_size_bytes: " << m_block_size_bytes << endl
143        << "  block_size_bits: " << m_block_size_bits << endl
144        << "  memory_size_bytes: " << m_memory_size_bytes << endl
145        << "  memory_size_bits: " << m_memory_size_bits << endl;
146}
147
148void
149RubySystem::printConfig(ostream& out)
150{
151    out << "\n================ Begin RubySystem Configuration Print ================\n\n";
152    printSystemConfig(out);
153    m_network_ptr->printConfig(out);
154    m_profiler_ptr->printConfig(out);
155    out << "\n================ End RubySystem Configuration Print ================\n\n";
156}
157
158void
159RubySystem::printStats(ostream& out)
160{
161    const time_t T = time(NULL);
162    tm *localTime = localtime(&T);
163    char buf[100];
164    strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime);
165
166    out << "Real time: " << buf << endl;
167
168    m_profiler_ptr->printStats(out);
169    m_network_ptr->printStats(out);
170}
171
172void
173RubySystem::writeCompressedTrace(uint8* raw_data, string filename,
174                                 uint64 uncompressed_trace_size)
175{
176    // Create the checkpoint file for the memory
177    string thefile = Checkpoint::dir() + "/" + filename.c_str();
178
179    int fd = creat(thefile.c_str(), 0664);
180    if (fd < 0) {
181        perror("creat");
182        fatal("Can't open memory trace file '%s'\n", filename);
183    }
184
185    gzFile compressedMemory = gzdopen(fd, "wb");
186    if (compressedMemory == NULL)
187        fatal("Insufficient memory to allocate compression state for %s\n",
188              filename);
189
190    if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) !=
191        uncompressed_trace_size) {
192        fatal("Write failed on memory trace file '%s'\n", filename);
193    }
194
195    if (gzclose(compressedMemory)) {
196        fatal("Close failed on memory trace file '%s'\n", filename);
197    }
198    delete raw_data;
199}
200
201void
202RubySystem::serialize(std::ostream &os)
203{
204    m_cooldown_enabled = true;
205
206    vector<Sequencer*> sequencer_map;
207    Sequencer* sequencer_ptr = NULL;
208    int cntrl_id = -1;
209
210
211    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
212        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
213        if (sequencer_ptr == NULL) {
214            sequencer_ptr = sequencer_map[cntrl];
215            cntrl_id = cntrl;
216        }
217    }
218
219    assert(sequencer_ptr != NULL);
220
221    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
222        if (sequencer_map[cntrl] == NULL) {
223            sequencer_map[cntrl] = sequencer_ptr;
224        }
225    }
226
227    // Create the CacheRecorder and record the cache trace
228    m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map);
229
230    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
231        m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder);
232    }
233
234    // save the current tick value
235    Tick curtick_original = curTick();
236    // save the event queue head
237    Event* eventq_head = eventq->replaceHead(NULL);
238
239    // Schedule an event to start cache cooldown
240    RubyEvent* e = new RubyEvent(this);
241    schedule(e,curTick());
242    simulate();
243
244    // Restore eventq head
245    eventq_head = eventq->replaceHead(eventq_head);
246    // Restore curTick
247    curTick(curtick_original);
248
249    uint8* raw_data = NULL;
250
251    if (m_mem_vec_ptr != NULL) {
252        uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data);
253
254        string memory_trace_file = name() + ".memory.gz";
255        writeCompressedTrace(raw_data, memory_trace_file,
256                             memory_trace_size);
257
258        SERIALIZE_SCALAR(memory_trace_file);
259        SERIALIZE_SCALAR(memory_trace_size);
260
261    } else {
262        for (int i = 0; i < m_sparse_memory_vector.size(); ++i) {
263            m_sparse_memory_vector[i]->recordBlocks(cntrl_id,
264                                                    m_cache_recorder);
265        }
266    }
267
268    // Aggergate the trace entries together into a single array
269    raw_data = new uint8_t[4096];
270    uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data,
271                                                                 4096);
272    string cache_trace_file = name() + ".cache.gz";
273    writeCompressedTrace(raw_data, cache_trace_file, cache_trace_size);
274
275    SERIALIZE_SCALAR(cache_trace_file);
276    SERIALIZE_SCALAR(cache_trace_size);
277
278    m_cooldown_enabled = false;
279}
280
281void
282RubySystem::readCompressedTrace(string filename, uint8*& raw_data,
283                                uint64& uncompressed_trace_size)
284{
285    // Read the trace file
286    gzFile compressedTrace;
287
288    // trace file
289    int fd = open(filename.c_str(), O_RDONLY);
290    if (fd < 0) {
291        perror("open");
292        fatal("Unable to open trace file %s", filename);
293    }
294
295    compressedTrace = gzdopen(fd, "rb");
296    if (compressedTrace == NULL) {
297        fatal("Insufficient memory to allocate compression state for %s\n",
298              filename);
299    }
300
301    raw_data = new uint8_t[uncompressed_trace_size];
302    if (gzread(compressedTrace, raw_data, uncompressed_trace_size) <
303            uncompressed_trace_size) {
304        fatal("Unable to read complete trace from file %s\n", filename);
305    }
306
307    if (gzclose(compressedTrace)) {
308        fatal("Failed to close cache trace file '%s'\n", filename);
309    }
310}
311
312void
313RubySystem::unserialize(Checkpoint *cp, const string &section)
314{
315    //
316    // The main purpose for clearing stats in the unserialize process is so
317    // that the profiler can correctly set its start time to the unserialized
318    // value of curTick()
319    //
320    clearStats();
321    uint8* uncompressed_trace = NULL;
322
323    if (m_mem_vec_ptr != NULL) {
324        string memory_trace_file;
325        uint64 memory_trace_size = 0;
326
327        UNSERIALIZE_SCALAR(memory_trace_file);
328        UNSERIALIZE_SCALAR(memory_trace_size);
329        memory_trace_file = cp->cptDir + "/" + memory_trace_file;
330
331        readCompressedTrace(memory_trace_file, uncompressed_trace,
332                            memory_trace_size);
333        m_mem_vec_ptr->populatePages(uncompressed_trace);
334
335        delete uncompressed_trace;
336        uncompressed_trace = NULL;
337    }
338
339    string cache_trace_file;
340    uint64 cache_trace_size = 0;
341
342    UNSERIALIZE_SCALAR(cache_trace_file);
343    UNSERIALIZE_SCALAR(cache_trace_size);
344    cache_trace_file = cp->cptDir + "/" + cache_trace_file;
345
346    readCompressedTrace(cache_trace_file, uncompressed_trace,
347                        cache_trace_size);
348    m_warmup_enabled = true;
349
350    vector<Sequencer*> sequencer_map;
351    Sequencer* t = NULL;
352    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
353        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
354        if(t == NULL) t = sequencer_map[cntrl];
355    }
356
357    assert(t != NULL);
358
359    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
360        if (sequencer_map[cntrl] == NULL) {
361            sequencer_map[cntrl] = t;
362        }
363    }
364
365    m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size,
366                                         sequencer_map);
367}
368
369void
370RubySystem::startup()
371{
372    if (m_warmup_enabled) {
373        // save the current tick value
374        Tick curtick_original = curTick();
375        // save the event queue head
376        Event* eventq_head = eventq->replaceHead(NULL);
377        // set curTick to 0
378        curTick(0);
379
380        // Schedule an event to start cache warmup
381        RubyEvent* e = new RubyEvent(this);
382        schedule(e,curTick());
383        simulate();
384
385        delete m_cache_recorder;
386        m_cache_recorder = NULL;
387        m_warmup_enabled = false;
388        // Restore eventq head
389        eventq_head = eventq->replaceHead(eventq_head);
390        // Restore curTick
391        curTick(curtick_original);
392    }
393}
394
395void
396RubySystem::RubyEvent::process()
397{
398    if (ruby_system->m_warmup_enabled) {
399        ruby_system->m_cache_recorder->enqueueNextFetchRequest();
400    }  else if (ruby_system->m_cooldown_enabled) {
401        ruby_system->m_cache_recorder->enqueueNextFlushRequest();
402    }
403}
404
405void
406RubySystem::clearStats() const
407{
408    m_profiler_ptr->clearStats();
409    m_network_ptr->clearStats();
410}
411
412#ifdef CHECK_COHERENCE
413// This code will check for cases if the given cache block is exclusive in
414// one node and shared in another-- a coherence violation
415//
416// To use, the SLICC specification must call sequencer.checkCoherence(address)
417// when the controller changes to a state with new permissions.  Do this
418// in setState.  The SLICC spec must also define methods "isBlockShared"
419// and "isBlockExclusive" that are specific to that protocol
420//
421void
422RubySystem::checkGlobalCoherenceInvariant(const Address& addr)
423{
424#if 0
425    NodeID exclusive = -1;
426    bool sharedDetected = false;
427    NodeID lastShared = -1;
428
429    for (int i = 0; i < m_chip_vector.size(); i++) {
430        if (m_chip_vector[i]->isBlockExclusive(addr)) {
431            if (exclusive != -1) {
432                // coherence violation
433                WARN_EXPR(exclusive);
434                WARN_EXPR(m_chip_vector[i]->getID());
435                WARN_EXPR(addr);
436                WARN_EXPR(g_eventQueue_ptr->getTime());
437                ERROR_MSG("Coherence Violation Detected -- 2 exclusive chips");
438            } else if (sharedDetected) {
439                WARN_EXPR(lastShared);
440                WARN_EXPR(m_chip_vector[i]->getID());
441                WARN_EXPR(addr);
442                WARN_EXPR(g_eventQueue_ptr->getTime());
443                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
444            } else {
445                exclusive = m_chip_vector[i]->getID();
446            }
447        } else if (m_chip_vector[i]->isBlockShared(addr)) {
448            sharedDetected = true;
449            lastShared = m_chip_vector[i]->getID();
450
451            if (exclusive != -1) {
452                WARN_EXPR(lastShared);
453                WARN_EXPR(exclusive);
454                WARN_EXPR(addr);
455                WARN_EXPR(g_eventQueue_ptr->getTime());
456                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
457            }
458        }
459    }
460#endif
461}
462#endif
463
464RubySystem *
465RubySystemParams::create()
466{
467    return new RubySystem(this);
468}
469
470/**
471 * virtual process function that is invoked when the callback
472 * queue is executed.
473 */
474void
475RubyExitCallback::process()
476{
477    std::ostream *os = simout.create(stats_filename);
478    RubySystem::printConfig(*os);
479    *os << endl;
480    RubySystem::printStats(*os);
481}
482