RubySystem.cc revision 9171 
1/*
2 * Copyright (c) 1999-2011 Mark D. Hill and David A. Wood
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <fcntl.h>
30#include <zlib.h>
31
32#include <cstdio>
33
34#include "base/intmath.hh"
35#include "base/output.hh"
36#include "debug/RubyCacheTrace.hh"
37#include "mem/ruby/common/Address.hh"
38#include "mem/ruby/network/Network.hh"
39#include "mem/ruby/profiler/Profiler.hh"
40#include "mem/ruby/system/System.hh"
41#include "sim/eventq.hh"
42#include "sim/simulate.hh"
43
44using namespace std;
45
46int RubySystem::m_random_seed;
47bool RubySystem::m_randomization;
48Tick RubySystem::m_clock;
49int RubySystem::m_block_size_bytes;
50int RubySystem::m_block_size_bits;
51uint64 RubySystem::m_memory_size_bytes;
52int RubySystem::m_memory_size_bits;
53
54Network* RubySystem::m_network_ptr;
55Profiler* RubySystem::m_profiler_ptr;
56MemoryVector* RubySystem::m_mem_vec_ptr;
57
58RubySystem::RubySystem(const Params *p)
59    : SimObject(p)
60{
61    if (g_system_ptr != NULL)
62        fatal("Only one RubySystem object currently allowed.\n");
63
64    m_random_seed = p->random_seed;
65    srandom(m_random_seed);
66    m_randomization = p->randomization;
67    m_clock = p->clock;
68
69    m_block_size_bytes = p->block_size_bytes;
70    assert(isPowerOf2(m_block_size_bytes));
71    m_block_size_bits = floorLog2(m_block_size_bytes);
72
73    m_memory_size_bytes = p->mem_size;
74    if (m_memory_size_bytes == 0) {
75        m_memory_size_bits = 0;
76    } else {
77        m_memory_size_bits = floorLog2(m_memory_size_bytes);
78    }
79
80    g_system_ptr = this;
81    if (p->no_mem_vec) {
82        m_mem_vec_ptr = NULL;
83    } else {
84        m_mem_vec_ptr = new MemoryVector;
85        m_mem_vec_ptr->resize(m_memory_size_bytes);
86    }
87
88    //
89    // Print ruby configuration and stats at exit
90    //
91    RubyExitCallback* rubyExitCB = new RubyExitCallback(p->stats_filename);
92    registerExitCallback(rubyExitCB);
93    m_warmup_enabled = false;
94    m_cooldown_enabled = false;
95}
96
97void
98RubySystem::init()
99{
100    m_profiler_ptr->clearStats();
101}
102
103void
104RubySystem::registerNetwork(Network* network_ptr)
105{
106  m_network_ptr = network_ptr;
107}
108
109void
110RubySystem::registerProfiler(Profiler* profiler_ptr)
111{
112  m_profiler_ptr = profiler_ptr;
113}
114
115void
116RubySystem::registerAbstractController(AbstractController* cntrl)
117{
118  m_abs_cntrl_vec.push_back(cntrl);
119}
120
121void
122RubySystem::registerSparseMemory(SparseMemory* s)
123{
124    m_sparse_memory_vector.push_back(s);
125}
126
127void
128RubySystem::registerMemController(MemoryControl *mc) {
129    m_memory_controller = mc;
130}
131
132RubySystem::~RubySystem()
133{
134    delete m_network_ptr;
135    delete m_profiler_ptr;
136    if (m_mem_vec_ptr)
137        delete m_mem_vec_ptr;
138}
139
140void
141RubySystem::printStats(ostream& out)
142{
143    const time_t T = time(NULL);
144    tm *localTime = localtime(&T);
145    char buf[100];
146    strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime);
147
148    out << "Real time: " << buf << endl;
149
150    m_profiler_ptr->printStats(out);
151    m_network_ptr->printStats(out);
152}
153
154void
155RubySystem::writeCompressedTrace(uint8* raw_data, string filename,
156                                 uint64 uncompressed_trace_size)
157{
158    // Create the checkpoint file for the memory
159    string thefile = Checkpoint::dir() + "/" + filename.c_str();
160
161    int fd = creat(thefile.c_str(), 0664);
162    if (fd < 0) {
163        perror("creat");
164        fatal("Can't open memory trace file '%s'\n", filename);
165    }
166
167    gzFile compressedMemory = gzdopen(fd, "wb");
168    if (compressedMemory == NULL)
169        fatal("Insufficient memory to allocate compression state for %s\n",
170              filename);
171
172    if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) !=
173        uncompressed_trace_size) {
174        fatal("Write failed on memory trace file '%s'\n", filename);
175    }
176
177    if (gzclose(compressedMemory)) {
178        fatal("Close failed on memory trace file '%s'\n", filename);
179    }
180    delete raw_data;
181}
182
183void
184RubySystem::serialize(std::ostream &os)
185{
186    m_cooldown_enabled = true;
187
188    vector<Sequencer*> sequencer_map;
189    Sequencer* sequencer_ptr = NULL;
190    int cntrl_id = -1;
191
192
193    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
194        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
195        if (sequencer_ptr == NULL) {
196            sequencer_ptr = sequencer_map[cntrl];
197            cntrl_id = cntrl;
198        }
199    }
200
201    assert(sequencer_ptr != NULL);
202
203    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
204        if (sequencer_map[cntrl] == NULL) {
205            sequencer_map[cntrl] = sequencer_ptr;
206        }
207    }
208
209    DPRINTF(RubyCacheTrace, "Recording Cache Trace\n");
210    // Create the CacheRecorder and record the cache trace
211    m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map);
212
213    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
214        m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder);
215    }
216
217    DPRINTF(RubyCacheTrace, "Cache Trace Complete\n");
218    // save the current tick value
219    Tick curtick_original = curTick();
220    // save the event queue head
221    Event* eventq_head = eventq->replaceHead(NULL);
222    DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n",
223            curtick_original);
224
225    // Schedule an event to start cache cooldown
226    DPRINTF(RubyCacheTrace, "Starting cache flush\n");
227    enqueueRubyEvent(curTick());
228    simulate();
229    DPRINTF(RubyCacheTrace, "Cache flush complete\n");
230
231    // Restore eventq head
232    eventq_head = eventq->replaceHead(eventq_head);
233    // Restore curTick
234    curTick(curtick_original);
235
236    uint8* raw_data = NULL;
237
238    if (m_mem_vec_ptr != NULL) {
239        uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data);
240
241        string memory_trace_file = name() + ".memory.gz";
242        writeCompressedTrace(raw_data, memory_trace_file,
243                             memory_trace_size);
244
245        SERIALIZE_SCALAR(memory_trace_file);
246        SERIALIZE_SCALAR(memory_trace_size);
247
248    } else {
249        for (int i = 0; i < m_sparse_memory_vector.size(); ++i) {
250            m_sparse_memory_vector[i]->recordBlocks(cntrl_id,
251                                                    m_cache_recorder);
252        }
253    }
254
255    // Aggergate the trace entries together into a single array
256    raw_data = new uint8_t[4096];
257    uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data,
258                                                                 4096);
259    string cache_trace_file = name() + ".cache.gz";
260    writeCompressedTrace(raw_data, cache_trace_file, cache_trace_size);
261
262    SERIALIZE_SCALAR(cache_trace_file);
263    SERIALIZE_SCALAR(cache_trace_size);
264
265    m_cooldown_enabled = false;
266}
267
268void
269RubySystem::readCompressedTrace(string filename, uint8*& raw_data,
270                                uint64& uncompressed_trace_size)
271{
272    // Read the trace file
273    gzFile compressedTrace;
274
275    // trace file
276    int fd = open(filename.c_str(), O_RDONLY);
277    if (fd < 0) {
278        perror("open");
279        fatal("Unable to open trace file %s", filename);
280    }
281
282    compressedTrace = gzdopen(fd, "rb");
283    if (compressedTrace == NULL) {
284        fatal("Insufficient memory to allocate compression state for %s\n",
285              filename);
286    }
287
288    raw_data = new uint8_t[uncompressed_trace_size];
289    if (gzread(compressedTrace, raw_data, uncompressed_trace_size) <
290            uncompressed_trace_size) {
291        fatal("Unable to read complete trace from file %s\n", filename);
292    }
293
294    if (gzclose(compressedTrace)) {
295        fatal("Failed to close cache trace file '%s'\n", filename);
296    }
297}
298
299void
300RubySystem::unserialize(Checkpoint *cp, const string &section)
301{
302    //
303    // The main purpose for clearing stats in the unserialize process is so
304    // that the profiler can correctly set its start time to the unserialized
305    // value of curTick()
306    //
307    clearStats();
308    uint8* uncompressed_trace = NULL;
309
310    if (m_mem_vec_ptr != NULL) {
311        string memory_trace_file;
312        uint64 memory_trace_size = 0;
313
314        UNSERIALIZE_SCALAR(memory_trace_file);
315        UNSERIALIZE_SCALAR(memory_trace_size);
316        memory_trace_file = cp->cptDir + "/" + memory_trace_file;
317
318        readCompressedTrace(memory_trace_file, uncompressed_trace,
319                            memory_trace_size);
320        m_mem_vec_ptr->populatePages(uncompressed_trace);
321
322        delete uncompressed_trace;
323        uncompressed_trace = NULL;
324    }
325
326    string cache_trace_file;
327    uint64 cache_trace_size = 0;
328
329    UNSERIALIZE_SCALAR(cache_trace_file);
330    UNSERIALIZE_SCALAR(cache_trace_size);
331    cache_trace_file = cp->cptDir + "/" + cache_trace_file;
332
333    readCompressedTrace(cache_trace_file, uncompressed_trace,
334                        cache_trace_size);
335    m_warmup_enabled = true;
336
337    vector<Sequencer*> sequencer_map;
338    Sequencer* t = NULL;
339    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
340        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
341        if(t == NULL) t = sequencer_map[cntrl];
342    }
343
344    assert(t != NULL);
345
346    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
347        if (sequencer_map[cntrl] == NULL) {
348            sequencer_map[cntrl] = t;
349        }
350    }
351
352    m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size,
353                                         sequencer_map);
354}
355
356void
357RubySystem::startup()
358{
359    if (m_warmup_enabled) {
360        // save the current tick value
361        Tick curtick_original = curTick();
362        // save the event queue head
363        Event* eventq_head = eventq->replaceHead(NULL);
364        // set curTick to 0
365        curTick(0);
366
367        // Schedule an event to start cache warmup
368        enqueueRubyEvent(curTick());
369        simulate();
370
371        delete m_cache_recorder;
372        m_cache_recorder = NULL;
373        m_warmup_enabled = false;
374        // reset DRAM so that it's not waiting for events on the old event
375        // queue
376        m_memory_controller->reset();
377        // Restore eventq head
378        eventq_head = eventq->replaceHead(eventq_head);
379        // Restore curTick
380        curTick(curtick_original);
381    }
382}
383
384void
385RubySystem::RubyEvent::process()
386{
387    if (ruby_system->m_warmup_enabled) {
388        ruby_system->m_cache_recorder->enqueueNextFetchRequest();
389    }  else if (ruby_system->m_cooldown_enabled) {
390        ruby_system->m_cache_recorder->enqueueNextFlushRequest();
391    }
392}
393
394void
395RubySystem::clearStats() const
396{
397    m_profiler_ptr->clearStats();
398    m_network_ptr->clearStats();
399}
400
401#ifdef CHECK_COHERENCE
402// This code will check for cases if the given cache block is exclusive in
403// one node and shared in another-- a coherence violation
404//
405// To use, the SLICC specification must call sequencer.checkCoherence(address)
406// when the controller changes to a state with new permissions.  Do this
407// in setState.  The SLICC spec must also define methods "isBlockShared"
408// and "isBlockExclusive" that are specific to that protocol
409//
410void
411RubySystem::checkGlobalCoherenceInvariant(const Address& addr)
412{
413#if 0
414    NodeID exclusive = -1;
415    bool sharedDetected = false;
416    NodeID lastShared = -1;
417
418    for (int i = 0; i < m_chip_vector.size(); i++) {
419        if (m_chip_vector[i]->isBlockExclusive(addr)) {
420            if (exclusive != -1) {
421                // coherence violation
422                WARN_EXPR(exclusive);
423                WARN_EXPR(m_chip_vector[i]->getID());
424                WARN_EXPR(addr);
425                WARN_EXPR(getTime());
426                ERROR_MSG("Coherence Violation Detected -- 2 exclusive chips");
427            } else if (sharedDetected) {
428                WARN_EXPR(lastShared);
429                WARN_EXPR(m_chip_vector[i]->getID());
430                WARN_EXPR(addr);
431                WARN_EXPR(getTime());
432                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
433            } else {
434                exclusive = m_chip_vector[i]->getID();
435            }
436        } else if (m_chip_vector[i]->isBlockShared(addr)) {
437            sharedDetected = true;
438            lastShared = m_chip_vector[i]->getID();
439
440            if (exclusive != -1) {
441                WARN_EXPR(lastShared);
442                WARN_EXPR(exclusive);
443                WARN_EXPR(addr);
444                WARN_EXPR(getTime());
445                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
446            }
447        }
448    }
449#endif
450}
451#endif
452
453RubySystem *
454RubySystemParams::create()
455{
456    return new RubySystem(this);
457}
458
459/**
460 * virtual process function that is invoked when the callback
461 * queue is executed.
462 */
463void
464RubyExitCallback::process()
465{
466    std::ostream *os = simout.create(stats_filename);
467    RubySystem::printStats(*os);
468}
469