RubySystem.cc revision 9171
15335Shines@cs.fsu.edu/*
24486Sbinkertn@umich.edu * Copyright (c) 1999-2011 Mark D. Hill and David A. Wood
34486Sbinkertn@umich.edu * All rights reserved.
44486Sbinkertn@umich.edu *
54486Sbinkertn@umich.edu * Redistribution and use in source and binary forms, with or without
64486Sbinkertn@umich.edu * modification, are permitted provided that the following conditions are
74486Sbinkertn@umich.edu * met: redistributions of source code must retain the above copyright
84486Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer;
94486Sbinkertn@umich.edu * redistributions in binary form must reproduce the above copyright
104486Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer in the
114486Sbinkertn@umich.edu * documentation and/or other materials provided with the distribution;
124486Sbinkertn@umich.edu * neither the name of the copyright holders nor the names of its
134486Sbinkertn@umich.edu * contributors may be used to endorse or promote products derived from
144486Sbinkertn@umich.edu * this software without specific prior written permission.
154486Sbinkertn@umich.edu *
164486Sbinkertn@umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
174486Sbinkertn@umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
184486Sbinkertn@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
194486Sbinkertn@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
204486Sbinkertn@umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
214486Sbinkertn@umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
224486Sbinkertn@umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
234486Sbinkertn@umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
244486Sbinkertn@umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
254486Sbinkertn@umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
264486Sbinkertn@umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
274486Sbinkertn@umich.edu */
284486Sbinkertn@umich.edu
296654Snate@binkert.org#include <fcntl.h>
306654Snate@binkert.org#include <zlib.h>
316654Snate@binkert.org
323102SN/A#include <cstdio>
333102SN/A
346654Snate@binkert.org#include "base/intmath.hh"
352998SN/A#include "base/output.hh"
364776Sgblack@eecs.umich.edu#include "debug/RubyCacheTrace.hh"
374776Sgblack@eecs.umich.edu#include "mem/ruby/common/Address.hh"
386654Snate@binkert.org#include "mem/ruby/network/Network.hh"
392667SN/A#include "mem/ruby/profiler/Profiler.hh"
404776Sgblack@eecs.umich.edu#include "mem/ruby/system/System.hh"
414776Sgblack@eecs.umich.edu#include "sim/eventq.hh"
426654Snate@binkert.org#include "sim/simulate.hh"
436023Snate@binkert.org
446654Snate@binkert.orgusing namespace std;
455647Sgblack@eecs.umich.edu
466654Snate@binkert.orgint RubySystem::m_random_seed;
476022Sgblack@eecs.umich.edubool RubySystem::m_randomization;
486654Snate@binkert.orgTick RubySystem::m_clock;
495647Sgblack@eecs.umich.eduint RubySystem::m_block_size_bytes;
506654Snate@binkert.orgint RubySystem::m_block_size_bits;
516022Sgblack@eecs.umich.eduuint64 RubySystem::m_memory_size_bytes;
526654Snate@binkert.orgint RubySystem::m_memory_size_bits;
535647Sgblack@eecs.umich.edu
546654Snate@binkert.orgNetwork* RubySystem::m_network_ptr;
556022Sgblack@eecs.umich.eduProfiler* RubySystem::m_profiler_ptr;
566654Snate@binkert.orgMemoryVector* RubySystem::m_mem_vec_ptr;
575647Sgblack@eecs.umich.edu
586654Snate@binkert.orgRubySystem::RubySystem(const Params *p)
596116Snate@binkert.org    : SimObject(p)
606654Snate@binkert.org{
615647Sgblack@eecs.umich.edu    if (g_system_ptr != NULL)
626691Stjones1@inf.ed.ac.uk        fatal("Only one RubySystem object currently allowed.\n");
636691Stjones1@inf.ed.ac.uk
646691Stjones1@inf.ed.ac.uk    m_random_seed = p->random_seed;
656691Stjones1@inf.ed.ac.uk    srandom(m_random_seed);
664486Sbinkertn@umich.edu    m_randomization = p->randomization;
675529Snate@binkert.org    m_clock = p->clock;
681366SN/A
691310SN/A    m_block_size_bytes = p->block_size_bytes;
701310SN/A    assert(isPowerOf2(m_block_size_bytes));
712901SN/A    m_block_size_bits = floorLog2(m_block_size_bytes);
725712Shsul@eecs.umich.edu
735529Snate@binkert.org    m_memory_size_bytes = p->mem_size;
745529Snate@binkert.org    if (m_memory_size_bytes == 0) {
755529Snate@binkert.org        m_memory_size_bits = 0;
765529Snate@binkert.org    } else {
775529Snate@binkert.org        m_memory_size_bits = floorLog2(m_memory_size_bytes);
785821Ssaidi@eecs.umich.edu    }
793170SN/A
805780Ssteve.reinhardt@amd.com    g_system_ptr = this;
815780Ssteve.reinhardt@amd.com    if (p->no_mem_vec) {
825780Ssteve.reinhardt@amd.com        m_mem_vec_ptr = NULL;
835780Ssteve.reinhardt@amd.com    } else {
845780Ssteve.reinhardt@amd.com        m_mem_vec_ptr = new MemoryVector;
856654Snate@binkert.org        m_mem_vec_ptr->resize(m_memory_size_bytes);
865529Snate@binkert.org    }
873620SN/A
881445SN/A    //
891445SN/A    // Print ruby configuration and stats at exit
901310SN/A    //
916654Snate@binkert.org    RubyExitCallback* rubyExitCB = new RubyExitCallback(p->stats_filename);
926022Sgblack@eecs.umich.edu    registerExitCallback(rubyExitCB);
936022Sgblack@eecs.umich.edu    m_warmup_enabled = false;
946654Snate@binkert.org    m_cooldown_enabled = false;
955647Sgblack@eecs.umich.edu}
965647Sgblack@eecs.umich.edu
976654Snate@binkert.orgvoid
986023Snate@binkert.orgRubySystem::init()
996023Snate@binkert.org{
1006654Snate@binkert.org    m_profiler_ptr->clearStats();
1015647Sgblack@eecs.umich.edu}
1025647Sgblack@eecs.umich.edu
1036654Snate@binkert.orgvoid
1046022Sgblack@eecs.umich.eduRubySystem::registerNetwork(Network* network_ptr)
1056022Sgblack@eecs.umich.edu{
1066654Snate@binkert.org  m_network_ptr = network_ptr;
1075658Sgblack@eecs.umich.edu}
1085648Sgblack@eecs.umich.edu
1095648Sgblack@eecs.umich.eduvoid
1106654Snate@binkert.orgRubySystem::registerProfiler(Profiler* profiler_ptr)
1116022Sgblack@eecs.umich.edu{
1126022Sgblack@eecs.umich.edu  m_profiler_ptr = profiler_ptr;
1136654Snate@binkert.org}
1145647Sgblack@eecs.umich.edu
1155647Sgblack@eecs.umich.eduvoid
1166654Snate@binkert.orgRubySystem::registerAbstractController(AbstractController* cntrl)
1176116Snate@binkert.org{
1186116Snate@binkert.org  m_abs_cntrl_vec.push_back(cntrl);
1196654Snate@binkert.org}
1205647Sgblack@eecs.umich.edu
1215647Sgblack@eecs.umich.eduvoid
1226691Stjones1@inf.ed.ac.ukRubySystem::registerSparseMemory(SparseMemory* s)
1236691Stjones1@inf.ed.ac.uk{
1246691Stjones1@inf.ed.ac.uk    m_sparse_memory_vector.push_back(s);
1256691Stjones1@inf.ed.ac.uk}
1266691Stjones1@inf.ed.ac.uk
1276691Stjones1@inf.ed.ac.ukvoid
1286691Stjones1@inf.ed.ac.ukRubySystem::registerMemController(MemoryControl *mc) {
1294997Sgblack@eecs.umich.edu    m_memory_controller = mc;
1304997Sgblack@eecs.umich.edu}
1316654Snate@binkert.org
1324997Sgblack@eecs.umich.eduRubySystem::~RubySystem()
1334997Sgblack@eecs.umich.edu{
1341310SN/A    delete m_network_ptr;
1351310SN/A    delete m_profiler_ptr;
1361310SN/A    if (m_mem_vec_ptr)
1371310SN/A        delete m_mem_vec_ptr;
1381310SN/A}
1391310SN/A
1401310SN/Avoid
1411310SN/ARubySystem::printStats(ostream& out)
1423878SN/A{
1433878SN/A    const time_t T = time(NULL);
1441310SN/A    tm *localTime = localtime(&T);
1451369SN/A    char buf[100];
1461310SN/A    strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime);
1471634SN/A
1484167SN/A    out << "Real time: " << buf << endl;
1494167SN/A
1502998SN/A    m_profiler_ptr->printStats(out);
1514776Sgblack@eecs.umich.edu    m_network_ptr->printStats(out);
1524776Sgblack@eecs.umich.edu}
1537876Sgblack@eecs.umich.edu
1547876Sgblack@eecs.umich.eduvoid
1557876Sgblack@eecs.umich.eduRubySystem::writeCompressedTrace(uint8* raw_data, string filename,
1567876Sgblack@eecs.umich.edu                                 uint64 uncompressed_trace_size)
1577876Sgblack@eecs.umich.edu{
1586654Snate@binkert.org    // Create the checkpoint file for the memory
1597876Sgblack@eecs.umich.edu    string thefile = Checkpoint::dir() + "/" + filename.c_str();
1602998SN/A
1617876Sgblack@eecs.umich.edu    int fd = creat(thefile.c_str(), 0664);
1627876Sgblack@eecs.umich.edu    if (fd < 0) {
1637876Sgblack@eecs.umich.edu        perror("creat");
1647404SAli.Saidi@ARM.com        fatal("Can't open memory trace file '%s'\n", filename);
1657876Sgblack@eecs.umich.edu    }
1667876Sgblack@eecs.umich.edu
1677876Sgblack@eecs.umich.edu    gzFile compressedMemory = gzdopen(fd, "wb");
1687876Sgblack@eecs.umich.edu    if (compressedMemory == NULL)
1697876Sgblack@eecs.umich.edu        fatal("Insufficient memory to allocate compression state for %s\n",
1707876Sgblack@eecs.umich.edu              filename);
1717876Sgblack@eecs.umich.edu
1727876Sgblack@eecs.umich.edu    if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) !=
1737876Sgblack@eecs.umich.edu        uncompressed_trace_size) {
1742998SN/A        fatal("Write failed on memory trace file '%s'\n", filename);
1757868Sgblack@eecs.umich.edu    }
1767876Sgblack@eecs.umich.edu
1772998SN/A    if (gzclose(compressedMemory)) {
1782998SN/A        fatal("Close failed on memory trace file '%s'\n", filename);
1792998SN/A    }
1802998SN/A    delete raw_data;
1817876Sgblack@eecs.umich.edu}
1827404SAli.Saidi@ARM.com
1837868Sgblack@eecs.umich.eduvoid
1847868Sgblack@eecs.umich.eduRubySystem::serialize(std::ostream &os)
1857868Sgblack@eecs.umich.edu{
1867868Sgblack@eecs.umich.edu    m_cooldown_enabled = true;
1877868Sgblack@eecs.umich.edu
1887876Sgblack@eecs.umich.edu    vector<Sequencer*> sequencer_map;
1897876Sgblack@eecs.umich.edu    Sequencer* sequencer_ptr = NULL;
1907868Sgblack@eecs.umich.edu    int cntrl_id = -1;
1917876Sgblack@eecs.umich.edu
1922998SN/A
1937868Sgblack@eecs.umich.edu    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
1947868Sgblack@eecs.umich.edu        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
1952998SN/A        if (sequencer_ptr == NULL) {
1967876Sgblack@eecs.umich.edu            sequencer_ptr = sequencer_map[cntrl];
1972998SN/A            cntrl_id = cntrl;
1983017SN/A        }
1997876Sgblack@eecs.umich.edu    }
2005222Sksewell@umich.edu
2016654Snate@binkert.org    assert(sequencer_ptr != NULL);
2025222Sksewell@umich.edu
2035222Sksewell@umich.edu    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
2045222Sksewell@umich.edu        if (sequencer_map[cntrl] == NULL) {
2055222Sksewell@umich.edu            sequencer_map[cntrl] = sequencer_ptr;
2065222Sksewell@umich.edu        }
2075222Sksewell@umich.edu    }
2085222Sksewell@umich.edu
2095222Sksewell@umich.edu    DPRINTF(RubyCacheTrace, "Recording Cache Trace\n");
2105222Sksewell@umich.edu    // Create the CacheRecorder and record the cache trace
2115222Sksewell@umich.edu    m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map);
2125222Sksewell@umich.edu
2135222Sksewell@umich.edu    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
2145222Sksewell@umich.edu        m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder);
2155222Sksewell@umich.edu    }
2165222Sksewell@umich.edu
2175222Sksewell@umich.edu    DPRINTF(RubyCacheTrace, "Cache Trace Complete\n");
2185222Sksewell@umich.edu    // save the current tick value
2195222Sksewell@umich.edu    Tick curtick_original = curTick();
2205222Sksewell@umich.edu    // save the event queue head
2215222Sksewell@umich.edu    Event* eventq_head = eventq->replaceHead(NULL);
2225222Sksewell@umich.edu    DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n",
2235222Sksewell@umich.edu            curtick_original);
2245222Sksewell@umich.edu
2255222Sksewell@umich.edu    // Schedule an event to start cache cooldown
2265222Sksewell@umich.edu    DPRINTF(RubyCacheTrace, "Starting cache flush\n");
2275222Sksewell@umich.edu    enqueueRubyEvent(curTick());
2285222Sksewell@umich.edu    simulate();
2295222Sksewell@umich.edu    DPRINTF(RubyCacheTrace, "Cache flush complete\n");
2305222Sksewell@umich.edu
2315222Sksewell@umich.edu    // Restore eventq head
2325222Sksewell@umich.edu    eventq_head = eventq->replaceHead(eventq_head);
2335222Sksewell@umich.edu    // Restore curTick
2345222Sksewell@umich.edu    curTick(curtick_original);
2355222Sksewell@umich.edu
2365222Sksewell@umich.edu    uint8* raw_data = NULL;
2375222Sksewell@umich.edu
2385222Sksewell@umich.edu    if (m_mem_vec_ptr != NULL) {
2395222Sksewell@umich.edu        uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data);
2405222Sksewell@umich.edu
2415222Sksewell@umich.edu        string memory_trace_file = name() + ".memory.gz";
2425222Sksewell@umich.edu        writeCompressedTrace(raw_data, memory_trace_file,
2435222Sksewell@umich.edu                             memory_trace_size);
2445222Sksewell@umich.edu
2455222Sksewell@umich.edu        SERIALIZE_SCALAR(memory_trace_file);
2465222Sksewell@umich.edu        SERIALIZE_SCALAR(memory_trace_size);
2475222Sksewell@umich.edu
2485222Sksewell@umich.edu    } else {
2495222Sksewell@umich.edu        for (int i = 0; i < m_sparse_memory_vector.size(); ++i) {
2505222Sksewell@umich.edu            m_sparse_memory_vector[i]->recordBlocks(cntrl_id,
2515222Sksewell@umich.edu                                                    m_cache_recorder);
2525222Sksewell@umich.edu        }
2535222Sksewell@umich.edu    }
2545222Sksewell@umich.edu
2555222Sksewell@umich.edu    // Aggergate the trace entries together into a single array
256    raw_data = new uint8_t[4096];
257    uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data,
258                                                                 4096);
259    string cache_trace_file = name() + ".cache.gz";
260    writeCompressedTrace(raw_data, cache_trace_file, cache_trace_size);
261
262    SERIALIZE_SCALAR(cache_trace_file);
263    SERIALIZE_SCALAR(cache_trace_size);
264
265    m_cooldown_enabled = false;
266}
267
268void
269RubySystem::readCompressedTrace(string filename, uint8*& raw_data,
270                                uint64& uncompressed_trace_size)
271{
272    // Read the trace file
273    gzFile compressedTrace;
274
275    // trace file
276    int fd = open(filename.c_str(), O_RDONLY);
277    if (fd < 0) {
278        perror("open");
279        fatal("Unable to open trace file %s", filename);
280    }
281
282    compressedTrace = gzdopen(fd, "rb");
283    if (compressedTrace == NULL) {
284        fatal("Insufficient memory to allocate compression state for %s\n",
285              filename);
286    }
287
288    raw_data = new uint8_t[uncompressed_trace_size];
289    if (gzread(compressedTrace, raw_data, uncompressed_trace_size) <
290            uncompressed_trace_size) {
291        fatal("Unable to read complete trace from file %s\n", filename);
292    }
293
294    if (gzclose(compressedTrace)) {
295        fatal("Failed to close cache trace file '%s'\n", filename);
296    }
297}
298
299void
300RubySystem::unserialize(Checkpoint *cp, const string &section)
301{
302    //
303    // The main purpose for clearing stats in the unserialize process is so
304    // that the profiler can correctly set its start time to the unserialized
305    // value of curTick()
306    //
307    clearStats();
308    uint8* uncompressed_trace = NULL;
309
310    if (m_mem_vec_ptr != NULL) {
311        string memory_trace_file;
312        uint64 memory_trace_size = 0;
313
314        UNSERIALIZE_SCALAR(memory_trace_file);
315        UNSERIALIZE_SCALAR(memory_trace_size);
316        memory_trace_file = cp->cptDir + "/" + memory_trace_file;
317
318        readCompressedTrace(memory_trace_file, uncompressed_trace,
319                            memory_trace_size);
320        m_mem_vec_ptr->populatePages(uncompressed_trace);
321
322        delete uncompressed_trace;
323        uncompressed_trace = NULL;
324    }
325
326    string cache_trace_file;
327    uint64 cache_trace_size = 0;
328
329    UNSERIALIZE_SCALAR(cache_trace_file);
330    UNSERIALIZE_SCALAR(cache_trace_size);
331    cache_trace_file = cp->cptDir + "/" + cache_trace_file;
332
333    readCompressedTrace(cache_trace_file, uncompressed_trace,
334                        cache_trace_size);
335    m_warmup_enabled = true;
336
337    vector<Sequencer*> sequencer_map;
338    Sequencer* t = NULL;
339    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
340        sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer());
341        if(t == NULL) t = sequencer_map[cntrl];
342    }
343
344    assert(t != NULL);
345
346    for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
347        if (sequencer_map[cntrl] == NULL) {
348            sequencer_map[cntrl] = t;
349        }
350    }
351
352    m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size,
353                                         sequencer_map);
354}
355
356void
357RubySystem::startup()
358{
359    if (m_warmup_enabled) {
360        // save the current tick value
361        Tick curtick_original = curTick();
362        // save the event queue head
363        Event* eventq_head = eventq->replaceHead(NULL);
364        // set curTick to 0
365        curTick(0);
366
367        // Schedule an event to start cache warmup
368        enqueueRubyEvent(curTick());
369        simulate();
370
371        delete m_cache_recorder;
372        m_cache_recorder = NULL;
373        m_warmup_enabled = false;
374        // reset DRAM so that it's not waiting for events on the old event
375        // queue
376        m_memory_controller->reset();
377        // Restore eventq head
378        eventq_head = eventq->replaceHead(eventq_head);
379        // Restore curTick
380        curTick(curtick_original);
381    }
382}
383
384void
385RubySystem::RubyEvent::process()
386{
387    if (ruby_system->m_warmup_enabled) {
388        ruby_system->m_cache_recorder->enqueueNextFetchRequest();
389    }  else if (ruby_system->m_cooldown_enabled) {
390        ruby_system->m_cache_recorder->enqueueNextFlushRequest();
391    }
392}
393
394void
395RubySystem::clearStats() const
396{
397    m_profiler_ptr->clearStats();
398    m_network_ptr->clearStats();
399}
400
401#ifdef CHECK_COHERENCE
402// This code will check for cases if the given cache block is exclusive in
403// one node and shared in another-- a coherence violation
404//
405// To use, the SLICC specification must call sequencer.checkCoherence(address)
406// when the controller changes to a state with new permissions.  Do this
407// in setState.  The SLICC spec must also define methods "isBlockShared"
408// and "isBlockExclusive" that are specific to that protocol
409//
410void
411RubySystem::checkGlobalCoherenceInvariant(const Address& addr)
412{
413#if 0
414    NodeID exclusive = -1;
415    bool sharedDetected = false;
416    NodeID lastShared = -1;
417
418    for (int i = 0; i < m_chip_vector.size(); i++) {
419        if (m_chip_vector[i]->isBlockExclusive(addr)) {
420            if (exclusive != -1) {
421                // coherence violation
422                WARN_EXPR(exclusive);
423                WARN_EXPR(m_chip_vector[i]->getID());
424                WARN_EXPR(addr);
425                WARN_EXPR(getTime());
426                ERROR_MSG("Coherence Violation Detected -- 2 exclusive chips");
427            } else if (sharedDetected) {
428                WARN_EXPR(lastShared);
429                WARN_EXPR(m_chip_vector[i]->getID());
430                WARN_EXPR(addr);
431                WARN_EXPR(getTime());
432                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
433            } else {
434                exclusive = m_chip_vector[i]->getID();
435            }
436        } else if (m_chip_vector[i]->isBlockShared(addr)) {
437            sharedDetected = true;
438            lastShared = m_chip_vector[i]->getID();
439
440            if (exclusive != -1) {
441                WARN_EXPR(lastShared);
442                WARN_EXPR(exclusive);
443                WARN_EXPR(addr);
444                WARN_EXPR(getTime());
445                ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared");
446            }
447        }
448    }
449#endif
450}
451#endif
452
453RubySystem *
454RubySystemParams::create()
455{
456    return new RubySystem(this);
457}
458
459/**
460 * virtual process function that is invoked when the callback
461 * queue is executed.
462 */
463void
464RubyExitCallback::process()
465{
466    std::ostream *os = simout.create(stats_filename);
467    RubySystem::printStats(*os);
468}
469