RubySystem.cc revision 9171
15335Shines@cs.fsu.edu/* 24486Sbinkertn@umich.edu * Copyright (c) 1999-2011 Mark D. Hill and David A. Wood 34486Sbinkertn@umich.edu * All rights reserved. 44486Sbinkertn@umich.edu * 54486Sbinkertn@umich.edu * Redistribution and use in source and binary forms, with or without 64486Sbinkertn@umich.edu * modification, are permitted provided that the following conditions are 74486Sbinkertn@umich.edu * met: redistributions of source code must retain the above copyright 84486Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer; 94486Sbinkertn@umich.edu * redistributions in binary form must reproduce the above copyright 104486Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer in the 114486Sbinkertn@umich.edu * documentation and/or other materials provided with the distribution; 124486Sbinkertn@umich.edu * neither the name of the copyright holders nor the names of its 134486Sbinkertn@umich.edu * contributors may be used to endorse or promote products derived from 144486Sbinkertn@umich.edu * this software without specific prior written permission. 154486Sbinkertn@umich.edu * 164486Sbinkertn@umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 174486Sbinkertn@umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 184486Sbinkertn@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 194486Sbinkertn@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 204486Sbinkertn@umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 214486Sbinkertn@umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 224486Sbinkertn@umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 234486Sbinkertn@umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 244486Sbinkertn@umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 254486Sbinkertn@umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 264486Sbinkertn@umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 274486Sbinkertn@umich.edu */ 284486Sbinkertn@umich.edu 296654Snate@binkert.org#include <fcntl.h> 306654Snate@binkert.org#include <zlib.h> 316654Snate@binkert.org 323102SN/A#include <cstdio> 333102SN/A 346654Snate@binkert.org#include "base/intmath.hh" 352998SN/A#include "base/output.hh" 364776Sgblack@eecs.umich.edu#include "debug/RubyCacheTrace.hh" 374776Sgblack@eecs.umich.edu#include "mem/ruby/common/Address.hh" 386654Snate@binkert.org#include "mem/ruby/network/Network.hh" 392667SN/A#include "mem/ruby/profiler/Profiler.hh" 404776Sgblack@eecs.umich.edu#include "mem/ruby/system/System.hh" 414776Sgblack@eecs.umich.edu#include "sim/eventq.hh" 426654Snate@binkert.org#include "sim/simulate.hh" 436023Snate@binkert.org 446654Snate@binkert.orgusing namespace std; 455647Sgblack@eecs.umich.edu 466654Snate@binkert.orgint RubySystem::m_random_seed; 476022Sgblack@eecs.umich.edubool RubySystem::m_randomization; 486654Snate@binkert.orgTick RubySystem::m_clock; 495647Sgblack@eecs.umich.eduint RubySystem::m_block_size_bytes; 506654Snate@binkert.orgint RubySystem::m_block_size_bits; 516022Sgblack@eecs.umich.eduuint64 RubySystem::m_memory_size_bytes; 526654Snate@binkert.orgint RubySystem::m_memory_size_bits; 535647Sgblack@eecs.umich.edu 546654Snate@binkert.orgNetwork* RubySystem::m_network_ptr; 556022Sgblack@eecs.umich.eduProfiler* RubySystem::m_profiler_ptr; 566654Snate@binkert.orgMemoryVector* RubySystem::m_mem_vec_ptr; 575647Sgblack@eecs.umich.edu 586654Snate@binkert.orgRubySystem::RubySystem(const Params *p) 596116Snate@binkert.org : SimObject(p) 606654Snate@binkert.org{ 615647Sgblack@eecs.umich.edu if (g_system_ptr != NULL) 626691Stjones1@inf.ed.ac.uk fatal("Only one RubySystem object currently allowed.\n"); 636691Stjones1@inf.ed.ac.uk 646691Stjones1@inf.ed.ac.uk m_random_seed = p->random_seed; 656691Stjones1@inf.ed.ac.uk srandom(m_random_seed); 664486Sbinkertn@umich.edu m_randomization = p->randomization; 675529Snate@binkert.org m_clock = p->clock; 681366SN/A 691310SN/A m_block_size_bytes = p->block_size_bytes; 701310SN/A assert(isPowerOf2(m_block_size_bytes)); 712901SN/A m_block_size_bits = floorLog2(m_block_size_bytes); 725712Shsul@eecs.umich.edu 735529Snate@binkert.org m_memory_size_bytes = p->mem_size; 745529Snate@binkert.org if (m_memory_size_bytes == 0) { 755529Snate@binkert.org m_memory_size_bits = 0; 765529Snate@binkert.org } else { 775529Snate@binkert.org m_memory_size_bits = floorLog2(m_memory_size_bytes); 785821Ssaidi@eecs.umich.edu } 793170SN/A 805780Ssteve.reinhardt@amd.com g_system_ptr = this; 815780Ssteve.reinhardt@amd.com if (p->no_mem_vec) { 825780Ssteve.reinhardt@amd.com m_mem_vec_ptr = NULL; 835780Ssteve.reinhardt@amd.com } else { 845780Ssteve.reinhardt@amd.com m_mem_vec_ptr = new MemoryVector; 856654Snate@binkert.org m_mem_vec_ptr->resize(m_memory_size_bytes); 865529Snate@binkert.org } 873620SN/A 881445SN/A // 891445SN/A // Print ruby configuration and stats at exit 901310SN/A // 916654Snate@binkert.org RubyExitCallback* rubyExitCB = new RubyExitCallback(p->stats_filename); 926022Sgblack@eecs.umich.edu registerExitCallback(rubyExitCB); 936022Sgblack@eecs.umich.edu m_warmup_enabled = false; 946654Snate@binkert.org m_cooldown_enabled = false; 955647Sgblack@eecs.umich.edu} 965647Sgblack@eecs.umich.edu 976654Snate@binkert.orgvoid 986023Snate@binkert.orgRubySystem::init() 996023Snate@binkert.org{ 1006654Snate@binkert.org m_profiler_ptr->clearStats(); 1015647Sgblack@eecs.umich.edu} 1025647Sgblack@eecs.umich.edu 1036654Snate@binkert.orgvoid 1046022Sgblack@eecs.umich.eduRubySystem::registerNetwork(Network* network_ptr) 1056022Sgblack@eecs.umich.edu{ 1066654Snate@binkert.org m_network_ptr = network_ptr; 1075658Sgblack@eecs.umich.edu} 1085648Sgblack@eecs.umich.edu 1095648Sgblack@eecs.umich.eduvoid 1106654Snate@binkert.orgRubySystem::registerProfiler(Profiler* profiler_ptr) 1116022Sgblack@eecs.umich.edu{ 1126022Sgblack@eecs.umich.edu m_profiler_ptr = profiler_ptr; 1136654Snate@binkert.org} 1145647Sgblack@eecs.umich.edu 1155647Sgblack@eecs.umich.eduvoid 1166654Snate@binkert.orgRubySystem::registerAbstractController(AbstractController* cntrl) 1176116Snate@binkert.org{ 1186116Snate@binkert.org m_abs_cntrl_vec.push_back(cntrl); 1196654Snate@binkert.org} 1205647Sgblack@eecs.umich.edu 1215647Sgblack@eecs.umich.eduvoid 1226691Stjones1@inf.ed.ac.ukRubySystem::registerSparseMemory(SparseMemory* s) 1236691Stjones1@inf.ed.ac.uk{ 1246691Stjones1@inf.ed.ac.uk m_sparse_memory_vector.push_back(s); 1256691Stjones1@inf.ed.ac.uk} 1266691Stjones1@inf.ed.ac.uk 1276691Stjones1@inf.ed.ac.ukvoid 1286691Stjones1@inf.ed.ac.ukRubySystem::registerMemController(MemoryControl *mc) { 1294997Sgblack@eecs.umich.edu m_memory_controller = mc; 1304997Sgblack@eecs.umich.edu} 1316654Snate@binkert.org 1324997Sgblack@eecs.umich.eduRubySystem::~RubySystem() 1334997Sgblack@eecs.umich.edu{ 1341310SN/A delete m_network_ptr; 1351310SN/A delete m_profiler_ptr; 1361310SN/A if (m_mem_vec_ptr) 1371310SN/A delete m_mem_vec_ptr; 1381310SN/A} 1391310SN/A 1401310SN/Avoid 1411310SN/ARubySystem::printStats(ostream& out) 1423878SN/A{ 1433878SN/A const time_t T = time(NULL); 1441310SN/A tm *localTime = localtime(&T); 1451369SN/A char buf[100]; 1461310SN/A strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime); 1471634SN/A 1484167SN/A out << "Real time: " << buf << endl; 1494167SN/A 1502998SN/A m_profiler_ptr->printStats(out); 1514776Sgblack@eecs.umich.edu m_network_ptr->printStats(out); 1524776Sgblack@eecs.umich.edu} 1537876Sgblack@eecs.umich.edu 1547876Sgblack@eecs.umich.eduvoid 1557876Sgblack@eecs.umich.eduRubySystem::writeCompressedTrace(uint8* raw_data, string filename, 1567876Sgblack@eecs.umich.edu uint64 uncompressed_trace_size) 1577876Sgblack@eecs.umich.edu{ 1586654Snate@binkert.org // Create the checkpoint file for the memory 1597876Sgblack@eecs.umich.edu string thefile = Checkpoint::dir() + "/" + filename.c_str(); 1602998SN/A 1617876Sgblack@eecs.umich.edu int fd = creat(thefile.c_str(), 0664); 1627876Sgblack@eecs.umich.edu if (fd < 0) { 1637876Sgblack@eecs.umich.edu perror("creat"); 1647404SAli.Saidi@ARM.com fatal("Can't open memory trace file '%s'\n", filename); 1657876Sgblack@eecs.umich.edu } 1667876Sgblack@eecs.umich.edu 1677876Sgblack@eecs.umich.edu gzFile compressedMemory = gzdopen(fd, "wb"); 1687876Sgblack@eecs.umich.edu if (compressedMemory == NULL) 1697876Sgblack@eecs.umich.edu fatal("Insufficient memory to allocate compression state for %s\n", 1707876Sgblack@eecs.umich.edu filename); 1717876Sgblack@eecs.umich.edu 1727876Sgblack@eecs.umich.edu if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) != 1737876Sgblack@eecs.umich.edu uncompressed_trace_size) { 1742998SN/A fatal("Write failed on memory trace file '%s'\n", filename); 1757868Sgblack@eecs.umich.edu } 1767876Sgblack@eecs.umich.edu 1772998SN/A if (gzclose(compressedMemory)) { 1782998SN/A fatal("Close failed on memory trace file '%s'\n", filename); 1792998SN/A } 1802998SN/A delete raw_data; 1817876Sgblack@eecs.umich.edu} 1827404SAli.Saidi@ARM.com 1837868Sgblack@eecs.umich.eduvoid 1847868Sgblack@eecs.umich.eduRubySystem::serialize(std::ostream &os) 1857868Sgblack@eecs.umich.edu{ 1867868Sgblack@eecs.umich.edu m_cooldown_enabled = true; 1877868Sgblack@eecs.umich.edu 1887876Sgblack@eecs.umich.edu vector<Sequencer*> sequencer_map; 1897876Sgblack@eecs.umich.edu Sequencer* sequencer_ptr = NULL; 1907868Sgblack@eecs.umich.edu int cntrl_id = -1; 1917876Sgblack@eecs.umich.edu 1922998SN/A 1937868Sgblack@eecs.umich.edu for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 1947868Sgblack@eecs.umich.edu sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); 1952998SN/A if (sequencer_ptr == NULL) { 1967876Sgblack@eecs.umich.edu sequencer_ptr = sequencer_map[cntrl]; 1972998SN/A cntrl_id = cntrl; 1983017SN/A } 1997876Sgblack@eecs.umich.edu } 2005222Sksewell@umich.edu 2016654Snate@binkert.org assert(sequencer_ptr != NULL); 2025222Sksewell@umich.edu 2035222Sksewell@umich.edu for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 2045222Sksewell@umich.edu if (sequencer_map[cntrl] == NULL) { 2055222Sksewell@umich.edu sequencer_map[cntrl] = sequencer_ptr; 2065222Sksewell@umich.edu } 2075222Sksewell@umich.edu } 2085222Sksewell@umich.edu 2095222Sksewell@umich.edu DPRINTF(RubyCacheTrace, "Recording Cache Trace\n"); 2105222Sksewell@umich.edu // Create the CacheRecorder and record the cache trace 2115222Sksewell@umich.edu m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map); 2125222Sksewell@umich.edu 2135222Sksewell@umich.edu for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 2145222Sksewell@umich.edu m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder); 2155222Sksewell@umich.edu } 2165222Sksewell@umich.edu 2175222Sksewell@umich.edu DPRINTF(RubyCacheTrace, "Cache Trace Complete\n"); 2185222Sksewell@umich.edu // save the current tick value 2195222Sksewell@umich.edu Tick curtick_original = curTick(); 2205222Sksewell@umich.edu // save the event queue head 2215222Sksewell@umich.edu Event* eventq_head = eventq->replaceHead(NULL); 2225222Sksewell@umich.edu DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n", 2235222Sksewell@umich.edu curtick_original); 2245222Sksewell@umich.edu 2255222Sksewell@umich.edu // Schedule an event to start cache cooldown 2265222Sksewell@umich.edu DPRINTF(RubyCacheTrace, "Starting cache flush\n"); 2275222Sksewell@umich.edu enqueueRubyEvent(curTick()); 2285222Sksewell@umich.edu simulate(); 2295222Sksewell@umich.edu DPRINTF(RubyCacheTrace, "Cache flush complete\n"); 2305222Sksewell@umich.edu 2315222Sksewell@umich.edu // Restore eventq head 2325222Sksewell@umich.edu eventq_head = eventq->replaceHead(eventq_head); 2335222Sksewell@umich.edu // Restore curTick 2345222Sksewell@umich.edu curTick(curtick_original); 2355222Sksewell@umich.edu 2365222Sksewell@umich.edu uint8* raw_data = NULL; 2375222Sksewell@umich.edu 2385222Sksewell@umich.edu if (m_mem_vec_ptr != NULL) { 2395222Sksewell@umich.edu uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data); 2405222Sksewell@umich.edu 2415222Sksewell@umich.edu string memory_trace_file = name() + ".memory.gz"; 2425222Sksewell@umich.edu writeCompressedTrace(raw_data, memory_trace_file, 2435222Sksewell@umich.edu memory_trace_size); 2445222Sksewell@umich.edu 2455222Sksewell@umich.edu SERIALIZE_SCALAR(memory_trace_file); 2465222Sksewell@umich.edu SERIALIZE_SCALAR(memory_trace_size); 2475222Sksewell@umich.edu 2485222Sksewell@umich.edu } else { 2495222Sksewell@umich.edu for (int i = 0; i < m_sparse_memory_vector.size(); ++i) { 2505222Sksewell@umich.edu m_sparse_memory_vector[i]->recordBlocks(cntrl_id, 2515222Sksewell@umich.edu m_cache_recorder); 2525222Sksewell@umich.edu } 2535222Sksewell@umich.edu } 2545222Sksewell@umich.edu 2555222Sksewell@umich.edu // Aggergate the trace entries together into a single array 256 raw_data = new uint8_t[4096]; 257 uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data, 258 4096); 259 string cache_trace_file = name() + ".cache.gz"; 260 writeCompressedTrace(raw_data, cache_trace_file, cache_trace_size); 261 262 SERIALIZE_SCALAR(cache_trace_file); 263 SERIALIZE_SCALAR(cache_trace_size); 264 265 m_cooldown_enabled = false; 266} 267 268void 269RubySystem::readCompressedTrace(string filename, uint8*& raw_data, 270 uint64& uncompressed_trace_size) 271{ 272 // Read the trace file 273 gzFile compressedTrace; 274 275 // trace file 276 int fd = open(filename.c_str(), O_RDONLY); 277 if (fd < 0) { 278 perror("open"); 279 fatal("Unable to open trace file %s", filename); 280 } 281 282 compressedTrace = gzdopen(fd, "rb"); 283 if (compressedTrace == NULL) { 284 fatal("Insufficient memory to allocate compression state for %s\n", 285 filename); 286 } 287 288 raw_data = new uint8_t[uncompressed_trace_size]; 289 if (gzread(compressedTrace, raw_data, uncompressed_trace_size) < 290 uncompressed_trace_size) { 291 fatal("Unable to read complete trace from file %s\n", filename); 292 } 293 294 if (gzclose(compressedTrace)) { 295 fatal("Failed to close cache trace file '%s'\n", filename); 296 } 297} 298 299void 300RubySystem::unserialize(Checkpoint *cp, const string §ion) 301{ 302 // 303 // The main purpose for clearing stats in the unserialize process is so 304 // that the profiler can correctly set its start time to the unserialized 305 // value of curTick() 306 // 307 clearStats(); 308 uint8* uncompressed_trace = NULL; 309 310 if (m_mem_vec_ptr != NULL) { 311 string memory_trace_file; 312 uint64 memory_trace_size = 0; 313 314 UNSERIALIZE_SCALAR(memory_trace_file); 315 UNSERIALIZE_SCALAR(memory_trace_size); 316 memory_trace_file = cp->cptDir + "/" + memory_trace_file; 317 318 readCompressedTrace(memory_trace_file, uncompressed_trace, 319 memory_trace_size); 320 m_mem_vec_ptr->populatePages(uncompressed_trace); 321 322 delete uncompressed_trace; 323 uncompressed_trace = NULL; 324 } 325 326 string cache_trace_file; 327 uint64 cache_trace_size = 0; 328 329 UNSERIALIZE_SCALAR(cache_trace_file); 330 UNSERIALIZE_SCALAR(cache_trace_size); 331 cache_trace_file = cp->cptDir + "/" + cache_trace_file; 332 333 readCompressedTrace(cache_trace_file, uncompressed_trace, 334 cache_trace_size); 335 m_warmup_enabled = true; 336 337 vector<Sequencer*> sequencer_map; 338 Sequencer* t = NULL; 339 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 340 sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); 341 if(t == NULL) t = sequencer_map[cntrl]; 342 } 343 344 assert(t != NULL); 345 346 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 347 if (sequencer_map[cntrl] == NULL) { 348 sequencer_map[cntrl] = t; 349 } 350 } 351 352 m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size, 353 sequencer_map); 354} 355 356void 357RubySystem::startup() 358{ 359 if (m_warmup_enabled) { 360 // save the current tick value 361 Tick curtick_original = curTick(); 362 // save the event queue head 363 Event* eventq_head = eventq->replaceHead(NULL); 364 // set curTick to 0 365 curTick(0); 366 367 // Schedule an event to start cache warmup 368 enqueueRubyEvent(curTick()); 369 simulate(); 370 371 delete m_cache_recorder; 372 m_cache_recorder = NULL; 373 m_warmup_enabled = false; 374 // reset DRAM so that it's not waiting for events on the old event 375 // queue 376 m_memory_controller->reset(); 377 // Restore eventq head 378 eventq_head = eventq->replaceHead(eventq_head); 379 // Restore curTick 380 curTick(curtick_original); 381 } 382} 383 384void 385RubySystem::RubyEvent::process() 386{ 387 if (ruby_system->m_warmup_enabled) { 388 ruby_system->m_cache_recorder->enqueueNextFetchRequest(); 389 } else if (ruby_system->m_cooldown_enabled) { 390 ruby_system->m_cache_recorder->enqueueNextFlushRequest(); 391 } 392} 393 394void 395RubySystem::clearStats() const 396{ 397 m_profiler_ptr->clearStats(); 398 m_network_ptr->clearStats(); 399} 400 401#ifdef CHECK_COHERENCE 402// This code will check for cases if the given cache block is exclusive in 403// one node and shared in another-- a coherence violation 404// 405// To use, the SLICC specification must call sequencer.checkCoherence(address) 406// when the controller changes to a state with new permissions. Do this 407// in setState. The SLICC spec must also define methods "isBlockShared" 408// and "isBlockExclusive" that are specific to that protocol 409// 410void 411RubySystem::checkGlobalCoherenceInvariant(const Address& addr) 412{ 413#if 0 414 NodeID exclusive = -1; 415 bool sharedDetected = false; 416 NodeID lastShared = -1; 417 418 for (int i = 0; i < m_chip_vector.size(); i++) { 419 if (m_chip_vector[i]->isBlockExclusive(addr)) { 420 if (exclusive != -1) { 421 // coherence violation 422 WARN_EXPR(exclusive); 423 WARN_EXPR(m_chip_vector[i]->getID()); 424 WARN_EXPR(addr); 425 WARN_EXPR(getTime()); 426 ERROR_MSG("Coherence Violation Detected -- 2 exclusive chips"); 427 } else if (sharedDetected) { 428 WARN_EXPR(lastShared); 429 WARN_EXPR(m_chip_vector[i]->getID()); 430 WARN_EXPR(addr); 431 WARN_EXPR(getTime()); 432 ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared"); 433 } else { 434 exclusive = m_chip_vector[i]->getID(); 435 } 436 } else if (m_chip_vector[i]->isBlockShared(addr)) { 437 sharedDetected = true; 438 lastShared = m_chip_vector[i]->getID(); 439 440 if (exclusive != -1) { 441 WARN_EXPR(lastShared); 442 WARN_EXPR(exclusive); 443 WARN_EXPR(addr); 444 WARN_EXPR(getTime()); 445 ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared"); 446 } 447 } 448 } 449#endif 450} 451#endif 452 453RubySystem * 454RubySystemParams::create() 455{ 456 return new RubySystem(this); 457} 458 459/** 460 * virtual process function that is invoked when the callback 461 * queue is executed. 462 */ 463void 464RubyExitCallback::process() 465{ 466 std::ostream *os = simout.create(stats_filename); 467 RubySystem::printStats(*os); 468} 469