RubySystem.cc revision 9208
1/* 2 * Copyright (c) 1999-2011 Mark D. Hill and David A. Wood 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are 7 * met: redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer; 9 * redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution; 12 * neither the name of the copyright holders nor the names of its 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <fcntl.h> 30#include <zlib.h> 31 32#include <cstdio> 33 34#include "base/intmath.hh" 35#include "base/output.hh" 36#include "debug/RubyCacheTrace.hh" 37#include "mem/ruby/common/Address.hh" 38#include "mem/ruby/network/Network.hh" 39#include "mem/ruby/profiler/Profiler.hh" 40#include "mem/ruby/system/System.hh" 41#include "sim/eventq.hh" 42#include "sim/simulate.hh" 43 44using namespace std; 45 46int RubySystem::m_random_seed; 47bool RubySystem::m_randomization; 48int RubySystem::m_block_size_bytes; 49int RubySystem::m_block_size_bits; 50uint64 RubySystem::m_memory_size_bytes; 51int RubySystem::m_memory_size_bits; 52 53Network* RubySystem::m_network_ptr; 54Profiler* RubySystem::m_profiler_ptr; 55MemoryVector* RubySystem::m_mem_vec_ptr; 56 57RubySystem::RubySystem(const Params *p) 58 : ClockedObject(p) 59{ 60 if (g_system_ptr != NULL) 61 fatal("Only one RubySystem object currently allowed.\n"); 62 63 m_random_seed = p->random_seed; 64 srandom(m_random_seed); 65 m_randomization = p->randomization; 66 67 m_block_size_bytes = p->block_size_bytes; 68 assert(isPowerOf2(m_block_size_bytes)); 69 m_block_size_bits = floorLog2(m_block_size_bytes); 70 71 m_memory_size_bytes = p->mem_size; 72 if (m_memory_size_bytes == 0) { 73 m_memory_size_bits = 0; 74 } else { 75 m_memory_size_bits = floorLog2(m_memory_size_bytes); 76 } 77 78 g_system_ptr = this; 79 if (p->no_mem_vec) { 80 m_mem_vec_ptr = NULL; 81 } else { 82 m_mem_vec_ptr = new MemoryVector; 83 m_mem_vec_ptr->resize(m_memory_size_bytes); 84 } 85 86 // 87 // Print ruby configuration and stats at exit 88 // 89 RubyExitCallback* rubyExitCB = new RubyExitCallback(p->stats_filename); 90 registerExitCallback(rubyExitCB); 91 m_warmup_enabled = false; 92 m_cooldown_enabled = false; 93} 94 95void 96RubySystem::init() 97{ 98 m_profiler_ptr->clearStats(); 99} 100 101void 102RubySystem::registerNetwork(Network* network_ptr) 103{ 104 m_network_ptr = network_ptr; 105} 106 107void 108RubySystem::registerProfiler(Profiler* profiler_ptr) 109{ 110 m_profiler_ptr = profiler_ptr; 111} 112 113void 114RubySystem::registerAbstractController(AbstractController* cntrl) 115{ 116 m_abs_cntrl_vec.push_back(cntrl); 117} 118 119void 120RubySystem::registerSparseMemory(SparseMemory* s) 121{ 122 m_sparse_memory_vector.push_back(s); 123} 124 125void 126RubySystem::registerMemController(MemoryControl *mc) { 127 m_memory_controller = mc; 128} 129 130RubySystem::~RubySystem() 131{ 132 delete m_network_ptr; 133 delete m_profiler_ptr; 134 if (m_mem_vec_ptr) 135 delete m_mem_vec_ptr; 136} 137 138void 139RubySystem::printStats(ostream& out) 140{ 141 const time_t T = time(NULL); 142 tm *localTime = localtime(&T); 143 char buf[100]; 144 strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime); 145 146 out << "Real time: " << buf << endl; 147 148 m_profiler_ptr->printStats(out); 149 m_network_ptr->printStats(out); 150} 151 152void 153RubySystem::writeCompressedTrace(uint8_t *raw_data, string filename, 154 uint64 uncompressed_trace_size) 155{ 156 // Create the checkpoint file for the memory 157 string thefile = Checkpoint::dir() + "/" + filename.c_str(); 158 159 int fd = creat(thefile.c_str(), 0664); 160 if (fd < 0) { 161 perror("creat"); 162 fatal("Can't open memory trace file '%s'\n", filename); 163 } 164 165 gzFile compressedMemory = gzdopen(fd, "wb"); 166 if (compressedMemory == NULL) 167 fatal("Insufficient memory to allocate compression state for %s\n", 168 filename); 169 170 if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) != 171 uncompressed_trace_size) { 172 fatal("Write failed on memory trace file '%s'\n", filename); 173 } 174 175 if (gzclose(compressedMemory)) { 176 fatal("Close failed on memory trace file '%s'\n", filename); 177 } 178 delete raw_data; 179} 180 181void 182RubySystem::serialize(std::ostream &os) 183{ 184 m_cooldown_enabled = true; 185 186 vector<Sequencer*> sequencer_map; 187 Sequencer* sequencer_ptr = NULL; 188 int cntrl_id = -1; 189 190 191 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 192 sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); 193 if (sequencer_ptr == NULL) { 194 sequencer_ptr = sequencer_map[cntrl]; 195 cntrl_id = cntrl; 196 } 197 } 198 199 assert(sequencer_ptr != NULL); 200 201 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 202 if (sequencer_map[cntrl] == NULL) { 203 sequencer_map[cntrl] = sequencer_ptr; 204 } 205 } 206 207 DPRINTF(RubyCacheTrace, "Recording Cache Trace\n"); 208 // Create the CacheRecorder and record the cache trace 209 m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map); 210 211 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 212 m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder); 213 } 214 215 DPRINTF(RubyCacheTrace, "Cache Trace Complete\n"); 216 // save the current tick value 217 Tick curtick_original = curTick(); 218 // save the event queue head 219 Event* eventq_head = eventq->replaceHead(NULL); 220 DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n", 221 curtick_original); 222 223 // Schedule an event to start cache cooldown 224 DPRINTF(RubyCacheTrace, "Starting cache flush\n"); 225 enqueueRubyEvent(curTick()); 226 simulate(); 227 DPRINTF(RubyCacheTrace, "Cache flush complete\n"); 228 229 // Restore eventq head 230 eventq_head = eventq->replaceHead(eventq_head); 231 // Restore curTick 232 curTick(curtick_original); 233 234 uint8_t *raw_data = NULL; 235 236 if (m_mem_vec_ptr != NULL) { 237 uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data); 238 239 string memory_trace_file = name() + ".memory.gz"; 240 writeCompressedTrace(raw_data, memory_trace_file, 241 memory_trace_size); 242 243 SERIALIZE_SCALAR(memory_trace_file); 244 SERIALIZE_SCALAR(memory_trace_size); 245 246 } else { 247 for (int i = 0; i < m_sparse_memory_vector.size(); ++i) { 248 m_sparse_memory_vector[i]->recordBlocks(cntrl_id, 249 m_cache_recorder); 250 } 251 } 252 253 // Aggergate the trace entries together into a single array 254 raw_data = new uint8_t[4096]; 255 uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data, 256 4096); 257 string cache_trace_file = name() + ".cache.gz"; 258 writeCompressedTrace(raw_data, cache_trace_file, cache_trace_size); 259 260 SERIALIZE_SCALAR(cache_trace_file); 261 SERIALIZE_SCALAR(cache_trace_size); 262 263 m_cooldown_enabled = false; 264} 265 266void 267RubySystem::readCompressedTrace(string filename, uint8_t *&raw_data, 268 uint64& uncompressed_trace_size) 269{ 270 // Read the trace file 271 gzFile compressedTrace; 272 273 // trace file 274 int fd = open(filename.c_str(), O_RDONLY); 275 if (fd < 0) { 276 perror("open"); 277 fatal("Unable to open trace file %s", filename); 278 } 279 280 compressedTrace = gzdopen(fd, "rb"); 281 if (compressedTrace == NULL) { 282 fatal("Insufficient memory to allocate compression state for %s\n", 283 filename); 284 } 285 286 raw_data = new uint8_t[uncompressed_trace_size]; 287 if (gzread(compressedTrace, raw_data, uncompressed_trace_size) < 288 uncompressed_trace_size) { 289 fatal("Unable to read complete trace from file %s\n", filename); 290 } 291 292 if (gzclose(compressedTrace)) { 293 fatal("Failed to close cache trace file '%s'\n", filename); 294 } 295} 296 297void 298RubySystem::unserialize(Checkpoint *cp, const string §ion) 299{ 300 // 301 // The main purpose for clearing stats in the unserialize process is so 302 // that the profiler can correctly set its start time to the unserialized 303 // value of curTick() 304 // 305 clearStats(); 306 uint8_t *uncompressed_trace = NULL; 307 308 if (m_mem_vec_ptr != NULL) { 309 string memory_trace_file; 310 uint64 memory_trace_size = 0; 311 312 UNSERIALIZE_SCALAR(memory_trace_file); 313 UNSERIALIZE_SCALAR(memory_trace_size); 314 memory_trace_file = cp->cptDir + "/" + memory_trace_file; 315 316 readCompressedTrace(memory_trace_file, uncompressed_trace, 317 memory_trace_size); 318 m_mem_vec_ptr->populatePages(uncompressed_trace); 319 320 delete uncompressed_trace; 321 uncompressed_trace = NULL; 322 } 323 324 string cache_trace_file; 325 uint64 cache_trace_size = 0; 326 327 UNSERIALIZE_SCALAR(cache_trace_file); 328 UNSERIALIZE_SCALAR(cache_trace_size); 329 cache_trace_file = cp->cptDir + "/" + cache_trace_file; 330 331 readCompressedTrace(cache_trace_file, uncompressed_trace, 332 cache_trace_size); 333 m_warmup_enabled = true; 334 335 vector<Sequencer*> sequencer_map; 336 Sequencer* t = NULL; 337 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 338 sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); 339 if(t == NULL) t = sequencer_map[cntrl]; 340 } 341 342 assert(t != NULL); 343 344 for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { 345 if (sequencer_map[cntrl] == NULL) { 346 sequencer_map[cntrl] = t; 347 } 348 } 349 350 m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size, 351 sequencer_map); 352} 353 354void 355RubySystem::startup() 356{ 357 if (m_warmup_enabled) { 358 // save the current tick value 359 Tick curtick_original = curTick(); 360 // save the event queue head 361 Event* eventq_head = eventq->replaceHead(NULL); 362 // set curTick to 0 363 curTick(0); 364 365 // Schedule an event to start cache warmup 366 enqueueRubyEvent(curTick()); 367 simulate(); 368 369 delete m_cache_recorder; 370 m_cache_recorder = NULL; 371 m_warmup_enabled = false; 372 // reset DRAM so that it's not waiting for events on the old event 373 // queue 374 m_memory_controller->reset(); 375 // Restore eventq head 376 eventq_head = eventq->replaceHead(eventq_head); 377 // Restore curTick 378 curTick(curtick_original); 379 } 380} 381 382void 383RubySystem::RubyEvent::process() 384{ 385 if (ruby_system->m_warmup_enabled) { 386 ruby_system->m_cache_recorder->enqueueNextFetchRequest(); 387 } else if (ruby_system->m_cooldown_enabled) { 388 ruby_system->m_cache_recorder->enqueueNextFlushRequest(); 389 } 390} 391 392void 393RubySystem::clearStats() const 394{ 395 m_profiler_ptr->clearStats(); 396 m_network_ptr->clearStats(); 397} 398 399#ifdef CHECK_COHERENCE 400// This code will check for cases if the given cache block is exclusive in 401// one node and shared in another-- a coherence violation 402// 403// To use, the SLICC specification must call sequencer.checkCoherence(address) 404// when the controller changes to a state with new permissions. Do this 405// in setState. The SLICC spec must also define methods "isBlockShared" 406// and "isBlockExclusive" that are specific to that protocol 407// 408void 409RubySystem::checkGlobalCoherenceInvariant(const Address& addr) 410{ 411#if 0 412 NodeID exclusive = -1; 413 bool sharedDetected = false; 414 NodeID lastShared = -1; 415 416 for (int i = 0; i < m_chip_vector.size(); i++) { 417 if (m_chip_vector[i]->isBlockExclusive(addr)) { 418 if (exclusive != -1) { 419 // coherence violation 420 WARN_EXPR(exclusive); 421 WARN_EXPR(m_chip_vector[i]->getID()); 422 WARN_EXPR(addr); 423 WARN_EXPR(getTime()); 424 ERROR_MSG("Coherence Violation Detected -- 2 exclusive chips"); 425 } else if (sharedDetected) { 426 WARN_EXPR(lastShared); 427 WARN_EXPR(m_chip_vector[i]->getID()); 428 WARN_EXPR(addr); 429 WARN_EXPR(getTime()); 430 ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared"); 431 } else { 432 exclusive = m_chip_vector[i]->getID(); 433 } 434 } else if (m_chip_vector[i]->isBlockShared(addr)) { 435 sharedDetected = true; 436 lastShared = m_chip_vector[i]->getID(); 437 438 if (exclusive != -1) { 439 WARN_EXPR(lastShared); 440 WARN_EXPR(exclusive); 441 WARN_EXPR(addr); 442 WARN_EXPR(getTime()); 443 ERROR_MSG("Coherence Violation Detected -- exclusive chip with >=1 shared"); 444 } 445 } 446 } 447#endif 448} 449#endif 450 451RubySystem * 452RubySystemParams::create() 453{ 454 return new RubySystem(this); 455} 456 457/** 458 * virtual process function that is invoked when the callback 459 * queue is executed. 460 */ 461void 462RubyExitCallback::process() 463{ 464 std::ostream *os = simout.create(stats_filename); 465 RubySystem::printStats(*os); 466} 467