rename_impl.hh revision 3798:ec59feae527b
1/* 2 * Copyright (c) 2004-2006 The Regents of The University of Michigan 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are 7 * met: redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer; 9 * redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution; 12 * neither the name of the copyright holders nor the names of its 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Authors: Kevin Lim 29 * Korey Sewell 30 */ 31 32#include <list> 33 34#include "arch/isa_traits.hh" 35#include "arch/regfile.hh" 36#include "config/full_system.hh" 37#include "cpu/o3/rename.hh" 38 39template <class Impl> 40DefaultRename<Impl>::DefaultRename(Params *params) 41 : iewToRenameDelay(params->iewToRenameDelay), 42 decodeToRenameDelay(params->decodeToRenameDelay), 43 commitToRenameDelay(params->commitToRenameDelay), 44 renameWidth(params->renameWidth), 45 commitWidth(params->commitWidth), 46 resumeSerialize(false), 47 resumeUnblocking(false), 48 numThreads(params->numberOfThreads), 49 maxPhysicalRegs(params->numPhysIntRegs + params->numPhysFloatRegs) 50{ 51 _status = Inactive; 52 53 for (int i=0; i< numThreads; i++) { 54 renameStatus[i] = Idle; 55 56 freeEntries[i].iqEntries = 0; 57 freeEntries[i].lsqEntries = 0; 58 freeEntries[i].robEntries = 0; 59 60 stalls[i].iew = false; 61 stalls[i].commit = false; 62 serializeInst[i] = NULL; 63 64 instsInProgress[i] = 0; 65 66 emptyROB[i] = true; 67 68 serializeOnNextInst[i] = false; 69 } 70 71 // @todo: Make into a parameter. 72 skidBufferMax = (2 * (iewToRenameDelay * params->decodeWidth)) + renameWidth; 73} 74 75template <class Impl> 76std::string 77DefaultRename<Impl>::name() const 78{ 79 return cpu->name() + ".rename"; 80} 81 82template <class Impl> 83void 84DefaultRename<Impl>::regStats() 85{ 86 renameSquashCycles 87 .name(name() + ".RENAME:SquashCycles") 88 .desc("Number of cycles rename is squashing") 89 .prereq(renameSquashCycles); 90 renameIdleCycles 91 .name(name() + ".RENAME:IdleCycles") 92 .desc("Number of cycles rename is idle") 93 .prereq(renameIdleCycles); 94 renameBlockCycles 95 .name(name() + ".RENAME:BlockCycles") 96 .desc("Number of cycles rename is blocking") 97 .prereq(renameBlockCycles); 98 renameSerializeStallCycles 99 .name(name() + ".RENAME:serializeStallCycles") 100 .desc("count of cycles rename stalled for serializing inst") 101 .flags(Stats::total); 102 renameRunCycles 103 .name(name() + ".RENAME:RunCycles") 104 .desc("Number of cycles rename is running") 105 .prereq(renameIdleCycles); 106 renameUnblockCycles 107 .name(name() + ".RENAME:UnblockCycles") 108 .desc("Number of cycles rename is unblocking") 109 .prereq(renameUnblockCycles); 110 renameRenamedInsts 111 .name(name() + ".RENAME:RenamedInsts") 112 .desc("Number of instructions processed by rename") 113 .prereq(renameRenamedInsts); 114 renameSquashedInsts 115 .name(name() + ".RENAME:SquashedInsts") 116 .desc("Number of squashed instructions processed by rename") 117 .prereq(renameSquashedInsts); 118 renameROBFullEvents 119 .name(name() + ".RENAME:ROBFullEvents") 120 .desc("Number of times rename has blocked due to ROB full") 121 .prereq(renameROBFullEvents); 122 renameIQFullEvents 123 .name(name() + ".RENAME:IQFullEvents") 124 .desc("Number of times rename has blocked due to IQ full") 125 .prereq(renameIQFullEvents); 126 renameLSQFullEvents 127 .name(name() + ".RENAME:LSQFullEvents") 128 .desc("Number of times rename has blocked due to LSQ full") 129 .prereq(renameLSQFullEvents); 130 renameFullRegistersEvents 131 .name(name() + ".RENAME:FullRegisterEvents") 132 .desc("Number of times there has been no free registers") 133 .prereq(renameFullRegistersEvents); 134 renameRenamedOperands 135 .name(name() + ".RENAME:RenamedOperands") 136 .desc("Number of destination operands rename has renamed") 137 .prereq(renameRenamedOperands); 138 renameRenameLookups 139 .name(name() + ".RENAME:RenameLookups") 140 .desc("Number of register rename lookups that rename has made") 141 .prereq(renameRenameLookups); 142 renameCommittedMaps 143 .name(name() + ".RENAME:CommittedMaps") 144 .desc("Number of HB maps that are committed") 145 .prereq(renameCommittedMaps); 146 renameUndoneMaps 147 .name(name() + ".RENAME:UndoneMaps") 148 .desc("Number of HB maps that are undone due to squashing") 149 .prereq(renameUndoneMaps); 150 renamedSerializing 151 .name(name() + ".RENAME:serializingInsts") 152 .desc("count of serializing insts renamed") 153 .flags(Stats::total) 154 ; 155 renamedTempSerializing 156 .name(name() + ".RENAME:tempSerializingInsts") 157 .desc("count of temporary serializing insts renamed") 158 .flags(Stats::total) 159 ; 160 renameSkidInsts 161 .name(name() + ".RENAME:skidInsts") 162 .desc("count of insts added to the skid buffer") 163 .flags(Stats::total) 164 ; 165} 166 167template <class Impl> 168void 169DefaultRename<Impl>::setCPU(O3CPU *cpu_ptr) 170{ 171 DPRINTF(Rename, "Setting CPU pointer.\n"); 172 cpu = cpu_ptr; 173} 174 175template <class Impl> 176void 177DefaultRename<Impl>::setTimeBuffer(TimeBuffer<TimeStruct> *tb_ptr) 178{ 179 DPRINTF(Rename, "Setting time buffer pointer.\n"); 180 timeBuffer = tb_ptr; 181 182 // Setup wire to read information from time buffer, from IEW stage. 183 fromIEW = timeBuffer->getWire(-iewToRenameDelay); 184 185 // Setup wire to read infromation from time buffer, from commit stage. 186 fromCommit = timeBuffer->getWire(-commitToRenameDelay); 187 188 // Setup wire to write information to previous stages. 189 toDecode = timeBuffer->getWire(0); 190} 191 192template <class Impl> 193void 194DefaultRename<Impl>::setRenameQueue(TimeBuffer<RenameStruct> *rq_ptr) 195{ 196 DPRINTF(Rename, "Setting rename queue pointer.\n"); 197 renameQueue = rq_ptr; 198 199 // Setup wire to write information to future stages. 200 toIEW = renameQueue->getWire(0); 201} 202 203template <class Impl> 204void 205DefaultRename<Impl>::setDecodeQueue(TimeBuffer<DecodeStruct> *dq_ptr) 206{ 207 DPRINTF(Rename, "Setting decode queue pointer.\n"); 208 decodeQueue = dq_ptr; 209 210 // Setup wire to get information from decode. 211 fromDecode = decodeQueue->getWire(-decodeToRenameDelay); 212} 213 214template <class Impl> 215void 216DefaultRename<Impl>::initStage() 217{ 218 // Grab the number of free entries directly from the stages. 219 for (int tid=0; tid < numThreads; tid++) { 220 freeEntries[tid].iqEntries = iew_ptr->instQueue.numFreeEntries(tid); 221 freeEntries[tid].lsqEntries = iew_ptr->ldstQueue.numFreeEntries(tid); 222 freeEntries[tid].robEntries = commit_ptr->numROBFreeEntries(tid); 223 emptyROB[tid] = true; 224 } 225} 226 227template<class Impl> 228void 229DefaultRename<Impl>::setActiveThreads(std::list<unsigned> *at_ptr) 230{ 231 DPRINTF(Rename, "Setting active threads list pointer.\n"); 232 activeThreads = at_ptr; 233} 234 235 236template <class Impl> 237void 238DefaultRename<Impl>::setRenameMap(RenameMap rm_ptr[]) 239{ 240 DPRINTF(Rename, "Setting rename map pointers.\n"); 241 242 for (int i=0; i<numThreads; i++) { 243 renameMap[i] = &rm_ptr[i]; 244 } 245} 246 247template <class Impl> 248void 249DefaultRename<Impl>::setFreeList(FreeList *fl_ptr) 250{ 251 DPRINTF(Rename, "Setting free list pointer.\n"); 252 freeList = fl_ptr; 253} 254 255template<class Impl> 256void 257DefaultRename<Impl>::setScoreboard(Scoreboard *_scoreboard) 258{ 259 DPRINTF(Rename, "Setting scoreboard pointer.\n"); 260 scoreboard = _scoreboard; 261} 262 263template <class Impl> 264bool 265DefaultRename<Impl>::drain() 266{ 267 // Rename is ready to switch out at any time. 268 cpu->signalDrained(); 269 return true; 270} 271 272template <class Impl> 273void 274DefaultRename<Impl>::switchOut() 275{ 276 // Clear any state, fix up the rename map. 277 for (int i = 0; i < numThreads; i++) { 278 typename std::list<RenameHistory>::iterator hb_it = 279 historyBuffer[i].begin(); 280 281 while (!historyBuffer[i].empty()) { 282 assert(hb_it != historyBuffer[i].end()); 283 284 DPRINTF(Rename, "[tid:%u]: Removing history entry with sequence " 285 "number %i.\n", i, (*hb_it).instSeqNum); 286 287 // Tell the rename map to set the architected register to the 288 // previous physical register that it was renamed to. 289 renameMap[i]->setEntry(hb_it->archReg, hb_it->prevPhysReg); 290 291 // Put the renamed physical register back on the free list. 292 freeList->addReg(hb_it->newPhysReg); 293 294 // Be sure to mark its register as ready if it's a misc register. 295 if (hb_it->newPhysReg >= maxPhysicalRegs) { 296 scoreboard->setReg(hb_it->newPhysReg); 297 } 298 299 historyBuffer[i].erase(hb_it++); 300 } 301 insts[i].clear(); 302 skidBuffer[i].clear(); 303 } 304} 305 306template <class Impl> 307void 308DefaultRename<Impl>::takeOverFrom() 309{ 310 _status = Inactive; 311 initStage(); 312 313 // Reset all state prior to taking over from the other CPU. 314 for (int i=0; i< numThreads; i++) { 315 renameStatus[i] = Idle; 316 317 stalls[i].iew = false; 318 stalls[i].commit = false; 319 serializeInst[i] = NULL; 320 321 instsInProgress[i] = 0; 322 323 emptyROB[i] = true; 324 325 serializeOnNextInst[i] = false; 326 } 327} 328 329template <class Impl> 330void 331DefaultRename<Impl>::squash(const InstSeqNum &squash_seq_num, unsigned tid) 332{ 333 DPRINTF(Rename, "[tid:%u]: Squashing instructions.\n",tid); 334 335 // Clear the stall signal if rename was blocked or unblocking before. 336 // If it still needs to block, the blocking should happen the next 337 // cycle and there should be space to hold everything due to the squash. 338 if (renameStatus[tid] == Blocked || 339 renameStatus[tid] == Unblocking) { 340 toDecode->renameUnblock[tid] = 1; 341 342 resumeSerialize = false; 343 serializeInst[tid] = NULL; 344 } else if (renameStatus[tid] == SerializeStall) { 345 if (serializeInst[tid]->seqNum <= squash_seq_num) { 346 DPRINTF(Rename, "Rename will resume serializing after squash\n"); 347 resumeSerialize = true; 348 assert(serializeInst[tid]); 349 } else { 350 resumeSerialize = false; 351 toDecode->renameUnblock[tid] = 1; 352 353 serializeInst[tid] = NULL; 354 } 355 } 356 357 // Set the status to Squashing. 358 renameStatus[tid] = Squashing; 359 360 // Squash any instructions from decode. 361 unsigned squashCount = 0; 362 363 for (int i=0; i<fromDecode->size; i++) { 364 if (fromDecode->insts[i]->threadNumber == tid && 365 fromDecode->insts[i]->seqNum > squash_seq_num) { 366 fromDecode->insts[i]->setSquashed(); 367 wroteToTimeBuffer = true; 368 squashCount++; 369 } 370 371 } 372 373 // Clear the instruction list and skid buffer in case they have any 374 // insts in them. Since we support multiple ISAs, we cant just: 375 // "insts[tid].clear();" or "skidBuffer[tid].clear()" since there is 376 // a possible delay slot inst for different architectures 377 // insts[tid].clear(); 378#if ISA_HAS_DELAY_SLOT 379 DPRINTF(Rename, "[tid:%i] Squashing incoming decode instructions until " 380 "[sn:%i].\n",tid, squash_seq_num); 381 ListIt ilist_it = insts[tid].begin(); 382 while (ilist_it != insts[tid].end()) { 383 if ((*ilist_it)->seqNum > squash_seq_num) { 384 (*ilist_it)->setSquashed(); 385 DPRINTF(Rename, "Squashing incoming decode instruction, " 386 "[tid:%i] [sn:%i] PC %08p.\n", tid, (*ilist_it)->seqNum, (*ilist_it)->PC); 387 } 388 ilist_it++; 389 } 390#else 391 insts[tid].clear(); 392#endif 393 394 // Clear the skid buffer in case it has any data in it. 395 // See comments above. 396 // skidBuffer[tid].clear(); 397#if ISA_HAS_DELAY_SLOT 398 DPRINTF(Rename, "[tid:%i] Squashing incoming skidbuffer instructions " 399 "until [sn:%i].\n", tid, squash_seq_num); 400 ListIt slist_it = skidBuffer[tid].begin(); 401 while (slist_it != skidBuffer[tid].end()) { 402 if ((*slist_it)->seqNum > squash_seq_num) { 403 (*slist_it)->setSquashed(); 404 DPRINTF(Rename, "Squashing skidbuffer instruction, [tid:%i] [sn:%i]" 405 "PC %08p.\n", tid, (*slist_it)->seqNum, (*slist_it)->PC); 406 } 407 slist_it++; 408 } 409 resumeUnblocking = (skidBuffer[tid].size() != 0); 410 DPRINTF(Rename, "Resume unblocking set to %s\n", 411 resumeUnblocking ? "true" : "false"); 412#else 413 skidBuffer[tid].clear(); 414#endif 415 doSquash(squash_seq_num, tid); 416} 417 418template <class Impl> 419void 420DefaultRename<Impl>::tick() 421{ 422 wroteToTimeBuffer = false; 423 424 blockThisCycle = false; 425 426 bool status_change = false; 427 428 toIEWIndex = 0; 429 430 sortInsts(); 431 432 std::list<unsigned>::iterator threads = (*activeThreads).begin(); 433 434 // Check stall and squash signals. 435 while (threads != (*activeThreads).end()) { 436 unsigned tid = *threads++; 437 438 DPRINTF(Rename, "Processing [tid:%i]\n", tid); 439 440 status_change = checkSignalsAndUpdate(tid) || status_change; 441 442 rename(status_change, tid); 443 } 444 445 if (status_change) { 446 updateStatus(); 447 } 448 449 if (wroteToTimeBuffer) { 450 DPRINTF(Activity, "Activity this cycle.\n"); 451 cpu->activityThisCycle(); 452 } 453 454 threads = (*activeThreads).begin(); 455 456 while (threads != (*activeThreads).end()) { 457 unsigned tid = *threads++; 458 459 // If we committed this cycle then doneSeqNum will be > 0 460 if (fromCommit->commitInfo[tid].doneSeqNum != 0 && 461 !fromCommit->commitInfo[tid].squash && 462 renameStatus[tid] != Squashing) { 463 464 removeFromHistory(fromCommit->commitInfo[tid].doneSeqNum, 465 tid); 466 } 467 } 468 469 // @todo: make into updateProgress function 470 for (int tid=0; tid < numThreads; tid++) { 471 instsInProgress[tid] -= fromIEW->iewInfo[tid].dispatched; 472 473 assert(instsInProgress[tid] >=0); 474 } 475 476} 477 478template<class Impl> 479void 480DefaultRename<Impl>::rename(bool &status_change, unsigned tid) 481{ 482 // If status is Running or idle, 483 // call renameInsts() 484 // If status is Unblocking, 485 // buffer any instructions coming from decode 486 // continue trying to empty skid buffer 487 // check if stall conditions have passed 488 489 if (renameStatus[tid] == Blocked) { 490 ++renameBlockCycles; 491 } else if (renameStatus[tid] == Squashing) { 492 ++renameSquashCycles; 493 } else if (renameStatus[tid] == SerializeStall) { 494 ++renameSerializeStallCycles; 495 // If we are currently in SerializeStall and resumeSerialize 496 // was set, then that means that we are resuming serializing 497 // this cycle. Tell the previous stages to block. 498 if (resumeSerialize) { 499 resumeSerialize = false; 500 block(tid); 501 toDecode->renameUnblock[tid] = false; 502 } 503 } else if (renameStatus[tid] == Unblocking) { 504 if (resumeUnblocking) { 505 block(tid); 506 resumeUnblocking = false; 507 toDecode->renameUnblock[tid] = false; 508 } 509 } 510 511 if (renameStatus[tid] == Running || 512 renameStatus[tid] == Idle) { 513 DPRINTF(Rename, "[tid:%u]: Not blocked, so attempting to run " 514 "stage.\n", tid); 515 516 renameInsts(tid); 517 } else if (renameStatus[tid] == Unblocking) { 518 renameInsts(tid); 519 520 if (validInsts()) { 521 // Add the current inputs to the skid buffer so they can be 522 // reprocessed when this stage unblocks. 523 skidInsert(tid); 524 } 525 526 // If we switched over to blocking, then there's a potential for 527 // an overall status change. 528 status_change = unblock(tid) || status_change || blockThisCycle; 529 } 530} 531 532template <class Impl> 533void 534DefaultRename<Impl>::renameInsts(unsigned tid) 535{ 536 // Instructions can be either in the skid buffer or the queue of 537 // instructions coming from decode, depending on the status. 538 int insts_available = renameStatus[tid] == Unblocking ? 539 skidBuffer[tid].size() : insts[tid].size(); 540 541 // Check the decode queue to see if instructions are available. 542 // If there are no available instructions to rename, then do nothing. 543 if (insts_available == 0) { 544 DPRINTF(Rename, "[tid:%u]: Nothing to do, breaking out early.\n", 545 tid); 546 // Should I change status to idle? 547 ++renameIdleCycles; 548 return; 549 } else if (renameStatus[tid] == Unblocking) { 550 ++renameUnblockCycles; 551 } else if (renameStatus[tid] == Running) { 552 ++renameRunCycles; 553 } 554 555 DynInstPtr inst; 556 557 // Will have to do a different calculation for the number of free 558 // entries. 559 int free_rob_entries = calcFreeROBEntries(tid); 560 int free_iq_entries = calcFreeIQEntries(tid); 561 int free_lsq_entries = calcFreeLSQEntries(tid); 562 int min_free_entries = free_rob_entries; 563 564 FullSource source = ROB; 565 566 if (free_iq_entries < min_free_entries) { 567 min_free_entries = free_iq_entries; 568 source = IQ; 569 } 570 571 if (free_lsq_entries < min_free_entries) { 572 min_free_entries = free_lsq_entries; 573 source = LSQ; 574 } 575 576 // Check if there's any space left. 577 if (min_free_entries <= 0) { 578 DPRINTF(Rename, "[tid:%u]: Blocking due to no free ROB/IQ/LSQ " 579 "entries.\n" 580 "ROB has %i free entries.\n" 581 "IQ has %i free entries.\n" 582 "LSQ has %i free entries.\n", 583 tid, 584 free_rob_entries, 585 free_iq_entries, 586 free_lsq_entries); 587 588 blockThisCycle = true; 589 590 block(tid); 591 592 incrFullStat(source); 593 594 return; 595 } else if (min_free_entries < insts_available) { 596 DPRINTF(Rename, "[tid:%u]: Will have to block this cycle." 597 "%i insts available, but only %i insts can be " 598 "renamed due to ROB/IQ/LSQ limits.\n", 599 tid, insts_available, min_free_entries); 600 601 insts_available = min_free_entries; 602 603 blockThisCycle = true; 604 605 incrFullStat(source); 606 } 607 608 InstQueue &insts_to_rename = renameStatus[tid] == Unblocking ? 609 skidBuffer[tid] : insts[tid]; 610 611 DPRINTF(Rename, "[tid:%u]: %i available instructions to " 612 "send iew.\n", tid, insts_available); 613 614 DPRINTF(Rename, "[tid:%u]: %i insts pipelining from Rename | %i insts " 615 "dispatched to IQ last cycle.\n", 616 tid, instsInProgress[tid], fromIEW->iewInfo[tid].dispatched); 617 618 // Handle serializing the next instruction if necessary. 619 if (serializeOnNextInst[tid]) { 620 if (emptyROB[tid] && instsInProgress[tid] == 0) { 621 // ROB already empty; no need to serialize. 622 serializeOnNextInst[tid] = false; 623 } else if (!insts_to_rename.empty()) { 624 insts_to_rename.front()->setSerializeBefore(); 625 } 626 } 627 628 int renamed_insts = 0; 629 630 while (insts_available > 0 && toIEWIndex < renameWidth) { 631 DPRINTF(Rename, "[tid:%u]: Sending instructions to IEW.\n", tid); 632 633 assert(!insts_to_rename.empty()); 634 635 inst = insts_to_rename.front(); 636 637 insts_to_rename.pop_front(); 638 639 if (renameStatus[tid] == Unblocking) { 640 DPRINTF(Rename,"[tid:%u]: Removing [sn:%lli] PC:%#x from rename " 641 "skidBuffer\n", 642 tid, inst->seqNum, inst->readPC()); 643 } 644 645 if (inst->isSquashed()) { 646 DPRINTF(Rename, "[tid:%u]: instruction %i with PC %#x is " 647 "squashed, skipping.\n", 648 tid, inst->seqNum, inst->readPC()); 649 650 ++renameSquashedInsts; 651 652 // Decrement how many instructions are available. 653 --insts_available; 654 655 continue; 656 } 657 658 DPRINTF(Rename, "[tid:%u]: Processing instruction [sn:%lli] with " 659 "PC %#x.\n", 660 tid, inst->seqNum, inst->readPC()); 661 662 // Handle serializeAfter/serializeBefore instructions. 663 // serializeAfter marks the next instruction as serializeBefore. 664 // serializeBefore makes the instruction wait in rename until the ROB 665 // is empty. 666 667 // In this model, IPR accesses are serialize before 668 // instructions, and store conditionals are serialize after 669 // instructions. This is mainly due to lack of support for 670 // out-of-order operations of either of those classes of 671 // instructions. 672 if ((inst->isIprAccess() || inst->isSerializeBefore()) && 673 !inst->isSerializeHandled()) { 674 DPRINTF(Rename, "Serialize before instruction encountered.\n"); 675 676 if (!inst->isTempSerializeBefore()) { 677 renamedSerializing++; 678 inst->setSerializeHandled(); 679 } else { 680 renamedTempSerializing++; 681 } 682 683 // Change status over to SerializeStall so that other stages know 684 // what this is blocked on. 685 renameStatus[tid] = SerializeStall; 686 687 serializeInst[tid] = inst; 688 689 blockThisCycle = true; 690 691 break; 692 } else if ((inst->isStoreConditional() || inst->isSerializeAfter()) && 693 !inst->isSerializeHandled()) { 694 DPRINTF(Rename, "Serialize after instruction encountered.\n"); 695 696 renamedSerializing++; 697 698 inst->setSerializeHandled(); 699 700 serializeAfter(insts_to_rename, tid); 701 } 702 703 // Check here to make sure there are enough destination registers 704 // to rename to. Otherwise block. 705 if (renameMap[tid]->numFreeEntries() < inst->numDestRegs()) { 706 DPRINTF(Rename, "Blocking due to lack of free " 707 "physical registers to rename to.\n"); 708 blockThisCycle = true; 709 710 ++renameFullRegistersEvents; 711 712 break; 713 } 714 715 renameSrcRegs(inst, inst->threadNumber); 716 717 renameDestRegs(inst, inst->threadNumber); 718 719 ++renamed_insts; 720 721 // Put instruction in rename queue. 722 toIEW->insts[toIEWIndex] = inst; 723 ++(toIEW->size); 724 725 // Increment which instruction we're on. 726 ++toIEWIndex; 727 728 // Decrement how many instructions are available. 729 --insts_available; 730 } 731 732 instsInProgress[tid] += renamed_insts; 733 renameRenamedInsts += renamed_insts; 734 735 // If we wrote to the time buffer, record this. 736 if (toIEWIndex) { 737 wroteToTimeBuffer = true; 738 } 739 740 // Check if there's any instructions left that haven't yet been renamed. 741 // If so then block. 742 if (insts_available) { 743 blockThisCycle = true; 744 } 745 746 if (blockThisCycle) { 747 block(tid); 748 toDecode->renameUnblock[tid] = false; 749 } 750} 751 752template<class Impl> 753void 754DefaultRename<Impl>::skidInsert(unsigned tid) 755{ 756 DynInstPtr inst = NULL; 757 758 while (!insts[tid].empty()) { 759 inst = insts[tid].front(); 760 761 insts[tid].pop_front(); 762 763 assert(tid == inst->threadNumber); 764 765 DPRINTF(Rename, "[tid:%u]: Inserting [sn:%lli] PC:%#x into Rename " 766 "skidBuffer\n", tid, inst->seqNum, inst->readPC()); 767 768 ++renameSkidInsts; 769 770 skidBuffer[tid].push_back(inst); 771 } 772 773 if (skidBuffer[tid].size() > skidBufferMax) 774 { 775 typename InstQueue::iterator it; 776 warn("Skidbuffer contents:\n"); 777 for(it = skidBuffer[tid].begin(); it != skidBuffer[tid].end(); it++) 778 { 779 warn("[tid:%u]: %s [sn:%i].\n", tid, 780 (*it)->staticInst->disassemble(inst->readPC()), 781 (*it)->seqNum); 782 } 783 panic("Skidbuffer Exceeded Max Size"); 784 } 785} 786 787template <class Impl> 788void 789DefaultRename<Impl>::sortInsts() 790{ 791 int insts_from_decode = fromDecode->size; 792#ifdef DEBUG 793#if !ISA_HAS_DELAY_SLOT 794 for (int i=0; i < numThreads; i++) 795 assert(insts[i].empty()); 796#endif 797#endif 798 for (int i = 0; i < insts_from_decode; ++i) { 799 DynInstPtr inst = fromDecode->insts[i]; 800 insts[inst->threadNumber].push_back(inst); 801 } 802} 803 804template<class Impl> 805bool 806DefaultRename<Impl>::skidsEmpty() 807{ 808 std::list<unsigned>::iterator threads = (*activeThreads).begin(); 809 810 while (threads != (*activeThreads).end()) { 811 if (!skidBuffer[*threads++].empty()) 812 return false; 813 } 814 815 return true; 816} 817 818template<class Impl> 819void 820DefaultRename<Impl>::updateStatus() 821{ 822 bool any_unblocking = false; 823 824 std::list<unsigned>::iterator threads = (*activeThreads).begin(); 825 826 threads = (*activeThreads).begin(); 827 828 while (threads != (*activeThreads).end()) { 829 unsigned tid = *threads++; 830 831 if (renameStatus[tid] == Unblocking) { 832 any_unblocking = true; 833 break; 834 } 835 } 836 837 // Rename will have activity if it's unblocking. 838 if (any_unblocking) { 839 if (_status == Inactive) { 840 _status = Active; 841 842 DPRINTF(Activity, "Activating stage.\n"); 843 844 cpu->activateStage(O3CPU::RenameIdx); 845 } 846 } else { 847 // If it's not unblocking, then rename will not have any internal 848 // activity. Switch it to inactive. 849 if (_status == Active) { 850 _status = Inactive; 851 DPRINTF(Activity, "Deactivating stage.\n"); 852 853 cpu->deactivateStage(O3CPU::RenameIdx); 854 } 855 } 856} 857 858template <class Impl> 859bool 860DefaultRename<Impl>::block(unsigned tid) 861{ 862 DPRINTF(Rename, "[tid:%u]: Blocking.\n", tid); 863 864 // Add the current inputs onto the skid buffer, so they can be 865 // reprocessed when this stage unblocks. 866 skidInsert(tid); 867 868 // Only signal backwards to block if the previous stages do not think 869 // rename is already blocked. 870 if (renameStatus[tid] != Blocked) { 871 // If resumeUnblocking is set, we unblocked during the squash, 872 // but now we're have unblocking status. We need to tell earlier 873 // stages to block. 874 if (resumeUnblocking || renameStatus[tid] != Unblocking) { 875 toDecode->renameBlock[tid] = true; 876 toDecode->renameUnblock[tid] = false; 877 wroteToTimeBuffer = true; 878 } 879 880 // Rename can not go from SerializeStall to Blocked, otherwise 881 // it would not know to complete the serialize stall. 882 if (renameStatus[tid] != SerializeStall) { 883 // Set status to Blocked. 884 renameStatus[tid] = Blocked; 885 return true; 886 } 887 } 888 889 return false; 890} 891 892template <class Impl> 893bool 894DefaultRename<Impl>::unblock(unsigned tid) 895{ 896 DPRINTF(Rename, "[tid:%u]: Trying to unblock.\n", tid); 897 898 // Rename is done unblocking if the skid buffer is empty. 899 if (skidBuffer[tid].empty() && renameStatus[tid] != SerializeStall) { 900 901 DPRINTF(Rename, "[tid:%u]: Done unblocking.\n", tid); 902 903 toDecode->renameUnblock[tid] = true; 904 wroteToTimeBuffer = true; 905 906 renameStatus[tid] = Running; 907 return true; 908 } 909 910 return false; 911} 912 913template <class Impl> 914void 915DefaultRename<Impl>::doSquash(const InstSeqNum &squashed_seq_num, unsigned tid) 916{ 917 typename std::list<RenameHistory>::iterator hb_it = 918 historyBuffer[tid].begin(); 919 920 // After a syscall squashes everything, the history buffer may be empty 921 // but the ROB may still be squashing instructions. 922 if (historyBuffer[tid].empty()) { 923 return; 924 } 925 926 // Go through the most recent instructions, undoing the mappings 927 // they did and freeing up the registers. 928 while (!historyBuffer[tid].empty() && 929 (*hb_it).instSeqNum > squashed_seq_num) { 930 assert(hb_it != historyBuffer[tid].end()); 931 932 DPRINTF(Rename, "[tid:%u]: Removing history entry with sequence " 933 "number %i.\n", tid, (*hb_it).instSeqNum); 934 935 // Tell the rename map to set the architected register to the 936 // previous physical register that it was renamed to. 937 renameMap[tid]->setEntry(hb_it->archReg, hb_it->prevPhysReg); 938 939 // Put the renamed physical register back on the free list. 940 freeList->addReg(hb_it->newPhysReg); 941 942 // Be sure to mark its register as ready if it's a misc register. 943 if (hb_it->newPhysReg >= maxPhysicalRegs) { 944 scoreboard->setReg(hb_it->newPhysReg); 945 } 946 947 historyBuffer[tid].erase(hb_it++); 948 949 ++renameUndoneMaps; 950 } 951} 952 953template<class Impl> 954void 955DefaultRename<Impl>::removeFromHistory(InstSeqNum inst_seq_num, unsigned tid) 956{ 957 DPRINTF(Rename, "[tid:%u]: Removing a committed instruction from the " 958 "history buffer %u (size=%i), until [sn:%lli].\n", 959 tid, tid, historyBuffer[tid].size(), inst_seq_num); 960 961 typename std::list<RenameHistory>::iterator hb_it = 962 historyBuffer[tid].end(); 963 964 --hb_it; 965 966 if (historyBuffer[tid].empty()) { 967 DPRINTF(Rename, "[tid:%u]: History buffer is empty.\n", tid); 968 return; 969 } else if (hb_it->instSeqNum > inst_seq_num) { 970 DPRINTF(Rename, "[tid:%u]: Old sequence number encountered. Ensure " 971 "that a syscall happened recently.\n", tid); 972 return; 973 } 974 975 // Commit all the renames up until (and including) the committed sequence 976 // number. Some or even all of the committed instructions may not have 977 // rename histories if they did not have destination registers that were 978 // renamed. 979 while (!historyBuffer[tid].empty() && 980 hb_it != historyBuffer[tid].end() && 981 (*hb_it).instSeqNum <= inst_seq_num) { 982 983 DPRINTF(Rename, "[tid:%u]: Freeing up older rename of reg %i, " 984 "[sn:%lli].\n", 985 tid, (*hb_it).prevPhysReg, (*hb_it).instSeqNum); 986 987 freeList->addReg((*hb_it).prevPhysReg); 988 ++renameCommittedMaps; 989 990 historyBuffer[tid].erase(hb_it--); 991 } 992} 993 994template <class Impl> 995inline void 996DefaultRename<Impl>::renameSrcRegs(DynInstPtr &inst,unsigned tid) 997{ 998 assert(renameMap[tid] != 0); 999 1000 unsigned num_src_regs = inst->numSrcRegs(); 1001 1002 // Get the architectual register numbers from the source and 1003 // destination operands, and redirect them to the right register. 1004 // Will need to mark dependencies though. 1005 for (int src_idx = 0; src_idx < num_src_regs; src_idx++) { 1006 RegIndex src_reg = inst->srcRegIdx(src_idx); 1007 RegIndex flat_src_reg = src_reg; 1008 if (src_reg < TheISA::FP_Base_DepTag) { 1009 flat_src_reg = TheISA::flattenIntIndex(inst->tcBase(), src_reg); 1010 DPRINTF(Rename, "Flattening index %d to %d.\n", (int)src_reg, (int)flat_src_reg); 1011 } 1012 inst->flattenSrcReg(src_idx, flat_src_reg); 1013 1014 // Look up the source registers to get the phys. register they've 1015 // been renamed to, and set the sources to those registers. 1016 PhysRegIndex renamed_reg = renameMap[tid]->lookup(flat_src_reg); 1017 1018 DPRINTF(Rename, "[tid:%u]: Looking up arch reg %i, got " 1019 "physical reg %i.\n", tid, (int)flat_src_reg, 1020 (int)renamed_reg); 1021 1022 inst->renameSrcReg(src_idx, renamed_reg); 1023 1024 // See if the register is ready or not. 1025 if (scoreboard->getReg(renamed_reg) == true) { 1026 DPRINTF(Rename, "[tid:%u]: Register is ready.\n", tid); 1027 1028 inst->markSrcRegReady(src_idx); 1029 } 1030 1031 ++renameRenameLookups; 1032 } 1033} 1034 1035template <class Impl> 1036inline void 1037DefaultRename<Impl>::renameDestRegs(DynInstPtr &inst,unsigned tid) 1038{ 1039 typename RenameMap::RenameInfo rename_result; 1040 1041 unsigned num_dest_regs = inst->numDestRegs(); 1042 1043 // Rename the destination registers. 1044 for (int dest_idx = 0; dest_idx < num_dest_regs; dest_idx++) { 1045 RegIndex dest_reg = inst->destRegIdx(dest_idx); 1046 RegIndex flat_dest_reg = dest_reg; 1047 if (dest_reg < TheISA::FP_Base_DepTag) { 1048 flat_dest_reg = TheISA::flattenIntIndex(inst->tcBase(), dest_reg); 1049 DPRINTF(Rename, "Flattening index %d to %d.\n", (int)dest_reg, (int)flat_dest_reg); 1050 } 1051 1052 inst->flattenDestReg(dest_idx, flat_dest_reg); 1053 1054 // Get the physical register that the destination will be 1055 // renamed to. 1056 rename_result = renameMap[tid]->rename(flat_dest_reg); 1057 1058 //Mark Scoreboard entry as not ready 1059 scoreboard->unsetReg(rename_result.first); 1060 1061 DPRINTF(Rename, "[tid:%u]: Renaming arch reg %i to physical " 1062 "reg %i.\n", tid, (int)flat_dest_reg, 1063 (int)rename_result.first); 1064 1065 // Record the rename information so that a history can be kept. 1066 RenameHistory hb_entry(inst->seqNum, flat_dest_reg, 1067 rename_result.first, 1068 rename_result.second); 1069 1070 historyBuffer[tid].push_front(hb_entry); 1071 1072 DPRINTF(Rename, "[tid:%u]: Adding instruction to history buffer " 1073 "(size=%i), [sn:%lli].\n",tid, 1074 historyBuffer[tid].size(), 1075 (*historyBuffer[tid].begin()).instSeqNum); 1076 1077 // Tell the instruction to rename the appropriate destination 1078 // register (dest_idx) to the new physical register 1079 // (rename_result.first), and record the previous physical 1080 // register that the same logical register was renamed to 1081 // (rename_result.second). 1082 inst->renameDestReg(dest_idx, 1083 rename_result.first, 1084 rename_result.second); 1085 1086 ++renameRenamedOperands; 1087 } 1088} 1089 1090template <class Impl> 1091inline int 1092DefaultRename<Impl>::calcFreeROBEntries(unsigned tid) 1093{ 1094 int num_free = freeEntries[tid].robEntries - 1095 (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatched); 1096 1097 //DPRINTF(Rename,"[tid:%i]: %i rob free\n",tid,num_free); 1098 1099 return num_free; 1100} 1101 1102template <class Impl> 1103inline int 1104DefaultRename<Impl>::calcFreeIQEntries(unsigned tid) 1105{ 1106 int num_free = freeEntries[tid].iqEntries - 1107 (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatched); 1108 1109 //DPRINTF(Rename,"[tid:%i]: %i iq free\n",tid,num_free); 1110 1111 return num_free; 1112} 1113 1114template <class Impl> 1115inline int 1116DefaultRename<Impl>::calcFreeLSQEntries(unsigned tid) 1117{ 1118 int num_free = freeEntries[tid].lsqEntries - 1119 (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatchedToLSQ); 1120 1121 //DPRINTF(Rename,"[tid:%i]: %i lsq free\n",tid,num_free); 1122 1123 return num_free; 1124} 1125 1126template <class Impl> 1127unsigned 1128DefaultRename<Impl>::validInsts() 1129{ 1130 unsigned inst_count = 0; 1131 1132 for (int i=0; i<fromDecode->size; i++) { 1133 if (!fromDecode->insts[i]->isSquashed()) 1134 inst_count++; 1135 } 1136 1137 return inst_count; 1138} 1139 1140template <class Impl> 1141void 1142DefaultRename<Impl>::readStallSignals(unsigned tid) 1143{ 1144 if (fromIEW->iewBlock[tid]) { 1145 stalls[tid].iew = true; 1146 } 1147 1148 if (fromIEW->iewUnblock[tid]) { 1149 assert(stalls[tid].iew); 1150 stalls[tid].iew = false; 1151 } 1152 1153 if (fromCommit->commitBlock[tid]) { 1154 stalls[tid].commit = true; 1155 } 1156 1157 if (fromCommit->commitUnblock[tid]) { 1158 assert(stalls[tid].commit); 1159 stalls[tid].commit = false; 1160 } 1161} 1162 1163template <class Impl> 1164bool 1165DefaultRename<Impl>::checkStall(unsigned tid) 1166{ 1167 bool ret_val = false; 1168 1169 if (stalls[tid].iew) { 1170 DPRINTF(Rename,"[tid:%i]: Stall from IEW stage detected.\n", tid); 1171 ret_val = true; 1172 } else if (stalls[tid].commit) { 1173 DPRINTF(Rename,"[tid:%i]: Stall from Commit stage detected.\n", tid); 1174 ret_val = true; 1175 } else if (calcFreeROBEntries(tid) <= 0) { 1176 DPRINTF(Rename,"[tid:%i]: Stall: ROB has 0 free entries.\n", tid); 1177 ret_val = true; 1178 } else if (calcFreeIQEntries(tid) <= 0) { 1179 DPRINTF(Rename,"[tid:%i]: Stall: IQ has 0 free entries.\n", tid); 1180 ret_val = true; 1181 } else if (calcFreeLSQEntries(tid) <= 0) { 1182 DPRINTF(Rename,"[tid:%i]: Stall: LSQ has 0 free entries.\n", tid); 1183 ret_val = true; 1184 } else if (renameMap[tid]->numFreeEntries() <= 0) { 1185 DPRINTF(Rename,"[tid:%i]: Stall: RenameMap has 0 free entries.\n", tid); 1186 ret_val = true; 1187 } else if (renameStatus[tid] == SerializeStall && 1188 (!emptyROB[tid] || instsInProgress[tid])) { 1189 DPRINTF(Rename,"[tid:%i]: Stall: Serialize stall and ROB is not " 1190 "empty.\n", 1191 tid); 1192 ret_val = true; 1193 } 1194 1195 return ret_val; 1196} 1197 1198template <class Impl> 1199void 1200DefaultRename<Impl>::readFreeEntries(unsigned tid) 1201{ 1202 bool updated = false; 1203 if (fromIEW->iewInfo[tid].usedIQ) { 1204 freeEntries[tid].iqEntries = 1205 fromIEW->iewInfo[tid].freeIQEntries; 1206 updated = true; 1207 } 1208 1209 if (fromIEW->iewInfo[tid].usedLSQ) { 1210 freeEntries[tid].lsqEntries = 1211 fromIEW->iewInfo[tid].freeLSQEntries; 1212 updated = true; 1213 } 1214 1215 if (fromCommit->commitInfo[tid].usedROB) { 1216 freeEntries[tid].robEntries = 1217 fromCommit->commitInfo[tid].freeROBEntries; 1218 emptyROB[tid] = fromCommit->commitInfo[tid].emptyROB; 1219 updated = true; 1220 } 1221 1222 DPRINTF(Rename, "[tid:%i]: Free IQ: %i, Free ROB: %i, Free LSQ: %i\n", 1223 tid, 1224 freeEntries[tid].iqEntries, 1225 freeEntries[tid].robEntries, 1226 freeEntries[tid].lsqEntries); 1227 1228 DPRINTF(Rename, "[tid:%i]: %i instructions not yet in ROB\n", 1229 tid, instsInProgress[tid]); 1230} 1231 1232template <class Impl> 1233bool 1234DefaultRename<Impl>::checkSignalsAndUpdate(unsigned tid) 1235{ 1236 // Check if there's a squash signal, squash if there is 1237 // Check stall signals, block if necessary. 1238 // If status was blocked 1239 // check if stall conditions have passed 1240 // if so then go to unblocking 1241 // If status was Squashing 1242 // check if squashing is not high. Switch to running this cycle. 1243 // If status was serialize stall 1244 // check if ROB is empty and no insts are in flight to the ROB 1245 1246 readFreeEntries(tid); 1247 readStallSignals(tid); 1248 1249 if (fromCommit->commitInfo[tid].squash) { 1250 DPRINTF(Rename, "[tid:%u]: Squashing instructions due to squash from " 1251 "commit.\n", tid); 1252 1253#if ISA_HAS_DELAY_SLOT 1254 InstSeqNum squashed_seq_num = fromCommit->commitInfo[tid].bdelayDoneSeqNum; 1255#else 1256 InstSeqNum squashed_seq_num = fromCommit->commitInfo[tid].doneSeqNum; 1257#endif 1258 1259 squash(squashed_seq_num, tid); 1260 1261 return true; 1262 } 1263 1264 if (fromCommit->commitInfo[tid].robSquashing) { 1265 DPRINTF(Rename, "[tid:%u]: ROB is still squashing.\n", tid); 1266 1267 renameStatus[tid] = Squashing; 1268 1269 return true; 1270 } 1271 1272 if (checkStall(tid)) { 1273 return block(tid); 1274 } 1275 1276 if (renameStatus[tid] == Blocked) { 1277 DPRINTF(Rename, "[tid:%u]: Done blocking, switching to unblocking.\n", 1278 tid); 1279 1280 renameStatus[tid] = Unblocking; 1281 1282 unblock(tid); 1283 1284 return true; 1285 } 1286 1287 if (renameStatus[tid] == Squashing) { 1288 // Switch status to running if rename isn't being told to block or 1289 // squash this cycle. 1290 if (resumeSerialize) { 1291 DPRINTF(Rename, "[tid:%u]: Done squashing, switching to serialize.\n", 1292 tid); 1293 1294 renameStatus[tid] = SerializeStall; 1295 return true; 1296 } else if (resumeUnblocking) { 1297 DPRINTF(Rename, "[tid:%u]: Done squashing, switching to unblocking.\n", 1298 tid); 1299 renameStatus[tid] = Unblocking; 1300 return true; 1301 } else { 1302 DPRINTF(Rename, "[tid:%u]: Done squashing, switching to running.\n", 1303 tid); 1304 1305 renameStatus[tid] = Running; 1306 return false; 1307 } 1308 } 1309 1310 if (renameStatus[tid] == SerializeStall) { 1311 // Stall ends once the ROB is free. 1312 DPRINTF(Rename, "[tid:%u]: Done with serialize stall, switching to " 1313 "unblocking.\n", tid); 1314 1315 DynInstPtr serial_inst = serializeInst[tid]; 1316 1317 renameStatus[tid] = Unblocking; 1318 1319 unblock(tid); 1320 1321 DPRINTF(Rename, "[tid:%u]: Processing instruction [%lli] with " 1322 "PC %#x.\n", 1323 tid, serial_inst->seqNum, serial_inst->readPC()); 1324 1325 // Put instruction into queue here. 1326 serial_inst->clearSerializeBefore(); 1327 1328 if (!skidBuffer[tid].empty()) { 1329 skidBuffer[tid].push_front(serial_inst); 1330 } else { 1331 insts[tid].push_front(serial_inst); 1332 } 1333 1334 DPRINTF(Rename, "[tid:%u]: Instruction must be processed by rename." 1335 " Adding to front of list.\n", tid); 1336 1337 serializeInst[tid] = NULL; 1338 1339 return true; 1340 } 1341 1342 // If we've reached this point, we have not gotten any signals that 1343 // cause rename to change its status. Rename remains the same as before. 1344 return false; 1345} 1346 1347template<class Impl> 1348void 1349DefaultRename<Impl>::serializeAfter(InstQueue &inst_list, 1350 unsigned tid) 1351{ 1352 if (inst_list.empty()) { 1353 // Mark a bit to say that I must serialize on the next instruction. 1354 serializeOnNextInst[tid] = true; 1355 return; 1356 } 1357 1358 // Set the next instruction as serializing. 1359 inst_list.front()->setSerializeBefore(); 1360} 1361 1362template <class Impl> 1363inline void 1364DefaultRename<Impl>::incrFullStat(const FullSource &source) 1365{ 1366 switch (source) { 1367 case ROB: 1368 ++renameROBFullEvents; 1369 break; 1370 case IQ: 1371 ++renameIQFullEvents; 1372 break; 1373 case LSQ: 1374 ++renameLSQFullEvents; 1375 break; 1376 default: 1377 panic("Rename full stall stat should be incremented for a reason!"); 1378 break; 1379 } 1380} 1381 1382template <class Impl> 1383void 1384DefaultRename<Impl>::dumpHistory() 1385{ 1386 typename std::list<RenameHistory>::iterator buf_it; 1387 1388 for (int i = 0; i < numThreads; i++) { 1389 1390 buf_it = historyBuffer[i].begin(); 1391 1392 while (buf_it != historyBuffer[i].end()) { 1393 cprintf("Seq num: %i\nArch reg: %i New phys reg: %i Old phys " 1394 "reg: %i\n", (*buf_it).instSeqNum, (int)(*buf_it).archReg, 1395 (int)(*buf_it).newPhysReg, (int)(*buf_it).prevPhysReg); 1396 1397 buf_it++; 1398 } 1399 } 1400} 1401