rename_impl.hh revision 7854
1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2006 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 *          Korey Sewell
42 */
43
44#include <list>
45
46#include "arch/isa_traits.hh"
47#include "arch/registers.hh"
48#include "config/full_system.hh"
49#include "config/the_isa.hh"
50#include "cpu/o3/rename.hh"
51#include "params/DerivO3CPU.hh"
52
53using namespace std;
54
55template <class Impl>
56DefaultRename<Impl>::DefaultRename(O3CPU *_cpu, DerivO3CPUParams *params)
57    : cpu(_cpu),
58      iewToRenameDelay(params->iewToRenameDelay),
59      decodeToRenameDelay(params->decodeToRenameDelay),
60      commitToRenameDelay(params->commitToRenameDelay),
61      renameWidth(params->renameWidth),
62      commitWidth(params->commitWidth),
63      resumeSerialize(false),
64      resumeUnblocking(false),
65      numThreads(params->numThreads),
66      maxPhysicalRegs(params->numPhysIntRegs + params->numPhysFloatRegs)
67{
68    _status = Inactive;
69
70    for (ThreadID tid = 0; tid < numThreads; tid++) {
71        renameStatus[tid] = Idle;
72
73        freeEntries[tid].iqEntries = 0;
74        freeEntries[tid].lsqEntries = 0;
75        freeEntries[tid].robEntries = 0;
76
77        stalls[tid].iew = false;
78        stalls[tid].commit = false;
79        serializeInst[tid] = NULL;
80
81        instsInProgress[tid] = 0;
82
83        emptyROB[tid] = true;
84
85        serializeOnNextInst[tid] = false;
86    }
87
88    // @todo: Make into a parameter.
89    skidBufferMax = (2 * (iewToRenameDelay * params->decodeWidth)) + renameWidth;
90}
91
92template <class Impl>
93std::string
94DefaultRename<Impl>::name() const
95{
96    return cpu->name() + ".rename";
97}
98
99template <class Impl>
100void
101DefaultRename<Impl>::regStats()
102{
103    renameSquashCycles
104        .name(name() + ".RENAME:SquashCycles")
105        .desc("Number of cycles rename is squashing")
106        .prereq(renameSquashCycles);
107    renameIdleCycles
108        .name(name() + ".RENAME:IdleCycles")
109        .desc("Number of cycles rename is idle")
110        .prereq(renameIdleCycles);
111    renameBlockCycles
112        .name(name() + ".RENAME:BlockCycles")
113        .desc("Number of cycles rename is blocking")
114        .prereq(renameBlockCycles);
115    renameSerializeStallCycles
116        .name(name() + ".RENAME:serializeStallCycles")
117        .desc("count of cycles rename stalled for serializing inst")
118        .flags(Stats::total);
119    renameRunCycles
120        .name(name() + ".RENAME:RunCycles")
121        .desc("Number of cycles rename is running")
122        .prereq(renameIdleCycles);
123    renameUnblockCycles
124        .name(name() + ".RENAME:UnblockCycles")
125        .desc("Number of cycles rename is unblocking")
126        .prereq(renameUnblockCycles);
127    renameRenamedInsts
128        .name(name() + ".RENAME:RenamedInsts")
129        .desc("Number of instructions processed by rename")
130        .prereq(renameRenamedInsts);
131    renameSquashedInsts
132        .name(name() + ".RENAME:SquashedInsts")
133        .desc("Number of squashed instructions processed by rename")
134        .prereq(renameSquashedInsts);
135    renameROBFullEvents
136        .name(name() + ".RENAME:ROBFullEvents")
137        .desc("Number of times rename has blocked due to ROB full")
138        .prereq(renameROBFullEvents);
139    renameIQFullEvents
140        .name(name() + ".RENAME:IQFullEvents")
141        .desc("Number of times rename has blocked due to IQ full")
142        .prereq(renameIQFullEvents);
143    renameLSQFullEvents
144        .name(name() + ".RENAME:LSQFullEvents")
145        .desc("Number of times rename has blocked due to LSQ full")
146        .prereq(renameLSQFullEvents);
147    renameFullRegistersEvents
148        .name(name() + ".RENAME:FullRegisterEvents")
149        .desc("Number of times there has been no free registers")
150        .prereq(renameFullRegistersEvents);
151    renameRenamedOperands
152        .name(name() + ".RENAME:RenamedOperands")
153        .desc("Number of destination operands rename has renamed")
154        .prereq(renameRenamedOperands);
155    renameRenameLookups
156        .name(name() + ".RENAME:RenameLookups")
157        .desc("Number of register rename lookups that rename has made")
158        .prereq(renameRenameLookups);
159    renameCommittedMaps
160        .name(name() + ".RENAME:CommittedMaps")
161        .desc("Number of HB maps that are committed")
162        .prereq(renameCommittedMaps);
163    renameUndoneMaps
164        .name(name() + ".RENAME:UndoneMaps")
165        .desc("Number of HB maps that are undone due to squashing")
166        .prereq(renameUndoneMaps);
167    renamedSerializing
168        .name(name() + ".RENAME:serializingInsts")
169        .desc("count of serializing insts renamed")
170        .flags(Stats::total)
171        ;
172    renamedTempSerializing
173        .name(name() + ".RENAME:tempSerializingInsts")
174        .desc("count of temporary serializing insts renamed")
175        .flags(Stats::total)
176        ;
177    renameSkidInsts
178        .name(name() + ".RENAME:skidInsts")
179        .desc("count of insts added to the skid buffer")
180        .flags(Stats::total)
181        ;
182}
183
184template <class Impl>
185void
186DefaultRename<Impl>::setTimeBuffer(TimeBuffer<TimeStruct> *tb_ptr)
187{
188    timeBuffer = tb_ptr;
189
190    // Setup wire to read information from time buffer, from IEW stage.
191    fromIEW = timeBuffer->getWire(-iewToRenameDelay);
192
193    // Setup wire to read infromation from time buffer, from commit stage.
194    fromCommit = timeBuffer->getWire(-commitToRenameDelay);
195
196    // Setup wire to write information to previous stages.
197    toDecode = timeBuffer->getWire(0);
198}
199
200template <class Impl>
201void
202DefaultRename<Impl>::setRenameQueue(TimeBuffer<RenameStruct> *rq_ptr)
203{
204    renameQueue = rq_ptr;
205
206    // Setup wire to write information to future stages.
207    toIEW = renameQueue->getWire(0);
208}
209
210template <class Impl>
211void
212DefaultRename<Impl>::setDecodeQueue(TimeBuffer<DecodeStruct> *dq_ptr)
213{
214    decodeQueue = dq_ptr;
215
216    // Setup wire to get information from decode.
217    fromDecode = decodeQueue->getWire(-decodeToRenameDelay);
218}
219
220template <class Impl>
221void
222DefaultRename<Impl>::initStage()
223{
224    // Grab the number of free entries directly from the stages.
225    for (ThreadID tid = 0; tid < numThreads; tid++) {
226        freeEntries[tid].iqEntries = iew_ptr->instQueue.numFreeEntries(tid);
227        freeEntries[tid].lsqEntries = iew_ptr->ldstQueue.numFreeEntries(tid);
228        freeEntries[tid].robEntries = commit_ptr->numROBFreeEntries(tid);
229        emptyROB[tid] = true;
230    }
231}
232
233template<class Impl>
234void
235DefaultRename<Impl>::setActiveThreads(list<ThreadID> *at_ptr)
236{
237    activeThreads = at_ptr;
238}
239
240
241template <class Impl>
242void
243DefaultRename<Impl>::setRenameMap(RenameMap rm_ptr[])
244{
245    for (ThreadID tid = 0; tid < numThreads; tid++)
246        renameMap[tid] = &rm_ptr[tid];
247}
248
249template <class Impl>
250void
251DefaultRename<Impl>::setFreeList(FreeList *fl_ptr)
252{
253    freeList = fl_ptr;
254}
255
256template<class Impl>
257void
258DefaultRename<Impl>::setScoreboard(Scoreboard *_scoreboard)
259{
260    scoreboard = _scoreboard;
261}
262
263template <class Impl>
264bool
265DefaultRename<Impl>::drain()
266{
267    // Rename is ready to switch out at any time.
268    cpu->signalDrained();
269    return true;
270}
271
272template <class Impl>
273void
274DefaultRename<Impl>::switchOut()
275{
276    // Clear any state, fix up the rename map.
277    for (ThreadID tid = 0; tid < numThreads; tid++) {
278        typename std::list<RenameHistory>::iterator hb_it =
279            historyBuffer[tid].begin();
280
281        while (!historyBuffer[tid].empty()) {
282            assert(hb_it != historyBuffer[tid].end());
283
284            DPRINTF(Rename, "[tid:%u]: Removing history entry with sequence "
285                    "number %i.\n", tid, (*hb_it).instSeqNum);
286
287            // Tell the rename map to set the architected register to the
288            // previous physical register that it was renamed to.
289            renameMap[tid]->setEntry(hb_it->archReg, hb_it->prevPhysReg);
290
291            // Put the renamed physical register back on the free list.
292            freeList->addReg(hb_it->newPhysReg);
293
294            // Be sure to mark its register as ready if it's a misc register.
295            if (hb_it->newPhysReg >= maxPhysicalRegs) {
296                scoreboard->setReg(hb_it->newPhysReg);
297            }
298
299            historyBuffer[tid].erase(hb_it++);
300        }
301        insts[tid].clear();
302        skidBuffer[tid].clear();
303    }
304}
305
306template <class Impl>
307void
308DefaultRename<Impl>::takeOverFrom()
309{
310    _status = Inactive;
311    initStage();
312
313    // Reset all state prior to taking over from the other CPU.
314    for (ThreadID tid = 0; tid < numThreads; tid++) {
315        renameStatus[tid] = Idle;
316
317        stalls[tid].iew = false;
318        stalls[tid].commit = false;
319        serializeInst[tid] = NULL;
320
321        instsInProgress[tid] = 0;
322
323        emptyROB[tid] = true;
324
325        serializeOnNextInst[tid] = false;
326    }
327}
328
329template <class Impl>
330void
331DefaultRename<Impl>::squash(const InstSeqNum &squash_seq_num, ThreadID tid)
332{
333    DPRINTF(Rename, "[tid:%u]: Squashing instructions.\n",tid);
334
335    // Clear the stall signal if rename was blocked or unblocking before.
336    // If it still needs to block, the blocking should happen the next
337    // cycle and there should be space to hold everything due to the squash.
338    if (renameStatus[tid] == Blocked ||
339        renameStatus[tid] == Unblocking) {
340        toDecode->renameUnblock[tid] = 1;
341
342        resumeSerialize = false;
343        serializeInst[tid] = NULL;
344    } else if (renameStatus[tid] == SerializeStall) {
345        if (serializeInst[tid]->seqNum <= squash_seq_num) {
346            DPRINTF(Rename, "Rename will resume serializing after squash\n");
347            resumeSerialize = true;
348            assert(serializeInst[tid]);
349        } else {
350            resumeSerialize = false;
351            toDecode->renameUnblock[tid] = 1;
352
353            serializeInst[tid] = NULL;
354        }
355    }
356
357    // Set the status to Squashing.
358    renameStatus[tid] = Squashing;
359
360    // Squash any instructions from decode.
361    unsigned squashCount = 0;
362
363    for (int i=0; i<fromDecode->size; i++) {
364        if (fromDecode->insts[i]->threadNumber == tid &&
365            fromDecode->insts[i]->seqNum > squash_seq_num) {
366            fromDecode->insts[i]->setSquashed();
367            wroteToTimeBuffer = true;
368            squashCount++;
369        }
370
371    }
372
373    // Clear the instruction list and skid buffer in case they have any
374    // insts in them.
375    insts[tid].clear();
376
377    // Clear the skid buffer in case it has any data in it.
378    skidBuffer[tid].clear();
379
380    doSquash(squash_seq_num, tid);
381}
382
383template <class Impl>
384void
385DefaultRename<Impl>::tick()
386{
387    wroteToTimeBuffer = false;
388
389    blockThisCycle = false;
390
391    bool status_change = false;
392
393    toIEWIndex = 0;
394
395    sortInsts();
396
397    list<ThreadID>::iterator threads = activeThreads->begin();
398    list<ThreadID>::iterator end = activeThreads->end();
399
400    // Check stall and squash signals.
401    while (threads != end) {
402        ThreadID tid = *threads++;
403
404        DPRINTF(Rename, "Processing [tid:%i]\n", tid);
405
406        status_change = checkSignalsAndUpdate(tid) || status_change;
407
408        rename(status_change, tid);
409    }
410
411    if (status_change) {
412        updateStatus();
413    }
414
415    if (wroteToTimeBuffer) {
416        DPRINTF(Activity, "Activity this cycle.\n");
417        cpu->activityThisCycle();
418    }
419
420    threads = activeThreads->begin();
421
422    while (threads != end) {
423        ThreadID tid = *threads++;
424
425        // If we committed this cycle then doneSeqNum will be > 0
426        if (fromCommit->commitInfo[tid].doneSeqNum != 0 &&
427            !fromCommit->commitInfo[tid].squash &&
428            renameStatus[tid] != Squashing) {
429
430            removeFromHistory(fromCommit->commitInfo[tid].doneSeqNum,
431                                  tid);
432        }
433    }
434
435    // @todo: make into updateProgress function
436    for (ThreadID tid = 0; tid < numThreads; tid++) {
437        instsInProgress[tid] -= fromIEW->iewInfo[tid].dispatched;
438
439        assert(instsInProgress[tid] >=0);
440    }
441
442}
443
444template<class Impl>
445void
446DefaultRename<Impl>::rename(bool &status_change, ThreadID tid)
447{
448    // If status is Running or idle,
449    //     call renameInsts()
450    // If status is Unblocking,
451    //     buffer any instructions coming from decode
452    //     continue trying to empty skid buffer
453    //     check if stall conditions have passed
454
455    if (renameStatus[tid] == Blocked) {
456        ++renameBlockCycles;
457    } else if (renameStatus[tid] == Squashing) {
458        ++renameSquashCycles;
459    } else if (renameStatus[tid] == SerializeStall) {
460        ++renameSerializeStallCycles;
461        // If we are currently in SerializeStall and resumeSerialize
462        // was set, then that means that we are resuming serializing
463        // this cycle.  Tell the previous stages to block.
464        if (resumeSerialize) {
465            resumeSerialize = false;
466            block(tid);
467            toDecode->renameUnblock[tid] = false;
468        }
469    } else if (renameStatus[tid] == Unblocking) {
470        if (resumeUnblocking) {
471            block(tid);
472            resumeUnblocking = false;
473            toDecode->renameUnblock[tid] = false;
474        }
475    }
476
477    if (renameStatus[tid] == Running ||
478        renameStatus[tid] == Idle) {
479        DPRINTF(Rename, "[tid:%u]: Not blocked, so attempting to run "
480                "stage.\n", tid);
481
482        renameInsts(tid);
483    } else if (renameStatus[tid] == Unblocking) {
484        renameInsts(tid);
485
486        if (validInsts()) {
487            // Add the current inputs to the skid buffer so they can be
488            // reprocessed when this stage unblocks.
489            skidInsert(tid);
490        }
491
492        // If we switched over to blocking, then there's a potential for
493        // an overall status change.
494        status_change = unblock(tid) || status_change || blockThisCycle;
495    }
496}
497
498template <class Impl>
499void
500DefaultRename<Impl>::renameInsts(ThreadID tid)
501{
502    // Instructions can be either in the skid buffer or the queue of
503    // instructions coming from decode, depending on the status.
504    int insts_available = renameStatus[tid] == Unblocking ?
505        skidBuffer[tid].size() : insts[tid].size();
506
507    // Check the decode queue to see if instructions are available.
508    // If there are no available instructions to rename, then do nothing.
509    if (insts_available == 0) {
510        DPRINTF(Rename, "[tid:%u]: Nothing to do, breaking out early.\n",
511                tid);
512        // Should I change status to idle?
513        ++renameIdleCycles;
514        return;
515    } else if (renameStatus[tid] == Unblocking) {
516        ++renameUnblockCycles;
517    } else if (renameStatus[tid] == Running) {
518        ++renameRunCycles;
519    }
520
521    DynInstPtr inst;
522
523    // Will have to do a different calculation for the number of free
524    // entries.
525    int free_rob_entries = calcFreeROBEntries(tid);
526    int free_iq_entries  = calcFreeIQEntries(tid);
527    int free_lsq_entries = calcFreeLSQEntries(tid);
528    int min_free_entries = free_rob_entries;
529
530    FullSource source = ROB;
531
532    if (free_iq_entries < min_free_entries) {
533        min_free_entries = free_iq_entries;
534        source = IQ;
535    }
536
537    if (free_lsq_entries < min_free_entries) {
538        min_free_entries = free_lsq_entries;
539        source = LSQ;
540    }
541
542    // Check if there's any space left.
543    if (min_free_entries <= 0) {
544        DPRINTF(Rename, "[tid:%u]: Blocking due to no free ROB/IQ/LSQ "
545                "entries.\n"
546                "ROB has %i free entries.\n"
547                "IQ has %i free entries.\n"
548                "LSQ has %i free entries.\n",
549                tid,
550                free_rob_entries,
551                free_iq_entries,
552                free_lsq_entries);
553
554        blockThisCycle = true;
555
556        block(tid);
557
558        incrFullStat(source);
559
560        return;
561    } else if (min_free_entries < insts_available) {
562        DPRINTF(Rename, "[tid:%u]: Will have to block this cycle."
563                "%i insts available, but only %i insts can be "
564                "renamed due to ROB/IQ/LSQ limits.\n",
565                tid, insts_available, min_free_entries);
566
567        insts_available = min_free_entries;
568
569        blockThisCycle = true;
570
571        incrFullStat(source);
572    }
573
574    InstQueue &insts_to_rename = renameStatus[tid] == Unblocking ?
575        skidBuffer[tid] : insts[tid];
576
577    DPRINTF(Rename, "[tid:%u]: %i available instructions to "
578            "send iew.\n", tid, insts_available);
579
580    DPRINTF(Rename, "[tid:%u]: %i insts pipelining from Rename | %i insts "
581            "dispatched to IQ last cycle.\n",
582            tid, instsInProgress[tid], fromIEW->iewInfo[tid].dispatched);
583
584    // Handle serializing the next instruction if necessary.
585    if (serializeOnNextInst[tid]) {
586        if (emptyROB[tid] && instsInProgress[tid] == 0) {
587            // ROB already empty; no need to serialize.
588            serializeOnNextInst[tid] = false;
589        } else if (!insts_to_rename.empty()) {
590            insts_to_rename.front()->setSerializeBefore();
591        }
592    }
593
594    int renamed_insts = 0;
595
596    while (insts_available > 0 &&  toIEWIndex < renameWidth) {
597        DPRINTF(Rename, "[tid:%u]: Sending instructions to IEW.\n", tid);
598
599        assert(!insts_to_rename.empty());
600
601        inst = insts_to_rename.front();
602
603        insts_to_rename.pop_front();
604
605        if (renameStatus[tid] == Unblocking) {
606            DPRINTF(Rename,"[tid:%u]: Removing [sn:%lli] PC:%s from rename "
607                    "skidBuffer\n", tid, inst->seqNum, inst->pcState());
608        }
609
610        if (inst->isSquashed()) {
611            DPRINTF(Rename, "[tid:%u]: instruction %i with PC %s is "
612                    "squashed, skipping.\n", tid, inst->seqNum,
613                    inst->pcState());
614
615            ++renameSquashedInsts;
616
617            // Decrement how many instructions are available.
618            --insts_available;
619
620            continue;
621        }
622
623        DPRINTF(Rename, "[tid:%u]: Processing instruction [sn:%lli] with "
624                "PC %s.\n", tid, inst->seqNum, inst->pcState());
625
626        // Handle serializeAfter/serializeBefore instructions.
627        // serializeAfter marks the next instruction as serializeBefore.
628        // serializeBefore makes the instruction wait in rename until the ROB
629        // is empty.
630
631        // In this model, IPR accesses are serialize before
632        // instructions, and store conditionals are serialize after
633        // instructions.  This is mainly due to lack of support for
634        // out-of-order operations of either of those classes of
635        // instructions.
636        if ((inst->isIprAccess() || inst->isSerializeBefore()) &&
637            !inst->isSerializeHandled()) {
638            DPRINTF(Rename, "Serialize before instruction encountered.\n");
639
640            if (!inst->isTempSerializeBefore()) {
641                renamedSerializing++;
642                inst->setSerializeHandled();
643            } else {
644                renamedTempSerializing++;
645            }
646
647            // Change status over to SerializeStall so that other stages know
648            // what this is blocked on.
649            renameStatus[tid] = SerializeStall;
650
651            serializeInst[tid] = inst;
652
653            blockThisCycle = true;
654
655            break;
656        } else if ((inst->isStoreConditional() || inst->isSerializeAfter()) &&
657                   !inst->isSerializeHandled()) {
658            DPRINTF(Rename, "Serialize after instruction encountered.\n");
659
660            renamedSerializing++;
661
662            inst->setSerializeHandled();
663
664            serializeAfter(insts_to_rename, tid);
665        }
666
667        // Check here to make sure there are enough destination registers
668        // to rename to.  Otherwise block.
669        if (renameMap[tid]->numFreeEntries() < inst->numDestRegs()) {
670            DPRINTF(Rename, "Blocking due to lack of free "
671                    "physical registers to rename to.\n");
672            blockThisCycle = true;
673            insts_to_rename.push_front(inst);
674            ++renameFullRegistersEvents;
675
676            break;
677        }
678
679        renameSrcRegs(inst, inst->threadNumber);
680
681        renameDestRegs(inst, inst->threadNumber);
682
683        ++renamed_insts;
684
685        // Put instruction in rename queue.
686        toIEW->insts[toIEWIndex] = inst;
687        ++(toIEW->size);
688
689        // Increment which instruction we're on.
690        ++toIEWIndex;
691
692        // Decrement how many instructions are available.
693        --insts_available;
694    }
695
696    instsInProgress[tid] += renamed_insts;
697    renameRenamedInsts += renamed_insts;
698
699    // If we wrote to the time buffer, record this.
700    if (toIEWIndex) {
701        wroteToTimeBuffer = true;
702    }
703
704    // Check if there's any instructions left that haven't yet been renamed.
705    // If so then block.
706    if (insts_available) {
707        blockThisCycle = true;
708    }
709
710    if (blockThisCycle) {
711        block(tid);
712        toDecode->renameUnblock[tid] = false;
713    }
714}
715
716template<class Impl>
717void
718DefaultRename<Impl>::skidInsert(ThreadID tid)
719{
720    DynInstPtr inst = NULL;
721
722    while (!insts[tid].empty()) {
723        inst = insts[tid].front();
724
725        insts[tid].pop_front();
726
727        assert(tid == inst->threadNumber);
728
729        DPRINTF(Rename, "[tid:%u]: Inserting [sn:%lli] PC: %s into Rename "
730                "skidBuffer\n", tid, inst->seqNum, inst->pcState());
731
732        ++renameSkidInsts;
733
734        skidBuffer[tid].push_back(inst);
735    }
736
737    if (skidBuffer[tid].size() > skidBufferMax)
738    {
739        typename InstQueue::iterator it;
740        warn("Skidbuffer contents:\n");
741        for(it = skidBuffer[tid].begin(); it != skidBuffer[tid].end(); it++)
742        {
743            warn("[tid:%u]: %s [sn:%i].\n", tid,
744                    (*it)->staticInst->disassemble(inst->instAddr()),
745                    (*it)->seqNum);
746        }
747        panic("Skidbuffer Exceeded Max Size");
748    }
749}
750
751template <class Impl>
752void
753DefaultRename<Impl>::sortInsts()
754{
755    int insts_from_decode = fromDecode->size;
756#ifdef DEBUG
757    for (ThreadID tid = 0; tid < numThreads; tid++)
758        assert(insts[tid].empty());
759#endif
760    for (int i = 0; i < insts_from_decode; ++i) {
761        DynInstPtr inst = fromDecode->insts[i];
762        insts[inst->threadNumber].push_back(inst);
763    }
764}
765
766template<class Impl>
767bool
768DefaultRename<Impl>::skidsEmpty()
769{
770    list<ThreadID>::iterator threads = activeThreads->begin();
771    list<ThreadID>::iterator end = activeThreads->end();
772
773    while (threads != end) {
774        ThreadID tid = *threads++;
775
776        if (!skidBuffer[tid].empty())
777            return false;
778    }
779
780    return true;
781}
782
783template<class Impl>
784void
785DefaultRename<Impl>::updateStatus()
786{
787    bool any_unblocking = false;
788
789    list<ThreadID>::iterator threads = activeThreads->begin();
790    list<ThreadID>::iterator end = activeThreads->end();
791
792    while (threads != end) {
793        ThreadID tid = *threads++;
794
795        if (renameStatus[tid] == Unblocking) {
796            any_unblocking = true;
797            break;
798        }
799    }
800
801    // Rename will have activity if it's unblocking.
802    if (any_unblocking) {
803        if (_status == Inactive) {
804            _status = Active;
805
806            DPRINTF(Activity, "Activating stage.\n");
807
808            cpu->activateStage(O3CPU::RenameIdx);
809        }
810    } else {
811        // If it's not unblocking, then rename will not have any internal
812        // activity.  Switch it to inactive.
813        if (_status == Active) {
814            _status = Inactive;
815            DPRINTF(Activity, "Deactivating stage.\n");
816
817            cpu->deactivateStage(O3CPU::RenameIdx);
818        }
819    }
820}
821
822template <class Impl>
823bool
824DefaultRename<Impl>::block(ThreadID tid)
825{
826    DPRINTF(Rename, "[tid:%u]: Blocking.\n", tid);
827
828    // Add the current inputs onto the skid buffer, so they can be
829    // reprocessed when this stage unblocks.
830    skidInsert(tid);
831
832    // Only signal backwards to block if the previous stages do not think
833    // rename is already blocked.
834    if (renameStatus[tid] != Blocked) {
835        // If resumeUnblocking is set, we unblocked during the squash,
836        // but now we're have unblocking status. We need to tell earlier
837        // stages to block.
838        if (resumeUnblocking || renameStatus[tid] != Unblocking) {
839            toDecode->renameBlock[tid] = true;
840            toDecode->renameUnblock[tid] = false;
841            wroteToTimeBuffer = true;
842        }
843
844        // Rename can not go from SerializeStall to Blocked, otherwise
845        // it would not know to complete the serialize stall.
846        if (renameStatus[tid] != SerializeStall) {
847            // Set status to Blocked.
848            renameStatus[tid] = Blocked;
849            return true;
850        }
851    }
852
853    return false;
854}
855
856template <class Impl>
857bool
858DefaultRename<Impl>::unblock(ThreadID tid)
859{
860    DPRINTF(Rename, "[tid:%u]: Trying to unblock.\n", tid);
861
862    // Rename is done unblocking if the skid buffer is empty.
863    if (skidBuffer[tid].empty() && renameStatus[tid] != SerializeStall) {
864
865        DPRINTF(Rename, "[tid:%u]: Done unblocking.\n", tid);
866
867        toDecode->renameUnblock[tid] = true;
868        wroteToTimeBuffer = true;
869
870        renameStatus[tid] = Running;
871        return true;
872    }
873
874    return false;
875}
876
877template <class Impl>
878void
879DefaultRename<Impl>::doSquash(const InstSeqNum &squashed_seq_num, ThreadID tid)
880{
881    typename std::list<RenameHistory>::iterator hb_it =
882        historyBuffer[tid].begin();
883
884    // After a syscall squashes everything, the history buffer may be empty
885    // but the ROB may still be squashing instructions.
886    if (historyBuffer[tid].empty()) {
887        return;
888    }
889
890    // Go through the most recent instructions, undoing the mappings
891    // they did and freeing up the registers.
892    while (!historyBuffer[tid].empty() &&
893           (*hb_it).instSeqNum > squashed_seq_num) {
894        assert(hb_it != historyBuffer[tid].end());
895
896        DPRINTF(Rename, "[tid:%u]: Removing history entry with sequence "
897                "number %i.\n", tid, (*hb_it).instSeqNum);
898
899        // Tell the rename map to set the architected register to the
900        // previous physical register that it was renamed to.
901        renameMap[tid]->setEntry(hb_it->archReg, hb_it->prevPhysReg);
902
903        // Put the renamed physical register back on the free list.
904        freeList->addReg(hb_it->newPhysReg);
905
906        // Be sure to mark its register as ready if it's a misc register.
907        if (hb_it->newPhysReg >= maxPhysicalRegs) {
908            scoreboard->setReg(hb_it->newPhysReg);
909        }
910
911        historyBuffer[tid].erase(hb_it++);
912
913        ++renameUndoneMaps;
914    }
915}
916
917template<class Impl>
918void
919DefaultRename<Impl>::removeFromHistory(InstSeqNum inst_seq_num, ThreadID tid)
920{
921    DPRINTF(Rename, "[tid:%u]: Removing a committed instruction from the "
922            "history buffer %u (size=%i), until [sn:%lli].\n",
923            tid, tid, historyBuffer[tid].size(), inst_seq_num);
924
925    typename std::list<RenameHistory>::iterator hb_it =
926        historyBuffer[tid].end();
927
928    --hb_it;
929
930    if (historyBuffer[tid].empty()) {
931        DPRINTF(Rename, "[tid:%u]: History buffer is empty.\n", tid);
932        return;
933    } else if (hb_it->instSeqNum > inst_seq_num) {
934        DPRINTF(Rename, "[tid:%u]: Old sequence number encountered.  Ensure "
935                "that a syscall happened recently.\n", tid);
936        return;
937    }
938
939    // Commit all the renames up until (and including) the committed sequence
940    // number. Some or even all of the committed instructions may not have
941    // rename histories if they did not have destination registers that were
942    // renamed.
943    while (!historyBuffer[tid].empty() &&
944           hb_it != historyBuffer[tid].end() &&
945           (*hb_it).instSeqNum <= inst_seq_num) {
946
947        DPRINTF(Rename, "[tid:%u]: Freeing up older rename of reg %i, "
948                "[sn:%lli].\n",
949                tid, (*hb_it).prevPhysReg, (*hb_it).instSeqNum);
950
951        freeList->addReg((*hb_it).prevPhysReg);
952        ++renameCommittedMaps;
953
954        historyBuffer[tid].erase(hb_it--);
955    }
956}
957
958template <class Impl>
959inline void
960DefaultRename<Impl>::renameSrcRegs(DynInstPtr &inst, ThreadID tid)
961{
962    assert(renameMap[tid] != 0);
963
964    unsigned num_src_regs = inst->numSrcRegs();
965
966    // Get the architectual register numbers from the source and
967    // destination operands, and redirect them to the right register.
968    // Will need to mark dependencies though.
969    for (int src_idx = 0; src_idx < num_src_regs; src_idx++) {
970        RegIndex src_reg = inst->srcRegIdx(src_idx);
971        RegIndex flat_src_reg = src_reg;
972        if (src_reg < TheISA::FP_Base_DepTag) {
973            flat_src_reg = inst->tcBase()->flattenIntIndex(src_reg);
974            DPRINTF(Rename, "Flattening index %d to %d.\n",
975                    (int)src_reg, (int)flat_src_reg);
976        } else if (src_reg < TheISA::Ctrl_Base_DepTag) {
977            src_reg = src_reg - TheISA::FP_Base_DepTag;
978            flat_src_reg = inst->tcBase()->flattenFloatIndex(src_reg);
979            DPRINTF(Rename, "Flattening index %d to %d.\n",
980                    (int)src_reg, (int)flat_src_reg);
981            flat_src_reg += TheISA::NumIntRegs;
982        } else if (src_reg < TheISA::Max_DepTag) {
983            flat_src_reg = src_reg - TheISA::Ctrl_Base_DepTag +
984                           TheISA::NumFloatRegs + TheISA::NumIntRegs;
985            DPRINTF(Rename, "Adjusting reg index from %d to %d.\n",
986                    src_reg, flat_src_reg);
987        } else {
988            panic("Reg index is out of bound: %d.", src_reg);
989        }
990
991        inst->flattenSrcReg(src_idx, flat_src_reg);
992
993        // Look up the source registers to get the phys. register they've
994        // been renamed to, and set the sources to those registers.
995        PhysRegIndex renamed_reg = renameMap[tid]->lookup(flat_src_reg);
996
997        DPRINTF(Rename, "[tid:%u]: Looking up arch reg %i, got "
998                "physical reg %i.\n", tid, (int)flat_src_reg,
999                (int)renamed_reg);
1000
1001        inst->renameSrcReg(src_idx, renamed_reg);
1002
1003        // See if the register is ready or not.
1004        if (scoreboard->getReg(renamed_reg) == true) {
1005            DPRINTF(Rename, "[tid:%u]: Register %d is ready.\n",
1006                    tid, renamed_reg);
1007
1008            inst->markSrcRegReady(src_idx);
1009        } else {
1010            DPRINTF(Rename, "[tid:%u]: Register %d is not ready.\n",
1011                    tid, renamed_reg);
1012        }
1013
1014        ++renameRenameLookups;
1015    }
1016}
1017
1018template <class Impl>
1019inline void
1020DefaultRename<Impl>::renameDestRegs(DynInstPtr &inst, ThreadID tid)
1021{
1022    typename RenameMap::RenameInfo rename_result;
1023
1024    unsigned num_dest_regs = inst->numDestRegs();
1025
1026    // Rename the destination registers.
1027    for (int dest_idx = 0; dest_idx < num_dest_regs; dest_idx++) {
1028        RegIndex dest_reg = inst->destRegIdx(dest_idx);
1029        RegIndex flat_dest_reg = dest_reg;
1030        if (dest_reg < TheISA::FP_Base_DepTag) {
1031            // Integer registers are flattened.
1032            flat_dest_reg = inst->tcBase()->flattenIntIndex(dest_reg);
1033            DPRINTF(Rename, "Flattening index %d to %d.\n",
1034                    (int)dest_reg, (int)flat_dest_reg);
1035        } else if (dest_reg < TheISA::Ctrl_Base_DepTag) {
1036            dest_reg = dest_reg - TheISA::FP_Base_DepTag;
1037            flat_dest_reg = inst->tcBase()->flattenFloatIndex(dest_reg);
1038            DPRINTF(Rename, "Flattening index %d to %d.\n",
1039                    (int)dest_reg, (int)flat_dest_reg);
1040            flat_dest_reg += TheISA::NumIntRegs;
1041        } else if (dest_reg < TheISA::Max_DepTag) {
1042            // Floating point and Miscellaneous registers need their indexes
1043            // adjusted to account for the expanded number of flattened int regs.
1044            flat_dest_reg = dest_reg - TheISA::Ctrl_Base_DepTag +
1045                            TheISA::NumIntRegs + TheISA::NumFloatRegs;
1046            DPRINTF(Rename, "Adjusting reg index from %d to %d.\n",
1047                    dest_reg, flat_dest_reg);
1048        } else {
1049            panic("Reg index is out of bound: %d.", dest_reg);
1050        }
1051
1052        inst->flattenDestReg(dest_idx, flat_dest_reg);
1053
1054        // Get the physical register that the destination will be
1055        // renamed to.
1056        rename_result = renameMap[tid]->rename(flat_dest_reg);
1057
1058        //Mark Scoreboard entry as not ready
1059        if (dest_reg < TheISA::Ctrl_Base_DepTag)
1060            scoreboard->unsetReg(rename_result.first);
1061
1062        DPRINTF(Rename, "[tid:%u]: Renaming arch reg %i to physical "
1063                "reg %i.\n", tid, (int)flat_dest_reg,
1064                (int)rename_result.first);
1065
1066        // Record the rename information so that a history can be kept.
1067        RenameHistory hb_entry(inst->seqNum, flat_dest_reg,
1068                               rename_result.first,
1069                               rename_result.second);
1070
1071        historyBuffer[tid].push_front(hb_entry);
1072
1073        DPRINTF(Rename, "[tid:%u]: Adding instruction to history buffer "
1074                "(size=%i), [sn:%lli].\n",tid,
1075                historyBuffer[tid].size(),
1076                (*historyBuffer[tid].begin()).instSeqNum);
1077
1078        // Tell the instruction to rename the appropriate destination
1079        // register (dest_idx) to the new physical register
1080        // (rename_result.first), and record the previous physical
1081        // register that the same logical register was renamed to
1082        // (rename_result.second).
1083        inst->renameDestReg(dest_idx,
1084                            rename_result.first,
1085                            rename_result.second);
1086
1087        ++renameRenamedOperands;
1088    }
1089}
1090
1091template <class Impl>
1092inline int
1093DefaultRename<Impl>::calcFreeROBEntries(ThreadID tid)
1094{
1095    int num_free = freeEntries[tid].robEntries -
1096                  (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatched);
1097
1098    //DPRINTF(Rename,"[tid:%i]: %i rob free\n",tid,num_free);
1099
1100    return num_free;
1101}
1102
1103template <class Impl>
1104inline int
1105DefaultRename<Impl>::calcFreeIQEntries(ThreadID tid)
1106{
1107    int num_free = freeEntries[tid].iqEntries -
1108                  (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatched);
1109
1110    //DPRINTF(Rename,"[tid:%i]: %i iq free\n",tid,num_free);
1111
1112    return num_free;
1113}
1114
1115template <class Impl>
1116inline int
1117DefaultRename<Impl>::calcFreeLSQEntries(ThreadID tid)
1118{
1119    int num_free = freeEntries[tid].lsqEntries -
1120                  (instsInProgress[tid] - fromIEW->iewInfo[tid].dispatchedToLSQ);
1121
1122    //DPRINTF(Rename,"[tid:%i]: %i lsq free\n",tid,num_free);
1123
1124    return num_free;
1125}
1126
1127template <class Impl>
1128unsigned
1129DefaultRename<Impl>::validInsts()
1130{
1131    unsigned inst_count = 0;
1132
1133    for (int i=0; i<fromDecode->size; i++) {
1134        if (!fromDecode->insts[i]->isSquashed())
1135            inst_count++;
1136    }
1137
1138    return inst_count;
1139}
1140
1141template <class Impl>
1142void
1143DefaultRename<Impl>::readStallSignals(ThreadID tid)
1144{
1145    if (fromIEW->iewBlock[tid]) {
1146        stalls[tid].iew = true;
1147    }
1148
1149    if (fromIEW->iewUnblock[tid]) {
1150        assert(stalls[tid].iew);
1151        stalls[tid].iew = false;
1152    }
1153
1154    if (fromCommit->commitBlock[tid]) {
1155        stalls[tid].commit = true;
1156    }
1157
1158    if (fromCommit->commitUnblock[tid]) {
1159        assert(stalls[tid].commit);
1160        stalls[tid].commit = false;
1161    }
1162}
1163
1164template <class Impl>
1165bool
1166DefaultRename<Impl>::checkStall(ThreadID tid)
1167{
1168    bool ret_val = false;
1169
1170    if (stalls[tid].iew) {
1171        DPRINTF(Rename,"[tid:%i]: Stall from IEW stage detected.\n", tid);
1172        ret_val = true;
1173    } else if (stalls[tid].commit) {
1174        DPRINTF(Rename,"[tid:%i]: Stall from Commit stage detected.\n", tid);
1175        ret_val = true;
1176    } else if (calcFreeROBEntries(tid) <= 0) {
1177        DPRINTF(Rename,"[tid:%i]: Stall: ROB has 0 free entries.\n", tid);
1178        ret_val = true;
1179    } else if (calcFreeIQEntries(tid) <= 0) {
1180        DPRINTF(Rename,"[tid:%i]: Stall: IQ has 0 free entries.\n", tid);
1181        ret_val = true;
1182    } else if (calcFreeLSQEntries(tid) <= 0) {
1183        DPRINTF(Rename,"[tid:%i]: Stall: LSQ has 0 free entries.\n", tid);
1184        ret_val = true;
1185    } else if (renameMap[tid]->numFreeEntries() <= 0) {
1186        DPRINTF(Rename,"[tid:%i]: Stall: RenameMap has 0 free entries.\n", tid);
1187        ret_val = true;
1188    } else if (renameStatus[tid] == SerializeStall &&
1189               (!emptyROB[tid] || instsInProgress[tid])) {
1190        DPRINTF(Rename,"[tid:%i]: Stall: Serialize stall and ROB is not "
1191                "empty.\n",
1192                tid);
1193        ret_val = true;
1194    }
1195
1196    return ret_val;
1197}
1198
1199template <class Impl>
1200void
1201DefaultRename<Impl>::readFreeEntries(ThreadID tid)
1202{
1203    bool updated = false;
1204    if (fromIEW->iewInfo[tid].usedIQ) {
1205        freeEntries[tid].iqEntries =
1206            fromIEW->iewInfo[tid].freeIQEntries;
1207        updated = true;
1208    }
1209
1210    if (fromIEW->iewInfo[tid].usedLSQ) {
1211        freeEntries[tid].lsqEntries =
1212            fromIEW->iewInfo[tid].freeLSQEntries;
1213        updated = true;
1214    }
1215
1216    if (fromCommit->commitInfo[tid].usedROB) {
1217        freeEntries[tid].robEntries =
1218            fromCommit->commitInfo[tid].freeROBEntries;
1219        emptyROB[tid] = fromCommit->commitInfo[tid].emptyROB;
1220        updated = true;
1221    }
1222
1223    DPRINTF(Rename, "[tid:%i]: Free IQ: %i, Free ROB: %i, Free LSQ: %i\n",
1224            tid,
1225            freeEntries[tid].iqEntries,
1226            freeEntries[tid].robEntries,
1227            freeEntries[tid].lsqEntries);
1228
1229    DPRINTF(Rename, "[tid:%i]: %i instructions not yet in ROB\n",
1230            tid, instsInProgress[tid]);
1231}
1232
1233template <class Impl>
1234bool
1235DefaultRename<Impl>::checkSignalsAndUpdate(ThreadID tid)
1236{
1237    // Check if there's a squash signal, squash if there is
1238    // Check stall signals, block if necessary.
1239    // If status was blocked
1240    //     check if stall conditions have passed
1241    //         if so then go to unblocking
1242    // If status was Squashing
1243    //     check if squashing is not high.  Switch to running this cycle.
1244    // If status was serialize stall
1245    //     check if ROB is empty and no insts are in flight to the ROB
1246
1247    readFreeEntries(tid);
1248    readStallSignals(tid);
1249
1250    if (fromCommit->commitInfo[tid].squash) {
1251        DPRINTF(Rename, "[tid:%u]: Squashing instructions due to squash from "
1252                "commit.\n", tid);
1253
1254        squash(fromCommit->commitInfo[tid].doneSeqNum, tid);
1255
1256        return true;
1257    }
1258
1259    if (fromCommit->commitInfo[tid].robSquashing) {
1260        DPRINTF(Rename, "[tid:%u]: ROB is still squashing.\n", tid);
1261
1262        renameStatus[tid] = Squashing;
1263
1264        return true;
1265    }
1266
1267    if (checkStall(tid)) {
1268        return block(tid);
1269    }
1270
1271    if (renameStatus[tid] == Blocked) {
1272        DPRINTF(Rename, "[tid:%u]: Done blocking, switching to unblocking.\n",
1273                tid);
1274
1275        renameStatus[tid] = Unblocking;
1276
1277        unblock(tid);
1278
1279        return true;
1280    }
1281
1282    if (renameStatus[tid] == Squashing) {
1283        // Switch status to running if rename isn't being told to block or
1284        // squash this cycle.
1285        if (resumeSerialize) {
1286            DPRINTF(Rename, "[tid:%u]: Done squashing, switching to serialize.\n",
1287                    tid);
1288
1289            renameStatus[tid] = SerializeStall;
1290            return true;
1291        } else if (resumeUnblocking) {
1292            DPRINTF(Rename, "[tid:%u]: Done squashing, switching to unblocking.\n",
1293                    tid);
1294            renameStatus[tid] = Unblocking;
1295            return true;
1296        } else {
1297            DPRINTF(Rename, "[tid:%u]: Done squashing, switching to running.\n",
1298                    tid);
1299
1300            renameStatus[tid] = Running;
1301            return false;
1302        }
1303    }
1304
1305    if (renameStatus[tid] == SerializeStall) {
1306        // Stall ends once the ROB is free.
1307        DPRINTF(Rename, "[tid:%u]: Done with serialize stall, switching to "
1308                "unblocking.\n", tid);
1309
1310        DynInstPtr serial_inst = serializeInst[tid];
1311
1312        renameStatus[tid] = Unblocking;
1313
1314        unblock(tid);
1315
1316        DPRINTF(Rename, "[tid:%u]: Processing instruction [%lli] with "
1317                "PC %s.\n", tid, serial_inst->seqNum, serial_inst->pcState());
1318
1319        // Put instruction into queue here.
1320        serial_inst->clearSerializeBefore();
1321
1322        if (!skidBuffer[tid].empty()) {
1323            skidBuffer[tid].push_front(serial_inst);
1324        } else {
1325            insts[tid].push_front(serial_inst);
1326        }
1327
1328        DPRINTF(Rename, "[tid:%u]: Instruction must be processed by rename."
1329                " Adding to front of list.\n", tid);
1330
1331        serializeInst[tid] = NULL;
1332
1333        return true;
1334    }
1335
1336    // If we've reached this point, we have not gotten any signals that
1337    // cause rename to change its status.  Rename remains the same as before.
1338    return false;
1339}
1340
1341template<class Impl>
1342void
1343DefaultRename<Impl>::serializeAfter(InstQueue &inst_list, ThreadID tid)
1344{
1345    if (inst_list.empty()) {
1346        // Mark a bit to say that I must serialize on the next instruction.
1347        serializeOnNextInst[tid] = true;
1348        return;
1349    }
1350
1351    // Set the next instruction as serializing.
1352    inst_list.front()->setSerializeBefore();
1353}
1354
1355template <class Impl>
1356inline void
1357DefaultRename<Impl>::incrFullStat(const FullSource &source)
1358{
1359    switch (source) {
1360      case ROB:
1361        ++renameROBFullEvents;
1362        break;
1363      case IQ:
1364        ++renameIQFullEvents;
1365        break;
1366      case LSQ:
1367        ++renameLSQFullEvents;
1368        break;
1369      default:
1370        panic("Rename full stall stat should be incremented for a reason!");
1371        break;
1372    }
1373}
1374
1375template <class Impl>
1376void
1377DefaultRename<Impl>::dumpHistory()
1378{
1379    typename std::list<RenameHistory>::iterator buf_it;
1380
1381    for (ThreadID tid = 0; tid < numThreads; tid++) {
1382
1383        buf_it = historyBuffer[tid].begin();
1384
1385        while (buf_it != historyBuffer[tid].end()) {
1386            cprintf("Seq num: %i\nArch reg: %i New phys reg: %i Old phys "
1387                    "reg: %i\n", (*buf_it).instSeqNum, (int)(*buf_it).archReg,
1388                    (int)(*buf_it).newPhysReg, (int)(*buf_it).prevPhysReg);
1389
1390            buf_it++;
1391        }
1392    }
1393}
1394