base.cc revision 11839
1/*
2 * Copyright (c) 2012, 2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66    : BaseCPU(params),
67      vm(*params->system->getKvmVM()),
68      _status(Idle),
69      dataPort(name() + ".dcache_port", this),
70      instPort(name() + ".icache_port", this),
71      alwaysSyncTC(params->alwaysSyncTC),
72      threadContextDirty(true),
73      kvmStateDirty(false),
74      vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75      _kvmRun(NULL), mmioRing(NULL),
76      pageSize(sysconf(_SC_PAGE_SIZE)),
77      tickEvent(*this),
78      activeInstPeriod(0),
79      perfControlledByTimer(params->usePerfOverflow),
80      hostFactor(params->hostFactor),
81      ctrInsts(0)
82{
83    if (pageSize == -1)
84        panic("KVM: Failed to determine host page size (%i)\n",
85              errno);
86
87    if (FullSystem)
88        thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
89                                  params->isa[0]);
90    else
91        thread = new SimpleThread(this, /* thread_num */ 0, params->system,
92                                  params->workload[0], params->itb,
93                                  params->dtb, params->isa[0]);
94
95    thread->setStatus(ThreadContext::Halted);
96    tc = thread->getTC();
97    threadContexts.push_back(tc);
98}
99
100BaseKvmCPU::~BaseKvmCPU()
101{
102    if (_kvmRun)
103        munmap(_kvmRun, vcpuMMapSize);
104    close(vcpuFD);
105}
106
107void
108BaseKvmCPU::init()
109{
110    BaseCPU::init();
111
112    if (numThreads != 1)
113        fatal("KVM: Multithreading not supported");
114
115    tc->initMemProxies(tc);
116
117    // initialize CPU, including PC
118    if (FullSystem && !switchedOut())
119        TheISA::initCPU(tc, tc->contextId());
120}
121
122void
123BaseKvmCPU::startup()
124{
125    const BaseKvmCPUParams * const p(
126        dynamic_cast<const BaseKvmCPUParams *>(params()));
127
128    Kvm &kvm(*vm.kvm);
129
130    BaseCPU::startup();
131
132    assert(vcpuFD == -1);
133
134    // Tell the VM that a CPU is about to start.
135    vm.cpuStartup();
136
137    // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
138    // not guaranteed that the parent KVM VM has initialized at that
139    // point. Initialize virtual CPUs here instead.
140    vcpuFD = vm.createVCPU(vcpuID);
141
142    // Map the KVM run structure */
143    vcpuMMapSize = kvm.getVCPUMMapSize();
144    _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
145                                     PROT_READ | PROT_WRITE, MAP_SHARED,
146                                     vcpuFD, 0);
147    if (_kvmRun == MAP_FAILED)
148        panic("KVM: Failed to map run data structure\n");
149
150    // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
151    // available. The offset into the KVM's communication page is
152    // provided by the coalesced MMIO capability.
153    int mmioOffset(kvm.capCoalescedMMIO());
154    if (!p->useCoalescedMMIO) {
155        inform("KVM: Coalesced MMIO disabled by config.\n");
156    } else if (mmioOffset) {
157        inform("KVM: Coalesced IO available\n");
158        mmioRing = (struct kvm_coalesced_mmio_ring *)(
159            (char *)_kvmRun + (mmioOffset * pageSize));
160    } else {
161        inform("KVM: Coalesced not supported by host OS\n");
162    }
163
164    thread->startup();
165
166    Event *startupEvent(
167        new EventWrapper<BaseKvmCPU,
168                         &BaseKvmCPU::startupThread>(this, true));
169    schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175    return (activeMMIOReqs || pendingMMIOPkts.size())
176        ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182    if (cpu->system->isAtomicMode()) {
183        Tick delay = sendAtomic(pkt);
184        delete pkt->req;
185        delete pkt;
186        return delay;
187    } else {
188        if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189            activeMMIOReqs++;
190        } else {
191            pendingMMIOPkts.push(pkt);
192        }
193        // Return value is irrelevant for timing-mode accesses.
194        return 0;
195    }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201    DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
203    delete pkt->req;
204    delete pkt;
205    activeMMIOReqs--;
206
207    // We can switch back into KVM when all pending and in-flight MMIO
208    // operations have completed.
209    if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210        DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211        cpu->finishMMIOPending();
212    }
213    return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219    DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221    assert(pendingMMIOPkts.size());
222
223    // Assuming that we can issue infinite requests this cycle is a bit
224    // unrealistic, but it's not worth modeling something more complex in
225    // KVM.
226    while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227        pendingMMIOPkts.pop();
228        activeMMIOReqs++;
229    }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235    assert(_status = RunningMMIOPending);
236    assert(!tickEvent.scheduled());
237
238    _status = RunningServiceCompletion;
239    schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245    // Do thread-specific initialization. We need to setup signal
246    // delivery for counters and timers from within the thread that
247    // will execute the event queue to ensure that signals are
248    // delivered to the right threads.
249    const BaseKvmCPUParams * const p(
250        dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252    vcpuThread = pthread_self();
253
254    // Setup signal handlers. This has to be done after the vCPU is
255    // created since it manipulates the vCPU signal mask.
256    setupSignalHandler();
257
258    setupCounters();
259
260    if (p->usePerfOverflow)
261        runTimer.reset(new PerfKvmTimer(hwCycles,
262                                        KVM_KICK_SIGNAL,
263                                        p->hostFactor,
264                                        p->hostFreq));
265    else
266        runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267                                         p->hostFactor,
268                                         p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275    using namespace Stats;
276
277    BaseCPU::regStats();
278
279    numInsts
280        .name(name() + ".committedInsts")
281        .desc("Number of instructions committed")
282        ;
283
284    numVMExits
285        .name(name() + ".numVMExits")
286        .desc("total number of KVM exits")
287        ;
288
289    numVMHalfEntries
290        .name(name() + ".numVMHalfEntries")
291        .desc("number of KVM entries to finalize pending operations")
292        ;
293
294    numExitSignal
295        .name(name() + ".numExitSignal")
296        .desc("exits due to signal delivery")
297        ;
298
299    numMMIO
300        .name(name() + ".numMMIO")
301        .desc("number of VM exits due to memory mapped IO")
302        ;
303
304    numCoalescedMMIO
305        .name(name() + ".numCoalescedMMIO")
306        .desc("number of coalesced memory mapped IO requests")
307        ;
308
309    numIO
310        .name(name() + ".numIO")
311        .desc("number of VM exits due to legacy IO")
312        ;
313
314    numHalt
315        .name(name() + ".numHalt")
316        .desc("number of VM exits due to wait for interrupt instructions")
317        ;
318
319    numInterrupts
320        .name(name() + ".numInterrupts")
321        .desc("number of interrupts delivered")
322        ;
323
324    numHypercalls
325        .name(name() + ".numHypercalls")
326        .desc("number of hypercalls")
327        ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333    if (DTRACE(Checkpoint)) {
334        DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335        dump();
336    }
337
338    assert(tid == 0);
339    assert(_status == Idle);
340    thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346    DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348    assert(tid == 0);
349    assert(_status == Idle);
350    thread->unserialize(cp);
351    threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357    if (switchedOut())
358        return DrainState::Drained;
359
360    DPRINTF(Drain, "BaseKvmCPU::drain\n");
361    switch (_status) {
362      case Running:
363        // The base KVM code is normally ready when it is in the
364        // Running state, but the architecture specific code might be
365        // of a different opinion. This may happen when the CPU been
366        // notified of an event that hasn't been accepted by the vCPU
367        // yet.
368        if (!archIsDrained())
369            return DrainState::Draining;
370
371        // The state of the CPU is consistent, so we don't need to do
372        // anything special to drain it. We simply de-schedule the
373        // tick event and enter the Idle state to prevent nasty things
374        // like MMIOs from happening.
375        if (tickEvent.scheduled())
376            deschedule(tickEvent);
377        _status = Idle;
378
379        /** FALLTHROUGH */
380      case Idle:
381        // Idle, no need to drain
382        assert(!tickEvent.scheduled());
383
384        // Sync the thread context here since we'll need it when we
385        // switch CPUs or checkpoint the CPU.
386        syncThreadContext();
387
388        return DrainState::Drained;
389
390      case RunningServiceCompletion:
391        // The CPU has just requested a service that was handled in
392        // the RunningService state, but the results have still not
393        // been reported to the CPU. Now, we /could/ probably just
394        // update the register state ourselves instead of letting KVM
395        // handle it, but that would be tricky. Instead, we enter KVM
396        // and let it do its stuff.
397        DPRINTF(Drain, "KVM CPU is waiting for service completion, "
398                "requesting drain.\n");
399        return DrainState::Draining;
400
401      case RunningMMIOPending:
402        // We need to drain since there are in-flight timing accesses
403        DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
404                "requesting drain.\n");
405        return DrainState::Draining;
406
407      case RunningService:
408        // We need to drain since the CPU is waiting for service (e.g., MMIOs)
409        DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
410        return DrainState::Draining;
411
412      default:
413        panic("KVM: Unhandled CPU state in drain()\n");
414        return DrainState::Drained;
415    }
416}
417
418void
419BaseKvmCPU::drainResume()
420{
421    assert(!tickEvent.scheduled());
422
423    // We might have been switched out. In that case, we don't need to
424    // do anything.
425    if (switchedOut())
426        return;
427
428    DPRINTF(Kvm, "drainResume\n");
429    verifyMemoryMode();
430
431    // The tick event is de-scheduled as a part of the draining
432    // process. Re-schedule it if the thread context is active.
433    if (tc->status() == ThreadContext::Active) {
434        schedule(tickEvent, nextCycle());
435        _status = Running;
436    } else {
437        _status = Idle;
438    }
439}
440
441void
442BaseKvmCPU::notifyFork()
443{
444    // We should have drained prior to forking, which means that the
445    // tick event shouldn't be scheduled and the CPU is idle.
446    assert(!tickEvent.scheduled());
447    assert(_status == Idle);
448
449    if (vcpuFD != -1) {
450        if (close(vcpuFD) == -1)
451            warn("kvm CPU: notifyFork failed to close vcpuFD\n");
452
453        if (_kvmRun)
454            munmap(_kvmRun, vcpuMMapSize);
455
456        vcpuFD = -1;
457        _kvmRun = NULL;
458
459        hwInstructions.detach();
460        hwCycles.detach();
461    }
462}
463
464void
465BaseKvmCPU::switchOut()
466{
467    DPRINTF(Kvm, "switchOut\n");
468
469    BaseCPU::switchOut();
470
471    // We should have drained prior to executing a switchOut, which
472    // means that the tick event shouldn't be scheduled and the CPU is
473    // idle.
474    assert(!tickEvent.scheduled());
475    assert(_status == Idle);
476}
477
478void
479BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
480{
481    DPRINTF(Kvm, "takeOverFrom\n");
482
483    BaseCPU::takeOverFrom(cpu);
484
485    // We should have drained prior to executing a switchOut, which
486    // means that the tick event shouldn't be scheduled and the CPU is
487    // idle.
488    assert(!tickEvent.scheduled());
489    assert(_status == Idle);
490    assert(threadContexts.size() == 1);
491
492    // Force an update of the KVM state here instead of flagging the
493    // TC as dirty. This is not ideal from a performance point of
494    // view, but it makes debugging easier as it allows meaningful KVM
495    // state to be dumped before and after a takeover.
496    updateKvmState();
497    threadContextDirty = false;
498}
499
500void
501BaseKvmCPU::verifyMemoryMode() const
502{
503    if (!(system->bypassCaches())) {
504        fatal("The KVM-based CPUs requires the memory system to be in the "
505              "'noncaching' mode.\n");
506    }
507}
508
509void
510BaseKvmCPU::wakeup(ThreadID tid)
511{
512    DPRINTF(Kvm, "wakeup()\n");
513    // This method might have been called from another
514    // context. Migrate to this SimObject's event queue when
515    // delivering the wakeup signal.
516    EventQueue::ScopedMigration migrate(eventQueue());
517
518    // Kick the vCPU to get it to come out of KVM.
519    kick();
520
521    if (thread->status() != ThreadContext::Suspended)
522        return;
523
524    thread->activate();
525}
526
527void
528BaseKvmCPU::activateContext(ThreadID thread_num)
529{
530    DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
531
532    assert(thread_num == 0);
533    assert(thread);
534
535    assert(_status == Idle);
536    assert(!tickEvent.scheduled());
537
538    numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
539
540    schedule(tickEvent, clockEdge(Cycles(0)));
541    _status = Running;
542}
543
544
545void
546BaseKvmCPU::suspendContext(ThreadID thread_num)
547{
548    DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
549
550    assert(thread_num == 0);
551    assert(thread);
552
553    if (_status == Idle)
554        return;
555
556    assert(_status == Running || _status == RunningServiceCompletion);
557
558    // The tick event may no be scheduled if the quest has requested
559    // the monitor to wait for interrupts. The normal CPU models can
560    // get their tick events descheduled by quiesce instructions, but
561    // that can't happen here.
562    if (tickEvent.scheduled())
563        deschedule(tickEvent);
564
565    _status = Idle;
566}
567
568void
569BaseKvmCPU::deallocateContext(ThreadID thread_num)
570{
571    // for now, these are equivalent
572    suspendContext(thread_num);
573}
574
575void
576BaseKvmCPU::haltContext(ThreadID thread_num)
577{
578    // for now, these are equivalent
579    suspendContext(thread_num);
580}
581
582ThreadContext *
583BaseKvmCPU::getContext(int tn)
584{
585    assert(tn == 0);
586    syncThreadContext();
587    return tc;
588}
589
590
591Counter
592BaseKvmCPU::totalInsts() const
593{
594    return ctrInsts;
595}
596
597Counter
598BaseKvmCPU::totalOps() const
599{
600    hack_once("Pretending totalOps is equivalent to totalInsts()\n");
601    return ctrInsts;
602}
603
604void
605BaseKvmCPU::dump() const
606{
607    inform("State dumping not implemented.");
608}
609
610void
611BaseKvmCPU::tick()
612{
613    Tick delay(0);
614    assert(_status != Idle && _status != RunningMMIOPending);
615
616    switch (_status) {
617      case RunningService:
618        // handleKvmExit() will determine the next state of the CPU
619        delay = handleKvmExit();
620
621        if (tryDrain())
622            _status = Idle;
623        break;
624
625      case RunningServiceCompletion:
626      case Running: {
627          const uint64_t nextInstEvent(
628              !comInstEventQueue[0]->empty() ?
629              comInstEventQueue[0]->nextTick() : UINT64_MAX);
630          // Enter into KVM and complete pending IO instructions if we
631          // have an instruction event pending.
632          const Tick ticksToExecute(
633              nextInstEvent > ctrInsts ?
634              curEventQueue()->nextTick() - curTick() : 0);
635
636          if (alwaysSyncTC)
637              threadContextDirty = true;
638
639          // We might need to update the KVM state.
640          syncKvmState();
641
642          // Setup any pending instruction count breakpoints using
643          // PerfEvent if we are going to execute more than just an IO
644          // completion.
645          if (ticksToExecute > 0)
646              setupInstStop();
647
648          DPRINTF(KvmRun, "Entering KVM...\n");
649          if (drainState() == DrainState::Draining) {
650              // Force an immediate exit from KVM after completing
651              // pending operations. The architecture-specific code
652              // takes care to run until it is in a state where it can
653              // safely be drained.
654              delay = kvmRunDrain();
655          } else {
656              delay = kvmRun(ticksToExecute);
657          }
658
659          // The CPU might have been suspended before entering into
660          // KVM. Assume that the CPU was suspended /before/ entering
661          // into KVM and skip the exit handling.
662          if (_status == Idle)
663              break;
664
665          // Entering into KVM implies that we'll have to reload the thread
666          // context from KVM if we want to access it. Flag the KVM state as
667          // dirty with respect to the cached thread context.
668          kvmStateDirty = true;
669
670          if (alwaysSyncTC)
671              syncThreadContext();
672
673          // Enter into the RunningService state unless the
674          // simulation was stopped by a timer.
675          if (_kvmRun->exit_reason !=  KVM_EXIT_INTR) {
676              _status = RunningService;
677          } else {
678              ++numExitSignal;
679              _status = Running;
680          }
681
682          // Service any pending instruction events. The vCPU should
683          // have exited in time for the event using the instruction
684          // counter configured by setupInstStop().
685          comInstEventQueue[0]->serviceEvents(ctrInsts);
686          system->instEventQueue.serviceEvents(system->totalNumInsts);
687
688          if (tryDrain())
689              _status = Idle;
690      } break;
691
692      default:
693        panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
694              _status);
695    }
696
697    // Schedule a new tick if we are still running
698    if (_status != Idle && _status != RunningMMIOPending)
699        schedule(tickEvent, clockEdge(ticksToCycles(delay)));
700}
701
702Tick
703BaseKvmCPU::kvmRunDrain()
704{
705    // By default, the only thing we need to drain is a pending IO
706    // operation which assumes that we are in the
707    // RunningServiceCompletion or RunningMMIOPending state.
708    assert(_status == RunningServiceCompletion ||
709           _status == RunningMMIOPending);
710
711    // Deliver the data from the pending IO operation and immediately
712    // exit.
713    return kvmRun(0);
714}
715
716uint64_t
717BaseKvmCPU::getHostCycles() const
718{
719    return hwCycles.read();
720}
721
722Tick
723BaseKvmCPU::kvmRun(Tick ticks)
724{
725    Tick ticksExecuted;
726    fatal_if(vcpuFD == -1,
727             "Trying to run a KVM CPU in a forked child process. "
728             "This is not supported.\n");
729    DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
730
731    if (ticks == 0) {
732        // Settings ticks == 0 is a special case which causes an entry
733        // into KVM that finishes pending operations (e.g., IO) and
734        // then immediately exits.
735        DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
736
737        ++numVMHalfEntries;
738
739        // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
740        // thread). The KVM control signal is masked while executing
741        // in gem5 and gets unmasked temporarily as when entering
742        // KVM. See setSignalMask() and setupSignalHandler().
743        kick();
744
745        // Start the vCPU. KVM will check for signals after completing
746        // pending operations (IO). Since the KVM_KICK_SIGNAL is
747        // pending, this forces an immediate exit to gem5 again. We
748        // don't bother to setup timers since this shouldn't actually
749        // execute any code (other than completing half-executed IO
750        // instructions) in the guest.
751        ioctlRun();
752
753        // We always execute at least one cycle to prevent the
754        // BaseKvmCPU::tick() to be rescheduled on the same tick
755        // twice.
756        ticksExecuted = clockPeriod();
757    } else {
758        // This method is executed as a result of a tick event. That
759        // means that the event queue will be locked when entering the
760        // method. We temporarily unlock the event queue to allow
761        // other threads to steal control of this thread to inject
762        // interrupts. They will typically lock the queue and then
763        // force an exit from KVM by kicking the vCPU.
764        EventQueue::ScopedRelease release(curEventQueue());
765
766        if (ticks < runTimer->resolution()) {
767            DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
768                    ticks, runTimer->resolution());
769            ticks = runTimer->resolution();
770        }
771
772        // Get hardware statistics after synchronizing contexts. The KVM
773        // state update might affect guest cycle counters.
774        uint64_t baseCycles(getHostCycles());
775        uint64_t baseInstrs(hwInstructions.read());
776
777        // Arm the run timer and start the cycle timer if it isn't
778        // controlled by the overflow timer. Starting/stopping the cycle
779        // timer automatically starts the other perf timers as they are in
780        // the same counter group.
781        runTimer->arm(ticks);
782        if (!perfControlledByTimer)
783            hwCycles.start();
784
785        ioctlRun();
786
787        runTimer->disarm();
788        if (!perfControlledByTimer)
789            hwCycles.stop();
790
791        // The control signal may have been delivered after we exited
792        // from KVM. It will be pending in that case since it is
793        // masked when we aren't executing in KVM. Discard it to make
794        // sure we don't deliver it immediately next time we try to
795        // enter into KVM.
796        discardPendingSignal(KVM_KICK_SIGNAL);
797
798        const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
799        const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
800        const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
801        ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
802
803        /* Update statistics */
804        numCycles += simCyclesExecuted;;
805        numInsts += instsExecuted;
806        ctrInsts += instsExecuted;
807        system->totalNumInsts += instsExecuted;
808
809        DPRINTF(KvmRun,
810                "KVM: Executed %i instructions in %i cycles "
811                "(%i ticks, sim cycles: %i).\n",
812                instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
813    }
814
815    ++numVMExits;
816
817    return ticksExecuted + flushCoalescedMMIO();
818}
819
820void
821BaseKvmCPU::kvmNonMaskableInterrupt()
822{
823    ++numInterrupts;
824    if (ioctl(KVM_NMI) == -1)
825        panic("KVM: Failed to deliver NMI to virtual CPU\n");
826}
827
828void
829BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
830{
831    ++numInterrupts;
832    if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
833        panic("KVM: Failed to deliver interrupt to virtual CPU\n");
834}
835
836void
837BaseKvmCPU::getRegisters(struct kvm_regs &regs) const
838{
839    if (ioctl(KVM_GET_REGS, &regs) == -1)
840        panic("KVM: Failed to get guest registers\n");
841}
842
843void
844BaseKvmCPU::setRegisters(const struct kvm_regs &regs)
845{
846    if (ioctl(KVM_SET_REGS, (void *)&regs) == -1)
847        panic("KVM: Failed to set guest registers\n");
848}
849
850void
851BaseKvmCPU::getSpecialRegisters(struct kvm_sregs &regs) const
852{
853    if (ioctl(KVM_GET_SREGS, &regs) == -1)
854        panic("KVM: Failed to get guest special registers\n");
855}
856
857void
858BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs &regs)
859{
860    if (ioctl(KVM_SET_SREGS, (void *)&regs) == -1)
861        panic("KVM: Failed to set guest special registers\n");
862}
863
864void
865BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
866{
867    if (ioctl(KVM_GET_FPU, &state) == -1)
868        panic("KVM: Failed to get guest FPU state\n");
869}
870
871void
872BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
873{
874    if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
875        panic("KVM: Failed to set guest FPU state\n");
876}
877
878
879void
880BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
881{
882#ifdef KVM_SET_ONE_REG
883    struct kvm_one_reg reg;
884    reg.id = id;
885    reg.addr = (uint64_t)addr;
886
887    if (ioctl(KVM_SET_ONE_REG, &reg) == -1) {
888        panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
889              id, errno);
890    }
891#else
892    panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
893#endif
894}
895
896void
897BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
898{
899#ifdef KVM_GET_ONE_REG
900    struct kvm_one_reg reg;
901    reg.id = id;
902    reg.addr = (uint64_t)addr;
903
904    if (ioctl(KVM_GET_ONE_REG, &reg) == -1) {
905        panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
906              id, errno);
907    }
908#else
909    panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
910#endif
911}
912
913std::string
914BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
915{
916#ifdef KVM_GET_ONE_REG
917    std::ostringstream ss;
918
919    ss.setf(std::ios::hex, std::ios::basefield);
920    ss.setf(std::ios::showbase);
921#define HANDLE_INTTYPE(len)                      \
922    case KVM_REG_SIZE_U ## len: {                \
923        uint ## len ## _t value;                 \
924        getOneReg(id, &value);                   \
925        ss << value;                             \
926    }  break
927
928#define HANDLE_ARRAY(len)                               \
929    case KVM_REG_SIZE_U ## len: {                       \
930        uint8_t value[len / 8];                         \
931        getOneReg(id, value);                           \
932        ccprintf(ss, "[0x%x", value[0]);                \
933        for (int i = 1; i < len  / 8; ++i)              \
934            ccprintf(ss, ", 0x%x", value[i]);           \
935        ccprintf(ss, "]");                              \
936      } break
937
938    switch (id & KVM_REG_SIZE_MASK) {
939        HANDLE_INTTYPE(8);
940        HANDLE_INTTYPE(16);
941        HANDLE_INTTYPE(32);
942        HANDLE_INTTYPE(64);
943        HANDLE_ARRAY(128);
944        HANDLE_ARRAY(256);
945        HANDLE_ARRAY(512);
946        HANDLE_ARRAY(1024);
947      default:
948        ss << "??";
949    }
950
951#undef HANDLE_INTTYPE
952#undef HANDLE_ARRAY
953
954    return ss.str();
955#else
956    panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
957#endif
958}
959
960void
961BaseKvmCPU::syncThreadContext()
962{
963    if (!kvmStateDirty)
964        return;
965
966    assert(!threadContextDirty);
967
968    updateThreadContext();
969    kvmStateDirty = false;
970}
971
972void
973BaseKvmCPU::syncKvmState()
974{
975    if (!threadContextDirty)
976        return;
977
978    assert(!kvmStateDirty);
979
980    updateKvmState();
981    threadContextDirty = false;
982}
983
984Tick
985BaseKvmCPU::handleKvmExit()
986{
987    DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
988    assert(_status == RunningService);
989
990    // Switch into the running state by default. Individual handlers
991    // can override this.
992    _status = Running;
993    switch (_kvmRun->exit_reason) {
994      case KVM_EXIT_UNKNOWN:
995        return handleKvmExitUnknown();
996
997      case KVM_EXIT_EXCEPTION:
998        return handleKvmExitException();
999
1000      case KVM_EXIT_IO:
1001      {
1002        ++numIO;
1003        Tick ticks = handleKvmExitIO();
1004        _status = dataPort.nextIOState();
1005        return ticks;
1006      }
1007
1008      case KVM_EXIT_HYPERCALL:
1009        ++numHypercalls;
1010        return handleKvmExitHypercall();
1011
1012      case KVM_EXIT_HLT:
1013        /* The guest has halted and is waiting for interrupts */
1014        DPRINTF(Kvm, "handleKvmExitHalt\n");
1015        ++numHalt;
1016
1017        // Suspend the thread until the next interrupt arrives
1018        thread->suspend();
1019
1020        // This is actually ignored since the thread is suspended.
1021        return 0;
1022
1023      case KVM_EXIT_MMIO:
1024      {
1025        /* Service memory mapped IO requests */
1026        DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1027                _kvmRun->mmio.is_write,
1028                _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1029
1030        ++numMMIO;
1031        Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1032                                  _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1033        // doMMIOAccess could have triggered a suspend, in which case we don't
1034        // want to overwrite the _status.
1035        if (_status != Idle)
1036            _status = dataPort.nextIOState();
1037        return ticks;
1038      }
1039
1040      case KVM_EXIT_IRQ_WINDOW_OPEN:
1041        return handleKvmExitIRQWindowOpen();
1042
1043      case KVM_EXIT_FAIL_ENTRY:
1044        return handleKvmExitFailEntry();
1045
1046      case KVM_EXIT_INTR:
1047        /* KVM was interrupted by a signal, restart it in the next
1048         * tick. */
1049        return 0;
1050
1051      case KVM_EXIT_INTERNAL_ERROR:
1052        panic("KVM: Internal error (suberror: %u)\n",
1053              _kvmRun->internal.suberror);
1054
1055      default:
1056        dump();
1057        panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1058    }
1059}
1060
1061Tick
1062BaseKvmCPU::handleKvmExitIO()
1063{
1064    panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1065          _kvmRun->io.direction, _kvmRun->io.size,
1066          _kvmRun->io.port, _kvmRun->io.count);
1067}
1068
1069Tick
1070BaseKvmCPU::handleKvmExitHypercall()
1071{
1072    panic("KVM: Unhandled hypercall\n");
1073}
1074
1075Tick
1076BaseKvmCPU::handleKvmExitIRQWindowOpen()
1077{
1078    warn("KVM: Unhandled IRQ window.\n");
1079    return 0;
1080}
1081
1082
1083Tick
1084BaseKvmCPU::handleKvmExitUnknown()
1085{
1086    dump();
1087    panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1088          _kvmRun->hw.hardware_exit_reason);
1089}
1090
1091Tick
1092BaseKvmCPU::handleKvmExitException()
1093{
1094    dump();
1095    panic("KVM: Got exception when starting vCPU "
1096          "(exception: %u, error_code: %u)\n",
1097          _kvmRun->ex.exception, _kvmRun->ex.error_code);
1098}
1099
1100Tick
1101BaseKvmCPU::handleKvmExitFailEntry()
1102{
1103    dump();
1104    panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1105          _kvmRun->fail_entry.hardware_entry_failure_reason);
1106}
1107
1108Tick
1109BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1110{
1111    ThreadContext *tc(thread->getTC());
1112    syncThreadContext();
1113
1114    RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1115                                      dataMasterId());
1116    mmio_req->setContext(tc->contextId());
1117    // Some architectures do need to massage physical addresses a bit
1118    // before they are inserted into the memory system. This enables
1119    // APIC accesses on x86 and m5ops where supported through a MMIO
1120    // interface.
1121    BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1122    Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1123    if (fault != NoFault)
1124        warn("Finalization of MMIO address failed: %s\n", fault->name());
1125
1126
1127    const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1128    PacketPtr pkt = new Packet(mmio_req, cmd);
1129    pkt->dataStatic(data);
1130
1131    if (mmio_req->isMmappedIpr()) {
1132        // We currently assume that there is no need to migrate to a
1133        // different event queue when doing IPRs. Currently, IPRs are
1134        // only used for m5ops, so it should be a valid assumption.
1135        const Cycles ipr_delay(write ?
1136                             TheISA::handleIprWrite(tc, pkt) :
1137                             TheISA::handleIprRead(tc, pkt));
1138        threadContextDirty = true;
1139        delete pkt->req;
1140        delete pkt;
1141        return clockPeriod() * ipr_delay;
1142    } else {
1143        // Temporarily lock and migrate to the event queue of the
1144        // VM. This queue is assumed to "own" all devices we need to
1145        // access if running in multi-core mode.
1146        EventQueue::ScopedMigration migrate(vm.eventQueue());
1147
1148        return dataPort.submitIO(pkt);
1149    }
1150}
1151
1152void
1153BaseKvmCPU::setSignalMask(const sigset_t *mask)
1154{
1155    std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1156
1157    if (mask) {
1158        kvm_mask.reset((struct kvm_signal_mask *)operator new(
1159                           sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1160        // The kernel and the user-space headers have different ideas
1161        // about the size of sigset_t. This seems like a massive hack,
1162        // but is actually what qemu does.
1163        assert(sizeof(*mask) >= 8);
1164        kvm_mask->len = 8;
1165        memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1166    }
1167
1168    if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1169        panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1170              errno);
1171}
1172
1173int
1174BaseKvmCPU::ioctl(int request, long p1) const
1175{
1176    if (vcpuFD == -1)
1177        panic("KVM: CPU ioctl called before initialization\n");
1178
1179    return ::ioctl(vcpuFD, request, p1);
1180}
1181
1182Tick
1183BaseKvmCPU::flushCoalescedMMIO()
1184{
1185    if (!mmioRing)
1186        return 0;
1187
1188    DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1189
1190    // TODO: We might need to do synchronization when we start to
1191    // support multiple CPUs
1192    Tick ticks(0);
1193    while (mmioRing->first != mmioRing->last) {
1194        struct kvm_coalesced_mmio &ent(
1195            mmioRing->coalesced_mmio[mmioRing->first]);
1196
1197        DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1198                ent.phys_addr, ent.len);
1199
1200        ++numCoalescedMMIO;
1201        ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1202
1203        mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1204    }
1205
1206    return ticks;
1207}
1208
1209/**
1210 * Dummy handler for KVM kick signals.
1211 *
1212 * @note This function is usually not called since the kernel doesn't
1213 * seem to deliver signals when the signal is only unmasked when
1214 * running in KVM. This doesn't matter though since we are only
1215 * interested in getting KVM to exit, which happens as expected. See
1216 * setupSignalHandler() and kvmRun() for details about KVM signal
1217 * handling.
1218 */
1219static void
1220onKickSignal(int signo, siginfo_t *si, void *data)
1221{
1222}
1223
1224void
1225BaseKvmCPU::setupSignalHandler()
1226{
1227    struct sigaction sa;
1228
1229    memset(&sa, 0, sizeof(sa));
1230    sa.sa_sigaction = onKickSignal;
1231    sa.sa_flags = SA_SIGINFO | SA_RESTART;
1232    if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1233        panic("KVM: Failed to setup vCPU timer signal handler\n");
1234
1235    sigset_t sigset;
1236    if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1237        panic("KVM: Failed get signal mask\n");
1238
1239    // Request KVM to setup the same signal mask as we're currently
1240    // running with except for the KVM control signal. We'll sometimes
1241    // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1242    // KVM after servicing IO requests. See kvmRun().
1243    sigdelset(&sigset, KVM_KICK_SIGNAL);
1244    setSignalMask(&sigset);
1245
1246    // Mask our control signals so they aren't delivered unless we're
1247    // actually executing inside KVM.
1248    sigaddset(&sigset, KVM_KICK_SIGNAL);
1249    if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1250        panic("KVM: Failed mask the KVM control signals\n");
1251}
1252
1253bool
1254BaseKvmCPU::discardPendingSignal(int signum) const
1255{
1256    int discardedSignal;
1257
1258    // Setting the timeout to zero causes sigtimedwait to return
1259    // immediately.
1260    struct timespec timeout;
1261    timeout.tv_sec = 0;
1262    timeout.tv_nsec = 0;
1263
1264    sigset_t sigset;
1265    sigemptyset(&sigset);
1266    sigaddset(&sigset, signum);
1267
1268    do {
1269        discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1270    } while (discardedSignal == -1 && errno == EINTR);
1271
1272    if (discardedSignal == signum)
1273        return true;
1274    else if (discardedSignal == -1 && errno == EAGAIN)
1275        return false;
1276    else
1277        panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1278              discardedSignal, errno);
1279}
1280
1281void
1282BaseKvmCPU::setupCounters()
1283{
1284    DPRINTF(Kvm, "Attaching cycle counter...\n");
1285    PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1286                                PERF_COUNT_HW_CPU_CYCLES);
1287    cfgCycles.disabled(true)
1288        .pinned(true);
1289
1290    // Try to exclude the host. We set both exclude_hv and
1291    // exclude_host since different architectures use slightly
1292    // different APIs in the kernel.
1293    cfgCycles.exclude_hv(true)
1294        .exclude_host(true);
1295
1296    if (perfControlledByTimer) {
1297        // We need to configure the cycles counter to send overflows
1298        // since we are going to use it to trigger timer signals that
1299        // trap back into m5 from KVM. In practice, this means that we
1300        // need to set some non-zero sample period that gets
1301        // overridden when the timer is armed.
1302        cfgCycles.wakeupEvents(1)
1303            .samplePeriod(42);
1304    }
1305
1306    hwCycles.attach(cfgCycles,
1307                    0); // TID (0 => currentThread)
1308
1309    setupInstCounter();
1310}
1311
1312bool
1313BaseKvmCPU::tryDrain()
1314{
1315    if (drainState() != DrainState::Draining)
1316        return false;
1317
1318    if (!archIsDrained()) {
1319        DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1320        return false;
1321    }
1322
1323    if (_status == Idle || _status == Running) {
1324        DPRINTF(Drain,
1325                "tryDrain: CPU transitioned into the Idle state, drain done\n");
1326        signalDrainDone();
1327        return true;
1328    } else {
1329        DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1330        return false;
1331    }
1332}
1333
1334void
1335BaseKvmCPU::ioctlRun()
1336{
1337    if (ioctl(KVM_RUN) == -1) {
1338        if (errno != EINTR)
1339            panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1340                  errno);
1341    }
1342}
1343
1344void
1345BaseKvmCPU::setupInstStop()
1346{
1347    if (comInstEventQueue[0]->empty()) {
1348        setupInstCounter(0);
1349    } else {
1350        const uint64_t next(comInstEventQueue[0]->nextTick());
1351
1352        assert(next > ctrInsts);
1353        setupInstCounter(next - ctrInsts);
1354    }
1355}
1356
1357void
1358BaseKvmCPU::setupInstCounter(uint64_t period)
1359{
1360    // No need to do anything if we aren't attaching for the first
1361    // time or the period isn't changing.
1362    if (period == activeInstPeriod && hwInstructions.attached())
1363        return;
1364
1365    PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1366                                         PERF_COUNT_HW_INSTRUCTIONS);
1367
1368    // Try to exclude the host. We set both exclude_hv and
1369    // exclude_host since different architectures use slightly
1370    // different APIs in the kernel.
1371    cfgInstructions.exclude_hv(true)
1372        .exclude_host(true);
1373
1374    if (period) {
1375        // Setup a sampling counter if that has been requested.
1376        cfgInstructions.wakeupEvents(1)
1377            .samplePeriod(period);
1378    }
1379
1380    // We need to detach and re-attach the counter to reliably change
1381    // sampling settings. See PerfKvmCounter::period() for details.
1382    if (hwInstructions.attached())
1383        hwInstructions.detach();
1384    assert(hwCycles.attached());
1385    hwInstructions.attach(cfgInstructions,
1386                          0, // TID (0 => currentThread)
1387                          hwCycles);
1388
1389    if (period)
1390        hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1391
1392    activeInstPeriod = period;
1393}
1394