base.cc revision 12155
1/*
2 * Copyright (c) 2012, 2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66    : BaseCPU(params),
67      vm(*params->system->getKvmVM()),
68      _status(Idle),
69      dataPort(name() + ".dcache_port", this),
70      instPort(name() + ".icache_port", this),
71      alwaysSyncTC(params->alwaysSyncTC),
72      threadContextDirty(true),
73      kvmStateDirty(false),
74      vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75      _kvmRun(NULL), mmioRing(NULL),
76      pageSize(sysconf(_SC_PAGE_SIZE)),
77      tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
78                false, Event::CPU_Tick_Pri),
79      activeInstPeriod(0),
80      perfControlledByTimer(params->usePerfOverflow),
81      hostFactor(params->hostFactor),
82      ctrInsts(0)
83{
84    if (pageSize == -1)
85        panic("KVM: Failed to determine host page size (%i)\n",
86              errno);
87
88    if (FullSystem)
89        thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90                                  params->isa[0]);
91    else
92        thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93                                  params->workload[0], params->itb,
94                                  params->dtb, params->isa[0]);
95
96    thread->setStatus(ThreadContext::Halted);
97    tc = thread->getTC();
98    threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103    if (_kvmRun)
104        munmap(_kvmRun, vcpuMMapSize);
105    close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111    BaseCPU::init();
112
113    if (numThreads != 1)
114        fatal("KVM: Multithreading not supported");
115
116    tc->initMemProxies(tc);
117
118    // initialize CPU, including PC
119    if (FullSystem && !switchedOut())
120        TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126    const BaseKvmCPUParams * const p(
127        dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129    Kvm &kvm(*vm.kvm);
130
131    BaseCPU::startup();
132
133    assert(vcpuFD == -1);
134
135    // Tell the VM that a CPU is about to start.
136    vm.cpuStartup();
137
138    // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139    // not guaranteed that the parent KVM VM has initialized at that
140    // point. Initialize virtual CPUs here instead.
141    vcpuFD = vm.createVCPU(vcpuID);
142
143    // Map the KVM run structure */
144    vcpuMMapSize = kvm.getVCPUMMapSize();
145    _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146                                     PROT_READ | PROT_WRITE, MAP_SHARED,
147                                     vcpuFD, 0);
148    if (_kvmRun == MAP_FAILED)
149        panic("KVM: Failed to map run data structure\n");
150
151    // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152    // available. The offset into the KVM's communication page is
153    // provided by the coalesced MMIO capability.
154    int mmioOffset(kvm.capCoalescedMMIO());
155    if (!p->useCoalescedMMIO) {
156        inform("KVM: Coalesced MMIO disabled by config.\n");
157    } else if (mmioOffset) {
158        inform("KVM: Coalesced IO available\n");
159        mmioRing = (struct kvm_coalesced_mmio_ring *)(
160            (char *)_kvmRun + (mmioOffset * pageSize));
161    } else {
162        inform("KVM: Coalesced not supported by host OS\n");
163    }
164
165    thread->startup();
166
167    Event *startupEvent(
168        new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
169    schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175    return (activeMMIOReqs || pendingMMIOPkts.size())
176        ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182    if (cpu->system->isAtomicMode()) {
183        Tick delay = sendAtomic(pkt);
184        delete pkt->req;
185        delete pkt;
186        return delay;
187    } else {
188        if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189            activeMMIOReqs++;
190        } else {
191            pendingMMIOPkts.push(pkt);
192        }
193        // Return value is irrelevant for timing-mode accesses.
194        return 0;
195    }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201    DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
203    delete pkt->req;
204    delete pkt;
205    activeMMIOReqs--;
206
207    // We can switch back into KVM when all pending and in-flight MMIO
208    // operations have completed.
209    if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210        DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211        cpu->finishMMIOPending();
212    }
213    return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219    DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221    assert(pendingMMIOPkts.size());
222
223    // Assuming that we can issue infinite requests this cycle is a bit
224    // unrealistic, but it's not worth modeling something more complex in
225    // KVM.
226    while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227        pendingMMIOPkts.pop();
228        activeMMIOReqs++;
229    }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235    assert(_status = RunningMMIOPending);
236    assert(!tickEvent.scheduled());
237
238    _status = RunningServiceCompletion;
239    schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245    // Do thread-specific initialization. We need to setup signal
246    // delivery for counters and timers from within the thread that
247    // will execute the event queue to ensure that signals are
248    // delivered to the right threads.
249    const BaseKvmCPUParams * const p(
250        dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252    vcpuThread = pthread_self();
253
254    // Setup signal handlers. This has to be done after the vCPU is
255    // created since it manipulates the vCPU signal mask.
256    setupSignalHandler();
257
258    setupCounters();
259
260    if (p->usePerfOverflow)
261        runTimer.reset(new PerfKvmTimer(hwCycles,
262                                        KVM_KICK_SIGNAL,
263                                        p->hostFactor,
264                                        p->hostFreq));
265    else
266        runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267                                         p->hostFactor,
268                                         p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275    using namespace Stats;
276
277    BaseCPU::regStats();
278
279    numInsts
280        .name(name() + ".committedInsts")
281        .desc("Number of instructions committed")
282        ;
283
284    numVMExits
285        .name(name() + ".numVMExits")
286        .desc("total number of KVM exits")
287        ;
288
289    numVMHalfEntries
290        .name(name() + ".numVMHalfEntries")
291        .desc("number of KVM entries to finalize pending operations")
292        ;
293
294    numExitSignal
295        .name(name() + ".numExitSignal")
296        .desc("exits due to signal delivery")
297        ;
298
299    numMMIO
300        .name(name() + ".numMMIO")
301        .desc("number of VM exits due to memory mapped IO")
302        ;
303
304    numCoalescedMMIO
305        .name(name() + ".numCoalescedMMIO")
306        .desc("number of coalesced memory mapped IO requests")
307        ;
308
309    numIO
310        .name(name() + ".numIO")
311        .desc("number of VM exits due to legacy IO")
312        ;
313
314    numHalt
315        .name(name() + ".numHalt")
316        .desc("number of VM exits due to wait for interrupt instructions")
317        ;
318
319    numInterrupts
320        .name(name() + ".numInterrupts")
321        .desc("number of interrupts delivered")
322        ;
323
324    numHypercalls
325        .name(name() + ".numHypercalls")
326        .desc("number of hypercalls")
327        ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333    if (DTRACE(Checkpoint)) {
334        DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335        dump();
336    }
337
338    assert(tid == 0);
339    assert(_status == Idle);
340    thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346    DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348    assert(tid == 0);
349    assert(_status == Idle);
350    thread->unserialize(cp);
351    threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357    if (switchedOut())
358        return DrainState::Drained;
359
360    DPRINTF(Drain, "BaseKvmCPU::drain\n");
361
362    // The event queue won't be locked when calling drain since that's
363    // not done from an event. Lock the event queue here to make sure
364    // that scoped migrations continue to work if we need to
365    // synchronize the thread context.
366    std::lock_guard<EventQueue> lock(*this->eventQueue());
367
368    switch (_status) {
369      case Running:
370        // The base KVM code is normally ready when it is in the
371        // Running state, but the architecture specific code might be
372        // of a different opinion. This may happen when the CPU been
373        // notified of an event that hasn't been accepted by the vCPU
374        // yet.
375        if (!archIsDrained())
376            return DrainState::Draining;
377
378        // The state of the CPU is consistent, so we don't need to do
379        // anything special to drain it. We simply de-schedule the
380        // tick event and enter the Idle state to prevent nasty things
381        // like MMIOs from happening.
382        if (tickEvent.scheduled())
383            deschedule(tickEvent);
384        _status = Idle;
385
386        /** FALLTHROUGH */
387      case Idle:
388        // Idle, no need to drain
389        assert(!tickEvent.scheduled());
390
391        // Sync the thread context here since we'll need it when we
392        // switch CPUs or checkpoint the CPU.
393        syncThreadContext();
394
395        return DrainState::Drained;
396
397      case RunningServiceCompletion:
398        // The CPU has just requested a service that was handled in
399        // the RunningService state, but the results have still not
400        // been reported to the CPU. Now, we /could/ probably just
401        // update the register state ourselves instead of letting KVM
402        // handle it, but that would be tricky. Instead, we enter KVM
403        // and let it do its stuff.
404        DPRINTF(Drain, "KVM CPU is waiting for service completion, "
405                "requesting drain.\n");
406        return DrainState::Draining;
407
408      case RunningMMIOPending:
409        // We need to drain since there are in-flight timing accesses
410        DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
411                "requesting drain.\n");
412        return DrainState::Draining;
413
414      case RunningService:
415        // We need to drain since the CPU is waiting for service (e.g., MMIOs)
416        DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
417        return DrainState::Draining;
418
419      default:
420        panic("KVM: Unhandled CPU state in drain()\n");
421        return DrainState::Drained;
422    }
423}
424
425void
426BaseKvmCPU::drainResume()
427{
428    assert(!tickEvent.scheduled());
429
430    // We might have been switched out. In that case, we don't need to
431    // do anything.
432    if (switchedOut())
433        return;
434
435    DPRINTF(Kvm, "drainResume\n");
436    verifyMemoryMode();
437
438    // The tick event is de-scheduled as a part of the draining
439    // process. Re-schedule it if the thread context is active.
440    if (tc->status() == ThreadContext::Active) {
441        schedule(tickEvent, nextCycle());
442        _status = Running;
443    } else {
444        _status = Idle;
445    }
446}
447
448void
449BaseKvmCPU::notifyFork()
450{
451    // We should have drained prior to forking, which means that the
452    // tick event shouldn't be scheduled and the CPU is idle.
453    assert(!tickEvent.scheduled());
454    assert(_status == Idle);
455
456    if (vcpuFD != -1) {
457        if (close(vcpuFD) == -1)
458            warn("kvm CPU: notifyFork failed to close vcpuFD\n");
459
460        if (_kvmRun)
461            munmap(_kvmRun, vcpuMMapSize);
462
463        vcpuFD = -1;
464        _kvmRun = NULL;
465
466        hwInstructions.detach();
467        hwCycles.detach();
468    }
469}
470
471void
472BaseKvmCPU::switchOut()
473{
474    DPRINTF(Kvm, "switchOut\n");
475
476    BaseCPU::switchOut();
477
478    // We should have drained prior to executing a switchOut, which
479    // means that the tick event shouldn't be scheduled and the CPU is
480    // idle.
481    assert(!tickEvent.scheduled());
482    assert(_status == Idle);
483}
484
485void
486BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
487{
488    DPRINTF(Kvm, "takeOverFrom\n");
489
490    BaseCPU::takeOverFrom(cpu);
491
492    // We should have drained prior to executing a switchOut, which
493    // means that the tick event shouldn't be scheduled and the CPU is
494    // idle.
495    assert(!tickEvent.scheduled());
496    assert(_status == Idle);
497    assert(threadContexts.size() == 1);
498
499    // Force an update of the KVM state here instead of flagging the
500    // TC as dirty. This is not ideal from a performance point of
501    // view, but it makes debugging easier as it allows meaningful KVM
502    // state to be dumped before and after a takeover.
503    updateKvmState();
504    threadContextDirty = false;
505}
506
507void
508BaseKvmCPU::verifyMemoryMode() const
509{
510    if (!(system->bypassCaches())) {
511        fatal("The KVM-based CPUs requires the memory system to be in the "
512              "'noncaching' mode.\n");
513    }
514}
515
516void
517BaseKvmCPU::wakeup(ThreadID tid)
518{
519    DPRINTF(Kvm, "wakeup()\n");
520    // This method might have been called from another
521    // context. Migrate to this SimObject's event queue when
522    // delivering the wakeup signal.
523    EventQueue::ScopedMigration migrate(eventQueue());
524
525    // Kick the vCPU to get it to come out of KVM.
526    kick();
527
528    if (thread->status() != ThreadContext::Suspended)
529        return;
530
531    thread->activate();
532}
533
534void
535BaseKvmCPU::activateContext(ThreadID thread_num)
536{
537    DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
538
539    assert(thread_num == 0);
540    assert(thread);
541
542    assert(_status == Idle);
543    assert(!tickEvent.scheduled());
544
545    numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
546
547    schedule(tickEvent, clockEdge(Cycles(0)));
548    _status = Running;
549}
550
551
552void
553BaseKvmCPU::suspendContext(ThreadID thread_num)
554{
555    DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
556
557    assert(thread_num == 0);
558    assert(thread);
559
560    if (_status == Idle)
561        return;
562
563    assert(_status == Running || _status == RunningServiceCompletion);
564
565    // The tick event may no be scheduled if the quest has requested
566    // the monitor to wait for interrupts. The normal CPU models can
567    // get their tick events descheduled by quiesce instructions, but
568    // that can't happen here.
569    if (tickEvent.scheduled())
570        deschedule(tickEvent);
571
572    _status = Idle;
573}
574
575void
576BaseKvmCPU::deallocateContext(ThreadID thread_num)
577{
578    // for now, these are equivalent
579    suspendContext(thread_num);
580}
581
582void
583BaseKvmCPU::haltContext(ThreadID thread_num)
584{
585    // for now, these are equivalent
586    suspendContext(thread_num);
587}
588
589ThreadContext *
590BaseKvmCPU::getContext(int tn)
591{
592    assert(tn == 0);
593    syncThreadContext();
594    return tc;
595}
596
597
598Counter
599BaseKvmCPU::totalInsts() const
600{
601    return ctrInsts;
602}
603
604Counter
605BaseKvmCPU::totalOps() const
606{
607    hack_once("Pretending totalOps is equivalent to totalInsts()\n");
608    return ctrInsts;
609}
610
611void
612BaseKvmCPU::dump() const
613{
614    inform("State dumping not implemented.");
615}
616
617void
618BaseKvmCPU::tick()
619{
620    Tick delay(0);
621    assert(_status != Idle && _status != RunningMMIOPending);
622
623    switch (_status) {
624      case RunningService:
625        // handleKvmExit() will determine the next state of the CPU
626        delay = handleKvmExit();
627
628        if (tryDrain())
629            _status = Idle;
630        break;
631
632      case RunningServiceCompletion:
633      case Running: {
634          const uint64_t nextInstEvent(
635              !comInstEventQueue[0]->empty() ?
636              comInstEventQueue[0]->nextTick() : UINT64_MAX);
637          // Enter into KVM and complete pending IO instructions if we
638          // have an instruction event pending.
639          const Tick ticksToExecute(
640              nextInstEvent > ctrInsts ?
641              curEventQueue()->nextTick() - curTick() : 0);
642
643          if (alwaysSyncTC)
644              threadContextDirty = true;
645
646          // We might need to update the KVM state.
647          syncKvmState();
648
649          // Setup any pending instruction count breakpoints using
650          // PerfEvent if we are going to execute more than just an IO
651          // completion.
652          if (ticksToExecute > 0)
653              setupInstStop();
654
655          DPRINTF(KvmRun, "Entering KVM...\n");
656          if (drainState() == DrainState::Draining) {
657              // Force an immediate exit from KVM after completing
658              // pending operations. The architecture-specific code
659              // takes care to run until it is in a state where it can
660              // safely be drained.
661              delay = kvmRunDrain();
662          } else {
663              delay = kvmRun(ticksToExecute);
664          }
665
666          // The CPU might have been suspended before entering into
667          // KVM. Assume that the CPU was suspended /before/ entering
668          // into KVM and skip the exit handling.
669          if (_status == Idle)
670              break;
671
672          // Entering into KVM implies that we'll have to reload the thread
673          // context from KVM if we want to access it. Flag the KVM state as
674          // dirty with respect to the cached thread context.
675          kvmStateDirty = true;
676
677          if (alwaysSyncTC)
678              syncThreadContext();
679
680          // Enter into the RunningService state unless the
681          // simulation was stopped by a timer.
682          if (_kvmRun->exit_reason !=  KVM_EXIT_INTR) {
683              _status = RunningService;
684          } else {
685              ++numExitSignal;
686              _status = Running;
687          }
688
689          // Service any pending instruction events. The vCPU should
690          // have exited in time for the event using the instruction
691          // counter configured by setupInstStop().
692          comInstEventQueue[0]->serviceEvents(ctrInsts);
693          system->instEventQueue.serviceEvents(system->totalNumInsts);
694
695          if (tryDrain())
696              _status = Idle;
697      } break;
698
699      default:
700        panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
701              _status);
702    }
703
704    // Schedule a new tick if we are still running
705    if (_status != Idle && _status != RunningMMIOPending)
706        schedule(tickEvent, clockEdge(ticksToCycles(delay)));
707}
708
709Tick
710BaseKvmCPU::kvmRunDrain()
711{
712    // By default, the only thing we need to drain is a pending IO
713    // operation which assumes that we are in the
714    // RunningServiceCompletion or RunningMMIOPending state.
715    assert(_status == RunningServiceCompletion ||
716           _status == RunningMMIOPending);
717
718    // Deliver the data from the pending IO operation and immediately
719    // exit.
720    return kvmRun(0);
721}
722
723uint64_t
724BaseKvmCPU::getHostCycles() const
725{
726    return hwCycles.read();
727}
728
729Tick
730BaseKvmCPU::kvmRun(Tick ticks)
731{
732    Tick ticksExecuted;
733    fatal_if(vcpuFD == -1,
734             "Trying to run a KVM CPU in a forked child process. "
735             "This is not supported.\n");
736    DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
737
738    if (ticks == 0) {
739        // Settings ticks == 0 is a special case which causes an entry
740        // into KVM that finishes pending operations (e.g., IO) and
741        // then immediately exits.
742        DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
743
744        ++numVMHalfEntries;
745
746        // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
747        // thread). The KVM control signal is masked while executing
748        // in gem5 and gets unmasked temporarily as when entering
749        // KVM. See setSignalMask() and setupSignalHandler().
750        kick();
751
752        // Start the vCPU. KVM will check for signals after completing
753        // pending operations (IO). Since the KVM_KICK_SIGNAL is
754        // pending, this forces an immediate exit to gem5 again. We
755        // don't bother to setup timers since this shouldn't actually
756        // execute any code (other than completing half-executed IO
757        // instructions) in the guest.
758        ioctlRun();
759
760        // We always execute at least one cycle to prevent the
761        // BaseKvmCPU::tick() to be rescheduled on the same tick
762        // twice.
763        ticksExecuted = clockPeriod();
764    } else {
765        // This method is executed as a result of a tick event. That
766        // means that the event queue will be locked when entering the
767        // method. We temporarily unlock the event queue to allow
768        // other threads to steal control of this thread to inject
769        // interrupts. They will typically lock the queue and then
770        // force an exit from KVM by kicking the vCPU.
771        EventQueue::ScopedRelease release(curEventQueue());
772
773        if (ticks < runTimer->resolution()) {
774            DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
775                    ticks, runTimer->resolution());
776            ticks = runTimer->resolution();
777        }
778
779        // Get hardware statistics after synchronizing contexts. The KVM
780        // state update might affect guest cycle counters.
781        uint64_t baseCycles(getHostCycles());
782        uint64_t baseInstrs(hwInstructions.read());
783
784        // Arm the run timer and start the cycle timer if it isn't
785        // controlled by the overflow timer. Starting/stopping the cycle
786        // timer automatically starts the other perf timers as they are in
787        // the same counter group.
788        runTimer->arm(ticks);
789        if (!perfControlledByTimer)
790            hwCycles.start();
791
792        ioctlRun();
793
794        runTimer->disarm();
795        if (!perfControlledByTimer)
796            hwCycles.stop();
797
798        // The control signal may have been delivered after we exited
799        // from KVM. It will be pending in that case since it is
800        // masked when we aren't executing in KVM. Discard it to make
801        // sure we don't deliver it immediately next time we try to
802        // enter into KVM.
803        discardPendingSignal(KVM_KICK_SIGNAL);
804
805        const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
806        const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
807        const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
808        ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
809
810        /* Update statistics */
811        numCycles += simCyclesExecuted;;
812        numInsts += instsExecuted;
813        ctrInsts += instsExecuted;
814        system->totalNumInsts += instsExecuted;
815
816        DPRINTF(KvmRun,
817                "KVM: Executed %i instructions in %i cycles "
818                "(%i ticks, sim cycles: %i).\n",
819                instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
820    }
821
822    ++numVMExits;
823
824    return ticksExecuted + flushCoalescedMMIO();
825}
826
827void
828BaseKvmCPU::kvmNonMaskableInterrupt()
829{
830    ++numInterrupts;
831    if (ioctl(KVM_NMI) == -1)
832        panic("KVM: Failed to deliver NMI to virtual CPU\n");
833}
834
835void
836BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
837{
838    ++numInterrupts;
839    if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
840        panic("KVM: Failed to deliver interrupt to virtual CPU\n");
841}
842
843void
844BaseKvmCPU::getRegisters(struct kvm_regs &regs) const
845{
846    if (ioctl(KVM_GET_REGS, &regs) == -1)
847        panic("KVM: Failed to get guest registers\n");
848}
849
850void
851BaseKvmCPU::setRegisters(const struct kvm_regs &regs)
852{
853    if (ioctl(KVM_SET_REGS, (void *)&regs) == -1)
854        panic("KVM: Failed to set guest registers\n");
855}
856
857void
858BaseKvmCPU::getSpecialRegisters(struct kvm_sregs &regs) const
859{
860    if (ioctl(KVM_GET_SREGS, &regs) == -1)
861        panic("KVM: Failed to get guest special registers\n");
862}
863
864void
865BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs &regs)
866{
867    if (ioctl(KVM_SET_SREGS, (void *)&regs) == -1)
868        panic("KVM: Failed to set guest special registers\n");
869}
870
871void
872BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
873{
874    if (ioctl(KVM_GET_FPU, &state) == -1)
875        panic("KVM: Failed to get guest FPU state\n");
876}
877
878void
879BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
880{
881    if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
882        panic("KVM: Failed to set guest FPU state\n");
883}
884
885
886void
887BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
888{
889#ifdef KVM_SET_ONE_REG
890    struct kvm_one_reg reg;
891    reg.id = id;
892    reg.addr = (uint64_t)addr;
893
894    if (ioctl(KVM_SET_ONE_REG, &reg) == -1) {
895        panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
896              id, errno);
897    }
898#else
899    panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
900#endif
901}
902
903void
904BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
905{
906#ifdef KVM_GET_ONE_REG
907    struct kvm_one_reg reg;
908    reg.id = id;
909    reg.addr = (uint64_t)addr;
910
911    if (ioctl(KVM_GET_ONE_REG, &reg) == -1) {
912        panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
913              id, errno);
914    }
915#else
916    panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
917#endif
918}
919
920std::string
921BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
922{
923#ifdef KVM_GET_ONE_REG
924    std::ostringstream ss;
925
926    ss.setf(std::ios::hex, std::ios::basefield);
927    ss.setf(std::ios::showbase);
928#define HANDLE_INTTYPE(len)                      \
929    case KVM_REG_SIZE_U ## len: {                \
930        uint ## len ## _t value;                 \
931        getOneReg(id, &value);                   \
932        ss << value;                             \
933    }  break
934
935#define HANDLE_ARRAY(len)                               \
936    case KVM_REG_SIZE_U ## len: {                       \
937        uint8_t value[len / 8];                         \
938        getOneReg(id, value);                           \
939        ccprintf(ss, "[0x%x", value[0]);                \
940        for (int i = 1; i < len  / 8; ++i)              \
941            ccprintf(ss, ", 0x%x", value[i]);           \
942        ccprintf(ss, "]");                              \
943      } break
944
945    switch (id & KVM_REG_SIZE_MASK) {
946        HANDLE_INTTYPE(8);
947        HANDLE_INTTYPE(16);
948        HANDLE_INTTYPE(32);
949        HANDLE_INTTYPE(64);
950        HANDLE_ARRAY(128);
951        HANDLE_ARRAY(256);
952        HANDLE_ARRAY(512);
953        HANDLE_ARRAY(1024);
954      default:
955        ss << "??";
956    }
957
958#undef HANDLE_INTTYPE
959#undef HANDLE_ARRAY
960
961    return ss.str();
962#else
963    panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
964#endif
965}
966
967void
968BaseKvmCPU::syncThreadContext()
969{
970    if (!kvmStateDirty)
971        return;
972
973    assert(!threadContextDirty);
974
975    updateThreadContext();
976    kvmStateDirty = false;
977}
978
979void
980BaseKvmCPU::syncKvmState()
981{
982    if (!threadContextDirty)
983        return;
984
985    assert(!kvmStateDirty);
986
987    updateKvmState();
988    threadContextDirty = false;
989}
990
991Tick
992BaseKvmCPU::handleKvmExit()
993{
994    DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
995    assert(_status == RunningService);
996
997    // Switch into the running state by default. Individual handlers
998    // can override this.
999    _status = Running;
1000    switch (_kvmRun->exit_reason) {
1001      case KVM_EXIT_UNKNOWN:
1002        return handleKvmExitUnknown();
1003
1004      case KVM_EXIT_EXCEPTION:
1005        return handleKvmExitException();
1006
1007      case KVM_EXIT_IO:
1008      {
1009        ++numIO;
1010        Tick ticks = handleKvmExitIO();
1011        _status = dataPort.nextIOState();
1012        return ticks;
1013      }
1014
1015      case KVM_EXIT_HYPERCALL:
1016        ++numHypercalls;
1017        return handleKvmExitHypercall();
1018
1019      case KVM_EXIT_HLT:
1020        /* The guest has halted and is waiting for interrupts */
1021        DPRINTF(Kvm, "handleKvmExitHalt\n");
1022        ++numHalt;
1023
1024        // Suspend the thread until the next interrupt arrives
1025        thread->suspend();
1026
1027        // This is actually ignored since the thread is suspended.
1028        return 0;
1029
1030      case KVM_EXIT_MMIO:
1031      {
1032        /* Service memory mapped IO requests */
1033        DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1034                _kvmRun->mmio.is_write,
1035                _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1036
1037        ++numMMIO;
1038        Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1039                                  _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1040        // doMMIOAccess could have triggered a suspend, in which case we don't
1041        // want to overwrite the _status.
1042        if (_status != Idle)
1043            _status = dataPort.nextIOState();
1044        return ticks;
1045      }
1046
1047      case KVM_EXIT_IRQ_WINDOW_OPEN:
1048        return handleKvmExitIRQWindowOpen();
1049
1050      case KVM_EXIT_FAIL_ENTRY:
1051        return handleKvmExitFailEntry();
1052
1053      case KVM_EXIT_INTR:
1054        /* KVM was interrupted by a signal, restart it in the next
1055         * tick. */
1056        return 0;
1057
1058      case KVM_EXIT_INTERNAL_ERROR:
1059        panic("KVM: Internal error (suberror: %u)\n",
1060              _kvmRun->internal.suberror);
1061
1062      default:
1063        dump();
1064        panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1065    }
1066}
1067
1068Tick
1069BaseKvmCPU::handleKvmExitIO()
1070{
1071    panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1072          _kvmRun->io.direction, _kvmRun->io.size,
1073          _kvmRun->io.port, _kvmRun->io.count);
1074}
1075
1076Tick
1077BaseKvmCPU::handleKvmExitHypercall()
1078{
1079    panic("KVM: Unhandled hypercall\n");
1080}
1081
1082Tick
1083BaseKvmCPU::handleKvmExitIRQWindowOpen()
1084{
1085    warn("KVM: Unhandled IRQ window.\n");
1086    return 0;
1087}
1088
1089
1090Tick
1091BaseKvmCPU::handleKvmExitUnknown()
1092{
1093    dump();
1094    panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1095          _kvmRun->hw.hardware_exit_reason);
1096}
1097
1098Tick
1099BaseKvmCPU::handleKvmExitException()
1100{
1101    dump();
1102    panic("KVM: Got exception when starting vCPU "
1103          "(exception: %u, error_code: %u)\n",
1104          _kvmRun->ex.exception, _kvmRun->ex.error_code);
1105}
1106
1107Tick
1108BaseKvmCPU::handleKvmExitFailEntry()
1109{
1110    dump();
1111    panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1112          _kvmRun->fail_entry.hardware_entry_failure_reason);
1113}
1114
1115Tick
1116BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1117{
1118    ThreadContext *tc(thread->getTC());
1119    syncThreadContext();
1120
1121    RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1122                                      dataMasterId());
1123    mmio_req->setContext(tc->contextId());
1124    // Some architectures do need to massage physical addresses a bit
1125    // before they are inserted into the memory system. This enables
1126    // APIC accesses on x86 and m5ops where supported through a MMIO
1127    // interface.
1128    BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1129    Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1130    if (fault != NoFault)
1131        warn("Finalization of MMIO address failed: %s\n", fault->name());
1132
1133
1134    const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1135    PacketPtr pkt = new Packet(mmio_req, cmd);
1136    pkt->dataStatic(data);
1137
1138    if (mmio_req->isMmappedIpr()) {
1139        // We currently assume that there is no need to migrate to a
1140        // different event queue when doing IPRs. Currently, IPRs are
1141        // only used for m5ops, so it should be a valid assumption.
1142        const Cycles ipr_delay(write ?
1143                             TheISA::handleIprWrite(tc, pkt) :
1144                             TheISA::handleIprRead(tc, pkt));
1145        threadContextDirty = true;
1146        delete pkt->req;
1147        delete pkt;
1148        return clockPeriod() * ipr_delay;
1149    } else {
1150        // Temporarily lock and migrate to the device event queue to
1151        // prevent races in multi-core mode.
1152        EventQueue::ScopedMigration migrate(deviceEventQueue());
1153
1154        return dataPort.submitIO(pkt);
1155    }
1156}
1157
1158void
1159BaseKvmCPU::setSignalMask(const sigset_t *mask)
1160{
1161    std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1162
1163    if (mask) {
1164        kvm_mask.reset((struct kvm_signal_mask *)operator new(
1165                           sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1166        // The kernel and the user-space headers have different ideas
1167        // about the size of sigset_t. This seems like a massive hack,
1168        // but is actually what qemu does.
1169        assert(sizeof(*mask) >= 8);
1170        kvm_mask->len = 8;
1171        memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1172    }
1173
1174    if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1175        panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1176              errno);
1177}
1178
1179int
1180BaseKvmCPU::ioctl(int request, long p1) const
1181{
1182    if (vcpuFD == -1)
1183        panic("KVM: CPU ioctl called before initialization\n");
1184
1185    return ::ioctl(vcpuFD, request, p1);
1186}
1187
1188Tick
1189BaseKvmCPU::flushCoalescedMMIO()
1190{
1191    if (!mmioRing)
1192        return 0;
1193
1194    DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1195
1196    // TODO: We might need to do synchronization when we start to
1197    // support multiple CPUs
1198    Tick ticks(0);
1199    while (mmioRing->first != mmioRing->last) {
1200        struct kvm_coalesced_mmio &ent(
1201            mmioRing->coalesced_mmio[mmioRing->first]);
1202
1203        DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1204                ent.phys_addr, ent.len);
1205
1206        ++numCoalescedMMIO;
1207        ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1208
1209        mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1210    }
1211
1212    return ticks;
1213}
1214
1215/**
1216 * Dummy handler for KVM kick signals.
1217 *
1218 * @note This function is usually not called since the kernel doesn't
1219 * seem to deliver signals when the signal is only unmasked when
1220 * running in KVM. This doesn't matter though since we are only
1221 * interested in getting KVM to exit, which happens as expected. See
1222 * setupSignalHandler() and kvmRun() for details about KVM signal
1223 * handling.
1224 */
1225static void
1226onKickSignal(int signo, siginfo_t *si, void *data)
1227{
1228}
1229
1230void
1231BaseKvmCPU::setupSignalHandler()
1232{
1233    struct sigaction sa;
1234
1235    memset(&sa, 0, sizeof(sa));
1236    sa.sa_sigaction = onKickSignal;
1237    sa.sa_flags = SA_SIGINFO | SA_RESTART;
1238    if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1239        panic("KVM: Failed to setup vCPU timer signal handler\n");
1240
1241    sigset_t sigset;
1242    if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1243        panic("KVM: Failed get signal mask\n");
1244
1245    // Request KVM to setup the same signal mask as we're currently
1246    // running with except for the KVM control signal. We'll sometimes
1247    // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1248    // KVM after servicing IO requests. See kvmRun().
1249    sigdelset(&sigset, KVM_KICK_SIGNAL);
1250    setSignalMask(&sigset);
1251
1252    // Mask our control signals so they aren't delivered unless we're
1253    // actually executing inside KVM.
1254    sigaddset(&sigset, KVM_KICK_SIGNAL);
1255    if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1256        panic("KVM: Failed mask the KVM control signals\n");
1257}
1258
1259bool
1260BaseKvmCPU::discardPendingSignal(int signum) const
1261{
1262    int discardedSignal;
1263
1264    // Setting the timeout to zero causes sigtimedwait to return
1265    // immediately.
1266    struct timespec timeout;
1267    timeout.tv_sec = 0;
1268    timeout.tv_nsec = 0;
1269
1270    sigset_t sigset;
1271    sigemptyset(&sigset);
1272    sigaddset(&sigset, signum);
1273
1274    do {
1275        discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1276    } while (discardedSignal == -1 && errno == EINTR);
1277
1278    if (discardedSignal == signum)
1279        return true;
1280    else if (discardedSignal == -1 && errno == EAGAIN)
1281        return false;
1282    else
1283        panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1284              discardedSignal, errno);
1285}
1286
1287void
1288BaseKvmCPU::setupCounters()
1289{
1290    DPRINTF(Kvm, "Attaching cycle counter...\n");
1291    PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1292                                PERF_COUNT_HW_CPU_CYCLES);
1293    cfgCycles.disabled(true)
1294        .pinned(true);
1295
1296    // Try to exclude the host. We set both exclude_hv and
1297    // exclude_host since different architectures use slightly
1298    // different APIs in the kernel.
1299    cfgCycles.exclude_hv(true)
1300        .exclude_host(true);
1301
1302    if (perfControlledByTimer) {
1303        // We need to configure the cycles counter to send overflows
1304        // since we are going to use it to trigger timer signals that
1305        // trap back into m5 from KVM. In practice, this means that we
1306        // need to set some non-zero sample period that gets
1307        // overridden when the timer is armed.
1308        cfgCycles.wakeupEvents(1)
1309            .samplePeriod(42);
1310    }
1311
1312    hwCycles.attach(cfgCycles,
1313                    0); // TID (0 => currentThread)
1314
1315    setupInstCounter();
1316}
1317
1318bool
1319BaseKvmCPU::tryDrain()
1320{
1321    if (drainState() != DrainState::Draining)
1322        return false;
1323
1324    if (!archIsDrained()) {
1325        DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1326        return false;
1327    }
1328
1329    if (_status == Idle || _status == Running) {
1330        DPRINTF(Drain,
1331                "tryDrain: CPU transitioned into the Idle state, drain done\n");
1332        signalDrainDone();
1333        return true;
1334    } else {
1335        DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1336        return false;
1337    }
1338}
1339
1340void
1341BaseKvmCPU::ioctlRun()
1342{
1343    if (ioctl(KVM_RUN) == -1) {
1344        if (errno != EINTR)
1345            panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1346                  errno);
1347    }
1348}
1349
1350void
1351BaseKvmCPU::setupInstStop()
1352{
1353    if (comInstEventQueue[0]->empty()) {
1354        setupInstCounter(0);
1355    } else {
1356        const uint64_t next(comInstEventQueue[0]->nextTick());
1357
1358        assert(next > ctrInsts);
1359        setupInstCounter(next - ctrInsts);
1360    }
1361}
1362
1363void
1364BaseKvmCPU::setupInstCounter(uint64_t period)
1365{
1366    // No need to do anything if we aren't attaching for the first
1367    // time or the period isn't changing.
1368    if (period == activeInstPeriod && hwInstructions.attached())
1369        return;
1370
1371    PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1372                                         PERF_COUNT_HW_INSTRUCTIONS);
1373
1374    // Try to exclude the host. We set both exclude_hv and
1375    // exclude_host since different architectures use slightly
1376    // different APIs in the kernel.
1377    cfgInstructions.exclude_hv(true)
1378        .exclude_host(true);
1379
1380    if (period) {
1381        // Setup a sampling counter if that has been requested.
1382        cfgInstructions.wakeupEvents(1)
1383            .samplePeriod(period);
1384    }
1385
1386    // We need to detach and re-attach the counter to reliably change
1387    // sampling settings. See PerfKvmCounter::period() for details.
1388    if (hwInstructions.attached())
1389        hwInstructions.detach();
1390    assert(hwCycles.attached());
1391    hwInstructions.attach(cfgInstructions,
1392                          0, // TID (0 => currentThread)
1393                          hwCycles);
1394
1395    if (period)
1396        hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1397
1398    activeInstPeriod = period;
1399}
1400