1/*
2 * Copyright (c) 2012, 2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include <linux/kvm.h>
41#include <sys/ioctl.h>
42#include <sys/mman.h>
43#include <unistd.h>
44
45#include <cerrno>
46#include <csignal>
47#include <ostream>
48
49#include "arch/mmapped_ipr.hh"
50#include "arch/utility.hh"
51#include "cpu/kvm/base.hh"
52#include "debug/Checkpoint.hh"
53#include "debug/Drain.hh"
54#include "debug/Kvm.hh"
55#include "debug/KvmIO.hh"
56#include "debug/KvmRun.hh"
57#include "params/BaseKvmCPU.hh"
58#include "sim/process.hh"
59#include "sim/system.hh"
60
61#include <signal.h>
62
63/* Used by some KVM macros */
64#define PAGE_SIZE pageSize
65
66BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
67 : BaseCPU(params),
68 vm(*params->kvmVM),
69 _status(Idle),
70 dataPort(name() + ".dcache_port", this),
71 instPort(name() + ".icache_port", this),
72 alwaysSyncTC(params->alwaysSyncTC),
73 threadContextDirty(true),
74 kvmStateDirty(false),
75 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
76 _kvmRun(NULL), mmioRing(NULL),
77 pageSize(sysconf(_SC_PAGE_SIZE)),
78 tickEvent(*this),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventWrapper<BaseKvmCPU,
169 &BaseKvmCPU::startupThread>(this, true));
170 schedule(startupEvent, curTick());
171}
172
173BaseKvmCPU::Status
174BaseKvmCPU::KVMCpuPort::nextIOState() const
175{
176 return (activeMMIOReqs || pendingMMIOPkts.size())
177 ? RunningMMIOPending : RunningServiceCompletion;
178}
179
180Tick
181BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
182{
183 if (cpu->system->isAtomicMode()) {
184 Tick delay = sendAtomic(pkt);
185 delete pkt->req;
186 delete pkt;
187 return delay;
188 } else {
189 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
190 activeMMIOReqs++;
191 } else {
192 pendingMMIOPkts.push(pkt);
193 }
194 // Return value is irrelevant for timing-mode accesses.
195 return 0;
196 }
197}
198
199bool
200BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
201{
202 DPRINTF(KvmIO, "KVM: Finished timing request\n");
203
204 delete pkt->req;
205 delete pkt;
206 activeMMIOReqs--;
207
208 // We can switch back into KVM when all pending and in-flight MMIO
209 // operations have completed.
210 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
211 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
212 cpu->finishMMIOPending();
213 }
214 return true;
215}
216
217void
218BaseKvmCPU::KVMCpuPort::recvReqRetry()
219{
220 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
221
222 assert(pendingMMIOPkts.size());
223
224 // Assuming that we can issue infinite requests this cycle is a bit
225 // unrealistic, but it's not worth modeling something more complex in
226 // KVM.
227 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
228 pendingMMIOPkts.pop();
229 activeMMIOReqs++;
230 }
231}
232
233void
234BaseKvmCPU::finishMMIOPending()
235{
236 assert(_status = RunningMMIOPending);
237 assert(!tickEvent.scheduled());
238
239 _status = RunningServiceCompletion;
240 schedule(tickEvent, nextCycle());
241}
242
243void
244BaseKvmCPU::startupThread()
245{
246 // Do thread-specific initialization. We need to setup signal
247 // delivery for counters and timers from within the thread that
248 // will execute the event queue to ensure that signals are
249 // delivered to the right threads.
250 const BaseKvmCPUParams * const p(
251 dynamic_cast<const BaseKvmCPUParams *>(params()));
252
253 vcpuThread = pthread_self();
254
255 // Setup signal handlers. This has to be done after the vCPU is
256 // created since it manipulates the vCPU signal mask.
257 setupSignalHandler();
258
259 setupCounters();
260
261 if (p->usePerfOverflow)
262 runTimer.reset(new PerfKvmTimer(hwCycles,
263 KVM_KICK_SIGNAL,
264 p->hostFactor,
265 p->hostFreq));
266 else
267 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
268 p->hostFactor,
269 p->hostFreq));
270
271}
272
273void
274BaseKvmCPU::regStats()
275{
276 using namespace Stats;
277
278 BaseCPU::regStats();
279
280 numInsts
281 .name(name() + ".committedInsts")
282 .desc("Number of instructions committed")
283 ;
284
285 numVMExits
286 .name(name() + ".numVMExits")
287 .desc("total number of KVM exits")
288 ;
289
290 numVMHalfEntries
291 .name(name() + ".numVMHalfEntries")
292 .desc("number of KVM entries to finalize pending operations")
293 ;
294
295 numExitSignal
296 .name(name() + ".numExitSignal")
297 .desc("exits due to signal delivery")
298 ;
299
300 numMMIO
301 .name(name() + ".numMMIO")
302 .desc("number of VM exits due to memory mapped IO")
303 ;
304
305 numCoalescedMMIO
306 .name(name() + ".numCoalescedMMIO")
307 .desc("number of coalesced memory mapped IO requests")
308 ;
309
310 numIO
311 .name(name() + ".numIO")
312 .desc("number of VM exits due to legacy IO")
313 ;
314
315 numHalt
316 .name(name() + ".numHalt")
317 .desc("number of VM exits due to wait for interrupt instructions")
318 ;
319
320 numInterrupts
321 .name(name() + ".numInterrupts")
322 .desc("number of interrupts delivered")
323 ;
324
325 numHypercalls
326 .name(name() + ".numHypercalls")
327 .desc("number of hypercalls")
328 ;
329}
330
331void
332BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
333{
334 if (DTRACE(Checkpoint)) {
335 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
336 dump();
337 }
338
339 assert(tid == 0);
340 assert(_status == Idle);
341 thread->serialize(cp);
342}
343
344void
345BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
346{
347 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
348
349 assert(tid == 0);
350 assert(_status == Idle);
351 thread->unserialize(cp);
352 threadContextDirty = true;
353}
354
355DrainState
356BaseKvmCPU::drain()
357{
358 if (switchedOut())
359 return DrainState::Drained;
360
361 DPRINTF(Drain, "BaseKvmCPU::drain\n");
362 switch (_status) {
363 case Running:
364 // The base KVM code is normally ready when it is in the
365 // Running state, but the architecture specific code might be
366 // of a different opinion. This may happen when the CPU been
367 // notified of an event that hasn't been accepted by the vCPU
368 // yet.
369 if (!archIsDrained())
370 return DrainState::Draining;
371
372 // The state of the CPU is consistent, so we don't need to do
373 // anything special to drain it. We simply de-schedule the
374 // tick event and enter the Idle state to prevent nasty things
375 // like MMIOs from happening.
376 if (tickEvent.scheduled())
377 deschedule(tickEvent);
378 _status = Idle;
379
380 /** FALLTHROUGH */
381 case Idle:
382 // Idle, no need to drain
383 assert(!tickEvent.scheduled());
384
385 // Sync the thread context here since we'll need it when we
386 // switch CPUs or checkpoint the CPU.
387 syncThreadContext();
388
389 return DrainState::Drained;
390
391 case RunningServiceCompletion:
392 // The CPU has just requested a service that was handled in
393 // the RunningService state, but the results have still not
394 // been reported to the CPU. Now, we /could/ probably just
395 // update the register state ourselves instead of letting KVM
396 // handle it, but that would be tricky. Instead, we enter KVM
397 // and let it do its stuff.
398 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
399 "requesting drain.\n");
400 return DrainState::Draining;
401
402 case RunningMMIOPending:
403 // We need to drain since there are in-flight timing accesses
404 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
405 "requesting drain.\n");
406 return DrainState::Draining;
407
408 case RunningService:
409 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
410 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
411 return DrainState::Draining;
412
413 default:
414 panic("KVM: Unhandled CPU state in drain()\n");
415 return DrainState::Drained;
416 }
417}
418
419void
420BaseKvmCPU::drainResume()
421{
422 assert(!tickEvent.scheduled());
423
424 // We might have been switched out. In that case, we don't need to
425 // do anything.
426 if (switchedOut())
427 return;
428
429 DPRINTF(Kvm, "drainResume\n");
430 verifyMemoryMode();
431
432 // The tick event is de-scheduled as a part of the draining
433 // process. Re-schedule it if the thread context is active.
434 if (tc->status() == ThreadContext::Active) {
435 schedule(tickEvent, nextCycle());
436 _status = Running;
437 } else {
438 _status = Idle;
439 }
440}
441
442void
443BaseKvmCPU::notifyFork()
444{
445 // We should have drained prior to forking, which means that the
446 // tick event shouldn't be scheduled and the CPU is idle.
447 assert(!tickEvent.scheduled());
448 assert(_status == Idle);
449
450 if (vcpuFD != -1) {
451 if (close(vcpuFD) == -1)
452 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
453
454 if (_kvmRun)
455 munmap(_kvmRun, vcpuMMapSize);
456
457 vcpuFD = -1;
458 _kvmRun = NULL;
459
460 hwInstructions.detach();
461 hwCycles.detach();
462 }
463}
464
465void
466BaseKvmCPU::switchOut()
467{
468 DPRINTF(Kvm, "switchOut\n");
469
470 BaseCPU::switchOut();
471
472 // We should have drained prior to executing a switchOut, which
473 // means that the tick event shouldn't be scheduled and the CPU is
474 // idle.
475 assert(!tickEvent.scheduled());
476 assert(_status == Idle);
477}
478
479void
480BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
481{
482 DPRINTF(Kvm, "takeOverFrom\n");
483
484 BaseCPU::takeOverFrom(cpu);
485
486 // We should have drained prior to executing a switchOut, which
487 // means that the tick event shouldn't be scheduled and the CPU is
488 // idle.
489 assert(!tickEvent.scheduled());
490 assert(_status == Idle);
491 assert(threadContexts.size() == 1);
492
493 // Force an update of the KVM state here instead of flagging the
494 // TC as dirty. This is not ideal from a performance point of
495 // view, but it makes debugging easier as it allows meaningful KVM
496 // state to be dumped before and after a takeover.
497 updateKvmState();
498 threadContextDirty = false;
499}
500
501void
502BaseKvmCPU::verifyMemoryMode() const
503{
504 if (!(system->bypassCaches())) {
505 fatal("The KVM-based CPUs requires the memory system to be in the "
506 "'noncaching' mode.\n");
507 }
508}
509
510void
511BaseKvmCPU::wakeup(ThreadID tid)
512{
513 DPRINTF(Kvm, "wakeup()\n");
514 // This method might have been called from another
515 // context. Migrate to this SimObject's event queue when
516 // delivering the wakeup signal.
517 EventQueue::ScopedMigration migrate(eventQueue());
518
519 // Kick the vCPU to get it to come out of KVM.
520 kick();
521
522 if (thread->status() != ThreadContext::Suspended)
523 return;
524
525 thread->activate();
526}
527
528void
529BaseKvmCPU::activateContext(ThreadID thread_num)
530{
531 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
532
533 assert(thread_num == 0);
534 assert(thread);
535
536 assert(_status == Idle);
537 assert(!tickEvent.scheduled());
538
539 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
540
541 schedule(tickEvent, clockEdge(Cycles(0)));
542 _status = Running;
543}
544
545
546void
547BaseKvmCPU::suspendContext(ThreadID thread_num)
548{
549 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
550
551 assert(thread_num == 0);
552 assert(thread);
553
554 if (_status == Idle)
555 return;
556
557 assert(_status == Running || _status == RunningServiceCompletion);
558
559 // The tick event may no be scheduled if the quest has requested
560 // the monitor to wait for interrupts. The normal CPU models can
561 // get their tick events descheduled by quiesce instructions, but
562 // that can't happen here.
563 if (tickEvent.scheduled())
564 deschedule(tickEvent);
565
566 _status = Idle;
567}
568
569void
570BaseKvmCPU::deallocateContext(ThreadID thread_num)
571{
572 // for now, these are equivalent
573 suspendContext(thread_num);
574}
575
576void
577BaseKvmCPU::haltContext(ThreadID thread_num)
578{
579 // for now, these are equivalent
580 suspendContext(thread_num);
581}
582
583ThreadContext *
584BaseKvmCPU::getContext(int tn)
585{
586 assert(tn == 0);
587 syncThreadContext();
588 return tc;
589}
590
591
592Counter
593BaseKvmCPU::totalInsts() const
594{
595 return ctrInsts;
596}
597
598Counter
599BaseKvmCPU::totalOps() const
600{
601 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
602 return ctrInsts;
603}
604
605void
606BaseKvmCPU::dump() const
607{
608 inform("State dumping not implemented.");
609}
610
611void
612BaseKvmCPU::tick()
613{
614 Tick delay(0);
615 assert(_status != Idle && _status != RunningMMIOPending);
616
617 switch (_status) {
618 case RunningService:
619 // handleKvmExit() will determine the next state of the CPU
620 delay = handleKvmExit();
621
622 if (tryDrain())
623 _status = Idle;
624 break;
625
626 case RunningServiceCompletion:
627 case Running: {
628 const uint64_t nextInstEvent(
629 !comInstEventQueue[0]->empty() ?
630 comInstEventQueue[0]->nextTick() : UINT64_MAX);
631 // Enter into KVM and complete pending IO instructions if we
632 // have an instruction event pending.
633 const Tick ticksToExecute(
634 nextInstEvent > ctrInsts ?
635 curEventQueue()->nextTick() - curTick() : 0);
636
637 if (alwaysSyncTC)
638 threadContextDirty = true;
639
640 // We might need to update the KVM state.
641 syncKvmState();
642
643 // Setup any pending instruction count breakpoints using
644 // PerfEvent if we are going to execute more than just an IO
645 // completion.
646 if (ticksToExecute > 0)
647 setupInstStop();
648
649 DPRINTF(KvmRun, "Entering KVM...\n");
650 if (drainState() == DrainState::Draining) {
651 // Force an immediate exit from KVM after completing
652 // pending operations. The architecture-specific code
653 // takes care to run until it is in a state where it can
654 // safely be drained.
655 delay = kvmRunDrain();
656 } else {
657 delay = kvmRun(ticksToExecute);
658 }
659
660 // The CPU might have been suspended before entering into
661 // KVM. Assume that the CPU was suspended /before/ entering
662 // into KVM and skip the exit handling.
663 if (_status == Idle)
664 break;
665
666 // Entering into KVM implies that we'll have to reload the thread
667 // context from KVM if we want to access it. Flag the KVM state as
668 // dirty with respect to the cached thread context.
669 kvmStateDirty = true;
670
671 if (alwaysSyncTC)
672 syncThreadContext();
673
674 // Enter into the RunningService state unless the
675 // simulation was stopped by a timer.
676 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
677 _status = RunningService;
678 } else {
679 ++numExitSignal;
680 _status = Running;
681 }
682
683 // Service any pending instruction events. The vCPU should
684 // have exited in time for the event using the instruction
685 // counter configured by setupInstStop().
686 comInstEventQueue[0]->serviceEvents(ctrInsts);
687 system->instEventQueue.serviceEvents(system->totalNumInsts);
688
689 if (tryDrain())
690 _status = Idle;
691 } break;
692
693 default:
694 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
695 _status);
696 }
697
698 // Schedule a new tick if we are still running
699 if (_status != Idle && _status != RunningMMIOPending)
700 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
701}
702
703Tick
704BaseKvmCPU::kvmRunDrain()
705{
706 // By default, the only thing we need to drain is a pending IO
707 // operation which assumes that we are in the
708 // RunningServiceCompletion or RunningMMIOPending state.
709 assert(_status == RunningServiceCompletion ||
710 _status == RunningMMIOPending);
711
712 // Deliver the data from the pending IO operation and immediately
713 // exit.
714 return kvmRun(0);
715}
716
717uint64_t
718BaseKvmCPU::getHostCycles() const
719{
720 return hwCycles.read();
721}
722
723Tick
724BaseKvmCPU::kvmRun(Tick ticks)
725{
726 Tick ticksExecuted;
727 fatal_if(vcpuFD == -1,
728 "Trying to run a KVM CPU in a forked child process. "
729 "This is not supported.\n");
730 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
731
732 if (ticks == 0) {
733 // Settings ticks == 0 is a special case which causes an entry
734 // into KVM that finishes pending operations (e.g., IO) and
735 // then immediately exits.
736 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
737
738 ++numVMHalfEntries;
739
740 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
741 // thread). The KVM control signal is masked while executing
742 // in gem5 and gets unmasked temporarily as when entering
743 // KVM. See setSignalMask() and setupSignalHandler().
744 kick();
745
746 // Start the vCPU. KVM will check for signals after completing
747 // pending operations (IO). Since the KVM_KICK_SIGNAL is
748 // pending, this forces an immediate exit to gem5 again. We
749 // don't bother to setup timers since this shouldn't actually
750 // execute any code (other than completing half-executed IO
751 // instructions) in the guest.
752 ioctlRun();
753
754 // We always execute at least one cycle to prevent the
755 // BaseKvmCPU::tick() to be rescheduled on the same tick
756 // twice.
757 ticksExecuted = clockPeriod();
758 } else {
759 // This method is executed as a result of a tick event. That
760 // means that the event queue will be locked when entering the
761 // method. We temporarily unlock the event queue to allow
762 // other threads to steal control of this thread to inject
763 // interrupts. They will typically lock the queue and then
764 // force an exit from KVM by kicking the vCPU.
765 EventQueue::ScopedRelease release(curEventQueue());
766
767 if (ticks < runTimer->resolution()) {
768 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
769 ticks, runTimer->resolution());
770 ticks = runTimer->resolution();
771 }
772
773 // Get hardware statistics after synchronizing contexts. The KVM
774 // state update might affect guest cycle counters.
775 uint64_t baseCycles(getHostCycles());
776 uint64_t baseInstrs(hwInstructions.read());
777
778 // Arm the run timer and start the cycle timer if it isn't
779 // controlled by the overflow timer. Starting/stopping the cycle
780 // timer automatically starts the other perf timers as they are in
781 // the same counter group.
782 runTimer->arm(ticks);
783 if (!perfControlledByTimer)
784 hwCycles.start();
785
786 ioctlRun();
787
788 runTimer->disarm();
789 if (!perfControlledByTimer)
790 hwCycles.stop();
791
792 // The control signal may have been delivered after we exited
793 // from KVM. It will be pending in that case since it is
794 // masked when we aren't executing in KVM. Discard it to make
795 // sure we don't deliver it immediately next time we try to
796 // enter into KVM.
797 discardPendingSignal(KVM_KICK_SIGNAL);
798
799 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
800 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
801 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
802 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
803
804 /* Update statistics */
805 numCycles += simCyclesExecuted;;
806 numInsts += instsExecuted;
807 ctrInsts += instsExecuted;
808 system->totalNumInsts += instsExecuted;
809
810 DPRINTF(KvmRun,
811 "KVM: Executed %i instructions in %i cycles "
812 "(%i ticks, sim cycles: %i).\n",
813 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
814 }
815
816 ++numVMExits;
817
818 return ticksExecuted + flushCoalescedMMIO();
819}
820
821void
822BaseKvmCPU::kvmNonMaskableInterrupt()
823{
824 ++numInterrupts;
825 if (ioctl(KVM_NMI) == -1)
826 panic("KVM: Failed to deliver NMI to virtual CPU\n");
827}
828
829void
830BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
831{
832 ++numInterrupts;
833 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
834 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
835}
836
837void
838BaseKvmCPU::getRegisters(struct kvm_regs &regs) const
839{
840 if (ioctl(KVM_GET_REGS, &regs) == -1)
841 panic("KVM: Failed to get guest registers\n");
842}
843
844void
845BaseKvmCPU::setRegisters(const struct kvm_regs &regs)
846{
847 if (ioctl(KVM_SET_REGS, (void *)&regs) == -1)
848 panic("KVM: Failed to set guest registers\n");
849}
850
851void
852BaseKvmCPU::getSpecialRegisters(struct kvm_sregs &regs) const
853{
854 if (ioctl(KVM_GET_SREGS, &regs) == -1)
855 panic("KVM: Failed to get guest special registers\n");
856}
857
858void
859BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs &regs)
860{
861 if (ioctl(KVM_SET_SREGS, (void *)&regs) == -1)
862 panic("KVM: Failed to set guest special registers\n");
863}
864
865void
866BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
867{
868 if (ioctl(KVM_GET_FPU, &state) == -1)
869 panic("KVM: Failed to get guest FPU state\n");
870}
871
872void
873BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
874{
875 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
876 panic("KVM: Failed to set guest FPU state\n");
877}
878
879
880void
881BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
882{
883#ifdef KVM_SET_ONE_REG
884 struct kvm_one_reg reg;
885 reg.id = id;
886 reg.addr = (uint64_t)addr;
887
888 if (ioctl(KVM_SET_ONE_REG, &reg) == -1) {
889 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
890 id, errno);
891 }
892#else
893 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
894#endif
895}
896
897void
898BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
899{
900#ifdef KVM_GET_ONE_REG
901 struct kvm_one_reg reg;
902 reg.id = id;
903 reg.addr = (uint64_t)addr;
904
905 if (ioctl(KVM_GET_ONE_REG, &reg) == -1) {
906 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
907 id, errno);
908 }
909#else
910 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
911#endif
912}
913
914std::string
915BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
916{
917#ifdef KVM_GET_ONE_REG
918 std::ostringstream ss;
919
920 ss.setf(std::ios::hex, std::ios::basefield);
921 ss.setf(std::ios::showbase);
922#define HANDLE_INTTYPE(len) \
923 case KVM_REG_SIZE_U ## len: { \
924 uint ## len ## _t value; \
925 getOneReg(id, &value); \
926 ss << value; \
927 } break
928
929#define HANDLE_ARRAY(len) \
930 case KVM_REG_SIZE_U ## len: { \
931 uint8_t value[len / 8]; \
932 getOneReg(id, value); \
933 ccprintf(ss, "[0x%x", value[0]); \
934 for (int i = 1; i < len / 8; ++i) \
935 ccprintf(ss, ", 0x%x", value[i]); \
936 ccprintf(ss, "]"); \
937 } break
938
939 switch (id & KVM_REG_SIZE_MASK) {
940 HANDLE_INTTYPE(8);
941 HANDLE_INTTYPE(16);
942 HANDLE_INTTYPE(32);
943 HANDLE_INTTYPE(64);
944 HANDLE_ARRAY(128);
945 HANDLE_ARRAY(256);
946 HANDLE_ARRAY(512);
947 HANDLE_ARRAY(1024);
948 default:
949 ss << "??";
950 }
951
952#undef HANDLE_INTTYPE
953#undef HANDLE_ARRAY
954
955 return ss.str();
956#else
957 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
958#endif
959}
960
961void
962BaseKvmCPU::syncThreadContext()
963{
964 if (!kvmStateDirty)
965 return;
966
967 assert(!threadContextDirty);
968
969 updateThreadContext();
970 kvmStateDirty = false;
971}
972
973void
974BaseKvmCPU::syncKvmState()
975{
976 if (!threadContextDirty)
977 return;
978
979 assert(!kvmStateDirty);
980
981 updateKvmState();
982 threadContextDirty = false;
983}
984
985Tick
986BaseKvmCPU::handleKvmExit()
987{
988 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
989 assert(_status == RunningService);
990
991 // Switch into the running state by default. Individual handlers
992 // can override this.
993 _status = Running;
994 switch (_kvmRun->exit_reason) {
995 case KVM_EXIT_UNKNOWN:
996 return handleKvmExitUnknown();
997
998 case KVM_EXIT_EXCEPTION:
999 return handleKvmExitException();
1000
1001 case KVM_EXIT_IO:
1002 {
1003 ++numIO;
1004 Tick ticks = handleKvmExitIO();
1005 _status = dataPort.nextIOState();
1006 return ticks;
1007 }
1008
1009 case KVM_EXIT_HYPERCALL:
1010 ++numHypercalls;
1011 return handleKvmExitHypercall();
1012
1013 case KVM_EXIT_HLT:
1014 /* The guest has halted and is waiting for interrupts */
1015 DPRINTF(Kvm, "handleKvmExitHalt\n");
1016 ++numHalt;
1017
1018 // Suspend the thread until the next interrupt arrives
1019 thread->suspend();
1020
1021 // This is actually ignored since the thread is suspended.
1022 return 0;
1023
1024 case KVM_EXIT_MMIO:
1025 {
1026 /* Service memory mapped IO requests */
1027 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1028 _kvmRun->mmio.is_write,
1029 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1030
1031 ++numMMIO;
1032 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1033 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1034 // doMMIOAccess could have triggered a suspend, in which case we don't
1035 // want to overwrite the _status.
1036 if (_status != Idle)
1037 _status = dataPort.nextIOState();
1038 return ticks;
1039 }
1040
1041 case KVM_EXIT_IRQ_WINDOW_OPEN:
1042 return handleKvmExitIRQWindowOpen();
1043
1044 case KVM_EXIT_FAIL_ENTRY:
1045 return handleKvmExitFailEntry();
1046
1047 case KVM_EXIT_INTR:
1048 /* KVM was interrupted by a signal, restart it in the next
1049 * tick. */
1050 return 0;
1051
1052 case KVM_EXIT_INTERNAL_ERROR:
1053 panic("KVM: Internal error (suberror: %u)\n",
1054 _kvmRun->internal.suberror);
1055
1056 default:
1057 dump();
1058 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1059 }
1060}
1061
1062Tick
1063BaseKvmCPU::handleKvmExitIO()
1064{
1065 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1066 _kvmRun->io.direction, _kvmRun->io.size,
1067 _kvmRun->io.port, _kvmRun->io.count);
1068}
1069
1070Tick
1071BaseKvmCPU::handleKvmExitHypercall()
1072{
1073 panic("KVM: Unhandled hypercall\n");
1074}
1075
1076Tick
1077BaseKvmCPU::handleKvmExitIRQWindowOpen()
1078{
1079 warn("KVM: Unhandled IRQ window.\n");
1080 return 0;
1081}
1082
1083
1084Tick
1085BaseKvmCPU::handleKvmExitUnknown()
1086{
1087 dump();
1088 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1089 _kvmRun->hw.hardware_exit_reason);
1090}
1091
1092Tick
1093BaseKvmCPU::handleKvmExitException()
1094{
1095 dump();
1096 panic("KVM: Got exception when starting vCPU "
1097 "(exception: %u, error_code: %u)\n",
1098 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1099}
1100
1101Tick
1102BaseKvmCPU::handleKvmExitFailEntry()
1103{
1104 dump();
1105 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1106 _kvmRun->fail_entry.hardware_entry_failure_reason);
1107}
1108
1109Tick
1110BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1111{
1112 ThreadContext *tc(thread->getTC());
1113 syncThreadContext();
1114
1115 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1116 dataMasterId());
1117 mmio_req->setContext(tc->contextId());
1118 // Some architectures do need to massage physical addresses a bit
1119 // before they are inserted into the memory system. This enables
1120 // APIC accesses on x86 and m5ops where supported through a MMIO
1121 // interface.
1122 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1123 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1124 if (fault != NoFault)
1125 warn("Finalization of MMIO address failed: %s\n", fault->name());
1126
1127
1128 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1129 PacketPtr pkt = new Packet(mmio_req, cmd);
1130 pkt->dataStatic(data);
1131
1132 if (mmio_req->isMmappedIpr()) {
1133 // We currently assume that there is no need to migrate to a
1134 // different event queue when doing IPRs. Currently, IPRs are
1135 // only used for m5ops, so it should be a valid assumption.
1136 const Cycles ipr_delay(write ?
1137 TheISA::handleIprWrite(tc, pkt) :
1138 TheISA::handleIprRead(tc, pkt));
1139 threadContextDirty = true;
1140 delete pkt->req;
1141 delete pkt;
1142 return clockPeriod() * ipr_delay;
1143 } else {
1144 // Temporarily lock and migrate to the event queue of the
1145 // VM. This queue is assumed to "own" all devices we need to
1146 // access if running in multi-core mode.
1147 EventQueue::ScopedMigration migrate(vm.eventQueue());
1148
1149 return dataPort.submitIO(pkt);
1150 }
1151}
1152
1153void
1154BaseKvmCPU::setSignalMask(const sigset_t *mask)
1155{
1156 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1157
1158 if (mask) {
1159 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1160 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1161 // The kernel and the user-space headers have different ideas
1162 // about the size of sigset_t. This seems like a massive hack,
1163 // but is actually what qemu does.
1164 assert(sizeof(*mask) >= 8);
1165 kvm_mask->len = 8;
1166 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1167 }
1168
1169 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1170 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1171 errno);
1172}
1173
1174int
1175BaseKvmCPU::ioctl(int request, long p1) const
1176{
1177 if (vcpuFD == -1)
1178 panic("KVM: CPU ioctl called before initialization\n");
1179
1180 return ::ioctl(vcpuFD, request, p1);
1181}
1182
1183Tick
1184BaseKvmCPU::flushCoalescedMMIO()
1185{
1186 if (!mmioRing)
1187 return 0;
1188
1189 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1190
1191 // TODO: We might need to do synchronization when we start to
1192 // support multiple CPUs
1193 Tick ticks(0);
1194 while (mmioRing->first != mmioRing->last) {
1195 struct kvm_coalesced_mmio &ent(
1196 mmioRing->coalesced_mmio[mmioRing->first]);
1197
1198 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1199 ent.phys_addr, ent.len);
1200
1201 ++numCoalescedMMIO;
1202 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1203
1204 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1205 }
1206
1207 return ticks;
1208}
1209
1210/**
1211 * Dummy handler for KVM kick signals.
1212 *
1213 * @note This function is usually not called since the kernel doesn't
1214 * seem to deliver signals when the signal is only unmasked when
1215 * running in KVM. This doesn't matter though since we are only
1216 * interested in getting KVM to exit, which happens as expected. See
1217 * setupSignalHandler() and kvmRun() for details about KVM signal
1218 * handling.
1219 */
1220static void
1221onKickSignal(int signo, siginfo_t *si, void *data)
1222{
1223}
1224
1225void
1226BaseKvmCPU::setupSignalHandler()
1227{
1228 struct sigaction sa;
1229
1230 memset(&sa, 0, sizeof(sa));
1231 sa.sa_sigaction = onKickSignal;
1232 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1233 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1234 panic("KVM: Failed to setup vCPU timer signal handler\n");
1235
1236 sigset_t sigset;
1237 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1238 panic("KVM: Failed get signal mask\n");
1239
1240 // Request KVM to setup the same signal mask as we're currently
1241 // running with except for the KVM control signal. We'll sometimes
1242 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1243 // KVM after servicing IO requests. See kvmRun().
1244 sigdelset(&sigset, KVM_KICK_SIGNAL);
1245 setSignalMask(&sigset);
1246
1247 // Mask our control signals so they aren't delivered unless we're
1248 // actually executing inside KVM.
1249 sigaddset(&sigset, KVM_KICK_SIGNAL);
1250 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1251 panic("KVM: Failed mask the KVM control signals\n");
1252}
1253
1254bool
1255BaseKvmCPU::discardPendingSignal(int signum) const
1256{
1257 int discardedSignal;
1258
1259 // Setting the timeout to zero causes sigtimedwait to return
1260 // immediately.
1261 struct timespec timeout;
1262 timeout.tv_sec = 0;
1263 timeout.tv_nsec = 0;
1264
1265 sigset_t sigset;
1266 sigemptyset(&sigset);
1267 sigaddset(&sigset, signum);
1268
1269 do {
1270 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1271 } while (discardedSignal == -1 && errno == EINTR);
1272
1273 if (discardedSignal == signum)
1274 return true;
1275 else if (discardedSignal == -1 && errno == EAGAIN)
1276 return false;
1277 else
1278 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1279 discardedSignal, errno);
1280}
1281
1282void
1283BaseKvmCPU::setupCounters()
1284{
1285 DPRINTF(Kvm, "Attaching cycle counter...\n");
1286 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1287 PERF_COUNT_HW_CPU_CYCLES);
1288 cfgCycles.disabled(true)
1289 .pinned(true);
1290
1291 // Try to exclude the host. We set both exclude_hv and
1292 // exclude_host since different architectures use slightly
1293 // different APIs in the kernel.
1294 cfgCycles.exclude_hv(true)
1295 .exclude_host(true);
1296
1297 if (perfControlledByTimer) {
1298 // We need to configure the cycles counter to send overflows
1299 // since we are going to use it to trigger timer signals that
1300 // trap back into m5 from KVM. In practice, this means that we
1301 // need to set some non-zero sample period that gets
1302 // overridden when the timer is armed.
1303 cfgCycles.wakeupEvents(1)
1304 .samplePeriod(42);
1305 }
1306
1307 hwCycles.attach(cfgCycles,
1308 0); // TID (0 => currentThread)
1309
1310 setupInstCounter();
1311}
1312
1313bool
1314BaseKvmCPU::tryDrain()
1315{
1316 if (drainState() != DrainState::Draining)
1317 return false;
1318
1319 if (!archIsDrained()) {
1320 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1321 return false;
1322 }
1323
1324 if (_status == Idle || _status == Running) {
1325 DPRINTF(Drain,
1326 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1327 signalDrainDone();
1328 return true;
1329 } else {
1330 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1331 return false;
1332 }
1333}
1334
1335void
1336BaseKvmCPU::ioctlRun()
1337{
1338 if (ioctl(KVM_RUN) == -1) {
1339 if (errno != EINTR)
1340 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1341 errno);
1342 }
1343}
1344
1345void
1346BaseKvmCPU::setupInstStop()
1347{
1348 if (comInstEventQueue[0]->empty()) {
1349 setupInstCounter(0);
1350 } else {
1351 const uint64_t next(comInstEventQueue[0]->nextTick());
1352
1353 assert(next > ctrInsts);
1354 setupInstCounter(next - ctrInsts);
1355 }
1356}
1357
1358void
1359BaseKvmCPU::setupInstCounter(uint64_t period)
1360{
1361 // No need to do anything if we aren't attaching for the first
1362 // time or the period isn't changing.
1363 if (period == activeInstPeriod && hwInstructions.attached())
1364 return;
1365
1366 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1367 PERF_COUNT_HW_INSTRUCTIONS);
1368
1369 // Try to exclude the host. We set both exclude_hv and
1370 // exclude_host since different architectures use slightly
1371 // different APIs in the kernel.
1372 cfgInstructions.exclude_hv(true)
1373 .exclude_host(true);
1374
1375 if (period) {
1376 // Setup a sampling counter if that has been requested.
1377 cfgInstructions.wakeupEvents(1)
1378 .samplePeriod(period);
1379 }
1380
1381 // We need to detach and re-attach the counter to reliably change
1382 // sampling settings. See PerfKvmCounter::period() for details.
1383 if (hwInstructions.attached())
1384 hwInstructions.detach();
1385 assert(hwCycles.attached());
1386 hwInstructions.attach(cfgInstructions,
1387 0, // TID (0 => currentThread)
1388 hwCycles);
1389
1390 if (period)
1391 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1392
1393 activeInstPeriod = period;
1394}