63/* Used by some KVM macros */
64#define PAGE_SIZE pageSize
65
66BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
67 : BaseCPU(params),
68 vm(*params->kvmVM),
69 _status(Idle),
70 dataPort(name() + ".dcache_port", this),
71 instPort(name() + ".icache_port", this),
72 alwaysSyncTC(params->alwaysSyncTC),
73 threadContextDirty(true),
74 kvmStateDirty(false),
75 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
76 _kvmRun(NULL), mmioRing(NULL),
77 pageSize(sysconf(_SC_PAGE_SIZE)),
78 tickEvent(*this),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventWrapper<BaseKvmCPU,
169 &BaseKvmCPU::startupThread>(this, true));
170 schedule(startupEvent, curTick());
171}
172
173BaseKvmCPU::Status
174BaseKvmCPU::KVMCpuPort::nextIOState() const
175{
176 return (activeMMIOReqs || pendingMMIOPkts.size())
177 ? RunningMMIOPending : RunningServiceCompletion;
178}
179
180Tick
181BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
182{
183 if (cpu->system->isAtomicMode()) {
184 Tick delay = sendAtomic(pkt);
185 delete pkt->req;
186 delete pkt;
187 return delay;
188 } else {
189 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
190 activeMMIOReqs++;
191 } else {
192 pendingMMIOPkts.push(pkt);
193 }
194 // Return value is irrelevant for timing-mode accesses.
195 return 0;
196 }
197}
198
199bool
200BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
201{
202 DPRINTF(KvmIO, "KVM: Finished timing request\n");
203
204 delete pkt->req;
205 delete pkt;
206 activeMMIOReqs--;
207
208 // We can switch back into KVM when all pending and in-flight MMIO
209 // operations have completed.
210 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
211 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
212 cpu->finishMMIOPending();
213 }
214 return true;
215}
216
217void
218BaseKvmCPU::KVMCpuPort::recvReqRetry()
219{
220 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
221
222 assert(pendingMMIOPkts.size());
223
224 // Assuming that we can issue infinite requests this cycle is a bit
225 // unrealistic, but it's not worth modeling something more complex in
226 // KVM.
227 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
228 pendingMMIOPkts.pop();
229 activeMMIOReqs++;
230 }
231}
232
233void
234BaseKvmCPU::finishMMIOPending()
235{
236 assert(_status = RunningMMIOPending);
237 assert(!tickEvent.scheduled());
238
239 _status = RunningServiceCompletion;
240 schedule(tickEvent, nextCycle());
241}
242
243void
244BaseKvmCPU::startupThread()
245{
246 // Do thread-specific initialization. We need to setup signal
247 // delivery for counters and timers from within the thread that
248 // will execute the event queue to ensure that signals are
249 // delivered to the right threads.
250 const BaseKvmCPUParams * const p(
251 dynamic_cast<const BaseKvmCPUParams *>(params()));
252
253 vcpuThread = pthread_self();
254
255 // Setup signal handlers. This has to be done after the vCPU is
256 // created since it manipulates the vCPU signal mask.
257 setupSignalHandler();
258
259 setupCounters();
260
261 if (p->usePerfOverflow)
262 runTimer.reset(new PerfKvmTimer(hwCycles,
263 KVM_KICK_SIGNAL,
264 p->hostFactor,
265 p->hostFreq));
266 else
267 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
268 p->hostFactor,
269 p->hostFreq));
270
271}
272
273void
274BaseKvmCPU::regStats()
275{
276 using namespace Stats;
277
278 BaseCPU::regStats();
279
280 numInsts
281 .name(name() + ".committedInsts")
282 .desc("Number of instructions committed")
283 ;
284
285 numVMExits
286 .name(name() + ".numVMExits")
287 .desc("total number of KVM exits")
288 ;
289
290 numVMHalfEntries
291 .name(name() + ".numVMHalfEntries")
292 .desc("number of KVM entries to finalize pending operations")
293 ;
294
295 numExitSignal
296 .name(name() + ".numExitSignal")
297 .desc("exits due to signal delivery")
298 ;
299
300 numMMIO
301 .name(name() + ".numMMIO")
302 .desc("number of VM exits due to memory mapped IO")
303 ;
304
305 numCoalescedMMIO
306 .name(name() + ".numCoalescedMMIO")
307 .desc("number of coalesced memory mapped IO requests")
308 ;
309
310 numIO
311 .name(name() + ".numIO")
312 .desc("number of VM exits due to legacy IO")
313 ;
314
315 numHalt
316 .name(name() + ".numHalt")
317 .desc("number of VM exits due to wait for interrupt instructions")
318 ;
319
320 numInterrupts
321 .name(name() + ".numInterrupts")
322 .desc("number of interrupts delivered")
323 ;
324
325 numHypercalls
326 .name(name() + ".numHypercalls")
327 .desc("number of hypercalls")
328 ;
329}
330
331void
332BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
333{
334 if (DTRACE(Checkpoint)) {
335 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
336 dump();
337 }
338
339 assert(tid == 0);
340 assert(_status == Idle);
341 thread->serialize(cp);
342}
343
344void
345BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
346{
347 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
348
349 assert(tid == 0);
350 assert(_status == Idle);
351 thread->unserialize(cp);
352 threadContextDirty = true;
353}
354
355DrainState
356BaseKvmCPU::drain()
357{
358 if (switchedOut())
359 return DrainState::Drained;
360
361 DPRINTF(Drain, "BaseKvmCPU::drain\n");
362 switch (_status) {
363 case Running:
364 // The base KVM code is normally ready when it is in the
365 // Running state, but the architecture specific code might be
366 // of a different opinion. This may happen when the CPU been
367 // notified of an event that hasn't been accepted by the vCPU
368 // yet.
369 if (!archIsDrained())
370 return DrainState::Draining;
371
372 // The state of the CPU is consistent, so we don't need to do
373 // anything special to drain it. We simply de-schedule the
374 // tick event and enter the Idle state to prevent nasty things
375 // like MMIOs from happening.
376 if (tickEvent.scheduled())
377 deschedule(tickEvent);
378 _status = Idle;
379
380 /** FALLTHROUGH */
381 case Idle:
382 // Idle, no need to drain
383 assert(!tickEvent.scheduled());
384
385 // Sync the thread context here since we'll need it when we
386 // switch CPUs or checkpoint the CPU.
387 syncThreadContext();
388
389 return DrainState::Drained;
390
391 case RunningServiceCompletion:
392 // The CPU has just requested a service that was handled in
393 // the RunningService state, but the results have still not
394 // been reported to the CPU. Now, we /could/ probably just
395 // update the register state ourselves instead of letting KVM
396 // handle it, but that would be tricky. Instead, we enter KVM
397 // and let it do its stuff.
398 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
399 "requesting drain.\n");
400 return DrainState::Draining;
401
402 case RunningMMIOPending:
403 // We need to drain since there are in-flight timing accesses
404 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
405 "requesting drain.\n");
406 return DrainState::Draining;
407
408 case RunningService:
409 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
410 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
411 return DrainState::Draining;
412
413 default:
414 panic("KVM: Unhandled CPU state in drain()\n");
415 return DrainState::Drained;
416 }
417}
418
419void
420BaseKvmCPU::drainResume()
421{
422 assert(!tickEvent.scheduled());
423
424 // We might have been switched out. In that case, we don't need to
425 // do anything.
426 if (switchedOut())
427 return;
428
429 DPRINTF(Kvm, "drainResume\n");
430 verifyMemoryMode();
431
432 // The tick event is de-scheduled as a part of the draining
433 // process. Re-schedule it if the thread context is active.
434 if (tc->status() == ThreadContext::Active) {
435 schedule(tickEvent, nextCycle());
436 _status = Running;
437 } else {
438 _status = Idle;
439 }
440}
441
442void
443BaseKvmCPU::notifyFork()
444{
445 // We should have drained prior to forking, which means that the
446 // tick event shouldn't be scheduled and the CPU is idle.
447 assert(!tickEvent.scheduled());
448 assert(_status == Idle);
449
450 if (vcpuFD != -1) {
451 if (close(vcpuFD) == -1)
452 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
453
454 if (_kvmRun)
455 munmap(_kvmRun, vcpuMMapSize);
456
457 vcpuFD = -1;
458 _kvmRun = NULL;
459
460 hwInstructions.detach();
461 hwCycles.detach();
462 }
463}
464
465void
466BaseKvmCPU::switchOut()
467{
468 DPRINTF(Kvm, "switchOut\n");
469
470 BaseCPU::switchOut();
471
472 // We should have drained prior to executing a switchOut, which
473 // means that the tick event shouldn't be scheduled and the CPU is
474 // idle.
475 assert(!tickEvent.scheduled());
476 assert(_status == Idle);
477}
478
479void
480BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
481{
482 DPRINTF(Kvm, "takeOverFrom\n");
483
484 BaseCPU::takeOverFrom(cpu);
485
486 // We should have drained prior to executing a switchOut, which
487 // means that the tick event shouldn't be scheduled and the CPU is
488 // idle.
489 assert(!tickEvent.scheduled());
490 assert(_status == Idle);
491 assert(threadContexts.size() == 1);
492
493 // Force an update of the KVM state here instead of flagging the
494 // TC as dirty. This is not ideal from a performance point of
495 // view, but it makes debugging easier as it allows meaningful KVM
496 // state to be dumped before and after a takeover.
497 updateKvmState();
498 threadContextDirty = false;
499}
500
501void
502BaseKvmCPU::verifyMemoryMode() const
503{
504 if (!(system->bypassCaches())) {
505 fatal("The KVM-based CPUs requires the memory system to be in the "
506 "'noncaching' mode.\n");
507 }
508}
509
510void
511BaseKvmCPU::wakeup(ThreadID tid)
512{
513 DPRINTF(Kvm, "wakeup()\n");
514 // This method might have been called from another
515 // context. Migrate to this SimObject's event queue when
516 // delivering the wakeup signal.
517 EventQueue::ScopedMigration migrate(eventQueue());
518
519 // Kick the vCPU to get it to come out of KVM.
520 kick();
521
522 if (thread->status() != ThreadContext::Suspended)
523 return;
524
525 thread->activate();
526}
527
528void
529BaseKvmCPU::activateContext(ThreadID thread_num)
530{
531 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
532
533 assert(thread_num == 0);
534 assert(thread);
535
536 assert(_status == Idle);
537 assert(!tickEvent.scheduled());
538
539 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
540
541 schedule(tickEvent, clockEdge(Cycles(0)));
542 _status = Running;
543}
544
545
546void
547BaseKvmCPU::suspendContext(ThreadID thread_num)
548{
549 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
550
551 assert(thread_num == 0);
552 assert(thread);
553
554 if (_status == Idle)
555 return;
556
557 assert(_status == Running || _status == RunningServiceCompletion);
558
559 // The tick event may no be scheduled if the quest has requested
560 // the monitor to wait for interrupts. The normal CPU models can
561 // get their tick events descheduled by quiesce instructions, but
562 // that can't happen here.
563 if (tickEvent.scheduled())
564 deschedule(tickEvent);
565
566 _status = Idle;
567}
568
569void
570BaseKvmCPU::deallocateContext(ThreadID thread_num)
571{
572 // for now, these are equivalent
573 suspendContext(thread_num);
574}
575
576void
577BaseKvmCPU::haltContext(ThreadID thread_num)
578{
579 // for now, these are equivalent
580 suspendContext(thread_num);
581}
582
583ThreadContext *
584BaseKvmCPU::getContext(int tn)
585{
586 assert(tn == 0);
587 syncThreadContext();
588 return tc;
589}
590
591
592Counter
593BaseKvmCPU::totalInsts() const
594{
595 return ctrInsts;
596}
597
598Counter
599BaseKvmCPU::totalOps() const
600{
601 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
602 return ctrInsts;
603}
604
605void
606BaseKvmCPU::dump() const
607{
608 inform("State dumping not implemented.");
609}
610
611void
612BaseKvmCPU::tick()
613{
614 Tick delay(0);
615 assert(_status != Idle && _status != RunningMMIOPending);
616
617 switch (_status) {
618 case RunningService:
619 // handleKvmExit() will determine the next state of the CPU
620 delay = handleKvmExit();
621
622 if (tryDrain())
623 _status = Idle;
624 break;
625
626 case RunningServiceCompletion:
627 case Running: {
628 const uint64_t nextInstEvent(
629 !comInstEventQueue[0]->empty() ?
630 comInstEventQueue[0]->nextTick() : UINT64_MAX);
631 // Enter into KVM and complete pending IO instructions if we
632 // have an instruction event pending.
633 const Tick ticksToExecute(
634 nextInstEvent > ctrInsts ?
635 curEventQueue()->nextTick() - curTick() : 0);
636
637 if (alwaysSyncTC)
638 threadContextDirty = true;
639
640 // We might need to update the KVM state.
641 syncKvmState();
642
643 // Setup any pending instruction count breakpoints using
644 // PerfEvent if we are going to execute more than just an IO
645 // completion.
646 if (ticksToExecute > 0)
647 setupInstStop();
648
649 DPRINTF(KvmRun, "Entering KVM...\n");
650 if (drainState() == DrainState::Draining) {
651 // Force an immediate exit from KVM after completing
652 // pending operations. The architecture-specific code
653 // takes care to run until it is in a state where it can
654 // safely be drained.
655 delay = kvmRunDrain();
656 } else {
657 delay = kvmRun(ticksToExecute);
658 }
659
660 // The CPU might have been suspended before entering into
661 // KVM. Assume that the CPU was suspended /before/ entering
662 // into KVM and skip the exit handling.
663 if (_status == Idle)
664 break;
665
666 // Entering into KVM implies that we'll have to reload the thread
667 // context from KVM if we want to access it. Flag the KVM state as
668 // dirty with respect to the cached thread context.
669 kvmStateDirty = true;
670
671 if (alwaysSyncTC)
672 syncThreadContext();
673
674 // Enter into the RunningService state unless the
675 // simulation was stopped by a timer.
676 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
677 _status = RunningService;
678 } else {
679 ++numExitSignal;
680 _status = Running;
681 }
682
683 // Service any pending instruction events. The vCPU should
684 // have exited in time for the event using the instruction
685 // counter configured by setupInstStop().
686 comInstEventQueue[0]->serviceEvents(ctrInsts);
687 system->instEventQueue.serviceEvents(system->totalNumInsts);
688
689 if (tryDrain())
690 _status = Idle;
691 } break;
692
693 default:
694 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
695 _status);
696 }
697
698 // Schedule a new tick if we are still running
699 if (_status != Idle && _status != RunningMMIOPending)
700 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
701}
702
703Tick
704BaseKvmCPU::kvmRunDrain()
705{
706 // By default, the only thing we need to drain is a pending IO
707 // operation which assumes that we are in the
708 // RunningServiceCompletion or RunningMMIOPending state.
709 assert(_status == RunningServiceCompletion ||
710 _status == RunningMMIOPending);
711
712 // Deliver the data from the pending IO operation and immediately
713 // exit.
714 return kvmRun(0);
715}
716
717uint64_t
718BaseKvmCPU::getHostCycles() const
719{
720 return hwCycles.read();
721}
722
723Tick
724BaseKvmCPU::kvmRun(Tick ticks)
725{
726 Tick ticksExecuted;
727 fatal_if(vcpuFD == -1,
728 "Trying to run a KVM CPU in a forked child process. "
729 "This is not supported.\n");
730 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
731
732 if (ticks == 0) {
733 // Settings ticks == 0 is a special case which causes an entry
734 // into KVM that finishes pending operations (e.g., IO) and
735 // then immediately exits.
736 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
737
738 ++numVMHalfEntries;
739
740 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
741 // thread). The KVM control signal is masked while executing
742 // in gem5 and gets unmasked temporarily as when entering
743 // KVM. See setSignalMask() and setupSignalHandler().
744 kick();
745
746 // Start the vCPU. KVM will check for signals after completing
747 // pending operations (IO). Since the KVM_KICK_SIGNAL is
748 // pending, this forces an immediate exit to gem5 again. We
749 // don't bother to setup timers since this shouldn't actually
750 // execute any code (other than completing half-executed IO
751 // instructions) in the guest.
752 ioctlRun();
753
754 // We always execute at least one cycle to prevent the
755 // BaseKvmCPU::tick() to be rescheduled on the same tick
756 // twice.
757 ticksExecuted = clockPeriod();
758 } else {
759 // This method is executed as a result of a tick event. That
760 // means that the event queue will be locked when entering the
761 // method. We temporarily unlock the event queue to allow
762 // other threads to steal control of this thread to inject
763 // interrupts. They will typically lock the queue and then
764 // force an exit from KVM by kicking the vCPU.
765 EventQueue::ScopedRelease release(curEventQueue());
766
767 if (ticks < runTimer->resolution()) {
768 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
769 ticks, runTimer->resolution());
770 ticks = runTimer->resolution();
771 }
772
773 // Get hardware statistics after synchronizing contexts. The KVM
774 // state update might affect guest cycle counters.
775 uint64_t baseCycles(getHostCycles());
776 uint64_t baseInstrs(hwInstructions.read());
777
778 // Arm the run timer and start the cycle timer if it isn't
779 // controlled by the overflow timer. Starting/stopping the cycle
780 // timer automatically starts the other perf timers as they are in
781 // the same counter group.
782 runTimer->arm(ticks);
783 if (!perfControlledByTimer)
784 hwCycles.start();
785
786 ioctlRun();
787
788 runTimer->disarm();
789 if (!perfControlledByTimer)
790 hwCycles.stop();
791
792 // The control signal may have been delivered after we exited
793 // from KVM. It will be pending in that case since it is
794 // masked when we aren't executing in KVM. Discard it to make
795 // sure we don't deliver it immediately next time we try to
796 // enter into KVM.
797 discardPendingSignal(KVM_KICK_SIGNAL);
798
799 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
800 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
801 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
802 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
803
804 /* Update statistics */
805 numCycles += simCyclesExecuted;;
806 numInsts += instsExecuted;
807 ctrInsts += instsExecuted;
808 system->totalNumInsts += instsExecuted;
809
810 DPRINTF(KvmRun,
811 "KVM: Executed %i instructions in %i cycles "
812 "(%i ticks, sim cycles: %i).\n",
813 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
814 }
815
816 ++numVMExits;
817
818 return ticksExecuted + flushCoalescedMMIO();
819}
820
821void
822BaseKvmCPU::kvmNonMaskableInterrupt()
823{
824 ++numInterrupts;
825 if (ioctl(KVM_NMI) == -1)
826 panic("KVM: Failed to deliver NMI to virtual CPU\n");
827}
828
829void
830BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
831{
832 ++numInterrupts;
833 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
834 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
835}
836
837void
838BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
839{
840 if (ioctl(KVM_GET_REGS, ®s) == -1)
841 panic("KVM: Failed to get guest registers\n");
842}
843
844void
845BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
846{
847 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
848 panic("KVM: Failed to set guest registers\n");
849}
850
851void
852BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
853{
854 if (ioctl(KVM_GET_SREGS, ®s) == -1)
855 panic("KVM: Failed to get guest special registers\n");
856}
857
858void
859BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
860{
861 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
862 panic("KVM: Failed to set guest special registers\n");
863}
864
865void
866BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
867{
868 if (ioctl(KVM_GET_FPU, &state) == -1)
869 panic("KVM: Failed to get guest FPU state\n");
870}
871
872void
873BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
874{
875 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
876 panic("KVM: Failed to set guest FPU state\n");
877}
878
879
880void
881BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
882{
883#ifdef KVM_SET_ONE_REG
884 struct kvm_one_reg reg;
885 reg.id = id;
886 reg.addr = (uint64_t)addr;
887
888 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
889 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
890 id, errno);
891 }
892#else
893 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
894#endif
895}
896
897void
898BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
899{
900#ifdef KVM_GET_ONE_REG
901 struct kvm_one_reg reg;
902 reg.id = id;
903 reg.addr = (uint64_t)addr;
904
905 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
906 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
907 id, errno);
908 }
909#else
910 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
911#endif
912}
913
914std::string
915BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
916{
917#ifdef KVM_GET_ONE_REG
918 std::ostringstream ss;
919
920 ss.setf(std::ios::hex, std::ios::basefield);
921 ss.setf(std::ios::showbase);
922#define HANDLE_INTTYPE(len) \
923 case KVM_REG_SIZE_U ## len: { \
924 uint ## len ## _t value; \
925 getOneReg(id, &value); \
926 ss << value; \
927 } break
928
929#define HANDLE_ARRAY(len) \
930 case KVM_REG_SIZE_U ## len: { \
931 uint8_t value[len / 8]; \
932 getOneReg(id, value); \
933 ccprintf(ss, "[0x%x", value[0]); \
934 for (int i = 1; i < len / 8; ++i) \
935 ccprintf(ss, ", 0x%x", value[i]); \
936 ccprintf(ss, "]"); \
937 } break
938
939 switch (id & KVM_REG_SIZE_MASK) {
940 HANDLE_INTTYPE(8);
941 HANDLE_INTTYPE(16);
942 HANDLE_INTTYPE(32);
943 HANDLE_INTTYPE(64);
944 HANDLE_ARRAY(128);
945 HANDLE_ARRAY(256);
946 HANDLE_ARRAY(512);
947 HANDLE_ARRAY(1024);
948 default:
949 ss << "??";
950 }
951
952#undef HANDLE_INTTYPE
953#undef HANDLE_ARRAY
954
955 return ss.str();
956#else
957 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
958#endif
959}
960
961void
962BaseKvmCPU::syncThreadContext()
963{
964 if (!kvmStateDirty)
965 return;
966
967 assert(!threadContextDirty);
968
969 updateThreadContext();
970 kvmStateDirty = false;
971}
972
973void
974BaseKvmCPU::syncKvmState()
975{
976 if (!threadContextDirty)
977 return;
978
979 assert(!kvmStateDirty);
980
981 updateKvmState();
982 threadContextDirty = false;
983}
984
985Tick
986BaseKvmCPU::handleKvmExit()
987{
988 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
989 assert(_status == RunningService);
990
991 // Switch into the running state by default. Individual handlers
992 // can override this.
993 _status = Running;
994 switch (_kvmRun->exit_reason) {
995 case KVM_EXIT_UNKNOWN:
996 return handleKvmExitUnknown();
997
998 case KVM_EXIT_EXCEPTION:
999 return handleKvmExitException();
1000
1001 case KVM_EXIT_IO:
1002 {
1003 ++numIO;
1004 Tick ticks = handleKvmExitIO();
1005 _status = dataPort.nextIOState();
1006 return ticks;
1007 }
1008
1009 case KVM_EXIT_HYPERCALL:
1010 ++numHypercalls;
1011 return handleKvmExitHypercall();
1012
1013 case KVM_EXIT_HLT:
1014 /* The guest has halted and is waiting for interrupts */
1015 DPRINTF(Kvm, "handleKvmExitHalt\n");
1016 ++numHalt;
1017
1018 // Suspend the thread until the next interrupt arrives
1019 thread->suspend();
1020
1021 // This is actually ignored since the thread is suspended.
1022 return 0;
1023
1024 case KVM_EXIT_MMIO:
1025 {
1026 /* Service memory mapped IO requests */
1027 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1028 _kvmRun->mmio.is_write,
1029 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1030
1031 ++numMMIO;
1032 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1033 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1034 // doMMIOAccess could have triggered a suspend, in which case we don't
1035 // want to overwrite the _status.
1036 if (_status != Idle)
1037 _status = dataPort.nextIOState();
1038 return ticks;
1039 }
1040
1041 case KVM_EXIT_IRQ_WINDOW_OPEN:
1042 return handleKvmExitIRQWindowOpen();
1043
1044 case KVM_EXIT_FAIL_ENTRY:
1045 return handleKvmExitFailEntry();
1046
1047 case KVM_EXIT_INTR:
1048 /* KVM was interrupted by a signal, restart it in the next
1049 * tick. */
1050 return 0;
1051
1052 case KVM_EXIT_INTERNAL_ERROR:
1053 panic("KVM: Internal error (suberror: %u)\n",
1054 _kvmRun->internal.suberror);
1055
1056 default:
1057 dump();
1058 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1059 }
1060}
1061
1062Tick
1063BaseKvmCPU::handleKvmExitIO()
1064{
1065 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1066 _kvmRun->io.direction, _kvmRun->io.size,
1067 _kvmRun->io.port, _kvmRun->io.count);
1068}
1069
1070Tick
1071BaseKvmCPU::handleKvmExitHypercall()
1072{
1073 panic("KVM: Unhandled hypercall\n");
1074}
1075
1076Tick
1077BaseKvmCPU::handleKvmExitIRQWindowOpen()
1078{
1079 warn("KVM: Unhandled IRQ window.\n");
1080 return 0;
1081}
1082
1083
1084Tick
1085BaseKvmCPU::handleKvmExitUnknown()
1086{
1087 dump();
1088 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1089 _kvmRun->hw.hardware_exit_reason);
1090}
1091
1092Tick
1093BaseKvmCPU::handleKvmExitException()
1094{
1095 dump();
1096 panic("KVM: Got exception when starting vCPU "
1097 "(exception: %u, error_code: %u)\n",
1098 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1099}
1100
1101Tick
1102BaseKvmCPU::handleKvmExitFailEntry()
1103{
1104 dump();
1105 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1106 _kvmRun->fail_entry.hardware_entry_failure_reason);
1107}
1108
1109Tick
1110BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1111{
1112 ThreadContext *tc(thread->getTC());
1113 syncThreadContext();
1114
1115 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1116 dataMasterId());
1117 mmio_req->setContext(tc->contextId());
1118 // Some architectures do need to massage physical addresses a bit
1119 // before they are inserted into the memory system. This enables
1120 // APIC accesses on x86 and m5ops where supported through a MMIO
1121 // interface.
1122 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1123 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1124 if (fault != NoFault)
1125 warn("Finalization of MMIO address failed: %s\n", fault->name());
1126
1127
1128 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1129 PacketPtr pkt = new Packet(mmio_req, cmd);
1130 pkt->dataStatic(data);
1131
1132 if (mmio_req->isMmappedIpr()) {
1133 // We currently assume that there is no need to migrate to a
1134 // different event queue when doing IPRs. Currently, IPRs are
1135 // only used for m5ops, so it should be a valid assumption.
1136 const Cycles ipr_delay(write ?
1137 TheISA::handleIprWrite(tc, pkt) :
1138 TheISA::handleIprRead(tc, pkt));
1139 threadContextDirty = true;
1140 delete pkt->req;
1141 delete pkt;
1142 return clockPeriod() * ipr_delay;
1143 } else {
1144 // Temporarily lock and migrate to the event queue of the
1145 // VM. This queue is assumed to "own" all devices we need to
1146 // access if running in multi-core mode.
1147 EventQueue::ScopedMigration migrate(vm.eventQueue());
1148
1149 return dataPort.submitIO(pkt);
1150 }
1151}
1152
1153void
1154BaseKvmCPU::setSignalMask(const sigset_t *mask)
1155{
1156 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1157
1158 if (mask) {
1159 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1160 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1161 // The kernel and the user-space headers have different ideas
1162 // about the size of sigset_t. This seems like a massive hack,
1163 // but is actually what qemu does.
1164 assert(sizeof(*mask) >= 8);
1165 kvm_mask->len = 8;
1166 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1167 }
1168
1169 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1170 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1171 errno);
1172}
1173
1174int
1175BaseKvmCPU::ioctl(int request, long p1) const
1176{
1177 if (vcpuFD == -1)
1178 panic("KVM: CPU ioctl called before initialization\n");
1179
1180 return ::ioctl(vcpuFD, request, p1);
1181}
1182
1183Tick
1184BaseKvmCPU::flushCoalescedMMIO()
1185{
1186 if (!mmioRing)
1187 return 0;
1188
1189 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1190
1191 // TODO: We might need to do synchronization when we start to
1192 // support multiple CPUs
1193 Tick ticks(0);
1194 while (mmioRing->first != mmioRing->last) {
1195 struct kvm_coalesced_mmio &ent(
1196 mmioRing->coalesced_mmio[mmioRing->first]);
1197
1198 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1199 ent.phys_addr, ent.len);
1200
1201 ++numCoalescedMMIO;
1202 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1203
1204 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1205 }
1206
1207 return ticks;
1208}
1209
1210/**
1211 * Dummy handler for KVM kick signals.
1212 *
1213 * @note This function is usually not called since the kernel doesn't
1214 * seem to deliver signals when the signal is only unmasked when
1215 * running in KVM. This doesn't matter though since we are only
1216 * interested in getting KVM to exit, which happens as expected. See
1217 * setupSignalHandler() and kvmRun() for details about KVM signal
1218 * handling.
1219 */
1220static void
1221onKickSignal(int signo, siginfo_t *si, void *data)
1222{
1223}
1224
1225void
1226BaseKvmCPU::setupSignalHandler()
1227{
1228 struct sigaction sa;
1229
1230 memset(&sa, 0, sizeof(sa));
1231 sa.sa_sigaction = onKickSignal;
1232 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1233 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1234 panic("KVM: Failed to setup vCPU timer signal handler\n");
1235
1236 sigset_t sigset;
1237 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1238 panic("KVM: Failed get signal mask\n");
1239
1240 // Request KVM to setup the same signal mask as we're currently
1241 // running with except for the KVM control signal. We'll sometimes
1242 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1243 // KVM after servicing IO requests. See kvmRun().
1244 sigdelset(&sigset, KVM_KICK_SIGNAL);
1245 setSignalMask(&sigset);
1246
1247 // Mask our control signals so they aren't delivered unless we're
1248 // actually executing inside KVM.
1249 sigaddset(&sigset, KVM_KICK_SIGNAL);
1250 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1251 panic("KVM: Failed mask the KVM control signals\n");
1252}
1253
1254bool
1255BaseKvmCPU::discardPendingSignal(int signum) const
1256{
1257 int discardedSignal;
1258
1259 // Setting the timeout to zero causes sigtimedwait to return
1260 // immediately.
1261 struct timespec timeout;
1262 timeout.tv_sec = 0;
1263 timeout.tv_nsec = 0;
1264
1265 sigset_t sigset;
1266 sigemptyset(&sigset);
1267 sigaddset(&sigset, signum);
1268
1269 do {
1270 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1271 } while (discardedSignal == -1 && errno == EINTR);
1272
1273 if (discardedSignal == signum)
1274 return true;
1275 else if (discardedSignal == -1 && errno == EAGAIN)
1276 return false;
1277 else
1278 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1279 discardedSignal, errno);
1280}
1281
1282void
1283BaseKvmCPU::setupCounters()
1284{
1285 DPRINTF(Kvm, "Attaching cycle counter...\n");
1286 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1287 PERF_COUNT_HW_CPU_CYCLES);
1288 cfgCycles.disabled(true)
1289 .pinned(true);
1290
1291 // Try to exclude the host. We set both exclude_hv and
1292 // exclude_host since different architectures use slightly
1293 // different APIs in the kernel.
1294 cfgCycles.exclude_hv(true)
1295 .exclude_host(true);
1296
1297 if (perfControlledByTimer) {
1298 // We need to configure the cycles counter to send overflows
1299 // since we are going to use it to trigger timer signals that
1300 // trap back into m5 from KVM. In practice, this means that we
1301 // need to set some non-zero sample period that gets
1302 // overridden when the timer is armed.
1303 cfgCycles.wakeupEvents(1)
1304 .samplePeriod(42);
1305 }
1306
1307 hwCycles.attach(cfgCycles,
1308 0); // TID (0 => currentThread)
1309
1310 setupInstCounter();
1311}
1312
1313bool
1314BaseKvmCPU::tryDrain()
1315{
1316 if (drainState() != DrainState::Draining)
1317 return false;
1318
1319 if (!archIsDrained()) {
1320 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1321 return false;
1322 }
1323
1324 if (_status == Idle || _status == Running) {
1325 DPRINTF(Drain,
1326 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1327 signalDrainDone();
1328 return true;
1329 } else {
1330 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1331 return false;
1332 }
1333}
1334
1335void
1336BaseKvmCPU::ioctlRun()
1337{
1338 if (ioctl(KVM_RUN) == -1) {
1339 if (errno != EINTR)
1340 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1341 errno);
1342 }
1343}
1344
1345void
1346BaseKvmCPU::setupInstStop()
1347{
1348 if (comInstEventQueue[0]->empty()) {
1349 setupInstCounter(0);
1350 } else {
1351 const uint64_t next(comInstEventQueue[0]->nextTick());
1352
1353 assert(next > ctrInsts);
1354 setupInstCounter(next - ctrInsts);
1355 }
1356}
1357
1358void
1359BaseKvmCPU::setupInstCounter(uint64_t period)
1360{
1361 // No need to do anything if we aren't attaching for the first
1362 // time or the period isn't changing.
1363 if (period == activeInstPeriod && hwInstructions.attached())
1364 return;
1365
1366 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1367 PERF_COUNT_HW_INSTRUCTIONS);
1368
1369 // Try to exclude the host. We set both exclude_hv and
1370 // exclude_host since different architectures use slightly
1371 // different APIs in the kernel.
1372 cfgInstructions.exclude_hv(true)
1373 .exclude_host(true);
1374
1375 if (period) {
1376 // Setup a sampling counter if that has been requested.
1377 cfgInstructions.wakeupEvents(1)
1378 .samplePeriod(period);
1379 }
1380
1381 // We need to detach and re-attach the counter to reliably change
1382 // sampling settings. See PerfKvmCounter::period() for details.
1383 if (hwInstructions.attached())
1384 hwInstructions.detach();
1385 assert(hwCycles.attached());
1386 hwInstructions.attach(cfgInstructions,
1387 0, // TID (0 => currentThread)
1388 hwCycles);
1389
1390 if (period)
1391 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1392
1393 activeInstPeriod = period;
1394}
| 62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66 : BaseCPU(params),
67 vm(*params->kvmVM),
68 _status(Idle),
69 dataPort(name() + ".dcache_port", this),
70 instPort(name() + ".icache_port", this),
71 alwaysSyncTC(params->alwaysSyncTC),
72 threadContextDirty(true),
73 kvmStateDirty(false),
74 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75 _kvmRun(NULL), mmioRing(NULL),
76 pageSize(sysconf(_SC_PAGE_SIZE)),
77 tickEvent(*this),
78 activeInstPeriod(0),
79 perfControlledByTimer(params->usePerfOverflow),
80 hostFactor(params->hostFactor),
81 ctrInsts(0)
82{
83 if (pageSize == -1)
84 panic("KVM: Failed to determine host page size (%i)\n",
85 errno);
86
87 if (FullSystem)
88 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
89 params->isa[0]);
90 else
91 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
92 params->workload[0], params->itb,
93 params->dtb, params->isa[0]);
94
95 thread->setStatus(ThreadContext::Halted);
96 tc = thread->getTC();
97 threadContexts.push_back(tc);
98}
99
100BaseKvmCPU::~BaseKvmCPU()
101{
102 if (_kvmRun)
103 munmap(_kvmRun, vcpuMMapSize);
104 close(vcpuFD);
105}
106
107void
108BaseKvmCPU::init()
109{
110 BaseCPU::init();
111
112 if (numThreads != 1)
113 fatal("KVM: Multithreading not supported");
114
115 tc->initMemProxies(tc);
116
117 // initialize CPU, including PC
118 if (FullSystem && !switchedOut())
119 TheISA::initCPU(tc, tc->contextId());
120}
121
122void
123BaseKvmCPU::startup()
124{
125 const BaseKvmCPUParams * const p(
126 dynamic_cast<const BaseKvmCPUParams *>(params()));
127
128 Kvm &kvm(*vm.kvm);
129
130 BaseCPU::startup();
131
132 assert(vcpuFD == -1);
133
134 // Tell the VM that a CPU is about to start.
135 vm.cpuStartup();
136
137 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
138 // not guaranteed that the parent KVM VM has initialized at that
139 // point. Initialize virtual CPUs here instead.
140 vcpuFD = vm.createVCPU(vcpuID);
141
142 // Map the KVM run structure */
143 vcpuMMapSize = kvm.getVCPUMMapSize();
144 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
145 PROT_READ | PROT_WRITE, MAP_SHARED,
146 vcpuFD, 0);
147 if (_kvmRun == MAP_FAILED)
148 panic("KVM: Failed to map run data structure\n");
149
150 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
151 // available. The offset into the KVM's communication page is
152 // provided by the coalesced MMIO capability.
153 int mmioOffset(kvm.capCoalescedMMIO());
154 if (!p->useCoalescedMMIO) {
155 inform("KVM: Coalesced MMIO disabled by config.\n");
156 } else if (mmioOffset) {
157 inform("KVM: Coalesced IO available\n");
158 mmioRing = (struct kvm_coalesced_mmio_ring *)(
159 (char *)_kvmRun + (mmioOffset * pageSize));
160 } else {
161 inform("KVM: Coalesced not supported by host OS\n");
162 }
163
164 thread->startup();
165
166 Event *startupEvent(
167 new EventWrapper<BaseKvmCPU,
168 &BaseKvmCPU::startupThread>(this, true));
169 schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175 return (activeMMIOReqs || pendingMMIOPkts.size())
176 ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182 if (cpu->system->isAtomicMode()) {
183 Tick delay = sendAtomic(pkt);
184 delete pkt->req;
185 delete pkt;
186 return delay;
187 } else {
188 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189 activeMMIOReqs++;
190 } else {
191 pendingMMIOPkts.push(pkt);
192 }
193 // Return value is irrelevant for timing-mode accesses.
194 return 0;
195 }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201 DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
203 delete pkt->req;
204 delete pkt;
205 activeMMIOReqs--;
206
207 // We can switch back into KVM when all pending and in-flight MMIO
208 // operations have completed.
209 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211 cpu->finishMMIOPending();
212 }
213 return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221 assert(pendingMMIOPkts.size());
222
223 // Assuming that we can issue infinite requests this cycle is a bit
224 // unrealistic, but it's not worth modeling something more complex in
225 // KVM.
226 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227 pendingMMIOPkts.pop();
228 activeMMIOReqs++;
229 }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235 assert(_status = RunningMMIOPending);
236 assert(!tickEvent.scheduled());
237
238 _status = RunningServiceCompletion;
239 schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245 // Do thread-specific initialization. We need to setup signal
246 // delivery for counters and timers from within the thread that
247 // will execute the event queue to ensure that signals are
248 // delivered to the right threads.
249 const BaseKvmCPUParams * const p(
250 dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252 vcpuThread = pthread_self();
253
254 // Setup signal handlers. This has to be done after the vCPU is
255 // created since it manipulates the vCPU signal mask.
256 setupSignalHandler();
257
258 setupCounters();
259
260 if (p->usePerfOverflow)
261 runTimer.reset(new PerfKvmTimer(hwCycles,
262 KVM_KICK_SIGNAL,
263 p->hostFactor,
264 p->hostFreq));
265 else
266 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267 p->hostFactor,
268 p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275 using namespace Stats;
276
277 BaseCPU::regStats();
278
279 numInsts
280 .name(name() + ".committedInsts")
281 .desc("Number of instructions committed")
282 ;
283
284 numVMExits
285 .name(name() + ".numVMExits")
286 .desc("total number of KVM exits")
287 ;
288
289 numVMHalfEntries
290 .name(name() + ".numVMHalfEntries")
291 .desc("number of KVM entries to finalize pending operations")
292 ;
293
294 numExitSignal
295 .name(name() + ".numExitSignal")
296 .desc("exits due to signal delivery")
297 ;
298
299 numMMIO
300 .name(name() + ".numMMIO")
301 .desc("number of VM exits due to memory mapped IO")
302 ;
303
304 numCoalescedMMIO
305 .name(name() + ".numCoalescedMMIO")
306 .desc("number of coalesced memory mapped IO requests")
307 ;
308
309 numIO
310 .name(name() + ".numIO")
311 .desc("number of VM exits due to legacy IO")
312 ;
313
314 numHalt
315 .name(name() + ".numHalt")
316 .desc("number of VM exits due to wait for interrupt instructions")
317 ;
318
319 numInterrupts
320 .name(name() + ".numInterrupts")
321 .desc("number of interrupts delivered")
322 ;
323
324 numHypercalls
325 .name(name() + ".numHypercalls")
326 .desc("number of hypercalls")
327 ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333 if (DTRACE(Checkpoint)) {
334 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335 dump();
336 }
337
338 assert(tid == 0);
339 assert(_status == Idle);
340 thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348 assert(tid == 0);
349 assert(_status == Idle);
350 thread->unserialize(cp);
351 threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357 if (switchedOut())
358 return DrainState::Drained;
359
360 DPRINTF(Drain, "BaseKvmCPU::drain\n");
361 switch (_status) {
362 case Running:
363 // The base KVM code is normally ready when it is in the
364 // Running state, but the architecture specific code might be
365 // of a different opinion. This may happen when the CPU been
366 // notified of an event that hasn't been accepted by the vCPU
367 // yet.
368 if (!archIsDrained())
369 return DrainState::Draining;
370
371 // The state of the CPU is consistent, so we don't need to do
372 // anything special to drain it. We simply de-schedule the
373 // tick event and enter the Idle state to prevent nasty things
374 // like MMIOs from happening.
375 if (tickEvent.scheduled())
376 deschedule(tickEvent);
377 _status = Idle;
378
379 /** FALLTHROUGH */
380 case Idle:
381 // Idle, no need to drain
382 assert(!tickEvent.scheduled());
383
384 // Sync the thread context here since we'll need it when we
385 // switch CPUs or checkpoint the CPU.
386 syncThreadContext();
387
388 return DrainState::Drained;
389
390 case RunningServiceCompletion:
391 // The CPU has just requested a service that was handled in
392 // the RunningService state, but the results have still not
393 // been reported to the CPU. Now, we /could/ probably just
394 // update the register state ourselves instead of letting KVM
395 // handle it, but that would be tricky. Instead, we enter KVM
396 // and let it do its stuff.
397 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
398 "requesting drain.\n");
399 return DrainState::Draining;
400
401 case RunningMMIOPending:
402 // We need to drain since there are in-flight timing accesses
403 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
404 "requesting drain.\n");
405 return DrainState::Draining;
406
407 case RunningService:
408 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
409 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
410 return DrainState::Draining;
411
412 default:
413 panic("KVM: Unhandled CPU state in drain()\n");
414 return DrainState::Drained;
415 }
416}
417
418void
419BaseKvmCPU::drainResume()
420{
421 assert(!tickEvent.scheduled());
422
423 // We might have been switched out. In that case, we don't need to
424 // do anything.
425 if (switchedOut())
426 return;
427
428 DPRINTF(Kvm, "drainResume\n");
429 verifyMemoryMode();
430
431 // The tick event is de-scheduled as a part of the draining
432 // process. Re-schedule it if the thread context is active.
433 if (tc->status() == ThreadContext::Active) {
434 schedule(tickEvent, nextCycle());
435 _status = Running;
436 } else {
437 _status = Idle;
438 }
439}
440
441void
442BaseKvmCPU::notifyFork()
443{
444 // We should have drained prior to forking, which means that the
445 // tick event shouldn't be scheduled and the CPU is idle.
446 assert(!tickEvent.scheduled());
447 assert(_status == Idle);
448
449 if (vcpuFD != -1) {
450 if (close(vcpuFD) == -1)
451 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
452
453 if (_kvmRun)
454 munmap(_kvmRun, vcpuMMapSize);
455
456 vcpuFD = -1;
457 _kvmRun = NULL;
458
459 hwInstructions.detach();
460 hwCycles.detach();
461 }
462}
463
464void
465BaseKvmCPU::switchOut()
466{
467 DPRINTF(Kvm, "switchOut\n");
468
469 BaseCPU::switchOut();
470
471 // We should have drained prior to executing a switchOut, which
472 // means that the tick event shouldn't be scheduled and the CPU is
473 // idle.
474 assert(!tickEvent.scheduled());
475 assert(_status == Idle);
476}
477
478void
479BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
480{
481 DPRINTF(Kvm, "takeOverFrom\n");
482
483 BaseCPU::takeOverFrom(cpu);
484
485 // We should have drained prior to executing a switchOut, which
486 // means that the tick event shouldn't be scheduled and the CPU is
487 // idle.
488 assert(!tickEvent.scheduled());
489 assert(_status == Idle);
490 assert(threadContexts.size() == 1);
491
492 // Force an update of the KVM state here instead of flagging the
493 // TC as dirty. This is not ideal from a performance point of
494 // view, but it makes debugging easier as it allows meaningful KVM
495 // state to be dumped before and after a takeover.
496 updateKvmState();
497 threadContextDirty = false;
498}
499
500void
501BaseKvmCPU::verifyMemoryMode() const
502{
503 if (!(system->bypassCaches())) {
504 fatal("The KVM-based CPUs requires the memory system to be in the "
505 "'noncaching' mode.\n");
506 }
507}
508
509void
510BaseKvmCPU::wakeup(ThreadID tid)
511{
512 DPRINTF(Kvm, "wakeup()\n");
513 // This method might have been called from another
514 // context. Migrate to this SimObject's event queue when
515 // delivering the wakeup signal.
516 EventQueue::ScopedMigration migrate(eventQueue());
517
518 // Kick the vCPU to get it to come out of KVM.
519 kick();
520
521 if (thread->status() != ThreadContext::Suspended)
522 return;
523
524 thread->activate();
525}
526
527void
528BaseKvmCPU::activateContext(ThreadID thread_num)
529{
530 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
531
532 assert(thread_num == 0);
533 assert(thread);
534
535 assert(_status == Idle);
536 assert(!tickEvent.scheduled());
537
538 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
539
540 schedule(tickEvent, clockEdge(Cycles(0)));
541 _status = Running;
542}
543
544
545void
546BaseKvmCPU::suspendContext(ThreadID thread_num)
547{
548 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
549
550 assert(thread_num == 0);
551 assert(thread);
552
553 if (_status == Idle)
554 return;
555
556 assert(_status == Running || _status == RunningServiceCompletion);
557
558 // The tick event may no be scheduled if the quest has requested
559 // the monitor to wait for interrupts. The normal CPU models can
560 // get their tick events descheduled by quiesce instructions, but
561 // that can't happen here.
562 if (tickEvent.scheduled())
563 deschedule(tickEvent);
564
565 _status = Idle;
566}
567
568void
569BaseKvmCPU::deallocateContext(ThreadID thread_num)
570{
571 // for now, these are equivalent
572 suspendContext(thread_num);
573}
574
575void
576BaseKvmCPU::haltContext(ThreadID thread_num)
577{
578 // for now, these are equivalent
579 suspendContext(thread_num);
580}
581
582ThreadContext *
583BaseKvmCPU::getContext(int tn)
584{
585 assert(tn == 0);
586 syncThreadContext();
587 return tc;
588}
589
590
591Counter
592BaseKvmCPU::totalInsts() const
593{
594 return ctrInsts;
595}
596
597Counter
598BaseKvmCPU::totalOps() const
599{
600 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
601 return ctrInsts;
602}
603
604void
605BaseKvmCPU::dump() const
606{
607 inform("State dumping not implemented.");
608}
609
610void
611BaseKvmCPU::tick()
612{
613 Tick delay(0);
614 assert(_status != Idle && _status != RunningMMIOPending);
615
616 switch (_status) {
617 case RunningService:
618 // handleKvmExit() will determine the next state of the CPU
619 delay = handleKvmExit();
620
621 if (tryDrain())
622 _status = Idle;
623 break;
624
625 case RunningServiceCompletion:
626 case Running: {
627 const uint64_t nextInstEvent(
628 !comInstEventQueue[0]->empty() ?
629 comInstEventQueue[0]->nextTick() : UINT64_MAX);
630 // Enter into KVM and complete pending IO instructions if we
631 // have an instruction event pending.
632 const Tick ticksToExecute(
633 nextInstEvent > ctrInsts ?
634 curEventQueue()->nextTick() - curTick() : 0);
635
636 if (alwaysSyncTC)
637 threadContextDirty = true;
638
639 // We might need to update the KVM state.
640 syncKvmState();
641
642 // Setup any pending instruction count breakpoints using
643 // PerfEvent if we are going to execute more than just an IO
644 // completion.
645 if (ticksToExecute > 0)
646 setupInstStop();
647
648 DPRINTF(KvmRun, "Entering KVM...\n");
649 if (drainState() == DrainState::Draining) {
650 // Force an immediate exit from KVM after completing
651 // pending operations. The architecture-specific code
652 // takes care to run until it is in a state where it can
653 // safely be drained.
654 delay = kvmRunDrain();
655 } else {
656 delay = kvmRun(ticksToExecute);
657 }
658
659 // The CPU might have been suspended before entering into
660 // KVM. Assume that the CPU was suspended /before/ entering
661 // into KVM and skip the exit handling.
662 if (_status == Idle)
663 break;
664
665 // Entering into KVM implies that we'll have to reload the thread
666 // context from KVM if we want to access it. Flag the KVM state as
667 // dirty with respect to the cached thread context.
668 kvmStateDirty = true;
669
670 if (alwaysSyncTC)
671 syncThreadContext();
672
673 // Enter into the RunningService state unless the
674 // simulation was stopped by a timer.
675 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
676 _status = RunningService;
677 } else {
678 ++numExitSignal;
679 _status = Running;
680 }
681
682 // Service any pending instruction events. The vCPU should
683 // have exited in time for the event using the instruction
684 // counter configured by setupInstStop().
685 comInstEventQueue[0]->serviceEvents(ctrInsts);
686 system->instEventQueue.serviceEvents(system->totalNumInsts);
687
688 if (tryDrain())
689 _status = Idle;
690 } break;
691
692 default:
693 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
694 _status);
695 }
696
697 // Schedule a new tick if we are still running
698 if (_status != Idle && _status != RunningMMIOPending)
699 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
700}
701
702Tick
703BaseKvmCPU::kvmRunDrain()
704{
705 // By default, the only thing we need to drain is a pending IO
706 // operation which assumes that we are in the
707 // RunningServiceCompletion or RunningMMIOPending state.
708 assert(_status == RunningServiceCompletion ||
709 _status == RunningMMIOPending);
710
711 // Deliver the data from the pending IO operation and immediately
712 // exit.
713 return kvmRun(0);
714}
715
716uint64_t
717BaseKvmCPU::getHostCycles() const
718{
719 return hwCycles.read();
720}
721
722Tick
723BaseKvmCPU::kvmRun(Tick ticks)
724{
725 Tick ticksExecuted;
726 fatal_if(vcpuFD == -1,
727 "Trying to run a KVM CPU in a forked child process. "
728 "This is not supported.\n");
729 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
730
731 if (ticks == 0) {
732 // Settings ticks == 0 is a special case which causes an entry
733 // into KVM that finishes pending operations (e.g., IO) and
734 // then immediately exits.
735 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
736
737 ++numVMHalfEntries;
738
739 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
740 // thread). The KVM control signal is masked while executing
741 // in gem5 and gets unmasked temporarily as when entering
742 // KVM. See setSignalMask() and setupSignalHandler().
743 kick();
744
745 // Start the vCPU. KVM will check for signals after completing
746 // pending operations (IO). Since the KVM_KICK_SIGNAL is
747 // pending, this forces an immediate exit to gem5 again. We
748 // don't bother to setup timers since this shouldn't actually
749 // execute any code (other than completing half-executed IO
750 // instructions) in the guest.
751 ioctlRun();
752
753 // We always execute at least one cycle to prevent the
754 // BaseKvmCPU::tick() to be rescheduled on the same tick
755 // twice.
756 ticksExecuted = clockPeriod();
757 } else {
758 // This method is executed as a result of a tick event. That
759 // means that the event queue will be locked when entering the
760 // method. We temporarily unlock the event queue to allow
761 // other threads to steal control of this thread to inject
762 // interrupts. They will typically lock the queue and then
763 // force an exit from KVM by kicking the vCPU.
764 EventQueue::ScopedRelease release(curEventQueue());
765
766 if (ticks < runTimer->resolution()) {
767 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
768 ticks, runTimer->resolution());
769 ticks = runTimer->resolution();
770 }
771
772 // Get hardware statistics after synchronizing contexts. The KVM
773 // state update might affect guest cycle counters.
774 uint64_t baseCycles(getHostCycles());
775 uint64_t baseInstrs(hwInstructions.read());
776
777 // Arm the run timer and start the cycle timer if it isn't
778 // controlled by the overflow timer. Starting/stopping the cycle
779 // timer automatically starts the other perf timers as they are in
780 // the same counter group.
781 runTimer->arm(ticks);
782 if (!perfControlledByTimer)
783 hwCycles.start();
784
785 ioctlRun();
786
787 runTimer->disarm();
788 if (!perfControlledByTimer)
789 hwCycles.stop();
790
791 // The control signal may have been delivered after we exited
792 // from KVM. It will be pending in that case since it is
793 // masked when we aren't executing in KVM. Discard it to make
794 // sure we don't deliver it immediately next time we try to
795 // enter into KVM.
796 discardPendingSignal(KVM_KICK_SIGNAL);
797
798 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
799 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
800 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
801 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
802
803 /* Update statistics */
804 numCycles += simCyclesExecuted;;
805 numInsts += instsExecuted;
806 ctrInsts += instsExecuted;
807 system->totalNumInsts += instsExecuted;
808
809 DPRINTF(KvmRun,
810 "KVM: Executed %i instructions in %i cycles "
811 "(%i ticks, sim cycles: %i).\n",
812 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
813 }
814
815 ++numVMExits;
816
817 return ticksExecuted + flushCoalescedMMIO();
818}
819
820void
821BaseKvmCPU::kvmNonMaskableInterrupt()
822{
823 ++numInterrupts;
824 if (ioctl(KVM_NMI) == -1)
825 panic("KVM: Failed to deliver NMI to virtual CPU\n");
826}
827
828void
829BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
830{
831 ++numInterrupts;
832 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
833 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
834}
835
836void
837BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
838{
839 if (ioctl(KVM_GET_REGS, ®s) == -1)
840 panic("KVM: Failed to get guest registers\n");
841}
842
843void
844BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
845{
846 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
847 panic("KVM: Failed to set guest registers\n");
848}
849
850void
851BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
852{
853 if (ioctl(KVM_GET_SREGS, ®s) == -1)
854 panic("KVM: Failed to get guest special registers\n");
855}
856
857void
858BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
859{
860 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
861 panic("KVM: Failed to set guest special registers\n");
862}
863
864void
865BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
866{
867 if (ioctl(KVM_GET_FPU, &state) == -1)
868 panic("KVM: Failed to get guest FPU state\n");
869}
870
871void
872BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
873{
874 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
875 panic("KVM: Failed to set guest FPU state\n");
876}
877
878
879void
880BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
881{
882#ifdef KVM_SET_ONE_REG
883 struct kvm_one_reg reg;
884 reg.id = id;
885 reg.addr = (uint64_t)addr;
886
887 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
888 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
889 id, errno);
890 }
891#else
892 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
893#endif
894}
895
896void
897BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
898{
899#ifdef KVM_GET_ONE_REG
900 struct kvm_one_reg reg;
901 reg.id = id;
902 reg.addr = (uint64_t)addr;
903
904 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
905 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
906 id, errno);
907 }
908#else
909 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
910#endif
911}
912
913std::string
914BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
915{
916#ifdef KVM_GET_ONE_REG
917 std::ostringstream ss;
918
919 ss.setf(std::ios::hex, std::ios::basefield);
920 ss.setf(std::ios::showbase);
921#define HANDLE_INTTYPE(len) \
922 case KVM_REG_SIZE_U ## len: { \
923 uint ## len ## _t value; \
924 getOneReg(id, &value); \
925 ss << value; \
926 } break
927
928#define HANDLE_ARRAY(len) \
929 case KVM_REG_SIZE_U ## len: { \
930 uint8_t value[len / 8]; \
931 getOneReg(id, value); \
932 ccprintf(ss, "[0x%x", value[0]); \
933 for (int i = 1; i < len / 8; ++i) \
934 ccprintf(ss, ", 0x%x", value[i]); \
935 ccprintf(ss, "]"); \
936 } break
937
938 switch (id & KVM_REG_SIZE_MASK) {
939 HANDLE_INTTYPE(8);
940 HANDLE_INTTYPE(16);
941 HANDLE_INTTYPE(32);
942 HANDLE_INTTYPE(64);
943 HANDLE_ARRAY(128);
944 HANDLE_ARRAY(256);
945 HANDLE_ARRAY(512);
946 HANDLE_ARRAY(1024);
947 default:
948 ss << "??";
949 }
950
951#undef HANDLE_INTTYPE
952#undef HANDLE_ARRAY
953
954 return ss.str();
955#else
956 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
957#endif
958}
959
960void
961BaseKvmCPU::syncThreadContext()
962{
963 if (!kvmStateDirty)
964 return;
965
966 assert(!threadContextDirty);
967
968 updateThreadContext();
969 kvmStateDirty = false;
970}
971
972void
973BaseKvmCPU::syncKvmState()
974{
975 if (!threadContextDirty)
976 return;
977
978 assert(!kvmStateDirty);
979
980 updateKvmState();
981 threadContextDirty = false;
982}
983
984Tick
985BaseKvmCPU::handleKvmExit()
986{
987 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
988 assert(_status == RunningService);
989
990 // Switch into the running state by default. Individual handlers
991 // can override this.
992 _status = Running;
993 switch (_kvmRun->exit_reason) {
994 case KVM_EXIT_UNKNOWN:
995 return handleKvmExitUnknown();
996
997 case KVM_EXIT_EXCEPTION:
998 return handleKvmExitException();
999
1000 case KVM_EXIT_IO:
1001 {
1002 ++numIO;
1003 Tick ticks = handleKvmExitIO();
1004 _status = dataPort.nextIOState();
1005 return ticks;
1006 }
1007
1008 case KVM_EXIT_HYPERCALL:
1009 ++numHypercalls;
1010 return handleKvmExitHypercall();
1011
1012 case KVM_EXIT_HLT:
1013 /* The guest has halted and is waiting for interrupts */
1014 DPRINTF(Kvm, "handleKvmExitHalt\n");
1015 ++numHalt;
1016
1017 // Suspend the thread until the next interrupt arrives
1018 thread->suspend();
1019
1020 // This is actually ignored since the thread is suspended.
1021 return 0;
1022
1023 case KVM_EXIT_MMIO:
1024 {
1025 /* Service memory mapped IO requests */
1026 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1027 _kvmRun->mmio.is_write,
1028 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1029
1030 ++numMMIO;
1031 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1032 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1033 // doMMIOAccess could have triggered a suspend, in which case we don't
1034 // want to overwrite the _status.
1035 if (_status != Idle)
1036 _status = dataPort.nextIOState();
1037 return ticks;
1038 }
1039
1040 case KVM_EXIT_IRQ_WINDOW_OPEN:
1041 return handleKvmExitIRQWindowOpen();
1042
1043 case KVM_EXIT_FAIL_ENTRY:
1044 return handleKvmExitFailEntry();
1045
1046 case KVM_EXIT_INTR:
1047 /* KVM was interrupted by a signal, restart it in the next
1048 * tick. */
1049 return 0;
1050
1051 case KVM_EXIT_INTERNAL_ERROR:
1052 panic("KVM: Internal error (suberror: %u)\n",
1053 _kvmRun->internal.suberror);
1054
1055 default:
1056 dump();
1057 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1058 }
1059}
1060
1061Tick
1062BaseKvmCPU::handleKvmExitIO()
1063{
1064 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1065 _kvmRun->io.direction, _kvmRun->io.size,
1066 _kvmRun->io.port, _kvmRun->io.count);
1067}
1068
1069Tick
1070BaseKvmCPU::handleKvmExitHypercall()
1071{
1072 panic("KVM: Unhandled hypercall\n");
1073}
1074
1075Tick
1076BaseKvmCPU::handleKvmExitIRQWindowOpen()
1077{
1078 warn("KVM: Unhandled IRQ window.\n");
1079 return 0;
1080}
1081
1082
1083Tick
1084BaseKvmCPU::handleKvmExitUnknown()
1085{
1086 dump();
1087 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1088 _kvmRun->hw.hardware_exit_reason);
1089}
1090
1091Tick
1092BaseKvmCPU::handleKvmExitException()
1093{
1094 dump();
1095 panic("KVM: Got exception when starting vCPU "
1096 "(exception: %u, error_code: %u)\n",
1097 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1098}
1099
1100Tick
1101BaseKvmCPU::handleKvmExitFailEntry()
1102{
1103 dump();
1104 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1105 _kvmRun->fail_entry.hardware_entry_failure_reason);
1106}
1107
1108Tick
1109BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1110{
1111 ThreadContext *tc(thread->getTC());
1112 syncThreadContext();
1113
1114 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1115 dataMasterId());
1116 mmio_req->setContext(tc->contextId());
1117 // Some architectures do need to massage physical addresses a bit
1118 // before they are inserted into the memory system. This enables
1119 // APIC accesses on x86 and m5ops where supported through a MMIO
1120 // interface.
1121 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1122 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1123 if (fault != NoFault)
1124 warn("Finalization of MMIO address failed: %s\n", fault->name());
1125
1126
1127 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1128 PacketPtr pkt = new Packet(mmio_req, cmd);
1129 pkt->dataStatic(data);
1130
1131 if (mmio_req->isMmappedIpr()) {
1132 // We currently assume that there is no need to migrate to a
1133 // different event queue when doing IPRs. Currently, IPRs are
1134 // only used for m5ops, so it should be a valid assumption.
1135 const Cycles ipr_delay(write ?
1136 TheISA::handleIprWrite(tc, pkt) :
1137 TheISA::handleIprRead(tc, pkt));
1138 threadContextDirty = true;
1139 delete pkt->req;
1140 delete pkt;
1141 return clockPeriod() * ipr_delay;
1142 } else {
1143 // Temporarily lock and migrate to the event queue of the
1144 // VM. This queue is assumed to "own" all devices we need to
1145 // access if running in multi-core mode.
1146 EventQueue::ScopedMigration migrate(vm.eventQueue());
1147
1148 return dataPort.submitIO(pkt);
1149 }
1150}
1151
1152void
1153BaseKvmCPU::setSignalMask(const sigset_t *mask)
1154{
1155 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1156
1157 if (mask) {
1158 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1159 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1160 // The kernel and the user-space headers have different ideas
1161 // about the size of sigset_t. This seems like a massive hack,
1162 // but is actually what qemu does.
1163 assert(sizeof(*mask) >= 8);
1164 kvm_mask->len = 8;
1165 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1166 }
1167
1168 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1169 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1170 errno);
1171}
1172
1173int
1174BaseKvmCPU::ioctl(int request, long p1) const
1175{
1176 if (vcpuFD == -1)
1177 panic("KVM: CPU ioctl called before initialization\n");
1178
1179 return ::ioctl(vcpuFD, request, p1);
1180}
1181
1182Tick
1183BaseKvmCPU::flushCoalescedMMIO()
1184{
1185 if (!mmioRing)
1186 return 0;
1187
1188 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1189
1190 // TODO: We might need to do synchronization when we start to
1191 // support multiple CPUs
1192 Tick ticks(0);
1193 while (mmioRing->first != mmioRing->last) {
1194 struct kvm_coalesced_mmio &ent(
1195 mmioRing->coalesced_mmio[mmioRing->first]);
1196
1197 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1198 ent.phys_addr, ent.len);
1199
1200 ++numCoalescedMMIO;
1201 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1202
1203 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1204 }
1205
1206 return ticks;
1207}
1208
1209/**
1210 * Dummy handler for KVM kick signals.
1211 *
1212 * @note This function is usually not called since the kernel doesn't
1213 * seem to deliver signals when the signal is only unmasked when
1214 * running in KVM. This doesn't matter though since we are only
1215 * interested in getting KVM to exit, which happens as expected. See
1216 * setupSignalHandler() and kvmRun() for details about KVM signal
1217 * handling.
1218 */
1219static void
1220onKickSignal(int signo, siginfo_t *si, void *data)
1221{
1222}
1223
1224void
1225BaseKvmCPU::setupSignalHandler()
1226{
1227 struct sigaction sa;
1228
1229 memset(&sa, 0, sizeof(sa));
1230 sa.sa_sigaction = onKickSignal;
1231 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1232 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1233 panic("KVM: Failed to setup vCPU timer signal handler\n");
1234
1235 sigset_t sigset;
1236 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1237 panic("KVM: Failed get signal mask\n");
1238
1239 // Request KVM to setup the same signal mask as we're currently
1240 // running with except for the KVM control signal. We'll sometimes
1241 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1242 // KVM after servicing IO requests. See kvmRun().
1243 sigdelset(&sigset, KVM_KICK_SIGNAL);
1244 setSignalMask(&sigset);
1245
1246 // Mask our control signals so they aren't delivered unless we're
1247 // actually executing inside KVM.
1248 sigaddset(&sigset, KVM_KICK_SIGNAL);
1249 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1250 panic("KVM: Failed mask the KVM control signals\n");
1251}
1252
1253bool
1254BaseKvmCPU::discardPendingSignal(int signum) const
1255{
1256 int discardedSignal;
1257
1258 // Setting the timeout to zero causes sigtimedwait to return
1259 // immediately.
1260 struct timespec timeout;
1261 timeout.tv_sec = 0;
1262 timeout.tv_nsec = 0;
1263
1264 sigset_t sigset;
1265 sigemptyset(&sigset);
1266 sigaddset(&sigset, signum);
1267
1268 do {
1269 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1270 } while (discardedSignal == -1 && errno == EINTR);
1271
1272 if (discardedSignal == signum)
1273 return true;
1274 else if (discardedSignal == -1 && errno == EAGAIN)
1275 return false;
1276 else
1277 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1278 discardedSignal, errno);
1279}
1280
1281void
1282BaseKvmCPU::setupCounters()
1283{
1284 DPRINTF(Kvm, "Attaching cycle counter...\n");
1285 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1286 PERF_COUNT_HW_CPU_CYCLES);
1287 cfgCycles.disabled(true)
1288 .pinned(true);
1289
1290 // Try to exclude the host. We set both exclude_hv and
1291 // exclude_host since different architectures use slightly
1292 // different APIs in the kernel.
1293 cfgCycles.exclude_hv(true)
1294 .exclude_host(true);
1295
1296 if (perfControlledByTimer) {
1297 // We need to configure the cycles counter to send overflows
1298 // since we are going to use it to trigger timer signals that
1299 // trap back into m5 from KVM. In practice, this means that we
1300 // need to set some non-zero sample period that gets
1301 // overridden when the timer is armed.
1302 cfgCycles.wakeupEvents(1)
1303 .samplePeriod(42);
1304 }
1305
1306 hwCycles.attach(cfgCycles,
1307 0); // TID (0 => currentThread)
1308
1309 setupInstCounter();
1310}
1311
1312bool
1313BaseKvmCPU::tryDrain()
1314{
1315 if (drainState() != DrainState::Draining)
1316 return false;
1317
1318 if (!archIsDrained()) {
1319 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1320 return false;
1321 }
1322
1323 if (_status == Idle || _status == Running) {
1324 DPRINTF(Drain,
1325 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1326 signalDrainDone();
1327 return true;
1328 } else {
1329 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1330 return false;
1331 }
1332}
1333
1334void
1335BaseKvmCPU::ioctlRun()
1336{
1337 if (ioctl(KVM_RUN) == -1) {
1338 if (errno != EINTR)
1339 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1340 errno);
1341 }
1342}
1343
1344void
1345BaseKvmCPU::setupInstStop()
1346{
1347 if (comInstEventQueue[0]->empty()) {
1348 setupInstCounter(0);
1349 } else {
1350 const uint64_t next(comInstEventQueue[0]->nextTick());
1351
1352 assert(next > ctrInsts);
1353 setupInstCounter(next - ctrInsts);
1354 }
1355}
1356
1357void
1358BaseKvmCPU::setupInstCounter(uint64_t period)
1359{
1360 // No need to do anything if we aren't attaching for the first
1361 // time or the period isn't changing.
1362 if (period == activeInstPeriod && hwInstructions.attached())
1363 return;
1364
1365 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1366 PERF_COUNT_HW_INSTRUCTIONS);
1367
1368 // Try to exclude the host. We set both exclude_hv and
1369 // exclude_host since different architectures use slightly
1370 // different APIs in the kernel.
1371 cfgInstructions.exclude_hv(true)
1372 .exclude_host(true);
1373
1374 if (period) {
1375 // Setup a sampling counter if that has been requested.
1376 cfgInstructions.wakeupEvents(1)
1377 .samplePeriod(period);
1378 }
1379
1380 // We need to detach and re-attach the counter to reliably change
1381 // sampling settings. See PerfKvmCounter::period() for details.
1382 if (hwInstructions.attached())
1383 hwInstructions.detach();
1384 assert(hwCycles.attached());
1385 hwInstructions.attach(cfgInstructions,
1386 0, // TID (0 => currentThread)
1387 hwCycles);
1388
1389 if (period)
1390 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1391
1392 activeInstPeriod = period;
1393}
|