3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66 : BaseCPU(params),
67 vm(*params->system->getKvmVM()),
68 _status(Idle),
69 dataPort(name() + ".dcache_port", this),
70 instPort(name() + ".icache_port", this),
71 alwaysSyncTC(params->alwaysSyncTC),
72 threadContextDirty(true),
73 kvmStateDirty(false),
74 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75 _kvmRun(NULL), mmioRing(NULL),
76 pageSize(sysconf(_SC_PAGE_SIZE)),
77 tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
78 false, Event::CPU_Tick_Pri),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
169 schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175 return (activeMMIOReqs || pendingMMIOPkts.size())
176 ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182 if (cpu->system->isAtomicMode()) {
183 Tick delay = sendAtomic(pkt);
184 delete pkt->req;
185 delete pkt;
186 return delay;
187 } else {
188 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189 activeMMIOReqs++;
190 } else {
191 pendingMMIOPkts.push(pkt);
192 }
193 // Return value is irrelevant for timing-mode accesses.
194 return 0;
195 }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201 DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
203 delete pkt->req;
204 delete pkt;
205 activeMMIOReqs--;
206
207 // We can switch back into KVM when all pending and in-flight MMIO
208 // operations have completed.
209 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211 cpu->finishMMIOPending();
212 }
213 return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221 assert(pendingMMIOPkts.size());
222
223 // Assuming that we can issue infinite requests this cycle is a bit
224 // unrealistic, but it's not worth modeling something more complex in
225 // KVM.
226 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227 pendingMMIOPkts.pop();
228 activeMMIOReqs++;
229 }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235 assert(_status = RunningMMIOPending);
236 assert(!tickEvent.scheduled());
237
238 _status = RunningServiceCompletion;
239 schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245 // Do thread-specific initialization. We need to setup signal
246 // delivery for counters and timers from within the thread that
247 // will execute the event queue to ensure that signals are
248 // delivered to the right threads.
249 const BaseKvmCPUParams * const p(
250 dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252 vcpuThread = pthread_self();
253
254 // Setup signal handlers. This has to be done after the vCPU is
255 // created since it manipulates the vCPU signal mask.
256 setupSignalHandler();
257
258 setupCounters();
259
260 if (p->usePerfOverflow)
261 runTimer.reset(new PerfKvmTimer(hwCycles,
262 KVM_KICK_SIGNAL,
263 p->hostFactor,
264 p->hostFreq));
265 else
266 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267 p->hostFactor,
268 p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275 using namespace Stats;
276
277 BaseCPU::regStats();
278
279 numInsts
280 .name(name() + ".committedInsts")
281 .desc("Number of instructions committed")
282 ;
283
284 numVMExits
285 .name(name() + ".numVMExits")
286 .desc("total number of KVM exits")
287 ;
288
289 numVMHalfEntries
290 .name(name() + ".numVMHalfEntries")
291 .desc("number of KVM entries to finalize pending operations")
292 ;
293
294 numExitSignal
295 .name(name() + ".numExitSignal")
296 .desc("exits due to signal delivery")
297 ;
298
299 numMMIO
300 .name(name() + ".numMMIO")
301 .desc("number of VM exits due to memory mapped IO")
302 ;
303
304 numCoalescedMMIO
305 .name(name() + ".numCoalescedMMIO")
306 .desc("number of coalesced memory mapped IO requests")
307 ;
308
309 numIO
310 .name(name() + ".numIO")
311 .desc("number of VM exits due to legacy IO")
312 ;
313
314 numHalt
315 .name(name() + ".numHalt")
316 .desc("number of VM exits due to wait for interrupt instructions")
317 ;
318
319 numInterrupts
320 .name(name() + ".numInterrupts")
321 .desc("number of interrupts delivered")
322 ;
323
324 numHypercalls
325 .name(name() + ".numHypercalls")
326 .desc("number of hypercalls")
327 ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333 if (DTRACE(Checkpoint)) {
334 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335 dump();
336 }
337
338 assert(tid == 0);
339 assert(_status == Idle);
340 thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348 assert(tid == 0);
349 assert(_status == Idle);
350 thread->unserialize(cp);
351 threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357 if (switchedOut())
358 return DrainState::Drained;
359
360 DPRINTF(Drain, "BaseKvmCPU::drain\n");
361
362 // The event queue won't be locked when calling drain since that's
363 // not done from an event. Lock the event queue here to make sure
364 // that scoped migrations continue to work if we need to
365 // synchronize the thread context.
366 std::lock_guard<EventQueue> lock(*this->eventQueue());
367
368 switch (_status) {
369 case Running:
370 // The base KVM code is normally ready when it is in the
371 // Running state, but the architecture specific code might be
372 // of a different opinion. This may happen when the CPU been
373 // notified of an event that hasn't been accepted by the vCPU
374 // yet.
375 if (!archIsDrained())
376 return DrainState::Draining;
377
378 // The state of the CPU is consistent, so we don't need to do
379 // anything special to drain it. We simply de-schedule the
380 // tick event and enter the Idle state to prevent nasty things
381 // like MMIOs from happening.
382 if (tickEvent.scheduled())
383 deschedule(tickEvent);
384 _status = Idle;
385
386 /** FALLTHROUGH */
387 case Idle:
388 // Idle, no need to drain
389 assert(!tickEvent.scheduled());
390
391 // Sync the thread context here since we'll need it when we
392 // switch CPUs or checkpoint the CPU.
393 syncThreadContext();
394
395 return DrainState::Drained;
396
397 case RunningServiceCompletion:
398 // The CPU has just requested a service that was handled in
399 // the RunningService state, but the results have still not
400 // been reported to the CPU. Now, we /could/ probably just
401 // update the register state ourselves instead of letting KVM
402 // handle it, but that would be tricky. Instead, we enter KVM
403 // and let it do its stuff.
404 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
405 "requesting drain.\n");
406 return DrainState::Draining;
407
408 case RunningMMIOPending:
409 // We need to drain since there are in-flight timing accesses
410 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
411 "requesting drain.\n");
412 return DrainState::Draining;
413
414 case RunningService:
415 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
416 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
417 return DrainState::Draining;
418
419 default:
420 panic("KVM: Unhandled CPU state in drain()\n");
421 return DrainState::Drained;
422 }
423}
424
425void
426BaseKvmCPU::drainResume()
427{
428 assert(!tickEvent.scheduled());
429
430 // We might have been switched out. In that case, we don't need to
431 // do anything.
432 if (switchedOut())
433 return;
434
435 DPRINTF(Kvm, "drainResume\n");
436 verifyMemoryMode();
437
438 // The tick event is de-scheduled as a part of the draining
439 // process. Re-schedule it if the thread context is active.
440 if (tc->status() == ThreadContext::Active) {
441 schedule(tickEvent, nextCycle());
442 _status = Running;
443 } else {
444 _status = Idle;
445 }
446}
447
448void
449BaseKvmCPU::notifyFork()
450{
451 // We should have drained prior to forking, which means that the
452 // tick event shouldn't be scheduled and the CPU is idle.
453 assert(!tickEvent.scheduled());
454 assert(_status == Idle);
455
456 if (vcpuFD != -1) {
457 if (close(vcpuFD) == -1)
458 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
459
460 if (_kvmRun)
461 munmap(_kvmRun, vcpuMMapSize);
462
463 vcpuFD = -1;
464 _kvmRun = NULL;
465
466 hwInstructions.detach();
467 hwCycles.detach();
468 }
469}
470
471void
472BaseKvmCPU::switchOut()
473{
474 DPRINTF(Kvm, "switchOut\n");
475
476 BaseCPU::switchOut();
477
478 // We should have drained prior to executing a switchOut, which
479 // means that the tick event shouldn't be scheduled and the CPU is
480 // idle.
481 assert(!tickEvent.scheduled());
482 assert(_status == Idle);
483}
484
485void
486BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
487{
488 DPRINTF(Kvm, "takeOverFrom\n");
489
490 BaseCPU::takeOverFrom(cpu);
491
492 // We should have drained prior to executing a switchOut, which
493 // means that the tick event shouldn't be scheduled and the CPU is
494 // idle.
495 assert(!tickEvent.scheduled());
496 assert(_status == Idle);
497 assert(threadContexts.size() == 1);
498
499 // Force an update of the KVM state here instead of flagging the
500 // TC as dirty. This is not ideal from a performance point of
501 // view, but it makes debugging easier as it allows meaningful KVM
502 // state to be dumped before and after a takeover.
503 updateKvmState();
504 threadContextDirty = false;
505}
506
507void
508BaseKvmCPU::verifyMemoryMode() const
509{
510 if (!(system->bypassCaches())) {
511 fatal("The KVM-based CPUs requires the memory system to be in the "
512 "'noncaching' mode.\n");
513 }
514}
515
516void
517BaseKvmCPU::wakeup(ThreadID tid)
518{
519 DPRINTF(Kvm, "wakeup()\n");
520 // This method might have been called from another
521 // context. Migrate to this SimObject's event queue when
522 // delivering the wakeup signal.
523 EventQueue::ScopedMigration migrate(eventQueue());
524
525 // Kick the vCPU to get it to come out of KVM.
526 kick();
527
528 if (thread->status() != ThreadContext::Suspended)
529 return;
530
531 thread->activate();
532}
533
534void
535BaseKvmCPU::activateContext(ThreadID thread_num)
536{
537 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
538
539 assert(thread_num == 0);
540 assert(thread);
541
542 assert(_status == Idle);
543 assert(!tickEvent.scheduled());
544
545 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
546
547 schedule(tickEvent, clockEdge(Cycles(0)));
548 _status = Running;
549}
550
551
552void
553BaseKvmCPU::suspendContext(ThreadID thread_num)
554{
555 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
556
557 assert(thread_num == 0);
558 assert(thread);
559
560 if (_status == Idle)
561 return;
562
563 assert(_status == Running || _status == RunningServiceCompletion);
564
565 // The tick event may no be scheduled if the quest has requested
566 // the monitor to wait for interrupts. The normal CPU models can
567 // get their tick events descheduled by quiesce instructions, but
568 // that can't happen here.
569 if (tickEvent.scheduled())
570 deschedule(tickEvent);
571
572 _status = Idle;
573}
574
575void
576BaseKvmCPU::deallocateContext(ThreadID thread_num)
577{
578 // for now, these are equivalent
579 suspendContext(thread_num);
580}
581
582void
583BaseKvmCPU::haltContext(ThreadID thread_num)
584{
585 // for now, these are equivalent
586 suspendContext(thread_num);
| 3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66 : BaseCPU(params),
67 vm(*params->system->getKvmVM()),
68 _status(Idle),
69 dataPort(name() + ".dcache_port", this),
70 instPort(name() + ".icache_port", this),
71 alwaysSyncTC(params->alwaysSyncTC),
72 threadContextDirty(true),
73 kvmStateDirty(false),
74 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75 _kvmRun(NULL), mmioRing(NULL),
76 pageSize(sysconf(_SC_PAGE_SIZE)),
77 tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
78 false, Event::CPU_Tick_Pri),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
169 schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175 return (activeMMIOReqs || pendingMMIOPkts.size())
176 ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182 if (cpu->system->isAtomicMode()) {
183 Tick delay = sendAtomic(pkt);
184 delete pkt->req;
185 delete pkt;
186 return delay;
187 } else {
188 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189 activeMMIOReqs++;
190 } else {
191 pendingMMIOPkts.push(pkt);
192 }
193 // Return value is irrelevant for timing-mode accesses.
194 return 0;
195 }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201 DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
203 delete pkt->req;
204 delete pkt;
205 activeMMIOReqs--;
206
207 // We can switch back into KVM when all pending and in-flight MMIO
208 // operations have completed.
209 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211 cpu->finishMMIOPending();
212 }
213 return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221 assert(pendingMMIOPkts.size());
222
223 // Assuming that we can issue infinite requests this cycle is a bit
224 // unrealistic, but it's not worth modeling something more complex in
225 // KVM.
226 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227 pendingMMIOPkts.pop();
228 activeMMIOReqs++;
229 }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235 assert(_status = RunningMMIOPending);
236 assert(!tickEvent.scheduled());
237
238 _status = RunningServiceCompletion;
239 schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245 // Do thread-specific initialization. We need to setup signal
246 // delivery for counters and timers from within the thread that
247 // will execute the event queue to ensure that signals are
248 // delivered to the right threads.
249 const BaseKvmCPUParams * const p(
250 dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252 vcpuThread = pthread_self();
253
254 // Setup signal handlers. This has to be done after the vCPU is
255 // created since it manipulates the vCPU signal mask.
256 setupSignalHandler();
257
258 setupCounters();
259
260 if (p->usePerfOverflow)
261 runTimer.reset(new PerfKvmTimer(hwCycles,
262 KVM_KICK_SIGNAL,
263 p->hostFactor,
264 p->hostFreq));
265 else
266 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267 p->hostFactor,
268 p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275 using namespace Stats;
276
277 BaseCPU::regStats();
278
279 numInsts
280 .name(name() + ".committedInsts")
281 .desc("Number of instructions committed")
282 ;
283
284 numVMExits
285 .name(name() + ".numVMExits")
286 .desc("total number of KVM exits")
287 ;
288
289 numVMHalfEntries
290 .name(name() + ".numVMHalfEntries")
291 .desc("number of KVM entries to finalize pending operations")
292 ;
293
294 numExitSignal
295 .name(name() + ".numExitSignal")
296 .desc("exits due to signal delivery")
297 ;
298
299 numMMIO
300 .name(name() + ".numMMIO")
301 .desc("number of VM exits due to memory mapped IO")
302 ;
303
304 numCoalescedMMIO
305 .name(name() + ".numCoalescedMMIO")
306 .desc("number of coalesced memory mapped IO requests")
307 ;
308
309 numIO
310 .name(name() + ".numIO")
311 .desc("number of VM exits due to legacy IO")
312 ;
313
314 numHalt
315 .name(name() + ".numHalt")
316 .desc("number of VM exits due to wait for interrupt instructions")
317 ;
318
319 numInterrupts
320 .name(name() + ".numInterrupts")
321 .desc("number of interrupts delivered")
322 ;
323
324 numHypercalls
325 .name(name() + ".numHypercalls")
326 .desc("number of hypercalls")
327 ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333 if (DTRACE(Checkpoint)) {
334 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335 dump();
336 }
337
338 assert(tid == 0);
339 assert(_status == Idle);
340 thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348 assert(tid == 0);
349 assert(_status == Idle);
350 thread->unserialize(cp);
351 threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357 if (switchedOut())
358 return DrainState::Drained;
359
360 DPRINTF(Drain, "BaseKvmCPU::drain\n");
361
362 // The event queue won't be locked when calling drain since that's
363 // not done from an event. Lock the event queue here to make sure
364 // that scoped migrations continue to work if we need to
365 // synchronize the thread context.
366 std::lock_guard<EventQueue> lock(*this->eventQueue());
367
368 switch (_status) {
369 case Running:
370 // The base KVM code is normally ready when it is in the
371 // Running state, but the architecture specific code might be
372 // of a different opinion. This may happen when the CPU been
373 // notified of an event that hasn't been accepted by the vCPU
374 // yet.
375 if (!archIsDrained())
376 return DrainState::Draining;
377
378 // The state of the CPU is consistent, so we don't need to do
379 // anything special to drain it. We simply de-schedule the
380 // tick event and enter the Idle state to prevent nasty things
381 // like MMIOs from happening.
382 if (tickEvent.scheduled())
383 deschedule(tickEvent);
384 _status = Idle;
385
386 /** FALLTHROUGH */
387 case Idle:
388 // Idle, no need to drain
389 assert(!tickEvent.scheduled());
390
391 // Sync the thread context here since we'll need it when we
392 // switch CPUs or checkpoint the CPU.
393 syncThreadContext();
394
395 return DrainState::Drained;
396
397 case RunningServiceCompletion:
398 // The CPU has just requested a service that was handled in
399 // the RunningService state, but the results have still not
400 // been reported to the CPU. Now, we /could/ probably just
401 // update the register state ourselves instead of letting KVM
402 // handle it, but that would be tricky. Instead, we enter KVM
403 // and let it do its stuff.
404 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
405 "requesting drain.\n");
406 return DrainState::Draining;
407
408 case RunningMMIOPending:
409 // We need to drain since there are in-flight timing accesses
410 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
411 "requesting drain.\n");
412 return DrainState::Draining;
413
414 case RunningService:
415 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
416 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
417 return DrainState::Draining;
418
419 default:
420 panic("KVM: Unhandled CPU state in drain()\n");
421 return DrainState::Drained;
422 }
423}
424
425void
426BaseKvmCPU::drainResume()
427{
428 assert(!tickEvent.scheduled());
429
430 // We might have been switched out. In that case, we don't need to
431 // do anything.
432 if (switchedOut())
433 return;
434
435 DPRINTF(Kvm, "drainResume\n");
436 verifyMemoryMode();
437
438 // The tick event is de-scheduled as a part of the draining
439 // process. Re-schedule it if the thread context is active.
440 if (tc->status() == ThreadContext::Active) {
441 schedule(tickEvent, nextCycle());
442 _status = Running;
443 } else {
444 _status = Idle;
445 }
446}
447
448void
449BaseKvmCPU::notifyFork()
450{
451 // We should have drained prior to forking, which means that the
452 // tick event shouldn't be scheduled and the CPU is idle.
453 assert(!tickEvent.scheduled());
454 assert(_status == Idle);
455
456 if (vcpuFD != -1) {
457 if (close(vcpuFD) == -1)
458 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
459
460 if (_kvmRun)
461 munmap(_kvmRun, vcpuMMapSize);
462
463 vcpuFD = -1;
464 _kvmRun = NULL;
465
466 hwInstructions.detach();
467 hwCycles.detach();
468 }
469}
470
471void
472BaseKvmCPU::switchOut()
473{
474 DPRINTF(Kvm, "switchOut\n");
475
476 BaseCPU::switchOut();
477
478 // We should have drained prior to executing a switchOut, which
479 // means that the tick event shouldn't be scheduled and the CPU is
480 // idle.
481 assert(!tickEvent.scheduled());
482 assert(_status == Idle);
483}
484
485void
486BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
487{
488 DPRINTF(Kvm, "takeOverFrom\n");
489
490 BaseCPU::takeOverFrom(cpu);
491
492 // We should have drained prior to executing a switchOut, which
493 // means that the tick event shouldn't be scheduled and the CPU is
494 // idle.
495 assert(!tickEvent.scheduled());
496 assert(_status == Idle);
497 assert(threadContexts.size() == 1);
498
499 // Force an update of the KVM state here instead of flagging the
500 // TC as dirty. This is not ideal from a performance point of
501 // view, but it makes debugging easier as it allows meaningful KVM
502 // state to be dumped before and after a takeover.
503 updateKvmState();
504 threadContextDirty = false;
505}
506
507void
508BaseKvmCPU::verifyMemoryMode() const
509{
510 if (!(system->bypassCaches())) {
511 fatal("The KVM-based CPUs requires the memory system to be in the "
512 "'noncaching' mode.\n");
513 }
514}
515
516void
517BaseKvmCPU::wakeup(ThreadID tid)
518{
519 DPRINTF(Kvm, "wakeup()\n");
520 // This method might have been called from another
521 // context. Migrate to this SimObject's event queue when
522 // delivering the wakeup signal.
523 EventQueue::ScopedMigration migrate(eventQueue());
524
525 // Kick the vCPU to get it to come out of KVM.
526 kick();
527
528 if (thread->status() != ThreadContext::Suspended)
529 return;
530
531 thread->activate();
532}
533
534void
535BaseKvmCPU::activateContext(ThreadID thread_num)
536{
537 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
538
539 assert(thread_num == 0);
540 assert(thread);
541
542 assert(_status == Idle);
543 assert(!tickEvent.scheduled());
544
545 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
546
547 schedule(tickEvent, clockEdge(Cycles(0)));
548 _status = Running;
549}
550
551
552void
553BaseKvmCPU::suspendContext(ThreadID thread_num)
554{
555 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
556
557 assert(thread_num == 0);
558 assert(thread);
559
560 if (_status == Idle)
561 return;
562
563 assert(_status == Running || _status == RunningServiceCompletion);
564
565 // The tick event may no be scheduled if the quest has requested
566 // the monitor to wait for interrupts. The normal CPU models can
567 // get their tick events descheduled by quiesce instructions, but
568 // that can't happen here.
569 if (tickEvent.scheduled())
570 deschedule(tickEvent);
571
572 _status = Idle;
573}
574
575void
576BaseKvmCPU::deallocateContext(ThreadID thread_num)
577{
578 // for now, these are equivalent
579 suspendContext(thread_num);
580}
581
582void
583BaseKvmCPU::haltContext(ThreadID thread_num)
584{
585 // for now, these are equivalent
586 suspendContext(thread_num);
|
587}
588
589ThreadContext *
590BaseKvmCPU::getContext(int tn)
591{
592 assert(tn == 0);
593 syncThreadContext();
594 return tc;
595}
596
597
598Counter
599BaseKvmCPU::totalInsts() const
600{
601 return ctrInsts;
602}
603
604Counter
605BaseKvmCPU::totalOps() const
606{
607 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
608 return ctrInsts;
609}
610
611void
612BaseKvmCPU::dump() const
613{
614 inform("State dumping not implemented.");
615}
616
617void
618BaseKvmCPU::tick()
619{
620 Tick delay(0);
621 assert(_status != Idle && _status != RunningMMIOPending);
622
623 switch (_status) {
624 case RunningService:
625 // handleKvmExit() will determine the next state of the CPU
626 delay = handleKvmExit();
627
628 if (tryDrain())
629 _status = Idle;
630 break;
631
632 case RunningServiceCompletion:
633 case Running: {
634 const uint64_t nextInstEvent(
635 !comInstEventQueue[0]->empty() ?
636 comInstEventQueue[0]->nextTick() : UINT64_MAX);
637 // Enter into KVM and complete pending IO instructions if we
638 // have an instruction event pending.
639 const Tick ticksToExecute(
640 nextInstEvent > ctrInsts ?
641 curEventQueue()->nextTick() - curTick() : 0);
642
643 if (alwaysSyncTC)
644 threadContextDirty = true;
645
646 // We might need to update the KVM state.
647 syncKvmState();
648
649 // Setup any pending instruction count breakpoints using
650 // PerfEvent if we are going to execute more than just an IO
651 // completion.
652 if (ticksToExecute > 0)
653 setupInstStop();
654
655 DPRINTF(KvmRun, "Entering KVM...\n");
656 if (drainState() == DrainState::Draining) {
657 // Force an immediate exit from KVM after completing
658 // pending operations. The architecture-specific code
659 // takes care to run until it is in a state where it can
660 // safely be drained.
661 delay = kvmRunDrain();
662 } else {
663 delay = kvmRun(ticksToExecute);
664 }
665
666 // The CPU might have been suspended before entering into
667 // KVM. Assume that the CPU was suspended /before/ entering
668 // into KVM and skip the exit handling.
669 if (_status == Idle)
670 break;
671
672 // Entering into KVM implies that we'll have to reload the thread
673 // context from KVM if we want to access it. Flag the KVM state as
674 // dirty with respect to the cached thread context.
675 kvmStateDirty = true;
676
677 if (alwaysSyncTC)
678 syncThreadContext();
679
680 // Enter into the RunningService state unless the
681 // simulation was stopped by a timer.
682 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
683 _status = RunningService;
684 } else {
685 ++numExitSignal;
686 _status = Running;
687 }
688
689 // Service any pending instruction events. The vCPU should
690 // have exited in time for the event using the instruction
691 // counter configured by setupInstStop().
692 comInstEventQueue[0]->serviceEvents(ctrInsts);
693 system->instEventQueue.serviceEvents(system->totalNumInsts);
694
695 if (tryDrain())
696 _status = Idle;
697 } break;
698
699 default:
700 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
701 _status);
702 }
703
704 // Schedule a new tick if we are still running
705 if (_status != Idle && _status != RunningMMIOPending)
706 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
707}
708
709Tick
710BaseKvmCPU::kvmRunDrain()
711{
712 // By default, the only thing we need to drain is a pending IO
713 // operation which assumes that we are in the
714 // RunningServiceCompletion or RunningMMIOPending state.
715 assert(_status == RunningServiceCompletion ||
716 _status == RunningMMIOPending);
717
718 // Deliver the data from the pending IO operation and immediately
719 // exit.
720 return kvmRun(0);
721}
722
723uint64_t
724BaseKvmCPU::getHostCycles() const
725{
726 return hwCycles.read();
727}
728
729Tick
730BaseKvmCPU::kvmRun(Tick ticks)
731{
732 Tick ticksExecuted;
733 fatal_if(vcpuFD == -1,
734 "Trying to run a KVM CPU in a forked child process. "
735 "This is not supported.\n");
736 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
737
738 if (ticks == 0) {
739 // Settings ticks == 0 is a special case which causes an entry
740 // into KVM that finishes pending operations (e.g., IO) and
741 // then immediately exits.
742 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
743
744 ++numVMHalfEntries;
745
746 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
747 // thread). The KVM control signal is masked while executing
748 // in gem5 and gets unmasked temporarily as when entering
749 // KVM. See setSignalMask() and setupSignalHandler().
750 kick();
751
752 // Start the vCPU. KVM will check for signals after completing
753 // pending operations (IO). Since the KVM_KICK_SIGNAL is
754 // pending, this forces an immediate exit to gem5 again. We
755 // don't bother to setup timers since this shouldn't actually
756 // execute any code (other than completing half-executed IO
757 // instructions) in the guest.
758 ioctlRun();
759
760 // We always execute at least one cycle to prevent the
761 // BaseKvmCPU::tick() to be rescheduled on the same tick
762 // twice.
763 ticksExecuted = clockPeriod();
764 } else {
765 // This method is executed as a result of a tick event. That
766 // means that the event queue will be locked when entering the
767 // method. We temporarily unlock the event queue to allow
768 // other threads to steal control of this thread to inject
769 // interrupts. They will typically lock the queue and then
770 // force an exit from KVM by kicking the vCPU.
771 EventQueue::ScopedRelease release(curEventQueue());
772
773 if (ticks < runTimer->resolution()) {
774 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
775 ticks, runTimer->resolution());
776 ticks = runTimer->resolution();
777 }
778
779 // Get hardware statistics after synchronizing contexts. The KVM
780 // state update might affect guest cycle counters.
781 uint64_t baseCycles(getHostCycles());
782 uint64_t baseInstrs(hwInstructions.read());
783
784 // Arm the run timer and start the cycle timer if it isn't
785 // controlled by the overflow timer. Starting/stopping the cycle
786 // timer automatically starts the other perf timers as they are in
787 // the same counter group.
788 runTimer->arm(ticks);
789 if (!perfControlledByTimer)
790 hwCycles.start();
791
792 ioctlRun();
793
794 runTimer->disarm();
795 if (!perfControlledByTimer)
796 hwCycles.stop();
797
798 // The control signal may have been delivered after we exited
799 // from KVM. It will be pending in that case since it is
800 // masked when we aren't executing in KVM. Discard it to make
801 // sure we don't deliver it immediately next time we try to
802 // enter into KVM.
803 discardPendingSignal(KVM_KICK_SIGNAL);
804
805 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
806 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
807 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
808 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
809
810 /* Update statistics */
811 numCycles += simCyclesExecuted;;
812 numInsts += instsExecuted;
813 ctrInsts += instsExecuted;
814 system->totalNumInsts += instsExecuted;
815
816 DPRINTF(KvmRun,
817 "KVM: Executed %i instructions in %i cycles "
818 "(%i ticks, sim cycles: %i).\n",
819 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
820 }
821
822 ++numVMExits;
823
824 return ticksExecuted + flushCoalescedMMIO();
825}
826
827void
828BaseKvmCPU::kvmNonMaskableInterrupt()
829{
830 ++numInterrupts;
831 if (ioctl(KVM_NMI) == -1)
832 panic("KVM: Failed to deliver NMI to virtual CPU\n");
833}
834
835void
836BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
837{
838 ++numInterrupts;
839 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
840 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
841}
842
843void
844BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
845{
846 if (ioctl(KVM_GET_REGS, ®s) == -1)
847 panic("KVM: Failed to get guest registers\n");
848}
849
850void
851BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
852{
853 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
854 panic("KVM: Failed to set guest registers\n");
855}
856
857void
858BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
859{
860 if (ioctl(KVM_GET_SREGS, ®s) == -1)
861 panic("KVM: Failed to get guest special registers\n");
862}
863
864void
865BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
866{
867 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
868 panic("KVM: Failed to set guest special registers\n");
869}
870
871void
872BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
873{
874 if (ioctl(KVM_GET_FPU, &state) == -1)
875 panic("KVM: Failed to get guest FPU state\n");
876}
877
878void
879BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
880{
881 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
882 panic("KVM: Failed to set guest FPU state\n");
883}
884
885
886void
887BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
888{
889#ifdef KVM_SET_ONE_REG
890 struct kvm_one_reg reg;
891 reg.id = id;
892 reg.addr = (uint64_t)addr;
893
894 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
895 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
896 id, errno);
897 }
898#else
899 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
900#endif
901}
902
903void
904BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
905{
906#ifdef KVM_GET_ONE_REG
907 struct kvm_one_reg reg;
908 reg.id = id;
909 reg.addr = (uint64_t)addr;
910
911 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
912 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
913 id, errno);
914 }
915#else
916 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
917#endif
918}
919
920std::string
921BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
922{
923#ifdef KVM_GET_ONE_REG
924 std::ostringstream ss;
925
926 ss.setf(std::ios::hex, std::ios::basefield);
927 ss.setf(std::ios::showbase);
928#define HANDLE_INTTYPE(len) \
929 case KVM_REG_SIZE_U ## len: { \
930 uint ## len ## _t value; \
931 getOneReg(id, &value); \
932 ss << value; \
933 } break
934
935#define HANDLE_ARRAY(len) \
936 case KVM_REG_SIZE_U ## len: { \
937 uint8_t value[len / 8]; \
938 getOneReg(id, value); \
939 ccprintf(ss, "[0x%x", value[0]); \
940 for (int i = 1; i < len / 8; ++i) \
941 ccprintf(ss, ", 0x%x", value[i]); \
942 ccprintf(ss, "]"); \
943 } break
944
945 switch (id & KVM_REG_SIZE_MASK) {
946 HANDLE_INTTYPE(8);
947 HANDLE_INTTYPE(16);
948 HANDLE_INTTYPE(32);
949 HANDLE_INTTYPE(64);
950 HANDLE_ARRAY(128);
951 HANDLE_ARRAY(256);
952 HANDLE_ARRAY(512);
953 HANDLE_ARRAY(1024);
954 default:
955 ss << "??";
956 }
957
958#undef HANDLE_INTTYPE
959#undef HANDLE_ARRAY
960
961 return ss.str();
962#else
963 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
964#endif
965}
966
967void
968BaseKvmCPU::syncThreadContext()
969{
970 if (!kvmStateDirty)
971 return;
972
973 assert(!threadContextDirty);
974
975 updateThreadContext();
976 kvmStateDirty = false;
977}
978
979void
980BaseKvmCPU::syncKvmState()
981{
982 if (!threadContextDirty)
983 return;
984
985 assert(!kvmStateDirty);
986
987 updateKvmState();
988 threadContextDirty = false;
989}
990
991Tick
992BaseKvmCPU::handleKvmExit()
993{
994 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
995 assert(_status == RunningService);
996
997 // Switch into the running state by default. Individual handlers
998 // can override this.
999 _status = Running;
1000 switch (_kvmRun->exit_reason) {
1001 case KVM_EXIT_UNKNOWN:
1002 return handleKvmExitUnknown();
1003
1004 case KVM_EXIT_EXCEPTION:
1005 return handleKvmExitException();
1006
1007 case KVM_EXIT_IO:
1008 {
1009 ++numIO;
1010 Tick ticks = handleKvmExitIO();
1011 _status = dataPort.nextIOState();
1012 return ticks;
1013 }
1014
1015 case KVM_EXIT_HYPERCALL:
1016 ++numHypercalls;
1017 return handleKvmExitHypercall();
1018
1019 case KVM_EXIT_HLT:
1020 /* The guest has halted and is waiting for interrupts */
1021 DPRINTF(Kvm, "handleKvmExitHalt\n");
1022 ++numHalt;
1023
1024 // Suspend the thread until the next interrupt arrives
1025 thread->suspend();
1026
1027 // This is actually ignored since the thread is suspended.
1028 return 0;
1029
1030 case KVM_EXIT_MMIO:
1031 {
1032 /* Service memory mapped IO requests */
1033 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1034 _kvmRun->mmio.is_write,
1035 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1036
1037 ++numMMIO;
1038 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1039 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1040 // doMMIOAccess could have triggered a suspend, in which case we don't
1041 // want to overwrite the _status.
1042 if (_status != Idle)
1043 _status = dataPort.nextIOState();
1044 return ticks;
1045 }
1046
1047 case KVM_EXIT_IRQ_WINDOW_OPEN:
1048 return handleKvmExitIRQWindowOpen();
1049
1050 case KVM_EXIT_FAIL_ENTRY:
1051 return handleKvmExitFailEntry();
1052
1053 case KVM_EXIT_INTR:
1054 /* KVM was interrupted by a signal, restart it in the next
1055 * tick. */
1056 return 0;
1057
1058 case KVM_EXIT_INTERNAL_ERROR:
1059 panic("KVM: Internal error (suberror: %u)\n",
1060 _kvmRun->internal.suberror);
1061
1062 default:
1063 dump();
1064 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1065 }
1066}
1067
1068Tick
1069BaseKvmCPU::handleKvmExitIO()
1070{
1071 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1072 _kvmRun->io.direction, _kvmRun->io.size,
1073 _kvmRun->io.port, _kvmRun->io.count);
1074}
1075
1076Tick
1077BaseKvmCPU::handleKvmExitHypercall()
1078{
1079 panic("KVM: Unhandled hypercall\n");
1080}
1081
1082Tick
1083BaseKvmCPU::handleKvmExitIRQWindowOpen()
1084{
1085 warn("KVM: Unhandled IRQ window.\n");
1086 return 0;
1087}
1088
1089
1090Tick
1091BaseKvmCPU::handleKvmExitUnknown()
1092{
1093 dump();
1094 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1095 _kvmRun->hw.hardware_exit_reason);
1096}
1097
1098Tick
1099BaseKvmCPU::handleKvmExitException()
1100{
1101 dump();
1102 panic("KVM: Got exception when starting vCPU "
1103 "(exception: %u, error_code: %u)\n",
1104 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1105}
1106
1107Tick
1108BaseKvmCPU::handleKvmExitFailEntry()
1109{
1110 dump();
1111 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1112 _kvmRun->fail_entry.hardware_entry_failure_reason);
1113}
1114
1115Tick
1116BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1117{
1118 ThreadContext *tc(thread->getTC());
1119 syncThreadContext();
1120
1121 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1122 dataMasterId());
1123 mmio_req->setContext(tc->contextId());
1124 // Some architectures do need to massage physical addresses a bit
1125 // before they are inserted into the memory system. This enables
1126 // APIC accesses on x86 and m5ops where supported through a MMIO
1127 // interface.
1128 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1129 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1130 if (fault != NoFault)
1131 warn("Finalization of MMIO address failed: %s\n", fault->name());
1132
1133
1134 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1135 PacketPtr pkt = new Packet(mmio_req, cmd);
1136 pkt->dataStatic(data);
1137
1138 if (mmio_req->isMmappedIpr()) {
1139 // We currently assume that there is no need to migrate to a
1140 // different event queue when doing IPRs. Currently, IPRs are
1141 // only used for m5ops, so it should be a valid assumption.
1142 const Cycles ipr_delay(write ?
1143 TheISA::handleIprWrite(tc, pkt) :
1144 TheISA::handleIprRead(tc, pkt));
1145 threadContextDirty = true;
1146 delete pkt->req;
1147 delete pkt;
1148 return clockPeriod() * ipr_delay;
1149 } else {
1150 // Temporarily lock and migrate to the device event queue to
1151 // prevent races in multi-core mode.
1152 EventQueue::ScopedMigration migrate(deviceEventQueue());
1153
1154 return dataPort.submitIO(pkt);
1155 }
1156}
1157
1158void
1159BaseKvmCPU::setSignalMask(const sigset_t *mask)
1160{
1161 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1162
1163 if (mask) {
1164 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1165 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1166 // The kernel and the user-space headers have different ideas
1167 // about the size of sigset_t. This seems like a massive hack,
1168 // but is actually what qemu does.
1169 assert(sizeof(*mask) >= 8);
1170 kvm_mask->len = 8;
1171 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1172 }
1173
1174 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1175 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1176 errno);
1177}
1178
1179int
1180BaseKvmCPU::ioctl(int request, long p1) const
1181{
1182 if (vcpuFD == -1)
1183 panic("KVM: CPU ioctl called before initialization\n");
1184
1185 return ::ioctl(vcpuFD, request, p1);
1186}
1187
1188Tick
1189BaseKvmCPU::flushCoalescedMMIO()
1190{
1191 if (!mmioRing)
1192 return 0;
1193
1194 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1195
1196 // TODO: We might need to do synchronization when we start to
1197 // support multiple CPUs
1198 Tick ticks(0);
1199 while (mmioRing->first != mmioRing->last) {
1200 struct kvm_coalesced_mmio &ent(
1201 mmioRing->coalesced_mmio[mmioRing->first]);
1202
1203 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1204 ent.phys_addr, ent.len);
1205
1206 ++numCoalescedMMIO;
1207 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1208
1209 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1210 }
1211
1212 return ticks;
1213}
1214
1215/**
1216 * Dummy handler for KVM kick signals.
1217 *
1218 * @note This function is usually not called since the kernel doesn't
1219 * seem to deliver signals when the signal is only unmasked when
1220 * running in KVM. This doesn't matter though since we are only
1221 * interested in getting KVM to exit, which happens as expected. See
1222 * setupSignalHandler() and kvmRun() for details about KVM signal
1223 * handling.
1224 */
1225static void
1226onKickSignal(int signo, siginfo_t *si, void *data)
1227{
1228}
1229
1230void
1231BaseKvmCPU::setupSignalHandler()
1232{
1233 struct sigaction sa;
1234
1235 memset(&sa, 0, sizeof(sa));
1236 sa.sa_sigaction = onKickSignal;
1237 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1238 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1239 panic("KVM: Failed to setup vCPU timer signal handler\n");
1240
1241 sigset_t sigset;
1242 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1243 panic("KVM: Failed get signal mask\n");
1244
1245 // Request KVM to setup the same signal mask as we're currently
1246 // running with except for the KVM control signal. We'll sometimes
1247 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1248 // KVM after servicing IO requests. See kvmRun().
1249 sigdelset(&sigset, KVM_KICK_SIGNAL);
1250 setSignalMask(&sigset);
1251
1252 // Mask our control signals so they aren't delivered unless we're
1253 // actually executing inside KVM.
1254 sigaddset(&sigset, KVM_KICK_SIGNAL);
1255 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1256 panic("KVM: Failed mask the KVM control signals\n");
1257}
1258
1259bool
1260BaseKvmCPU::discardPendingSignal(int signum) const
1261{
1262 int discardedSignal;
1263
1264 // Setting the timeout to zero causes sigtimedwait to return
1265 // immediately.
1266 struct timespec timeout;
1267 timeout.tv_sec = 0;
1268 timeout.tv_nsec = 0;
1269
1270 sigset_t sigset;
1271 sigemptyset(&sigset);
1272 sigaddset(&sigset, signum);
1273
1274 do {
1275 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1276 } while (discardedSignal == -1 && errno == EINTR);
1277
1278 if (discardedSignal == signum)
1279 return true;
1280 else if (discardedSignal == -1 && errno == EAGAIN)
1281 return false;
1282 else
1283 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1284 discardedSignal, errno);
1285}
1286
1287void
1288BaseKvmCPU::setupCounters()
1289{
1290 DPRINTF(Kvm, "Attaching cycle counter...\n");
1291 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1292 PERF_COUNT_HW_CPU_CYCLES);
1293 cfgCycles.disabled(true)
1294 .pinned(true);
1295
1296 // Try to exclude the host. We set both exclude_hv and
1297 // exclude_host since different architectures use slightly
1298 // different APIs in the kernel.
1299 cfgCycles.exclude_hv(true)
1300 .exclude_host(true);
1301
1302 if (perfControlledByTimer) {
1303 // We need to configure the cycles counter to send overflows
1304 // since we are going to use it to trigger timer signals that
1305 // trap back into m5 from KVM. In practice, this means that we
1306 // need to set some non-zero sample period that gets
1307 // overridden when the timer is armed.
1308 cfgCycles.wakeupEvents(1)
1309 .samplePeriod(42);
1310 }
1311
1312 hwCycles.attach(cfgCycles,
1313 0); // TID (0 => currentThread)
1314
1315 setupInstCounter();
1316}
1317
1318bool
1319BaseKvmCPU::tryDrain()
1320{
1321 if (drainState() != DrainState::Draining)
1322 return false;
1323
1324 if (!archIsDrained()) {
1325 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1326 return false;
1327 }
1328
1329 if (_status == Idle || _status == Running) {
1330 DPRINTF(Drain,
1331 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1332 signalDrainDone();
1333 return true;
1334 } else {
1335 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1336 return false;
1337 }
1338}
1339
1340void
1341BaseKvmCPU::ioctlRun()
1342{
1343 if (ioctl(KVM_RUN) == -1) {
1344 if (errno != EINTR)
1345 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1346 errno);
1347 }
1348}
1349
1350void
1351BaseKvmCPU::setupInstStop()
1352{
1353 if (comInstEventQueue[0]->empty()) {
1354 setupInstCounter(0);
1355 } else {
1356 const uint64_t next(comInstEventQueue[0]->nextTick());
1357
1358 assert(next > ctrInsts);
1359 setupInstCounter(next - ctrInsts);
1360 }
1361}
1362
1363void
1364BaseKvmCPU::setupInstCounter(uint64_t period)
1365{
1366 // No need to do anything if we aren't attaching for the first
1367 // time or the period isn't changing.
1368 if (period == activeInstPeriod && hwInstructions.attached())
1369 return;
1370
1371 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1372 PERF_COUNT_HW_INSTRUCTIONS);
1373
1374 // Try to exclude the host. We set both exclude_hv and
1375 // exclude_host since different architectures use slightly
1376 // different APIs in the kernel.
1377 cfgInstructions.exclude_hv(true)
1378 .exclude_host(true);
1379
1380 if (period) {
1381 // Setup a sampling counter if that has been requested.
1382 cfgInstructions.wakeupEvents(1)
1383 .samplePeriod(period);
1384 }
1385
1386 // We need to detach and re-attach the counter to reliably change
1387 // sampling settings. See PerfKvmCounter::period() for details.
1388 if (hwInstructions.attached())
1389 hwInstructions.detach();
1390 assert(hwCycles.attached());
1391 hwInstructions.attach(cfgInstructions,
1392 0, // TID (0 => currentThread)
1393 hwCycles);
1394
1395 if (period)
1396 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1397
1398 activeInstPeriod = period;
1399}
| 588}
589
590ThreadContext *
591BaseKvmCPU::getContext(int tn)
592{
593 assert(tn == 0);
594 syncThreadContext();
595 return tc;
596}
597
598
599Counter
600BaseKvmCPU::totalInsts() const
601{
602 return ctrInsts;
603}
604
605Counter
606BaseKvmCPU::totalOps() const
607{
608 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
609 return ctrInsts;
610}
611
612void
613BaseKvmCPU::dump() const
614{
615 inform("State dumping not implemented.");
616}
617
618void
619BaseKvmCPU::tick()
620{
621 Tick delay(0);
622 assert(_status != Idle && _status != RunningMMIOPending);
623
624 switch (_status) {
625 case RunningService:
626 // handleKvmExit() will determine the next state of the CPU
627 delay = handleKvmExit();
628
629 if (tryDrain())
630 _status = Idle;
631 break;
632
633 case RunningServiceCompletion:
634 case Running: {
635 const uint64_t nextInstEvent(
636 !comInstEventQueue[0]->empty() ?
637 comInstEventQueue[0]->nextTick() : UINT64_MAX);
638 // Enter into KVM and complete pending IO instructions if we
639 // have an instruction event pending.
640 const Tick ticksToExecute(
641 nextInstEvent > ctrInsts ?
642 curEventQueue()->nextTick() - curTick() : 0);
643
644 if (alwaysSyncTC)
645 threadContextDirty = true;
646
647 // We might need to update the KVM state.
648 syncKvmState();
649
650 // Setup any pending instruction count breakpoints using
651 // PerfEvent if we are going to execute more than just an IO
652 // completion.
653 if (ticksToExecute > 0)
654 setupInstStop();
655
656 DPRINTF(KvmRun, "Entering KVM...\n");
657 if (drainState() == DrainState::Draining) {
658 // Force an immediate exit from KVM after completing
659 // pending operations. The architecture-specific code
660 // takes care to run until it is in a state where it can
661 // safely be drained.
662 delay = kvmRunDrain();
663 } else {
664 delay = kvmRun(ticksToExecute);
665 }
666
667 // The CPU might have been suspended before entering into
668 // KVM. Assume that the CPU was suspended /before/ entering
669 // into KVM and skip the exit handling.
670 if (_status == Idle)
671 break;
672
673 // Entering into KVM implies that we'll have to reload the thread
674 // context from KVM if we want to access it. Flag the KVM state as
675 // dirty with respect to the cached thread context.
676 kvmStateDirty = true;
677
678 if (alwaysSyncTC)
679 syncThreadContext();
680
681 // Enter into the RunningService state unless the
682 // simulation was stopped by a timer.
683 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
684 _status = RunningService;
685 } else {
686 ++numExitSignal;
687 _status = Running;
688 }
689
690 // Service any pending instruction events. The vCPU should
691 // have exited in time for the event using the instruction
692 // counter configured by setupInstStop().
693 comInstEventQueue[0]->serviceEvents(ctrInsts);
694 system->instEventQueue.serviceEvents(system->totalNumInsts);
695
696 if (tryDrain())
697 _status = Idle;
698 } break;
699
700 default:
701 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
702 _status);
703 }
704
705 // Schedule a new tick if we are still running
706 if (_status != Idle && _status != RunningMMIOPending)
707 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
708}
709
710Tick
711BaseKvmCPU::kvmRunDrain()
712{
713 // By default, the only thing we need to drain is a pending IO
714 // operation which assumes that we are in the
715 // RunningServiceCompletion or RunningMMIOPending state.
716 assert(_status == RunningServiceCompletion ||
717 _status == RunningMMIOPending);
718
719 // Deliver the data from the pending IO operation and immediately
720 // exit.
721 return kvmRun(0);
722}
723
724uint64_t
725BaseKvmCPU::getHostCycles() const
726{
727 return hwCycles.read();
728}
729
730Tick
731BaseKvmCPU::kvmRun(Tick ticks)
732{
733 Tick ticksExecuted;
734 fatal_if(vcpuFD == -1,
735 "Trying to run a KVM CPU in a forked child process. "
736 "This is not supported.\n");
737 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
738
739 if (ticks == 0) {
740 // Settings ticks == 0 is a special case which causes an entry
741 // into KVM that finishes pending operations (e.g., IO) and
742 // then immediately exits.
743 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
744
745 ++numVMHalfEntries;
746
747 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
748 // thread). The KVM control signal is masked while executing
749 // in gem5 and gets unmasked temporarily as when entering
750 // KVM. See setSignalMask() and setupSignalHandler().
751 kick();
752
753 // Start the vCPU. KVM will check for signals after completing
754 // pending operations (IO). Since the KVM_KICK_SIGNAL is
755 // pending, this forces an immediate exit to gem5 again. We
756 // don't bother to setup timers since this shouldn't actually
757 // execute any code (other than completing half-executed IO
758 // instructions) in the guest.
759 ioctlRun();
760
761 // We always execute at least one cycle to prevent the
762 // BaseKvmCPU::tick() to be rescheduled on the same tick
763 // twice.
764 ticksExecuted = clockPeriod();
765 } else {
766 // This method is executed as a result of a tick event. That
767 // means that the event queue will be locked when entering the
768 // method. We temporarily unlock the event queue to allow
769 // other threads to steal control of this thread to inject
770 // interrupts. They will typically lock the queue and then
771 // force an exit from KVM by kicking the vCPU.
772 EventQueue::ScopedRelease release(curEventQueue());
773
774 if (ticks < runTimer->resolution()) {
775 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
776 ticks, runTimer->resolution());
777 ticks = runTimer->resolution();
778 }
779
780 // Get hardware statistics after synchronizing contexts. The KVM
781 // state update might affect guest cycle counters.
782 uint64_t baseCycles(getHostCycles());
783 uint64_t baseInstrs(hwInstructions.read());
784
785 // Arm the run timer and start the cycle timer if it isn't
786 // controlled by the overflow timer. Starting/stopping the cycle
787 // timer automatically starts the other perf timers as they are in
788 // the same counter group.
789 runTimer->arm(ticks);
790 if (!perfControlledByTimer)
791 hwCycles.start();
792
793 ioctlRun();
794
795 runTimer->disarm();
796 if (!perfControlledByTimer)
797 hwCycles.stop();
798
799 // The control signal may have been delivered after we exited
800 // from KVM. It will be pending in that case since it is
801 // masked when we aren't executing in KVM. Discard it to make
802 // sure we don't deliver it immediately next time we try to
803 // enter into KVM.
804 discardPendingSignal(KVM_KICK_SIGNAL);
805
806 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
807 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
808 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
809 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
810
811 /* Update statistics */
812 numCycles += simCyclesExecuted;;
813 numInsts += instsExecuted;
814 ctrInsts += instsExecuted;
815 system->totalNumInsts += instsExecuted;
816
817 DPRINTF(KvmRun,
818 "KVM: Executed %i instructions in %i cycles "
819 "(%i ticks, sim cycles: %i).\n",
820 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
821 }
822
823 ++numVMExits;
824
825 return ticksExecuted + flushCoalescedMMIO();
826}
827
828void
829BaseKvmCPU::kvmNonMaskableInterrupt()
830{
831 ++numInterrupts;
832 if (ioctl(KVM_NMI) == -1)
833 panic("KVM: Failed to deliver NMI to virtual CPU\n");
834}
835
836void
837BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
838{
839 ++numInterrupts;
840 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
841 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
842}
843
844void
845BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
846{
847 if (ioctl(KVM_GET_REGS, ®s) == -1)
848 panic("KVM: Failed to get guest registers\n");
849}
850
851void
852BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
853{
854 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
855 panic("KVM: Failed to set guest registers\n");
856}
857
858void
859BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
860{
861 if (ioctl(KVM_GET_SREGS, ®s) == -1)
862 panic("KVM: Failed to get guest special registers\n");
863}
864
865void
866BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
867{
868 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
869 panic("KVM: Failed to set guest special registers\n");
870}
871
872void
873BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
874{
875 if (ioctl(KVM_GET_FPU, &state) == -1)
876 panic("KVM: Failed to get guest FPU state\n");
877}
878
879void
880BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
881{
882 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
883 panic("KVM: Failed to set guest FPU state\n");
884}
885
886
887void
888BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
889{
890#ifdef KVM_SET_ONE_REG
891 struct kvm_one_reg reg;
892 reg.id = id;
893 reg.addr = (uint64_t)addr;
894
895 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
896 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
897 id, errno);
898 }
899#else
900 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
901#endif
902}
903
904void
905BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
906{
907#ifdef KVM_GET_ONE_REG
908 struct kvm_one_reg reg;
909 reg.id = id;
910 reg.addr = (uint64_t)addr;
911
912 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
913 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
914 id, errno);
915 }
916#else
917 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
918#endif
919}
920
921std::string
922BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
923{
924#ifdef KVM_GET_ONE_REG
925 std::ostringstream ss;
926
927 ss.setf(std::ios::hex, std::ios::basefield);
928 ss.setf(std::ios::showbase);
929#define HANDLE_INTTYPE(len) \
930 case KVM_REG_SIZE_U ## len: { \
931 uint ## len ## _t value; \
932 getOneReg(id, &value); \
933 ss << value; \
934 } break
935
936#define HANDLE_ARRAY(len) \
937 case KVM_REG_SIZE_U ## len: { \
938 uint8_t value[len / 8]; \
939 getOneReg(id, value); \
940 ccprintf(ss, "[0x%x", value[0]); \
941 for (int i = 1; i < len / 8; ++i) \
942 ccprintf(ss, ", 0x%x", value[i]); \
943 ccprintf(ss, "]"); \
944 } break
945
946 switch (id & KVM_REG_SIZE_MASK) {
947 HANDLE_INTTYPE(8);
948 HANDLE_INTTYPE(16);
949 HANDLE_INTTYPE(32);
950 HANDLE_INTTYPE(64);
951 HANDLE_ARRAY(128);
952 HANDLE_ARRAY(256);
953 HANDLE_ARRAY(512);
954 HANDLE_ARRAY(1024);
955 default:
956 ss << "??";
957 }
958
959#undef HANDLE_INTTYPE
960#undef HANDLE_ARRAY
961
962 return ss.str();
963#else
964 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
965#endif
966}
967
968void
969BaseKvmCPU::syncThreadContext()
970{
971 if (!kvmStateDirty)
972 return;
973
974 assert(!threadContextDirty);
975
976 updateThreadContext();
977 kvmStateDirty = false;
978}
979
980void
981BaseKvmCPU::syncKvmState()
982{
983 if (!threadContextDirty)
984 return;
985
986 assert(!kvmStateDirty);
987
988 updateKvmState();
989 threadContextDirty = false;
990}
991
992Tick
993BaseKvmCPU::handleKvmExit()
994{
995 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
996 assert(_status == RunningService);
997
998 // Switch into the running state by default. Individual handlers
999 // can override this.
1000 _status = Running;
1001 switch (_kvmRun->exit_reason) {
1002 case KVM_EXIT_UNKNOWN:
1003 return handleKvmExitUnknown();
1004
1005 case KVM_EXIT_EXCEPTION:
1006 return handleKvmExitException();
1007
1008 case KVM_EXIT_IO:
1009 {
1010 ++numIO;
1011 Tick ticks = handleKvmExitIO();
1012 _status = dataPort.nextIOState();
1013 return ticks;
1014 }
1015
1016 case KVM_EXIT_HYPERCALL:
1017 ++numHypercalls;
1018 return handleKvmExitHypercall();
1019
1020 case KVM_EXIT_HLT:
1021 /* The guest has halted and is waiting for interrupts */
1022 DPRINTF(Kvm, "handleKvmExitHalt\n");
1023 ++numHalt;
1024
1025 // Suspend the thread until the next interrupt arrives
1026 thread->suspend();
1027
1028 // This is actually ignored since the thread is suspended.
1029 return 0;
1030
1031 case KVM_EXIT_MMIO:
1032 {
1033 /* Service memory mapped IO requests */
1034 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1035 _kvmRun->mmio.is_write,
1036 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1037
1038 ++numMMIO;
1039 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1040 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1041 // doMMIOAccess could have triggered a suspend, in which case we don't
1042 // want to overwrite the _status.
1043 if (_status != Idle)
1044 _status = dataPort.nextIOState();
1045 return ticks;
1046 }
1047
1048 case KVM_EXIT_IRQ_WINDOW_OPEN:
1049 return handleKvmExitIRQWindowOpen();
1050
1051 case KVM_EXIT_FAIL_ENTRY:
1052 return handleKvmExitFailEntry();
1053
1054 case KVM_EXIT_INTR:
1055 /* KVM was interrupted by a signal, restart it in the next
1056 * tick. */
1057 return 0;
1058
1059 case KVM_EXIT_INTERNAL_ERROR:
1060 panic("KVM: Internal error (suberror: %u)\n",
1061 _kvmRun->internal.suberror);
1062
1063 default:
1064 dump();
1065 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1066 }
1067}
1068
1069Tick
1070BaseKvmCPU::handleKvmExitIO()
1071{
1072 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1073 _kvmRun->io.direction, _kvmRun->io.size,
1074 _kvmRun->io.port, _kvmRun->io.count);
1075}
1076
1077Tick
1078BaseKvmCPU::handleKvmExitHypercall()
1079{
1080 panic("KVM: Unhandled hypercall\n");
1081}
1082
1083Tick
1084BaseKvmCPU::handleKvmExitIRQWindowOpen()
1085{
1086 warn("KVM: Unhandled IRQ window.\n");
1087 return 0;
1088}
1089
1090
1091Tick
1092BaseKvmCPU::handleKvmExitUnknown()
1093{
1094 dump();
1095 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1096 _kvmRun->hw.hardware_exit_reason);
1097}
1098
1099Tick
1100BaseKvmCPU::handleKvmExitException()
1101{
1102 dump();
1103 panic("KVM: Got exception when starting vCPU "
1104 "(exception: %u, error_code: %u)\n",
1105 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1106}
1107
1108Tick
1109BaseKvmCPU::handleKvmExitFailEntry()
1110{
1111 dump();
1112 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1113 _kvmRun->fail_entry.hardware_entry_failure_reason);
1114}
1115
1116Tick
1117BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1118{
1119 ThreadContext *tc(thread->getTC());
1120 syncThreadContext();
1121
1122 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1123 dataMasterId());
1124 mmio_req->setContext(tc->contextId());
1125 // Some architectures do need to massage physical addresses a bit
1126 // before they are inserted into the memory system. This enables
1127 // APIC accesses on x86 and m5ops where supported through a MMIO
1128 // interface.
1129 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1130 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1131 if (fault != NoFault)
1132 warn("Finalization of MMIO address failed: %s\n", fault->name());
1133
1134
1135 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1136 PacketPtr pkt = new Packet(mmio_req, cmd);
1137 pkt->dataStatic(data);
1138
1139 if (mmio_req->isMmappedIpr()) {
1140 // We currently assume that there is no need to migrate to a
1141 // different event queue when doing IPRs. Currently, IPRs are
1142 // only used for m5ops, so it should be a valid assumption.
1143 const Cycles ipr_delay(write ?
1144 TheISA::handleIprWrite(tc, pkt) :
1145 TheISA::handleIprRead(tc, pkt));
1146 threadContextDirty = true;
1147 delete pkt->req;
1148 delete pkt;
1149 return clockPeriod() * ipr_delay;
1150 } else {
1151 // Temporarily lock and migrate to the device event queue to
1152 // prevent races in multi-core mode.
1153 EventQueue::ScopedMigration migrate(deviceEventQueue());
1154
1155 return dataPort.submitIO(pkt);
1156 }
1157}
1158
1159void
1160BaseKvmCPU::setSignalMask(const sigset_t *mask)
1161{
1162 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1163
1164 if (mask) {
1165 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1166 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1167 // The kernel and the user-space headers have different ideas
1168 // about the size of sigset_t. This seems like a massive hack,
1169 // but is actually what qemu does.
1170 assert(sizeof(*mask) >= 8);
1171 kvm_mask->len = 8;
1172 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1173 }
1174
1175 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1176 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1177 errno);
1178}
1179
1180int
1181BaseKvmCPU::ioctl(int request, long p1) const
1182{
1183 if (vcpuFD == -1)
1184 panic("KVM: CPU ioctl called before initialization\n");
1185
1186 return ::ioctl(vcpuFD, request, p1);
1187}
1188
1189Tick
1190BaseKvmCPU::flushCoalescedMMIO()
1191{
1192 if (!mmioRing)
1193 return 0;
1194
1195 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1196
1197 // TODO: We might need to do synchronization when we start to
1198 // support multiple CPUs
1199 Tick ticks(0);
1200 while (mmioRing->first != mmioRing->last) {
1201 struct kvm_coalesced_mmio &ent(
1202 mmioRing->coalesced_mmio[mmioRing->first]);
1203
1204 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1205 ent.phys_addr, ent.len);
1206
1207 ++numCoalescedMMIO;
1208 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1209
1210 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1211 }
1212
1213 return ticks;
1214}
1215
1216/**
1217 * Dummy handler for KVM kick signals.
1218 *
1219 * @note This function is usually not called since the kernel doesn't
1220 * seem to deliver signals when the signal is only unmasked when
1221 * running in KVM. This doesn't matter though since we are only
1222 * interested in getting KVM to exit, which happens as expected. See
1223 * setupSignalHandler() and kvmRun() for details about KVM signal
1224 * handling.
1225 */
1226static void
1227onKickSignal(int signo, siginfo_t *si, void *data)
1228{
1229}
1230
1231void
1232BaseKvmCPU::setupSignalHandler()
1233{
1234 struct sigaction sa;
1235
1236 memset(&sa, 0, sizeof(sa));
1237 sa.sa_sigaction = onKickSignal;
1238 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1239 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1240 panic("KVM: Failed to setup vCPU timer signal handler\n");
1241
1242 sigset_t sigset;
1243 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1244 panic("KVM: Failed get signal mask\n");
1245
1246 // Request KVM to setup the same signal mask as we're currently
1247 // running with except for the KVM control signal. We'll sometimes
1248 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1249 // KVM after servicing IO requests. See kvmRun().
1250 sigdelset(&sigset, KVM_KICK_SIGNAL);
1251 setSignalMask(&sigset);
1252
1253 // Mask our control signals so they aren't delivered unless we're
1254 // actually executing inside KVM.
1255 sigaddset(&sigset, KVM_KICK_SIGNAL);
1256 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1257 panic("KVM: Failed mask the KVM control signals\n");
1258}
1259
1260bool
1261BaseKvmCPU::discardPendingSignal(int signum) const
1262{
1263 int discardedSignal;
1264
1265 // Setting the timeout to zero causes sigtimedwait to return
1266 // immediately.
1267 struct timespec timeout;
1268 timeout.tv_sec = 0;
1269 timeout.tv_nsec = 0;
1270
1271 sigset_t sigset;
1272 sigemptyset(&sigset);
1273 sigaddset(&sigset, signum);
1274
1275 do {
1276 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1277 } while (discardedSignal == -1 && errno == EINTR);
1278
1279 if (discardedSignal == signum)
1280 return true;
1281 else if (discardedSignal == -1 && errno == EAGAIN)
1282 return false;
1283 else
1284 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1285 discardedSignal, errno);
1286}
1287
1288void
1289BaseKvmCPU::setupCounters()
1290{
1291 DPRINTF(Kvm, "Attaching cycle counter...\n");
1292 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1293 PERF_COUNT_HW_CPU_CYCLES);
1294 cfgCycles.disabled(true)
1295 .pinned(true);
1296
1297 // Try to exclude the host. We set both exclude_hv and
1298 // exclude_host since different architectures use slightly
1299 // different APIs in the kernel.
1300 cfgCycles.exclude_hv(true)
1301 .exclude_host(true);
1302
1303 if (perfControlledByTimer) {
1304 // We need to configure the cycles counter to send overflows
1305 // since we are going to use it to trigger timer signals that
1306 // trap back into m5 from KVM. In practice, this means that we
1307 // need to set some non-zero sample period that gets
1308 // overridden when the timer is armed.
1309 cfgCycles.wakeupEvents(1)
1310 .samplePeriod(42);
1311 }
1312
1313 hwCycles.attach(cfgCycles,
1314 0); // TID (0 => currentThread)
1315
1316 setupInstCounter();
1317}
1318
1319bool
1320BaseKvmCPU::tryDrain()
1321{
1322 if (drainState() != DrainState::Draining)
1323 return false;
1324
1325 if (!archIsDrained()) {
1326 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1327 return false;
1328 }
1329
1330 if (_status == Idle || _status == Running) {
1331 DPRINTF(Drain,
1332 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1333 signalDrainDone();
1334 return true;
1335 } else {
1336 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1337 return false;
1338 }
1339}
1340
1341void
1342BaseKvmCPU::ioctlRun()
1343{
1344 if (ioctl(KVM_RUN) == -1) {
1345 if (errno != EINTR)
1346 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1347 errno);
1348 }
1349}
1350
1351void
1352BaseKvmCPU::setupInstStop()
1353{
1354 if (comInstEventQueue[0]->empty()) {
1355 setupInstCounter(0);
1356 } else {
1357 const uint64_t next(comInstEventQueue[0]->nextTick());
1358
1359 assert(next > ctrInsts);
1360 setupInstCounter(next - ctrInsts);
1361 }
1362}
1363
1364void
1365BaseKvmCPU::setupInstCounter(uint64_t period)
1366{
1367 // No need to do anything if we aren't attaching for the first
1368 // time or the period isn't changing.
1369 if (period == activeInstPeriod && hwInstructions.attached())
1370 return;
1371
1372 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1373 PERF_COUNT_HW_INSTRUCTIONS);
1374
1375 // Try to exclude the host. We set both exclude_hv and
1376 // exclude_host since different architectures use slightly
1377 // different APIs in the kernel.
1378 cfgInstructions.exclude_hv(true)
1379 .exclude_host(true);
1380
1381 if (period) {
1382 // Setup a sampling counter if that has been requested.
1383 cfgInstructions.wakeupEvents(1)
1384 .samplePeriod(period);
1385 }
1386
1387 // We need to detach and re-attach the counter to reliably change
1388 // sampling settings. See PerfKvmCounter::period() for details.
1389 if (hwInstructions.attached())
1390 hwInstructions.detach();
1391 assert(hwCycles.attached());
1392 hwInstructions.attach(cfgInstructions,
1393 0, // TID (0 => currentThread)
1394 hwCycles);
1395
1396 if (period)
1397 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1398
1399 activeInstPeriod = period;
1400}
|