1/*
2 * Copyright (c) 2012, 2015, 2017 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66 : BaseCPU(params),
67 vm(*params->system->getKvmVM()),
68 _status(Idle),
69 dataPort(name() + ".dcache_port", this),
70 instPort(name() + ".icache_port", this),
71 alwaysSyncTC(params->alwaysSyncTC),
72 threadContextDirty(true),
73 kvmStateDirty(false),
74 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75 _kvmRun(NULL), mmioRing(NULL),
76 pageSize(sysconf(_SC_PAGE_SIZE)),
77 tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
78 false, Event::CPU_Tick_Pri),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
169 schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175 return (activeMMIOReqs || pendingMMIOPkts.size())
176 ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182 if (cpu->system->isAtomicMode()) {
183 Tick delay = sendAtomic(pkt);
| 1/*
2 * Copyright (c) 2012, 2015, 2017 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "cpu/kvm/base.hh"
41
42#include <linux/kvm.h>
43#include <sys/ioctl.h>
44#include <sys/mman.h>
45#include <unistd.h>
46
47#include <cerrno>
48#include <csignal>
49#include <ostream>
50
51#include "arch/mmapped_ipr.hh"
52#include "arch/utility.hh"
53#include "debug/Checkpoint.hh"
54#include "debug/Drain.hh"
55#include "debug/Kvm.hh"
56#include "debug/KvmIO.hh"
57#include "debug/KvmRun.hh"
58#include "params/BaseKvmCPU.hh"
59#include "sim/process.hh"
60#include "sim/system.hh"
61
62/* Used by some KVM macros */
63#define PAGE_SIZE pageSize
64
65BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
66 : BaseCPU(params),
67 vm(*params->system->getKvmVM()),
68 _status(Idle),
69 dataPort(name() + ".dcache_port", this),
70 instPort(name() + ".icache_port", this),
71 alwaysSyncTC(params->alwaysSyncTC),
72 threadContextDirty(true),
73 kvmStateDirty(false),
74 vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
75 _kvmRun(NULL), mmioRing(NULL),
76 pageSize(sysconf(_SC_PAGE_SIZE)),
77 tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
78 false, Event::CPU_Tick_Pri),
79 activeInstPeriod(0),
80 perfControlledByTimer(params->usePerfOverflow),
81 hostFactor(params->hostFactor),
82 ctrInsts(0)
83{
84 if (pageSize == -1)
85 panic("KVM: Failed to determine host page size (%i)\n",
86 errno);
87
88 if (FullSystem)
89 thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb,
90 params->isa[0]);
91 else
92 thread = new SimpleThread(this, /* thread_num */ 0, params->system,
93 params->workload[0], params->itb,
94 params->dtb, params->isa[0]);
95
96 thread->setStatus(ThreadContext::Halted);
97 tc = thread->getTC();
98 threadContexts.push_back(tc);
99}
100
101BaseKvmCPU::~BaseKvmCPU()
102{
103 if (_kvmRun)
104 munmap(_kvmRun, vcpuMMapSize);
105 close(vcpuFD);
106}
107
108void
109BaseKvmCPU::init()
110{
111 BaseCPU::init();
112
113 if (numThreads != 1)
114 fatal("KVM: Multithreading not supported");
115
116 tc->initMemProxies(tc);
117
118 // initialize CPU, including PC
119 if (FullSystem && !switchedOut())
120 TheISA::initCPU(tc, tc->contextId());
121}
122
123void
124BaseKvmCPU::startup()
125{
126 const BaseKvmCPUParams * const p(
127 dynamic_cast<const BaseKvmCPUParams *>(params()));
128
129 Kvm &kvm(*vm.kvm);
130
131 BaseCPU::startup();
132
133 assert(vcpuFD == -1);
134
135 // Tell the VM that a CPU is about to start.
136 vm.cpuStartup();
137
138 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
139 // not guaranteed that the parent KVM VM has initialized at that
140 // point. Initialize virtual CPUs here instead.
141 vcpuFD = vm.createVCPU(vcpuID);
142
143 // Map the KVM run structure */
144 vcpuMMapSize = kvm.getVCPUMMapSize();
145 _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
146 PROT_READ | PROT_WRITE, MAP_SHARED,
147 vcpuFD, 0);
148 if (_kvmRun == MAP_FAILED)
149 panic("KVM: Failed to map run data structure\n");
150
151 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
152 // available. The offset into the KVM's communication page is
153 // provided by the coalesced MMIO capability.
154 int mmioOffset(kvm.capCoalescedMMIO());
155 if (!p->useCoalescedMMIO) {
156 inform("KVM: Coalesced MMIO disabled by config.\n");
157 } else if (mmioOffset) {
158 inform("KVM: Coalesced IO available\n");
159 mmioRing = (struct kvm_coalesced_mmio_ring *)(
160 (char *)_kvmRun + (mmioOffset * pageSize));
161 } else {
162 inform("KVM: Coalesced not supported by host OS\n");
163 }
164
165 thread->startup();
166
167 Event *startupEvent(
168 new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
169 schedule(startupEvent, curTick());
170}
171
172BaseKvmCPU::Status
173BaseKvmCPU::KVMCpuPort::nextIOState() const
174{
175 return (activeMMIOReqs || pendingMMIOPkts.size())
176 ? RunningMMIOPending : RunningServiceCompletion;
177}
178
179Tick
180BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt)
181{
182 if (cpu->system->isAtomicMode()) {
183 Tick delay = sendAtomic(pkt);
|
184 delete pkt->req;
| |
185 delete pkt;
186 return delay;
187 } else {
188 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
189 activeMMIOReqs++;
190 } else {
191 pendingMMIOPkts.push(pkt);
192 }
193 // Return value is irrelevant for timing-mode accesses.
194 return 0;
195 }
196}
197
198bool
199BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
200{
201 DPRINTF(KvmIO, "KVM: Finished timing request\n");
202
| 184 delete pkt;
185 return delay;
186 } else {
187 if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) {
188 activeMMIOReqs++;
189 } else {
190 pendingMMIOPkts.push(pkt);
191 }
192 // Return value is irrelevant for timing-mode accesses.
193 return 0;
194 }
195}
196
197bool
198BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt)
199{
200 DPRINTF(KvmIO, "KVM: Finished timing request\n");
201
|
203 delete pkt->req;
| |
204 delete pkt;
205 activeMMIOReqs--;
206
207 // We can switch back into KVM when all pending and in-flight MMIO
208 // operations have completed.
209 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
210 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
211 cpu->finishMMIOPending();
212 }
213 return true;
214}
215
216void
217BaseKvmCPU::KVMCpuPort::recvReqRetry()
218{
219 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
220
221 assert(pendingMMIOPkts.size());
222
223 // Assuming that we can issue infinite requests this cycle is a bit
224 // unrealistic, but it's not worth modeling something more complex in
225 // KVM.
226 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
227 pendingMMIOPkts.pop();
228 activeMMIOReqs++;
229 }
230}
231
232void
233BaseKvmCPU::finishMMIOPending()
234{
235 assert(_status = RunningMMIOPending);
236 assert(!tickEvent.scheduled());
237
238 _status = RunningServiceCompletion;
239 schedule(tickEvent, nextCycle());
240}
241
242void
243BaseKvmCPU::startupThread()
244{
245 // Do thread-specific initialization. We need to setup signal
246 // delivery for counters and timers from within the thread that
247 // will execute the event queue to ensure that signals are
248 // delivered to the right threads.
249 const BaseKvmCPUParams * const p(
250 dynamic_cast<const BaseKvmCPUParams *>(params()));
251
252 vcpuThread = pthread_self();
253
254 // Setup signal handlers. This has to be done after the vCPU is
255 // created since it manipulates the vCPU signal mask.
256 setupSignalHandler();
257
258 setupCounters();
259
260 if (p->usePerfOverflow)
261 runTimer.reset(new PerfKvmTimer(hwCycles,
262 KVM_KICK_SIGNAL,
263 p->hostFactor,
264 p->hostFreq));
265 else
266 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
267 p->hostFactor,
268 p->hostFreq));
269
270}
271
272void
273BaseKvmCPU::regStats()
274{
275 using namespace Stats;
276
277 BaseCPU::regStats();
278
279 numInsts
280 .name(name() + ".committedInsts")
281 .desc("Number of instructions committed")
282 ;
283
284 numVMExits
285 .name(name() + ".numVMExits")
286 .desc("total number of KVM exits")
287 ;
288
289 numVMHalfEntries
290 .name(name() + ".numVMHalfEntries")
291 .desc("number of KVM entries to finalize pending operations")
292 ;
293
294 numExitSignal
295 .name(name() + ".numExitSignal")
296 .desc("exits due to signal delivery")
297 ;
298
299 numMMIO
300 .name(name() + ".numMMIO")
301 .desc("number of VM exits due to memory mapped IO")
302 ;
303
304 numCoalescedMMIO
305 .name(name() + ".numCoalescedMMIO")
306 .desc("number of coalesced memory mapped IO requests")
307 ;
308
309 numIO
310 .name(name() + ".numIO")
311 .desc("number of VM exits due to legacy IO")
312 ;
313
314 numHalt
315 .name(name() + ".numHalt")
316 .desc("number of VM exits due to wait for interrupt instructions")
317 ;
318
319 numInterrupts
320 .name(name() + ".numInterrupts")
321 .desc("number of interrupts delivered")
322 ;
323
324 numHypercalls
325 .name(name() + ".numHypercalls")
326 .desc("number of hypercalls")
327 ;
328}
329
330void
331BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
332{
333 if (DTRACE(Checkpoint)) {
334 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
335 dump();
336 }
337
338 assert(tid == 0);
339 assert(_status == Idle);
340 thread->serialize(cp);
341}
342
343void
344BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
345{
346 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
347
348 assert(tid == 0);
349 assert(_status == Idle);
350 thread->unserialize(cp);
351 threadContextDirty = true;
352}
353
354DrainState
355BaseKvmCPU::drain()
356{
357 if (switchedOut())
358 return DrainState::Drained;
359
360 DPRINTF(Drain, "BaseKvmCPU::drain\n");
361
362 // The event queue won't be locked when calling drain since that's
363 // not done from an event. Lock the event queue here to make sure
364 // that scoped migrations continue to work if we need to
365 // synchronize the thread context.
366 std::lock_guard<EventQueue> lock(*this->eventQueue());
367
368 switch (_status) {
369 case Running:
370 // The base KVM code is normally ready when it is in the
371 // Running state, but the architecture specific code might be
372 // of a different opinion. This may happen when the CPU been
373 // notified of an event that hasn't been accepted by the vCPU
374 // yet.
375 if (!archIsDrained())
376 return DrainState::Draining;
377
378 // The state of the CPU is consistent, so we don't need to do
379 // anything special to drain it. We simply de-schedule the
380 // tick event and enter the Idle state to prevent nasty things
381 // like MMIOs from happening.
382 if (tickEvent.scheduled())
383 deschedule(tickEvent);
384 _status = Idle;
385
386 M5_FALLTHROUGH;
387 case Idle:
388 // Idle, no need to drain
389 assert(!tickEvent.scheduled());
390
391 // Sync the thread context here since we'll need it when we
392 // switch CPUs or checkpoint the CPU.
393 syncThreadContext();
394
395 return DrainState::Drained;
396
397 case RunningServiceCompletion:
398 // The CPU has just requested a service that was handled in
399 // the RunningService state, but the results have still not
400 // been reported to the CPU. Now, we /could/ probably just
401 // update the register state ourselves instead of letting KVM
402 // handle it, but that would be tricky. Instead, we enter KVM
403 // and let it do its stuff.
404 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
405 "requesting drain.\n");
406 return DrainState::Draining;
407
408 case RunningMMIOPending:
409 // We need to drain since there are in-flight timing accesses
410 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
411 "requesting drain.\n");
412 return DrainState::Draining;
413
414 case RunningService:
415 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
416 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
417 return DrainState::Draining;
418
419 default:
420 panic("KVM: Unhandled CPU state in drain()\n");
421 return DrainState::Drained;
422 }
423}
424
425void
426BaseKvmCPU::drainResume()
427{
428 assert(!tickEvent.scheduled());
429
430 // We might have been switched out. In that case, we don't need to
431 // do anything.
432 if (switchedOut())
433 return;
434
435 DPRINTF(Kvm, "drainResume\n");
436 verifyMemoryMode();
437
438 // The tick event is de-scheduled as a part of the draining
439 // process. Re-schedule it if the thread context is active.
440 if (tc->status() == ThreadContext::Active) {
441 schedule(tickEvent, nextCycle());
442 _status = Running;
443 } else {
444 _status = Idle;
445 }
446}
447
448void
449BaseKvmCPU::notifyFork()
450{
451 // We should have drained prior to forking, which means that the
452 // tick event shouldn't be scheduled and the CPU is idle.
453 assert(!tickEvent.scheduled());
454 assert(_status == Idle);
455
456 if (vcpuFD != -1) {
457 if (close(vcpuFD) == -1)
458 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
459
460 if (_kvmRun)
461 munmap(_kvmRun, vcpuMMapSize);
462
463 vcpuFD = -1;
464 _kvmRun = NULL;
465
466 hwInstructions.detach();
467 hwCycles.detach();
468 }
469}
470
471void
472BaseKvmCPU::switchOut()
473{
474 DPRINTF(Kvm, "switchOut\n");
475
476 BaseCPU::switchOut();
477
478 // We should have drained prior to executing a switchOut, which
479 // means that the tick event shouldn't be scheduled and the CPU is
480 // idle.
481 assert(!tickEvent.scheduled());
482 assert(_status == Idle);
483}
484
485void
486BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
487{
488 DPRINTF(Kvm, "takeOverFrom\n");
489
490 BaseCPU::takeOverFrom(cpu);
491
492 // We should have drained prior to executing a switchOut, which
493 // means that the tick event shouldn't be scheduled and the CPU is
494 // idle.
495 assert(!tickEvent.scheduled());
496 assert(_status == Idle);
497 assert(threadContexts.size() == 1);
498
499 // Force an update of the KVM state here instead of flagging the
500 // TC as dirty. This is not ideal from a performance point of
501 // view, but it makes debugging easier as it allows meaningful KVM
502 // state to be dumped before and after a takeover.
503 updateKvmState();
504 threadContextDirty = false;
505}
506
507void
508BaseKvmCPU::verifyMemoryMode() const
509{
510 if (!(system->bypassCaches())) {
511 fatal("The KVM-based CPUs requires the memory system to be in the "
512 "'noncaching' mode.\n");
513 }
514}
515
516void
517BaseKvmCPU::wakeup(ThreadID tid)
518{
519 DPRINTF(Kvm, "wakeup()\n");
520 // This method might have been called from another
521 // context. Migrate to this SimObject's event queue when
522 // delivering the wakeup signal.
523 EventQueue::ScopedMigration migrate(eventQueue());
524
525 // Kick the vCPU to get it to come out of KVM.
526 kick();
527
528 if (thread->status() != ThreadContext::Suspended)
529 return;
530
531 thread->activate();
532}
533
534void
535BaseKvmCPU::activateContext(ThreadID thread_num)
536{
537 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
538
539 assert(thread_num == 0);
540 assert(thread);
541
542 assert(_status == Idle);
543 assert(!tickEvent.scheduled());
544
545 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
546
547 schedule(tickEvent, clockEdge(Cycles(0)));
548 _status = Running;
549}
550
551
552void
553BaseKvmCPU::suspendContext(ThreadID thread_num)
554{
555 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
556
557 assert(thread_num == 0);
558 assert(thread);
559
560 if (_status == Idle)
561 return;
562
563 assert(_status == Running || _status == RunningServiceCompletion);
564
565 // The tick event may no be scheduled if the quest has requested
566 // the monitor to wait for interrupts. The normal CPU models can
567 // get their tick events descheduled by quiesce instructions, but
568 // that can't happen here.
569 if (tickEvent.scheduled())
570 deschedule(tickEvent);
571
572 _status = Idle;
573}
574
575void
576BaseKvmCPU::deallocateContext(ThreadID thread_num)
577{
578 // for now, these are equivalent
579 suspendContext(thread_num);
580}
581
582void
583BaseKvmCPU::haltContext(ThreadID thread_num)
584{
585 // for now, these are equivalent
586 suspendContext(thread_num);
587 updateCycleCounters(BaseCPU::CPU_STATE_SLEEP);
588}
589
590ThreadContext *
591BaseKvmCPU::getContext(int tn)
592{
593 assert(tn == 0);
594 syncThreadContext();
595 return tc;
596}
597
598
599Counter
600BaseKvmCPU::totalInsts() const
601{
602 return ctrInsts;
603}
604
605Counter
606BaseKvmCPU::totalOps() const
607{
608 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
609 return ctrInsts;
610}
611
612void
613BaseKvmCPU::dump() const
614{
615 inform("State dumping not implemented.");
616}
617
618void
619BaseKvmCPU::tick()
620{
621 Tick delay(0);
622 assert(_status != Idle && _status != RunningMMIOPending);
623
624 switch (_status) {
625 case RunningService:
626 // handleKvmExit() will determine the next state of the CPU
627 delay = handleKvmExit();
628
629 if (tryDrain())
630 _status = Idle;
631 break;
632
633 case RunningServiceCompletion:
634 case Running: {
635 const uint64_t nextInstEvent(
636 !comInstEventQueue[0]->empty() ?
637 comInstEventQueue[0]->nextTick() : UINT64_MAX);
638 // Enter into KVM and complete pending IO instructions if we
639 // have an instruction event pending.
640 const Tick ticksToExecute(
641 nextInstEvent > ctrInsts ?
642 curEventQueue()->nextTick() - curTick() : 0);
643
644 if (alwaysSyncTC)
645 threadContextDirty = true;
646
647 // We might need to update the KVM state.
648 syncKvmState();
649
650 // Setup any pending instruction count breakpoints using
651 // PerfEvent if we are going to execute more than just an IO
652 // completion.
653 if (ticksToExecute > 0)
654 setupInstStop();
655
656 DPRINTF(KvmRun, "Entering KVM...\n");
657 if (drainState() == DrainState::Draining) {
658 // Force an immediate exit from KVM after completing
659 // pending operations. The architecture-specific code
660 // takes care to run until it is in a state where it can
661 // safely be drained.
662 delay = kvmRunDrain();
663 } else {
664 delay = kvmRun(ticksToExecute);
665 }
666
667 // The CPU might have been suspended before entering into
668 // KVM. Assume that the CPU was suspended /before/ entering
669 // into KVM and skip the exit handling.
670 if (_status == Idle)
671 break;
672
673 // Entering into KVM implies that we'll have to reload the thread
674 // context from KVM if we want to access it. Flag the KVM state as
675 // dirty with respect to the cached thread context.
676 kvmStateDirty = true;
677
678 if (alwaysSyncTC)
679 syncThreadContext();
680
681 // Enter into the RunningService state unless the
682 // simulation was stopped by a timer.
683 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
684 _status = RunningService;
685 } else {
686 ++numExitSignal;
687 _status = Running;
688 }
689
690 // Service any pending instruction events. The vCPU should
691 // have exited in time for the event using the instruction
692 // counter configured by setupInstStop().
693 comInstEventQueue[0]->serviceEvents(ctrInsts);
694 system->instEventQueue.serviceEvents(system->totalNumInsts);
695
696 if (tryDrain())
697 _status = Idle;
698 } break;
699
700 default:
701 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
702 _status);
703 }
704
705 // Schedule a new tick if we are still running
706 if (_status != Idle && _status != RunningMMIOPending)
707 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
708}
709
710Tick
711BaseKvmCPU::kvmRunDrain()
712{
713 // By default, the only thing we need to drain is a pending IO
714 // operation which assumes that we are in the
715 // RunningServiceCompletion or RunningMMIOPending state.
716 assert(_status == RunningServiceCompletion ||
717 _status == RunningMMIOPending);
718
719 // Deliver the data from the pending IO operation and immediately
720 // exit.
721 return kvmRun(0);
722}
723
724uint64_t
725BaseKvmCPU::getHostCycles() const
726{
727 return hwCycles.read();
728}
729
730Tick
731BaseKvmCPU::kvmRun(Tick ticks)
732{
733 Tick ticksExecuted;
734 fatal_if(vcpuFD == -1,
735 "Trying to run a KVM CPU in a forked child process. "
736 "This is not supported.\n");
737 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
738
739 if (ticks == 0) {
740 // Settings ticks == 0 is a special case which causes an entry
741 // into KVM that finishes pending operations (e.g., IO) and
742 // then immediately exits.
743 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
744
745 ++numVMHalfEntries;
746
747 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
748 // thread). The KVM control signal is masked while executing
749 // in gem5 and gets unmasked temporarily as when entering
750 // KVM. See setSignalMask() and setupSignalHandler().
751 kick();
752
753 // Start the vCPU. KVM will check for signals after completing
754 // pending operations (IO). Since the KVM_KICK_SIGNAL is
755 // pending, this forces an immediate exit to gem5 again. We
756 // don't bother to setup timers since this shouldn't actually
757 // execute any code (other than completing half-executed IO
758 // instructions) in the guest.
759 ioctlRun();
760
761 // We always execute at least one cycle to prevent the
762 // BaseKvmCPU::tick() to be rescheduled on the same tick
763 // twice.
764 ticksExecuted = clockPeriod();
765 } else {
766 // This method is executed as a result of a tick event. That
767 // means that the event queue will be locked when entering the
768 // method. We temporarily unlock the event queue to allow
769 // other threads to steal control of this thread to inject
770 // interrupts. They will typically lock the queue and then
771 // force an exit from KVM by kicking the vCPU.
772 EventQueue::ScopedRelease release(curEventQueue());
773
774 if (ticks < runTimer->resolution()) {
775 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
776 ticks, runTimer->resolution());
777 ticks = runTimer->resolution();
778 }
779
780 // Get hardware statistics after synchronizing contexts. The KVM
781 // state update might affect guest cycle counters.
782 uint64_t baseCycles(getHostCycles());
783 uint64_t baseInstrs(hwInstructions.read());
784
785 // Arm the run timer and start the cycle timer if it isn't
786 // controlled by the overflow timer. Starting/stopping the cycle
787 // timer automatically starts the other perf timers as they are in
788 // the same counter group.
789 runTimer->arm(ticks);
790 if (!perfControlledByTimer)
791 hwCycles.start();
792
793 ioctlRun();
794
795 runTimer->disarm();
796 if (!perfControlledByTimer)
797 hwCycles.stop();
798
799 // The control signal may have been delivered after we exited
800 // from KVM. It will be pending in that case since it is
801 // masked when we aren't executing in KVM. Discard it to make
802 // sure we don't deliver it immediately next time we try to
803 // enter into KVM.
804 discardPendingSignal(KVM_KICK_SIGNAL);
805
806 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
807 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
808 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
809 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
810
811 /* Update statistics */
812 numCycles += simCyclesExecuted;;
813 numInsts += instsExecuted;
814 ctrInsts += instsExecuted;
815 system->totalNumInsts += instsExecuted;
816
817 DPRINTF(KvmRun,
818 "KVM: Executed %i instructions in %i cycles "
819 "(%i ticks, sim cycles: %i).\n",
820 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
821 }
822
823 ++numVMExits;
824
825 return ticksExecuted + flushCoalescedMMIO();
826}
827
828void
829BaseKvmCPU::kvmNonMaskableInterrupt()
830{
831 ++numInterrupts;
832 if (ioctl(KVM_NMI) == -1)
833 panic("KVM: Failed to deliver NMI to virtual CPU\n");
834}
835
836void
837BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
838{
839 ++numInterrupts;
840 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
841 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
842}
843
844void
845BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
846{
847 if (ioctl(KVM_GET_REGS, ®s) == -1)
848 panic("KVM: Failed to get guest registers\n");
849}
850
851void
852BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
853{
854 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
855 panic("KVM: Failed to set guest registers\n");
856}
857
858void
859BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
860{
861 if (ioctl(KVM_GET_SREGS, ®s) == -1)
862 panic("KVM: Failed to get guest special registers\n");
863}
864
865void
866BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
867{
868 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
869 panic("KVM: Failed to set guest special registers\n");
870}
871
872void
873BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
874{
875 if (ioctl(KVM_GET_FPU, &state) == -1)
876 panic("KVM: Failed to get guest FPU state\n");
877}
878
879void
880BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
881{
882 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
883 panic("KVM: Failed to set guest FPU state\n");
884}
885
886
887void
888BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
889{
890#ifdef KVM_SET_ONE_REG
891 struct kvm_one_reg reg;
892 reg.id = id;
893 reg.addr = (uint64_t)addr;
894
895 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
896 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
897 id, errno);
898 }
899#else
900 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
901#endif
902}
903
904void
905BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
906{
907#ifdef KVM_GET_ONE_REG
908 struct kvm_one_reg reg;
909 reg.id = id;
910 reg.addr = (uint64_t)addr;
911
912 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
913 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
914 id, errno);
915 }
916#else
917 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
918#endif
919}
920
921std::string
922BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
923{
924#ifdef KVM_GET_ONE_REG
925 std::ostringstream ss;
926
927 ss.setf(std::ios::hex, std::ios::basefield);
928 ss.setf(std::ios::showbase);
929#define HANDLE_INTTYPE(len) \
930 case KVM_REG_SIZE_U ## len: { \
931 uint ## len ## _t value; \
932 getOneReg(id, &value); \
933 ss << value; \
934 } break
935
936#define HANDLE_ARRAY(len) \
937 case KVM_REG_SIZE_U ## len: { \
938 uint8_t value[len / 8]; \
939 getOneReg(id, value); \
940 ccprintf(ss, "[0x%x", value[0]); \
941 for (int i = 1; i < len / 8; ++i) \
942 ccprintf(ss, ", 0x%x", value[i]); \
943 ccprintf(ss, "]"); \
944 } break
945
946 switch (id & KVM_REG_SIZE_MASK) {
947 HANDLE_INTTYPE(8);
948 HANDLE_INTTYPE(16);
949 HANDLE_INTTYPE(32);
950 HANDLE_INTTYPE(64);
951 HANDLE_ARRAY(128);
952 HANDLE_ARRAY(256);
953 HANDLE_ARRAY(512);
954 HANDLE_ARRAY(1024);
955 default:
956 ss << "??";
957 }
958
959#undef HANDLE_INTTYPE
960#undef HANDLE_ARRAY
961
962 return ss.str();
963#else
964 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
965#endif
966}
967
968void
969BaseKvmCPU::syncThreadContext()
970{
971 if (!kvmStateDirty)
972 return;
973
974 assert(!threadContextDirty);
975
976 updateThreadContext();
977 kvmStateDirty = false;
978}
979
980void
981BaseKvmCPU::syncKvmState()
982{
983 if (!threadContextDirty)
984 return;
985
986 assert(!kvmStateDirty);
987
988 updateKvmState();
989 threadContextDirty = false;
990}
991
992Tick
993BaseKvmCPU::handleKvmExit()
994{
995 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
996 assert(_status == RunningService);
997
998 // Switch into the running state by default. Individual handlers
999 // can override this.
1000 _status = Running;
1001 switch (_kvmRun->exit_reason) {
1002 case KVM_EXIT_UNKNOWN:
1003 return handleKvmExitUnknown();
1004
1005 case KVM_EXIT_EXCEPTION:
1006 return handleKvmExitException();
1007
1008 case KVM_EXIT_IO:
1009 {
1010 ++numIO;
1011 Tick ticks = handleKvmExitIO();
1012 _status = dataPort.nextIOState();
1013 return ticks;
1014 }
1015
1016 case KVM_EXIT_HYPERCALL:
1017 ++numHypercalls;
1018 return handleKvmExitHypercall();
1019
1020 case KVM_EXIT_HLT:
1021 /* The guest has halted and is waiting for interrupts */
1022 DPRINTF(Kvm, "handleKvmExitHalt\n");
1023 ++numHalt;
1024
1025 // Suspend the thread until the next interrupt arrives
1026 thread->suspend();
1027
1028 // This is actually ignored since the thread is suspended.
1029 return 0;
1030
1031 case KVM_EXIT_MMIO:
1032 {
1033 /* Service memory mapped IO requests */
1034 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1035 _kvmRun->mmio.is_write,
1036 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1037
1038 ++numMMIO;
1039 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1040 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1041 // doMMIOAccess could have triggered a suspend, in which case we don't
1042 // want to overwrite the _status.
1043 if (_status != Idle)
1044 _status = dataPort.nextIOState();
1045 return ticks;
1046 }
1047
1048 case KVM_EXIT_IRQ_WINDOW_OPEN:
1049 return handleKvmExitIRQWindowOpen();
1050
1051 case KVM_EXIT_FAIL_ENTRY:
1052 return handleKvmExitFailEntry();
1053
1054 case KVM_EXIT_INTR:
1055 /* KVM was interrupted by a signal, restart it in the next
1056 * tick. */
1057 return 0;
1058
1059 case KVM_EXIT_INTERNAL_ERROR:
1060 panic("KVM: Internal error (suberror: %u)\n",
1061 _kvmRun->internal.suberror);
1062
1063 default:
1064 dump();
1065 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1066 }
1067}
1068
1069Tick
1070BaseKvmCPU::handleKvmExitIO()
1071{
1072 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1073 _kvmRun->io.direction, _kvmRun->io.size,
1074 _kvmRun->io.port, _kvmRun->io.count);
1075}
1076
1077Tick
1078BaseKvmCPU::handleKvmExitHypercall()
1079{
1080 panic("KVM: Unhandled hypercall\n");
1081}
1082
1083Tick
1084BaseKvmCPU::handleKvmExitIRQWindowOpen()
1085{
1086 warn("KVM: Unhandled IRQ window.\n");
1087 return 0;
1088}
1089
1090
1091Tick
1092BaseKvmCPU::handleKvmExitUnknown()
1093{
1094 dump();
1095 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1096 _kvmRun->hw.hardware_exit_reason);
1097}
1098
1099Tick
1100BaseKvmCPU::handleKvmExitException()
1101{
1102 dump();
1103 panic("KVM: Got exception when starting vCPU "
1104 "(exception: %u, error_code: %u)\n",
1105 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1106}
1107
1108Tick
1109BaseKvmCPU::handleKvmExitFailEntry()
1110{
1111 dump();
1112 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1113 _kvmRun->fail_entry.hardware_entry_failure_reason);
1114}
1115
1116Tick
1117BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1118{
1119 ThreadContext *tc(thread->getTC());
1120 syncThreadContext();
1121
| 202 delete pkt;
203 activeMMIOReqs--;
204
205 // We can switch back into KVM when all pending and in-flight MMIO
206 // operations have completed.
207 if (!(activeMMIOReqs || pendingMMIOPkts.size())) {
208 DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n");
209 cpu->finishMMIOPending();
210 }
211 return true;
212}
213
214void
215BaseKvmCPU::KVMCpuPort::recvReqRetry()
216{
217 DPRINTF(KvmIO, "KVM: Retry for timing request\n");
218
219 assert(pendingMMIOPkts.size());
220
221 // Assuming that we can issue infinite requests this cycle is a bit
222 // unrealistic, but it's not worth modeling something more complex in
223 // KVM.
224 while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) {
225 pendingMMIOPkts.pop();
226 activeMMIOReqs++;
227 }
228}
229
230void
231BaseKvmCPU::finishMMIOPending()
232{
233 assert(_status = RunningMMIOPending);
234 assert(!tickEvent.scheduled());
235
236 _status = RunningServiceCompletion;
237 schedule(tickEvent, nextCycle());
238}
239
240void
241BaseKvmCPU::startupThread()
242{
243 // Do thread-specific initialization. We need to setup signal
244 // delivery for counters and timers from within the thread that
245 // will execute the event queue to ensure that signals are
246 // delivered to the right threads.
247 const BaseKvmCPUParams * const p(
248 dynamic_cast<const BaseKvmCPUParams *>(params()));
249
250 vcpuThread = pthread_self();
251
252 // Setup signal handlers. This has to be done after the vCPU is
253 // created since it manipulates the vCPU signal mask.
254 setupSignalHandler();
255
256 setupCounters();
257
258 if (p->usePerfOverflow)
259 runTimer.reset(new PerfKvmTimer(hwCycles,
260 KVM_KICK_SIGNAL,
261 p->hostFactor,
262 p->hostFreq));
263 else
264 runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC,
265 p->hostFactor,
266 p->hostFreq));
267
268}
269
270void
271BaseKvmCPU::regStats()
272{
273 using namespace Stats;
274
275 BaseCPU::regStats();
276
277 numInsts
278 .name(name() + ".committedInsts")
279 .desc("Number of instructions committed")
280 ;
281
282 numVMExits
283 .name(name() + ".numVMExits")
284 .desc("total number of KVM exits")
285 ;
286
287 numVMHalfEntries
288 .name(name() + ".numVMHalfEntries")
289 .desc("number of KVM entries to finalize pending operations")
290 ;
291
292 numExitSignal
293 .name(name() + ".numExitSignal")
294 .desc("exits due to signal delivery")
295 ;
296
297 numMMIO
298 .name(name() + ".numMMIO")
299 .desc("number of VM exits due to memory mapped IO")
300 ;
301
302 numCoalescedMMIO
303 .name(name() + ".numCoalescedMMIO")
304 .desc("number of coalesced memory mapped IO requests")
305 ;
306
307 numIO
308 .name(name() + ".numIO")
309 .desc("number of VM exits due to legacy IO")
310 ;
311
312 numHalt
313 .name(name() + ".numHalt")
314 .desc("number of VM exits due to wait for interrupt instructions")
315 ;
316
317 numInterrupts
318 .name(name() + ".numInterrupts")
319 .desc("number of interrupts delivered")
320 ;
321
322 numHypercalls
323 .name(name() + ".numHypercalls")
324 .desc("number of hypercalls")
325 ;
326}
327
328void
329BaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const
330{
331 if (DTRACE(Checkpoint)) {
332 DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid);
333 dump();
334 }
335
336 assert(tid == 0);
337 assert(_status == Idle);
338 thread->serialize(cp);
339}
340
341void
342BaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid)
343{
344 DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid);
345
346 assert(tid == 0);
347 assert(_status == Idle);
348 thread->unserialize(cp);
349 threadContextDirty = true;
350}
351
352DrainState
353BaseKvmCPU::drain()
354{
355 if (switchedOut())
356 return DrainState::Drained;
357
358 DPRINTF(Drain, "BaseKvmCPU::drain\n");
359
360 // The event queue won't be locked when calling drain since that's
361 // not done from an event. Lock the event queue here to make sure
362 // that scoped migrations continue to work if we need to
363 // synchronize the thread context.
364 std::lock_guard<EventQueue> lock(*this->eventQueue());
365
366 switch (_status) {
367 case Running:
368 // The base KVM code is normally ready when it is in the
369 // Running state, but the architecture specific code might be
370 // of a different opinion. This may happen when the CPU been
371 // notified of an event that hasn't been accepted by the vCPU
372 // yet.
373 if (!archIsDrained())
374 return DrainState::Draining;
375
376 // The state of the CPU is consistent, so we don't need to do
377 // anything special to drain it. We simply de-schedule the
378 // tick event and enter the Idle state to prevent nasty things
379 // like MMIOs from happening.
380 if (tickEvent.scheduled())
381 deschedule(tickEvent);
382 _status = Idle;
383
384 M5_FALLTHROUGH;
385 case Idle:
386 // Idle, no need to drain
387 assert(!tickEvent.scheduled());
388
389 // Sync the thread context here since we'll need it when we
390 // switch CPUs or checkpoint the CPU.
391 syncThreadContext();
392
393 return DrainState::Drained;
394
395 case RunningServiceCompletion:
396 // The CPU has just requested a service that was handled in
397 // the RunningService state, but the results have still not
398 // been reported to the CPU. Now, we /could/ probably just
399 // update the register state ourselves instead of letting KVM
400 // handle it, but that would be tricky. Instead, we enter KVM
401 // and let it do its stuff.
402 DPRINTF(Drain, "KVM CPU is waiting for service completion, "
403 "requesting drain.\n");
404 return DrainState::Draining;
405
406 case RunningMMIOPending:
407 // We need to drain since there are in-flight timing accesses
408 DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, "
409 "requesting drain.\n");
410 return DrainState::Draining;
411
412 case RunningService:
413 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
414 DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
415 return DrainState::Draining;
416
417 default:
418 panic("KVM: Unhandled CPU state in drain()\n");
419 return DrainState::Drained;
420 }
421}
422
423void
424BaseKvmCPU::drainResume()
425{
426 assert(!tickEvent.scheduled());
427
428 // We might have been switched out. In that case, we don't need to
429 // do anything.
430 if (switchedOut())
431 return;
432
433 DPRINTF(Kvm, "drainResume\n");
434 verifyMemoryMode();
435
436 // The tick event is de-scheduled as a part of the draining
437 // process. Re-schedule it if the thread context is active.
438 if (tc->status() == ThreadContext::Active) {
439 schedule(tickEvent, nextCycle());
440 _status = Running;
441 } else {
442 _status = Idle;
443 }
444}
445
446void
447BaseKvmCPU::notifyFork()
448{
449 // We should have drained prior to forking, which means that the
450 // tick event shouldn't be scheduled and the CPU is idle.
451 assert(!tickEvent.scheduled());
452 assert(_status == Idle);
453
454 if (vcpuFD != -1) {
455 if (close(vcpuFD) == -1)
456 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
457
458 if (_kvmRun)
459 munmap(_kvmRun, vcpuMMapSize);
460
461 vcpuFD = -1;
462 _kvmRun = NULL;
463
464 hwInstructions.detach();
465 hwCycles.detach();
466 }
467}
468
469void
470BaseKvmCPU::switchOut()
471{
472 DPRINTF(Kvm, "switchOut\n");
473
474 BaseCPU::switchOut();
475
476 // We should have drained prior to executing a switchOut, which
477 // means that the tick event shouldn't be scheduled and the CPU is
478 // idle.
479 assert(!tickEvent.scheduled());
480 assert(_status == Idle);
481}
482
483void
484BaseKvmCPU::takeOverFrom(BaseCPU *cpu)
485{
486 DPRINTF(Kvm, "takeOverFrom\n");
487
488 BaseCPU::takeOverFrom(cpu);
489
490 // We should have drained prior to executing a switchOut, which
491 // means that the tick event shouldn't be scheduled and the CPU is
492 // idle.
493 assert(!tickEvent.scheduled());
494 assert(_status == Idle);
495 assert(threadContexts.size() == 1);
496
497 // Force an update of the KVM state here instead of flagging the
498 // TC as dirty. This is not ideal from a performance point of
499 // view, but it makes debugging easier as it allows meaningful KVM
500 // state to be dumped before and after a takeover.
501 updateKvmState();
502 threadContextDirty = false;
503}
504
505void
506BaseKvmCPU::verifyMemoryMode() const
507{
508 if (!(system->bypassCaches())) {
509 fatal("The KVM-based CPUs requires the memory system to be in the "
510 "'noncaching' mode.\n");
511 }
512}
513
514void
515BaseKvmCPU::wakeup(ThreadID tid)
516{
517 DPRINTF(Kvm, "wakeup()\n");
518 // This method might have been called from another
519 // context. Migrate to this SimObject's event queue when
520 // delivering the wakeup signal.
521 EventQueue::ScopedMigration migrate(eventQueue());
522
523 // Kick the vCPU to get it to come out of KVM.
524 kick();
525
526 if (thread->status() != ThreadContext::Suspended)
527 return;
528
529 thread->activate();
530}
531
532void
533BaseKvmCPU::activateContext(ThreadID thread_num)
534{
535 DPRINTF(Kvm, "ActivateContext %d\n", thread_num);
536
537 assert(thread_num == 0);
538 assert(thread);
539
540 assert(_status == Idle);
541 assert(!tickEvent.scheduled());
542
543 numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend);
544
545 schedule(tickEvent, clockEdge(Cycles(0)));
546 _status = Running;
547}
548
549
550void
551BaseKvmCPU::suspendContext(ThreadID thread_num)
552{
553 DPRINTF(Kvm, "SuspendContext %d\n", thread_num);
554
555 assert(thread_num == 0);
556 assert(thread);
557
558 if (_status == Idle)
559 return;
560
561 assert(_status == Running || _status == RunningServiceCompletion);
562
563 // The tick event may no be scheduled if the quest has requested
564 // the monitor to wait for interrupts. The normal CPU models can
565 // get their tick events descheduled by quiesce instructions, but
566 // that can't happen here.
567 if (tickEvent.scheduled())
568 deschedule(tickEvent);
569
570 _status = Idle;
571}
572
573void
574BaseKvmCPU::deallocateContext(ThreadID thread_num)
575{
576 // for now, these are equivalent
577 suspendContext(thread_num);
578}
579
580void
581BaseKvmCPU::haltContext(ThreadID thread_num)
582{
583 // for now, these are equivalent
584 suspendContext(thread_num);
585 updateCycleCounters(BaseCPU::CPU_STATE_SLEEP);
586}
587
588ThreadContext *
589BaseKvmCPU::getContext(int tn)
590{
591 assert(tn == 0);
592 syncThreadContext();
593 return tc;
594}
595
596
597Counter
598BaseKvmCPU::totalInsts() const
599{
600 return ctrInsts;
601}
602
603Counter
604BaseKvmCPU::totalOps() const
605{
606 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
607 return ctrInsts;
608}
609
610void
611BaseKvmCPU::dump() const
612{
613 inform("State dumping not implemented.");
614}
615
616void
617BaseKvmCPU::tick()
618{
619 Tick delay(0);
620 assert(_status != Idle && _status != RunningMMIOPending);
621
622 switch (_status) {
623 case RunningService:
624 // handleKvmExit() will determine the next state of the CPU
625 delay = handleKvmExit();
626
627 if (tryDrain())
628 _status = Idle;
629 break;
630
631 case RunningServiceCompletion:
632 case Running: {
633 const uint64_t nextInstEvent(
634 !comInstEventQueue[0]->empty() ?
635 comInstEventQueue[0]->nextTick() : UINT64_MAX);
636 // Enter into KVM and complete pending IO instructions if we
637 // have an instruction event pending.
638 const Tick ticksToExecute(
639 nextInstEvent > ctrInsts ?
640 curEventQueue()->nextTick() - curTick() : 0);
641
642 if (alwaysSyncTC)
643 threadContextDirty = true;
644
645 // We might need to update the KVM state.
646 syncKvmState();
647
648 // Setup any pending instruction count breakpoints using
649 // PerfEvent if we are going to execute more than just an IO
650 // completion.
651 if (ticksToExecute > 0)
652 setupInstStop();
653
654 DPRINTF(KvmRun, "Entering KVM...\n");
655 if (drainState() == DrainState::Draining) {
656 // Force an immediate exit from KVM after completing
657 // pending operations. The architecture-specific code
658 // takes care to run until it is in a state where it can
659 // safely be drained.
660 delay = kvmRunDrain();
661 } else {
662 delay = kvmRun(ticksToExecute);
663 }
664
665 // The CPU might have been suspended before entering into
666 // KVM. Assume that the CPU was suspended /before/ entering
667 // into KVM and skip the exit handling.
668 if (_status == Idle)
669 break;
670
671 // Entering into KVM implies that we'll have to reload the thread
672 // context from KVM if we want to access it. Flag the KVM state as
673 // dirty with respect to the cached thread context.
674 kvmStateDirty = true;
675
676 if (alwaysSyncTC)
677 syncThreadContext();
678
679 // Enter into the RunningService state unless the
680 // simulation was stopped by a timer.
681 if (_kvmRun->exit_reason != KVM_EXIT_INTR) {
682 _status = RunningService;
683 } else {
684 ++numExitSignal;
685 _status = Running;
686 }
687
688 // Service any pending instruction events. The vCPU should
689 // have exited in time for the event using the instruction
690 // counter configured by setupInstStop().
691 comInstEventQueue[0]->serviceEvents(ctrInsts);
692 system->instEventQueue.serviceEvents(system->totalNumInsts);
693
694 if (tryDrain())
695 _status = Idle;
696 } break;
697
698 default:
699 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
700 _status);
701 }
702
703 // Schedule a new tick if we are still running
704 if (_status != Idle && _status != RunningMMIOPending)
705 schedule(tickEvent, clockEdge(ticksToCycles(delay)));
706}
707
708Tick
709BaseKvmCPU::kvmRunDrain()
710{
711 // By default, the only thing we need to drain is a pending IO
712 // operation which assumes that we are in the
713 // RunningServiceCompletion or RunningMMIOPending state.
714 assert(_status == RunningServiceCompletion ||
715 _status == RunningMMIOPending);
716
717 // Deliver the data from the pending IO operation and immediately
718 // exit.
719 return kvmRun(0);
720}
721
722uint64_t
723BaseKvmCPU::getHostCycles() const
724{
725 return hwCycles.read();
726}
727
728Tick
729BaseKvmCPU::kvmRun(Tick ticks)
730{
731 Tick ticksExecuted;
732 fatal_if(vcpuFD == -1,
733 "Trying to run a KVM CPU in a forked child process. "
734 "This is not supported.\n");
735 DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
736
737 if (ticks == 0) {
738 // Settings ticks == 0 is a special case which causes an entry
739 // into KVM that finishes pending operations (e.g., IO) and
740 // then immediately exits.
741 DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n");
742
743 ++numVMHalfEntries;
744
745 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
746 // thread). The KVM control signal is masked while executing
747 // in gem5 and gets unmasked temporarily as when entering
748 // KVM. See setSignalMask() and setupSignalHandler().
749 kick();
750
751 // Start the vCPU. KVM will check for signals after completing
752 // pending operations (IO). Since the KVM_KICK_SIGNAL is
753 // pending, this forces an immediate exit to gem5 again. We
754 // don't bother to setup timers since this shouldn't actually
755 // execute any code (other than completing half-executed IO
756 // instructions) in the guest.
757 ioctlRun();
758
759 // We always execute at least one cycle to prevent the
760 // BaseKvmCPU::tick() to be rescheduled on the same tick
761 // twice.
762 ticksExecuted = clockPeriod();
763 } else {
764 // This method is executed as a result of a tick event. That
765 // means that the event queue will be locked when entering the
766 // method. We temporarily unlock the event queue to allow
767 // other threads to steal control of this thread to inject
768 // interrupts. They will typically lock the queue and then
769 // force an exit from KVM by kicking the vCPU.
770 EventQueue::ScopedRelease release(curEventQueue());
771
772 if (ticks < runTimer->resolution()) {
773 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
774 ticks, runTimer->resolution());
775 ticks = runTimer->resolution();
776 }
777
778 // Get hardware statistics after synchronizing contexts. The KVM
779 // state update might affect guest cycle counters.
780 uint64_t baseCycles(getHostCycles());
781 uint64_t baseInstrs(hwInstructions.read());
782
783 // Arm the run timer and start the cycle timer if it isn't
784 // controlled by the overflow timer. Starting/stopping the cycle
785 // timer automatically starts the other perf timers as they are in
786 // the same counter group.
787 runTimer->arm(ticks);
788 if (!perfControlledByTimer)
789 hwCycles.start();
790
791 ioctlRun();
792
793 runTimer->disarm();
794 if (!perfControlledByTimer)
795 hwCycles.stop();
796
797 // The control signal may have been delivered after we exited
798 // from KVM. It will be pending in that case since it is
799 // masked when we aren't executing in KVM. Discard it to make
800 // sure we don't deliver it immediately next time we try to
801 // enter into KVM.
802 discardPendingSignal(KVM_KICK_SIGNAL);
803
804 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles);
805 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor);
806 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs);
807 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted);
808
809 /* Update statistics */
810 numCycles += simCyclesExecuted;;
811 numInsts += instsExecuted;
812 ctrInsts += instsExecuted;
813 system->totalNumInsts += instsExecuted;
814
815 DPRINTF(KvmRun,
816 "KVM: Executed %i instructions in %i cycles "
817 "(%i ticks, sim cycles: %i).\n",
818 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted);
819 }
820
821 ++numVMExits;
822
823 return ticksExecuted + flushCoalescedMMIO();
824}
825
826void
827BaseKvmCPU::kvmNonMaskableInterrupt()
828{
829 ++numInterrupts;
830 if (ioctl(KVM_NMI) == -1)
831 panic("KVM: Failed to deliver NMI to virtual CPU\n");
832}
833
834void
835BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt)
836{
837 ++numInterrupts;
838 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1)
839 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
840}
841
842void
843BaseKvmCPU::getRegisters(struct kvm_regs ®s) const
844{
845 if (ioctl(KVM_GET_REGS, ®s) == -1)
846 panic("KVM: Failed to get guest registers\n");
847}
848
849void
850BaseKvmCPU::setRegisters(const struct kvm_regs ®s)
851{
852 if (ioctl(KVM_SET_REGS, (void *)®s) == -1)
853 panic("KVM: Failed to set guest registers\n");
854}
855
856void
857BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const
858{
859 if (ioctl(KVM_GET_SREGS, ®s) == -1)
860 panic("KVM: Failed to get guest special registers\n");
861}
862
863void
864BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s)
865{
866 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1)
867 panic("KVM: Failed to set guest special registers\n");
868}
869
870void
871BaseKvmCPU::getFPUState(struct kvm_fpu &state) const
872{
873 if (ioctl(KVM_GET_FPU, &state) == -1)
874 panic("KVM: Failed to get guest FPU state\n");
875}
876
877void
878BaseKvmCPU::setFPUState(const struct kvm_fpu &state)
879{
880 if (ioctl(KVM_SET_FPU, (void *)&state) == -1)
881 panic("KVM: Failed to set guest FPU state\n");
882}
883
884
885void
886BaseKvmCPU::setOneReg(uint64_t id, const void *addr)
887{
888#ifdef KVM_SET_ONE_REG
889 struct kvm_one_reg reg;
890 reg.id = id;
891 reg.addr = (uint64_t)addr;
892
893 if (ioctl(KVM_SET_ONE_REG, ®) == -1) {
894 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
895 id, errno);
896 }
897#else
898 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
899#endif
900}
901
902void
903BaseKvmCPU::getOneReg(uint64_t id, void *addr) const
904{
905#ifdef KVM_GET_ONE_REG
906 struct kvm_one_reg reg;
907 reg.id = id;
908 reg.addr = (uint64_t)addr;
909
910 if (ioctl(KVM_GET_ONE_REG, ®) == -1) {
911 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
912 id, errno);
913 }
914#else
915 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
916#endif
917}
918
919std::string
920BaseKvmCPU::getAndFormatOneReg(uint64_t id) const
921{
922#ifdef KVM_GET_ONE_REG
923 std::ostringstream ss;
924
925 ss.setf(std::ios::hex, std::ios::basefield);
926 ss.setf(std::ios::showbase);
927#define HANDLE_INTTYPE(len) \
928 case KVM_REG_SIZE_U ## len: { \
929 uint ## len ## _t value; \
930 getOneReg(id, &value); \
931 ss << value; \
932 } break
933
934#define HANDLE_ARRAY(len) \
935 case KVM_REG_SIZE_U ## len: { \
936 uint8_t value[len / 8]; \
937 getOneReg(id, value); \
938 ccprintf(ss, "[0x%x", value[0]); \
939 for (int i = 1; i < len / 8; ++i) \
940 ccprintf(ss, ", 0x%x", value[i]); \
941 ccprintf(ss, "]"); \
942 } break
943
944 switch (id & KVM_REG_SIZE_MASK) {
945 HANDLE_INTTYPE(8);
946 HANDLE_INTTYPE(16);
947 HANDLE_INTTYPE(32);
948 HANDLE_INTTYPE(64);
949 HANDLE_ARRAY(128);
950 HANDLE_ARRAY(256);
951 HANDLE_ARRAY(512);
952 HANDLE_ARRAY(1024);
953 default:
954 ss << "??";
955 }
956
957#undef HANDLE_INTTYPE
958#undef HANDLE_ARRAY
959
960 return ss.str();
961#else
962 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
963#endif
964}
965
966void
967BaseKvmCPU::syncThreadContext()
968{
969 if (!kvmStateDirty)
970 return;
971
972 assert(!threadContextDirty);
973
974 updateThreadContext();
975 kvmStateDirty = false;
976}
977
978void
979BaseKvmCPU::syncKvmState()
980{
981 if (!threadContextDirty)
982 return;
983
984 assert(!kvmStateDirty);
985
986 updateKvmState();
987 threadContextDirty = false;
988}
989
990Tick
991BaseKvmCPU::handleKvmExit()
992{
993 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason);
994 assert(_status == RunningService);
995
996 // Switch into the running state by default. Individual handlers
997 // can override this.
998 _status = Running;
999 switch (_kvmRun->exit_reason) {
1000 case KVM_EXIT_UNKNOWN:
1001 return handleKvmExitUnknown();
1002
1003 case KVM_EXIT_EXCEPTION:
1004 return handleKvmExitException();
1005
1006 case KVM_EXIT_IO:
1007 {
1008 ++numIO;
1009 Tick ticks = handleKvmExitIO();
1010 _status = dataPort.nextIOState();
1011 return ticks;
1012 }
1013
1014 case KVM_EXIT_HYPERCALL:
1015 ++numHypercalls;
1016 return handleKvmExitHypercall();
1017
1018 case KVM_EXIT_HLT:
1019 /* The guest has halted and is waiting for interrupts */
1020 DPRINTF(Kvm, "handleKvmExitHalt\n");
1021 ++numHalt;
1022
1023 // Suspend the thread until the next interrupt arrives
1024 thread->suspend();
1025
1026 // This is actually ignored since the thread is suspended.
1027 return 0;
1028
1029 case KVM_EXIT_MMIO:
1030 {
1031 /* Service memory mapped IO requests */
1032 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
1033 _kvmRun->mmio.is_write,
1034 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len);
1035
1036 ++numMMIO;
1037 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data,
1038 _kvmRun->mmio.len, _kvmRun->mmio.is_write);
1039 // doMMIOAccess could have triggered a suspend, in which case we don't
1040 // want to overwrite the _status.
1041 if (_status != Idle)
1042 _status = dataPort.nextIOState();
1043 return ticks;
1044 }
1045
1046 case KVM_EXIT_IRQ_WINDOW_OPEN:
1047 return handleKvmExitIRQWindowOpen();
1048
1049 case KVM_EXIT_FAIL_ENTRY:
1050 return handleKvmExitFailEntry();
1051
1052 case KVM_EXIT_INTR:
1053 /* KVM was interrupted by a signal, restart it in the next
1054 * tick. */
1055 return 0;
1056
1057 case KVM_EXIT_INTERNAL_ERROR:
1058 panic("KVM: Internal error (suberror: %u)\n",
1059 _kvmRun->internal.suberror);
1060
1061 default:
1062 dump();
1063 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason);
1064 }
1065}
1066
1067Tick
1068BaseKvmCPU::handleKvmExitIO()
1069{
1070 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1071 _kvmRun->io.direction, _kvmRun->io.size,
1072 _kvmRun->io.port, _kvmRun->io.count);
1073}
1074
1075Tick
1076BaseKvmCPU::handleKvmExitHypercall()
1077{
1078 panic("KVM: Unhandled hypercall\n");
1079}
1080
1081Tick
1082BaseKvmCPU::handleKvmExitIRQWindowOpen()
1083{
1084 warn("KVM: Unhandled IRQ window.\n");
1085 return 0;
1086}
1087
1088
1089Tick
1090BaseKvmCPU::handleKvmExitUnknown()
1091{
1092 dump();
1093 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1094 _kvmRun->hw.hardware_exit_reason);
1095}
1096
1097Tick
1098BaseKvmCPU::handleKvmExitException()
1099{
1100 dump();
1101 panic("KVM: Got exception when starting vCPU "
1102 "(exception: %u, error_code: %u)\n",
1103 _kvmRun->ex.exception, _kvmRun->ex.error_code);
1104}
1105
1106Tick
1107BaseKvmCPU::handleKvmExitFailEntry()
1108{
1109 dump();
1110 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1111 _kvmRun->fail_entry.hardware_entry_failure_reason);
1112}
1113
1114Tick
1115BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write)
1116{
1117 ThreadContext *tc(thread->getTC());
1118 syncThreadContext();
1119
|
1122 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE,
1123 dataMasterId());
| 1120 RequestPtr mmio_req = std::make_shared<Request>(
1121 paddr, size, Request::UNCACHEABLE, dataMasterId());
1122
|
1124 mmio_req->setContext(tc->contextId());
1125 // Some architectures do need to massage physical addresses a bit
1126 // before they are inserted into the memory system. This enables
1127 // APIC accesses on x86 and m5ops where supported through a MMIO
1128 // interface.
1129 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1130 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1131 if (fault != NoFault)
1132 warn("Finalization of MMIO address failed: %s\n", fault->name());
1133
1134
1135 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1136 PacketPtr pkt = new Packet(mmio_req, cmd);
1137 pkt->dataStatic(data);
1138
1139 if (mmio_req->isMmappedIpr()) {
1140 // We currently assume that there is no need to migrate to a
1141 // different event queue when doing IPRs. Currently, IPRs are
1142 // only used for m5ops, so it should be a valid assumption.
1143 const Cycles ipr_delay(write ?
1144 TheISA::handleIprWrite(tc, pkt) :
1145 TheISA::handleIprRead(tc, pkt));
1146 threadContextDirty = true;
| 1123 mmio_req->setContext(tc->contextId());
1124 // Some architectures do need to massage physical addresses a bit
1125 // before they are inserted into the memory system. This enables
1126 // APIC accesses on x86 and m5ops where supported through a MMIO
1127 // interface.
1128 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read);
1129 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode));
1130 if (fault != NoFault)
1131 warn("Finalization of MMIO address failed: %s\n", fault->name());
1132
1133
1134 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq);
1135 PacketPtr pkt = new Packet(mmio_req, cmd);
1136 pkt->dataStatic(data);
1137
1138 if (mmio_req->isMmappedIpr()) {
1139 // We currently assume that there is no need to migrate to a
1140 // different event queue when doing IPRs. Currently, IPRs are
1141 // only used for m5ops, so it should be a valid assumption.
1142 const Cycles ipr_delay(write ?
1143 TheISA::handleIprWrite(tc, pkt) :
1144 TheISA::handleIprRead(tc, pkt));
1145 threadContextDirty = true;
|
1147 delete pkt->req;
| |
1148 delete pkt;
1149 return clockPeriod() * ipr_delay;
1150 } else {
1151 // Temporarily lock and migrate to the device event queue to
1152 // prevent races in multi-core mode.
1153 EventQueue::ScopedMigration migrate(deviceEventQueue());
1154
1155 return dataPort.submitIO(pkt);
1156 }
1157}
1158
1159void
1160BaseKvmCPU::setSignalMask(const sigset_t *mask)
1161{
1162 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1163
1164 if (mask) {
1165 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1166 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1167 // The kernel and the user-space headers have different ideas
1168 // about the size of sigset_t. This seems like a massive hack,
1169 // but is actually what qemu does.
1170 assert(sizeof(*mask) >= 8);
1171 kvm_mask->len = 8;
1172 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1173 }
1174
1175 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1176 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1177 errno);
1178}
1179
1180int
1181BaseKvmCPU::ioctl(int request, long p1) const
1182{
1183 if (vcpuFD == -1)
1184 panic("KVM: CPU ioctl called before initialization\n");
1185
1186 return ::ioctl(vcpuFD, request, p1);
1187}
1188
1189Tick
1190BaseKvmCPU::flushCoalescedMMIO()
1191{
1192 if (!mmioRing)
1193 return 0;
1194
1195 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1196
1197 // TODO: We might need to do synchronization when we start to
1198 // support multiple CPUs
1199 Tick ticks(0);
1200 while (mmioRing->first != mmioRing->last) {
1201 struct kvm_coalesced_mmio &ent(
1202 mmioRing->coalesced_mmio[mmioRing->first]);
1203
1204 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1205 ent.phys_addr, ent.len);
1206
1207 ++numCoalescedMMIO;
1208 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1209
1210 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1211 }
1212
1213 return ticks;
1214}
1215
1216/**
1217 * Dummy handler for KVM kick signals.
1218 *
1219 * @note This function is usually not called since the kernel doesn't
1220 * seem to deliver signals when the signal is only unmasked when
1221 * running in KVM. This doesn't matter though since we are only
1222 * interested in getting KVM to exit, which happens as expected. See
1223 * setupSignalHandler() and kvmRun() for details about KVM signal
1224 * handling.
1225 */
1226static void
1227onKickSignal(int signo, siginfo_t *si, void *data)
1228{
1229}
1230
1231void
1232BaseKvmCPU::setupSignalHandler()
1233{
1234 struct sigaction sa;
1235
1236 memset(&sa, 0, sizeof(sa));
1237 sa.sa_sigaction = onKickSignal;
1238 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1239 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1240 panic("KVM: Failed to setup vCPU timer signal handler\n");
1241
1242 sigset_t sigset;
1243 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1244 panic("KVM: Failed get signal mask\n");
1245
1246 // Request KVM to setup the same signal mask as we're currently
1247 // running with except for the KVM control signal. We'll sometimes
1248 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1249 // KVM after servicing IO requests. See kvmRun().
1250 sigdelset(&sigset, KVM_KICK_SIGNAL);
1251 setSignalMask(&sigset);
1252
1253 // Mask our control signals so they aren't delivered unless we're
1254 // actually executing inside KVM.
1255 sigaddset(&sigset, KVM_KICK_SIGNAL);
1256 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1257 panic("KVM: Failed mask the KVM control signals\n");
1258}
1259
1260bool
1261BaseKvmCPU::discardPendingSignal(int signum) const
1262{
1263 int discardedSignal;
1264
1265 // Setting the timeout to zero causes sigtimedwait to return
1266 // immediately.
1267 struct timespec timeout;
1268 timeout.tv_sec = 0;
1269 timeout.tv_nsec = 0;
1270
1271 sigset_t sigset;
1272 sigemptyset(&sigset);
1273 sigaddset(&sigset, signum);
1274
1275 do {
1276 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1277 } while (discardedSignal == -1 && errno == EINTR);
1278
1279 if (discardedSignal == signum)
1280 return true;
1281 else if (discardedSignal == -1 && errno == EAGAIN)
1282 return false;
1283 else
1284 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1285 discardedSignal, errno);
1286}
1287
1288void
1289BaseKvmCPU::setupCounters()
1290{
1291 DPRINTF(Kvm, "Attaching cycle counter...\n");
1292 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1293 PERF_COUNT_HW_CPU_CYCLES);
1294 cfgCycles.disabled(true)
1295 .pinned(true);
1296
1297 // Try to exclude the host. We set both exclude_hv and
1298 // exclude_host since different architectures use slightly
1299 // different APIs in the kernel.
1300 cfgCycles.exclude_hv(true)
1301 .exclude_host(true);
1302
1303 if (perfControlledByTimer) {
1304 // We need to configure the cycles counter to send overflows
1305 // since we are going to use it to trigger timer signals that
1306 // trap back into m5 from KVM. In practice, this means that we
1307 // need to set some non-zero sample period that gets
1308 // overridden when the timer is armed.
1309 cfgCycles.wakeupEvents(1)
1310 .samplePeriod(42);
1311 }
1312
1313 hwCycles.attach(cfgCycles,
1314 0); // TID (0 => currentThread)
1315
1316 setupInstCounter();
1317}
1318
1319bool
1320BaseKvmCPU::tryDrain()
1321{
1322 if (drainState() != DrainState::Draining)
1323 return false;
1324
1325 if (!archIsDrained()) {
1326 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1327 return false;
1328 }
1329
1330 if (_status == Idle || _status == Running) {
1331 DPRINTF(Drain,
1332 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1333 signalDrainDone();
1334 return true;
1335 } else {
1336 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1337 return false;
1338 }
1339}
1340
1341void
1342BaseKvmCPU::ioctlRun()
1343{
1344 if (ioctl(KVM_RUN) == -1) {
1345 if (errno != EINTR)
1346 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1347 errno);
1348 }
1349}
1350
1351void
1352BaseKvmCPU::setupInstStop()
1353{
1354 if (comInstEventQueue[0]->empty()) {
1355 setupInstCounter(0);
1356 } else {
1357 const uint64_t next(comInstEventQueue[0]->nextTick());
1358
1359 assert(next > ctrInsts);
1360 setupInstCounter(next - ctrInsts);
1361 }
1362}
1363
1364void
1365BaseKvmCPU::setupInstCounter(uint64_t period)
1366{
1367 // No need to do anything if we aren't attaching for the first
1368 // time or the period isn't changing.
1369 if (period == activeInstPeriod && hwInstructions.attached())
1370 return;
1371
1372 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1373 PERF_COUNT_HW_INSTRUCTIONS);
1374
1375 // Try to exclude the host. We set both exclude_hv and
1376 // exclude_host since different architectures use slightly
1377 // different APIs in the kernel.
1378 cfgInstructions.exclude_hv(true)
1379 .exclude_host(true);
1380
1381 if (period) {
1382 // Setup a sampling counter if that has been requested.
1383 cfgInstructions.wakeupEvents(1)
1384 .samplePeriod(period);
1385 }
1386
1387 // We need to detach and re-attach the counter to reliably change
1388 // sampling settings. See PerfKvmCounter::period() for details.
1389 if (hwInstructions.attached())
1390 hwInstructions.detach();
1391 assert(hwCycles.attached());
1392 hwInstructions.attach(cfgInstructions,
1393 0, // TID (0 => currentThread)
1394 hwCycles);
1395
1396 if (period)
1397 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1398
1399 activeInstPeriod = period;
1400}
| 1146 delete pkt;
1147 return clockPeriod() * ipr_delay;
1148 } else {
1149 // Temporarily lock and migrate to the device event queue to
1150 // prevent races in multi-core mode.
1151 EventQueue::ScopedMigration migrate(deviceEventQueue());
1152
1153 return dataPort.submitIO(pkt);
1154 }
1155}
1156
1157void
1158BaseKvmCPU::setSignalMask(const sigset_t *mask)
1159{
1160 std::unique_ptr<struct kvm_signal_mask> kvm_mask;
1161
1162 if (mask) {
1163 kvm_mask.reset((struct kvm_signal_mask *)operator new(
1164 sizeof(struct kvm_signal_mask) + sizeof(*mask)));
1165 // The kernel and the user-space headers have different ideas
1166 // about the size of sigset_t. This seems like a massive hack,
1167 // but is actually what qemu does.
1168 assert(sizeof(*mask) >= 8);
1169 kvm_mask->len = 8;
1170 memcpy(kvm_mask->sigset, mask, kvm_mask->len);
1171 }
1172
1173 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1)
1174 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1175 errno);
1176}
1177
1178int
1179BaseKvmCPU::ioctl(int request, long p1) const
1180{
1181 if (vcpuFD == -1)
1182 panic("KVM: CPU ioctl called before initialization\n");
1183
1184 return ::ioctl(vcpuFD, request, p1);
1185}
1186
1187Tick
1188BaseKvmCPU::flushCoalescedMMIO()
1189{
1190 if (!mmioRing)
1191 return 0;
1192
1193 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n");
1194
1195 // TODO: We might need to do synchronization when we start to
1196 // support multiple CPUs
1197 Tick ticks(0);
1198 while (mmioRing->first != mmioRing->last) {
1199 struct kvm_coalesced_mmio &ent(
1200 mmioRing->coalesced_mmio[mmioRing->first]);
1201
1202 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1203 ent.phys_addr, ent.len);
1204
1205 ++numCoalescedMMIO;
1206 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true);
1207
1208 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX;
1209 }
1210
1211 return ticks;
1212}
1213
1214/**
1215 * Dummy handler for KVM kick signals.
1216 *
1217 * @note This function is usually not called since the kernel doesn't
1218 * seem to deliver signals when the signal is only unmasked when
1219 * running in KVM. This doesn't matter though since we are only
1220 * interested in getting KVM to exit, which happens as expected. See
1221 * setupSignalHandler() and kvmRun() for details about KVM signal
1222 * handling.
1223 */
1224static void
1225onKickSignal(int signo, siginfo_t *si, void *data)
1226{
1227}
1228
1229void
1230BaseKvmCPU::setupSignalHandler()
1231{
1232 struct sigaction sa;
1233
1234 memset(&sa, 0, sizeof(sa));
1235 sa.sa_sigaction = onKickSignal;
1236 sa.sa_flags = SA_SIGINFO | SA_RESTART;
1237 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1)
1238 panic("KVM: Failed to setup vCPU timer signal handler\n");
1239
1240 sigset_t sigset;
1241 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
1242 panic("KVM: Failed get signal mask\n");
1243
1244 // Request KVM to setup the same signal mask as we're currently
1245 // running with except for the KVM control signal. We'll sometimes
1246 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1247 // KVM after servicing IO requests. See kvmRun().
1248 sigdelset(&sigset, KVM_KICK_SIGNAL);
1249 setSignalMask(&sigset);
1250
1251 // Mask our control signals so they aren't delivered unless we're
1252 // actually executing inside KVM.
1253 sigaddset(&sigset, KVM_KICK_SIGNAL);
1254 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
1255 panic("KVM: Failed mask the KVM control signals\n");
1256}
1257
1258bool
1259BaseKvmCPU::discardPendingSignal(int signum) const
1260{
1261 int discardedSignal;
1262
1263 // Setting the timeout to zero causes sigtimedwait to return
1264 // immediately.
1265 struct timespec timeout;
1266 timeout.tv_sec = 0;
1267 timeout.tv_nsec = 0;
1268
1269 sigset_t sigset;
1270 sigemptyset(&sigset);
1271 sigaddset(&sigset, signum);
1272
1273 do {
1274 discardedSignal = sigtimedwait(&sigset, NULL, &timeout);
1275 } while (discardedSignal == -1 && errno == EINTR);
1276
1277 if (discardedSignal == signum)
1278 return true;
1279 else if (discardedSignal == -1 && errno == EAGAIN)
1280 return false;
1281 else
1282 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1283 discardedSignal, errno);
1284}
1285
1286void
1287BaseKvmCPU::setupCounters()
1288{
1289 DPRINTF(Kvm, "Attaching cycle counter...\n");
1290 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE,
1291 PERF_COUNT_HW_CPU_CYCLES);
1292 cfgCycles.disabled(true)
1293 .pinned(true);
1294
1295 // Try to exclude the host. We set both exclude_hv and
1296 // exclude_host since different architectures use slightly
1297 // different APIs in the kernel.
1298 cfgCycles.exclude_hv(true)
1299 .exclude_host(true);
1300
1301 if (perfControlledByTimer) {
1302 // We need to configure the cycles counter to send overflows
1303 // since we are going to use it to trigger timer signals that
1304 // trap back into m5 from KVM. In practice, this means that we
1305 // need to set some non-zero sample period that gets
1306 // overridden when the timer is armed.
1307 cfgCycles.wakeupEvents(1)
1308 .samplePeriod(42);
1309 }
1310
1311 hwCycles.attach(cfgCycles,
1312 0); // TID (0 => currentThread)
1313
1314 setupInstCounter();
1315}
1316
1317bool
1318BaseKvmCPU::tryDrain()
1319{
1320 if (drainState() != DrainState::Draining)
1321 return false;
1322
1323 if (!archIsDrained()) {
1324 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n");
1325 return false;
1326 }
1327
1328 if (_status == Idle || _status == Running) {
1329 DPRINTF(Drain,
1330 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1331 signalDrainDone();
1332 return true;
1333 } else {
1334 DPRINTF(Drain, "tryDrain: CPU not ready.\n");
1335 return false;
1336 }
1337}
1338
1339void
1340BaseKvmCPU::ioctlRun()
1341{
1342 if (ioctl(KVM_RUN) == -1) {
1343 if (errno != EINTR)
1344 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1345 errno);
1346 }
1347}
1348
1349void
1350BaseKvmCPU::setupInstStop()
1351{
1352 if (comInstEventQueue[0]->empty()) {
1353 setupInstCounter(0);
1354 } else {
1355 const uint64_t next(comInstEventQueue[0]->nextTick());
1356
1357 assert(next > ctrInsts);
1358 setupInstCounter(next - ctrInsts);
1359 }
1360}
1361
1362void
1363BaseKvmCPU::setupInstCounter(uint64_t period)
1364{
1365 // No need to do anything if we aren't attaching for the first
1366 // time or the period isn't changing.
1367 if (period == activeInstPeriod && hwInstructions.attached())
1368 return;
1369
1370 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE,
1371 PERF_COUNT_HW_INSTRUCTIONS);
1372
1373 // Try to exclude the host. We set both exclude_hv and
1374 // exclude_host since different architectures use slightly
1375 // different APIs in the kernel.
1376 cfgInstructions.exclude_hv(true)
1377 .exclude_host(true);
1378
1379 if (period) {
1380 // Setup a sampling counter if that has been requested.
1381 cfgInstructions.wakeupEvents(1)
1382 .samplePeriod(period);
1383 }
1384
1385 // We need to detach and re-attach the counter to reliably change
1386 // sampling settings. See PerfKvmCounter::period() for details.
1387 if (hwInstructions.attached())
1388 hwInstructions.detach();
1389 assert(hwCycles.attached());
1390 hwInstructions.attach(cfgInstructions,
1391 0, // TID (0 => currentThread)
1392 hwCycles);
1393
1394 if (period)
1395 hwInstructions.enableSignals(KVM_KICK_SIGNAL);
1396
1397 activeInstPeriod = period;
1398}
|