1/*
2 * Copyright 2014 Google, Inc.
3 * Copyright (c) 2012, 2015 ARM Limited
4 * All rights reserved
5 *
6 * The license below extends only to copyright in the software and shall
7 * not be construed as granting a license to any other intellectual
8 * property including but not limited to intellectual property relating
9 * to a hardware implementation of the functionality of the software
10 * licensed hereunder.  You may use the software subject to the license
11 * terms below provided that you ensure that this notice is replicated
12 * unmodified and in its entirety in all distributions of the software,
13 * modified or unmodified, in source code or in binary form.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions are
17 * met: redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer;
19 * redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution;
22 * neither the name of the copyright holders nor the names of its
23 * contributors may be used to endorse or promote products derived from
24 * this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 *
38 * Authors: Andreas Sandberg
39 */
40
41#include "cpu/kvm/vm.hh"
42
43#include <fcntl.h>
44#include <linux/kvm.h>
45#include <sys/ioctl.h>
46#include <sys/stat.h>
47#include <sys/types.h>
48#include <unistd.h>
49
50#include <cerrno>
51#include <memory>
52
53#include "cpu/kvm/base.hh"
54#include "debug/Kvm.hh"
55#include "params/KvmVM.hh"
56#include "sim/system.hh"
57
58#define EXPECTED_KVM_API_VERSION 12
59
60#if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
61#error Unsupported KVM version
62#endif
63
64Kvm *Kvm::instance = NULL;
65
66Kvm::Kvm()
67    : kvmFD(-1), apiVersion(-1), vcpuMMapSize(0)
68{
69    kvmFD = ::open("/dev/kvm", O_RDWR);
70    if (kvmFD == -1)
71        fatal("KVM: Failed to open /dev/kvm\n");
72
73    apiVersion = ioctl(KVM_GET_API_VERSION);
74    if (apiVersion != EXPECTED_KVM_API_VERSION)
75        fatal("KVM: Incompatible API version\n");
76
77    vcpuMMapSize = ioctl(KVM_GET_VCPU_MMAP_SIZE);
78    if (vcpuMMapSize == -1)
79        panic("KVM: Failed to get virtual CPU MMAP size\n");
80}
81
82Kvm::~Kvm()
83{
84    close(kvmFD);
85}
86
87Kvm *
88Kvm::create()
89{
90    if (!instance)
91        instance = new Kvm();
92
93    return instance;
94}
95
96bool
97Kvm::capUserMemory() const
98{
99    return checkExtension(KVM_CAP_USER_MEMORY) != 0;
100}
101
102bool
103Kvm::capSetTSSAddress() const
104{
105    return checkExtension(KVM_CAP_SET_TSS_ADDR) != 0;
106}
107
108bool
109Kvm::capExtendedCPUID() const
110{
111    return checkExtension(KVM_CAP_EXT_CPUID) != 0;
112}
113
114bool
115Kvm::capUserNMI() const
116{
117#ifdef KVM_CAP_USER_NMI
118    return checkExtension(KVM_CAP_USER_NMI) != 0;
119#else
120    return false;
121#endif
122}
123
124int
125Kvm::capCoalescedMMIO() const
126{
127    return checkExtension(KVM_CAP_COALESCED_MMIO);
128}
129
130int
131Kvm::capNumMemSlots() const
132{
133#ifdef KVM_CAP_NR_MEMSLOTS
134    return checkExtension(KVM_CAP_NR_MEMSLOTS);
135#else
136    return 0;
137#endif
138}
139
140bool
141Kvm::capOneReg() const
142{
143#ifdef KVM_CAP_ONE_REG
144    return checkExtension(KVM_CAP_ONE_REG) != 0;
145#else
146    return false;
147#endif
148}
149
150bool
151Kvm::capIRQChip() const
152{
153    return checkExtension(KVM_CAP_IRQCHIP) != 0;
154}
155
156bool
157Kvm::capVCPUEvents() const
158{
159#ifdef KVM_CAP_VCPU_EVENTS
160    return checkExtension(KVM_CAP_VCPU_EVENTS) != 0;
161#else
162    return false;
163#endif
164}
165
166bool
167Kvm::capDebugRegs() const
168{
169#ifdef KVM_CAP_DEBUGREGS
170    return checkExtension(KVM_CAP_DEBUGREGS) != 0;
171#else
172    return false;
173#endif
174}
175
176bool
177Kvm::capXCRs() const
178{
179#ifdef KVM_CAP_XCRS
180    return checkExtension(KVM_CAP_XCRS) != 0;
181#else
182    return false;
183#endif
184}
185
186bool
187Kvm::capXSave() const
188{
189#ifdef KVM_CAP_XSAVE
190    return checkExtension(KVM_CAP_XSAVE) != 0;
191#else
192    return false;
193#endif
194}
195
196
197#if defined(__i386__) || defined(__x86_64__)
198bool
199Kvm::getSupportedCPUID(struct kvm_cpuid2 &cpuid) const
200{
201    if (ioctl(KVM_GET_SUPPORTED_CPUID, (void *)&cpuid) == -1) {
202        if (errno == E2BIG)
203            return false;
204        else
205            panic("KVM: Failed to get supported CPUID (errno: %i)\n", errno);
206    } else
207        return true;
208}
209
210const Kvm::CPUIDVector &
211Kvm::getSupportedCPUID() const
212{
213    if (supportedCPUIDCache.empty()) {
214        std::unique_ptr<struct kvm_cpuid2> cpuid;
215        int i(1);
216        do {
217            cpuid.reset((struct kvm_cpuid2 *)operator new(
218                            sizeof(kvm_cpuid2) + i * sizeof(kvm_cpuid_entry2)));
219
220            cpuid->nent = i;
221            ++i;
222        } while (!getSupportedCPUID(*cpuid));
223        supportedCPUIDCache.assign(cpuid->entries,
224                                   cpuid->entries + cpuid->nent);
225    }
226
227    return supportedCPUIDCache;
228}
229
230bool
231Kvm::getSupportedMSRs(struct kvm_msr_list &msrs) const
232{
233    if (ioctl(KVM_GET_MSR_INDEX_LIST, (void *)&msrs) == -1) {
234        if (errno == E2BIG)
235            return false;
236        else
237            panic("KVM: Failed to get supported CPUID (errno: %i)\n", errno);
238    } else
239        return true;
240}
241
242const Kvm::MSRIndexVector &
243Kvm::getSupportedMSRs() const
244{
245    if (supportedMSRCache.empty()) {
246        std::unique_ptr<struct kvm_msr_list> msrs;
247        int i(0);
248        do {
249            msrs.reset((struct kvm_msr_list *)operator new(
250                           sizeof(kvm_msr_list) + i * sizeof(uint32_t)));
251
252            msrs->nmsrs = i;
253            ++i;
254        } while (!getSupportedMSRs(*msrs));
255        supportedMSRCache.assign(msrs->indices, msrs->indices + msrs->nmsrs);
256    }
257
258    return supportedMSRCache;
259}
260
261#endif // x86-specific
262
263
264int
265Kvm::checkExtension(int extension) const
266{
267    int ret = ioctl(KVM_CHECK_EXTENSION, extension);
268    if (ret == -1)
269        panic("KVM: ioctl failed when checking for extension\n");
270    return ret;
271}
272
273int
274Kvm::ioctl(int request, long p1) const
275{
276    assert(kvmFD != -1);
277
278    return ::ioctl(kvmFD, request, p1);
279}
280
281int
282Kvm::createVM()
283{
284    int vmFD;
285
286    vmFD = ioctl(KVM_CREATE_VM);
287    if (vmFD == -1)
288        panic("Failed to create KVM VM\n");
289
290    return vmFD;
291}
292
293
294KvmVM::KvmVM(KvmVMParams *params)
295    : SimObject(params),
296      kvm(new Kvm()), system(nullptr),
297      vmFD(kvm->createVM()),
298      started(false),
299      nextVCPUID(0)
300{
301    maxMemorySlot = kvm->capNumMemSlots();
302    /* If we couldn't determine how memory slots there are, guess 32. */
303    if (!maxMemorySlot)
304        maxMemorySlot = 32;
305    /* Setup the coalesced MMIO regions */
306    for (int i = 0; i < params->coalescedMMIO.size(); ++i)
307        coalesceMMIO(params->coalescedMMIO[i]);
308}
309
310KvmVM::~KvmVM()
311{
312    if (vmFD != -1)
313        close(vmFD);
314
315    if (kvm)
316        delete kvm;
317}
318
319void
320KvmVM::notifyFork()
321{
322    if (vmFD != -1) {
323        if (close(vmFD) == -1)
324            warn("kvm VM: notifyFork failed to close vmFD\n");
325
326        vmFD = -1;
327
328        delete kvm;
329        kvm = NULL;
330    }
331}
332
333void
334KvmVM::cpuStartup()
335{
336    if (started)
337        return;
338    started = true;
339
340    delayedStartup();
341}
342
343void
344KvmVM::delayedStartup()
345{
346    assert(system); // set by the system during its construction
347    const std::vector<BackingStoreEntry> &memories(
348        system->getPhysMem().getBackingStore());
349
350    DPRINTF(Kvm, "Mapping %i memory region(s)\n", memories.size());
351    for (int slot(0); slot < memories.size(); ++slot) {
352        if (!memories[slot].kvmMap) {
353            DPRINTF(Kvm, "Skipping region marked as not usable by KVM\n");
354            continue;
355        }
356
357        const AddrRange &range(memories[slot].range);
358        void *pmem(memories[slot].pmem);
359
360        if (pmem) {
361            DPRINTF(Kvm, "Mapping region: 0x%p -> 0x%llx [size: 0x%llx]\n",
362                    pmem, range.start(), range.size());
363
364            if (range.interleaved()) {
365                panic("Tried to map an interleaved memory range into "
366                      "a KVM VM.\n");
367            }
368
369            const MemSlot slot = allocMemSlot(range.size());
370            setupMemSlot(slot, pmem, range.start(), 0/* flags */);
371        } else {
372            DPRINTF(Kvm, "Zero-region not mapped: [0x%llx]\n", range.start());
373            hack("KVM: Zero memory handled as IO\n");
374        }
375    }
376}
377
378const KvmVM::MemSlot
379KvmVM::allocMemSlot(uint64_t size)
380{
381    if (!size)
382        panic("Memory slots must have non-zero size.\n");
383
384    std::vector<MemorySlot>::iterator pos;
385    for (pos = memorySlots.begin(); pos != memorySlots.end(); pos++) {
386        if (!pos->size) {
387            pos->size = size;
388            pos->active = false;
389            return pos->slot;
390        }
391    }
392
393    uint32_t nextSlot = memorySlots.size();
394    if (nextSlot > maxMemorySlot)
395        panic("Out of memory slots.\n");
396
397    MemorySlot slot;
398    slot.size = size;
399    slot.slot = nextSlot;
400    slot.active = false;
401
402    memorySlots.push_back(slot);
403    return MemSlot(slot.slot);
404}
405
406void
407KvmVM::setupMemSlot(const KvmVM::MemSlot num, void *host_addr, Addr guest,
408                    uint32_t flags)
409{
410    MemorySlot &slot = memorySlots.at(num.num);
411    slot.active = true;
412    setUserMemoryRegion(num.num, host_addr, guest, slot.size, flags);
413}
414
415void
416KvmVM::disableMemSlot(const KvmVM::MemSlot num)
417{
418    MemorySlot &slot = memorySlots.at(num.num);
419    if (slot.active)
420        setUserMemoryRegion(num.num, NULL, 0, 0, 0);
421    slot.active = false;
422}
423
424void
425KvmVM::freeMemSlot(const KvmVM::MemSlot num)
426{
427    disableMemSlot(num.num);
428    MemorySlot &slot = memorySlots.at(num.num);
429    slot.size = 0;
430}
431
432void
433KvmVM::setUserMemoryRegion(uint32_t slot,
434                           void *host_addr, Addr guest_addr,
435                           uint64_t len, uint32_t flags)
436{
437    struct kvm_userspace_memory_region m;
438
439    memset(&m, 0, sizeof(m));
440    m.slot = slot;
441    m.flags = flags;
442    m.guest_phys_addr = (uint64_t)guest_addr;
443    m.memory_size = len;
444    m.userspace_addr = (__u64)host_addr;
445
446    if (ioctl(KVM_SET_USER_MEMORY_REGION, (void *)&m) == -1) {
447        panic("Failed to setup KVM memory region:\n"
448              "\tHost Address: 0x%p\n"
449              "\tGuest Address: 0x%llx\n",
450              "\tSize: %ll\n",
451              "\tFlags: 0x%x\n",
452              m.userspace_addr, m.guest_phys_addr,
453              m.memory_size, m.flags);
454    }
455}
456
457void
458KvmVM::coalesceMMIO(const AddrRange &range)
459{
460    coalesceMMIO(range.start(), range.size());
461}
462
463void
464KvmVM::coalesceMMIO(Addr start, int size)
465{
466    struct kvm_coalesced_mmio_zone zone;
467
468    zone.addr = start;
469    zone.size = size;
470    zone.pad = 0;
471
472    DPRINTF(Kvm, "KVM: Registering coalesced MMIO region [0x%x, 0x%x]\n",
473            zone.addr, zone.addr + zone.size - 1);
474    if (ioctl(KVM_REGISTER_COALESCED_MMIO, (void *)&zone) == -1)
475        panic("KVM: Failed to register coalesced MMIO region (%i)\n",
476              errno);
477}
478
479void
480KvmVM::setTSSAddress(Addr tss_address)
481{
482    if (ioctl(KVM_SET_TSS_ADDR, (unsigned long)tss_address) == -1)
483        panic("KVM: Failed to set VM TSS address\n");
484}
485
486void
487KvmVM::createIRQChip()
488{
489    if (_hasKernelIRQChip)
490        panic("KvmVM::createIRQChip called twice.\n");
491
492    if (ioctl(KVM_CREATE_IRQCHIP) != -1) {
493        _hasKernelIRQChip = true;
494    } else {
495        warn("KVM: Failed to create in-kernel IRQ chip (errno: %i)\n",
496             errno);
497        _hasKernelIRQChip = false;
498    }
499}
500
501void
502KvmVM::setIRQLine(uint32_t irq, bool high)
503{
504    struct kvm_irq_level kvm_level;
505
506    kvm_level.irq = irq;
507    kvm_level.level = high ? 1 : 0;
508
509    if (ioctl(KVM_IRQ_LINE, &kvm_level) == -1)
510        panic("KVM: Failed to set IRQ line level (errno: %i)\n",
511              errno);
512}
513
514int
515KvmVM::createDevice(uint32_t type, uint32_t flags)
516{
517#if defined(KVM_CREATE_DEVICE)
518    struct kvm_create_device dev = { type, 0, flags };
519
520    if (ioctl(KVM_CREATE_DEVICE, &dev) == -1) {
521        panic("KVM: Failed to create device (errno: %i)\n",
522              errno);
523    }
524
525    return dev.fd;
526#else
527    panic("Kernel headers don't support KVM_CREATE_DEVICE\n");
528#endif
529}
530
531void
532KvmVM::setSystem(System *s)
533{
534    panic_if(system != nullptr, "setSystem() can only be called once");
535    panic_if(s == nullptr, "setSystem() called with null System*");
536    system = s;
537}
538
539long
540KvmVM::contextIdToVCpuId(ContextID ctx) const
541{
542    assert(system != nullptr);
543    return dynamic_cast<BaseKvmCPU*>
544        (system->getThreadContext(ctx)->getCpuPtr())->getVCpuID();
545}
546
547int
548KvmVM::createVCPU(long vcpuID)
549{
550    int fd;
551
552    fd = ioctl(KVM_CREATE_VCPU, vcpuID);
553    if (fd == -1)
554        panic("KVM: Failed to create virtual CPU");
555
556    return fd;
557}
558
559long
560KvmVM::allocVCPUID()
561{
562    return nextVCPUID++;
563}
564
565#if defined(__aarch64__)
566void
567KvmVM::kvmArmPreferredTarget(struct kvm_vcpu_init &target) const
568{
569    if (ioctl(KVM_ARM_PREFERRED_TARGET, &target) == -1) {
570        panic("KVM: Failed to get ARM preferred CPU target (errno: %i)\n",
571              errno);
572    }
573}
574#endif
575
576int
577KvmVM::ioctl(int request, long p1) const
578{
579    assert(vmFD != -1);
580
581    return ::ioctl(vmFD, request, p1);
582}
583
584
585KvmVM *
586KvmVMParams::create()
587{
588    static bool created = false;
589    if (created)
590        warn_once("Use of multiple KvmVMs is currently untested!\n");
591
592    created = true;
593
594    return new KvmVM(this);
595}
596