base.cc revision 11793:ef606668d247
15361Srstrong@cs.ucsd.edu/* 23671Sbinkertn@umich.edu * Copyright (c) 2012, 2015 ARM Limited 33671Sbinkertn@umich.edu * All rights reserved 43671Sbinkertn@umich.edu * 53671Sbinkertn@umich.edu * The license below extends only to copyright in the software and shall 63671Sbinkertn@umich.edu * not be construed as granting a license to any other intellectual 73671Sbinkertn@umich.edu * property including but not limited to intellectual property relating 83671Sbinkertn@umich.edu * to a hardware implementation of the functionality of the software 93671Sbinkertn@umich.edu * licensed hereunder. You may use the software subject to the license 103671Sbinkertn@umich.edu * terms below provided that you ensure that this notice is replicated 113671Sbinkertn@umich.edu * unmodified and in its entirety in all distributions of the software, 123671Sbinkertn@umich.edu * modified or unmodified, in source code or in binary form. 133671Sbinkertn@umich.edu * 143671Sbinkertn@umich.edu * Redistribution and use in source and binary forms, with or without 153671Sbinkertn@umich.edu * modification, are permitted provided that the following conditions are 163671Sbinkertn@umich.edu * met: redistributions of source code must retain the above copyright 173671Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer; 183671Sbinkertn@umich.edu * redistributions in binary form must reproduce the above copyright 193671Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer in the 203671Sbinkertn@umich.edu * documentation and/or other materials provided with the distribution; 213671Sbinkertn@umich.edu * neither the name of the copyright holders nor the names of its 223671Sbinkertn@umich.edu * contributors may be used to endorse or promote products derived from 233671Sbinkertn@umich.edu * this software without specific prior written permission. 243671Sbinkertn@umich.edu * 253671Sbinkertn@umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 263671Sbinkertn@umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 273671Sbinkertn@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 283671Sbinkertn@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 293671Sbinkertn@umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 303671Sbinkertn@umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 313671Sbinkertn@umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 323671Sbinkertn@umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 333671Sbinkertn@umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 346028Ssteve.reinhardt@amd.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 353671Sbinkertn@umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 363671Sbinkertn@umich.edu * 373671Sbinkertn@umich.edu * Authors: Andreas Sandberg 383671Sbinkertn@umich.edu */ 393671Sbinkertn@umich.edu 403671Sbinkertn@umich.edu#include "cpu/kvm/base.hh" 413671Sbinkertn@umich.edu 423671Sbinkertn@umich.edu#include <linux/kvm.h> 433671Sbinkertn@umich.edu#include <sys/ioctl.h> 443671Sbinkertn@umich.edu#include <sys/mman.h> 453671Sbinkertn@umich.edu#include <unistd.h> 463671Sbinkertn@umich.edu 473671Sbinkertn@umich.edu#include <cerrno> 483671Sbinkertn@umich.edu#include <csignal> 493671Sbinkertn@umich.edu#include <ostream> 503671Sbinkertn@umich.edu 513671Sbinkertn@umich.edu#include "arch/mmapped_ipr.hh" 523671Sbinkertn@umich.edu#include "arch/utility.hh" 533671Sbinkertn@umich.edu#include "debug/Checkpoint.hh" 543671Sbinkertn@umich.edu#include "debug/Drain.hh" 553671Sbinkertn@umich.edu#include "debug/Kvm.hh" 563671Sbinkertn@umich.edu#include "debug/KvmIO.hh" 573671Sbinkertn@umich.edu#include "debug/KvmRun.hh" 583671Sbinkertn@umich.edu#include "params/BaseKvmCPU.hh" 593671Sbinkertn@umich.edu#include "sim/process.hh" 603671Sbinkertn@umich.edu#include "sim/system.hh" 613671Sbinkertn@umich.edu 623671Sbinkertn@umich.edu/* Used by some KVM macros */ 633671Sbinkertn@umich.edu#define PAGE_SIZE pageSize 643671Sbinkertn@umich.edu 653671Sbinkertn@umich.eduBaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params) 663671Sbinkertn@umich.edu : BaseCPU(params), 673671Sbinkertn@umich.edu vm(*params->kvmVM), 683671Sbinkertn@umich.edu _status(Idle), 693671Sbinkertn@umich.edu dataPort(name() + ".dcache_port", this), 703671Sbinkertn@umich.edu instPort(name() + ".icache_port", this), 713671Sbinkertn@umich.edu alwaysSyncTC(params->alwaysSyncTC), 723671Sbinkertn@umich.edu threadContextDirty(true), 733671Sbinkertn@umich.edu kvmStateDirty(false), 743671Sbinkertn@umich.edu vcpuID(vm.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0), 753671Sbinkertn@umich.edu _kvmRun(NULL), mmioRing(NULL), 763671Sbinkertn@umich.edu pageSize(sysconf(_SC_PAGE_SIZE)), 773671Sbinkertn@umich.edu tickEvent(*this), 783671Sbinkertn@umich.edu activeInstPeriod(0), 793671Sbinkertn@umich.edu perfControlledByTimer(params->usePerfOverflow), 803671Sbinkertn@umich.edu hostFactor(params->hostFactor), 813671Sbinkertn@umich.edu ctrInsts(0) 823671Sbinkertn@umich.edu{ 833671Sbinkertn@umich.edu if (pageSize == -1) 843671Sbinkertn@umich.edu panic("KVM: Failed to determine host page size (%i)\n", 853671Sbinkertn@umich.edu errno); 863671Sbinkertn@umich.edu 873671Sbinkertn@umich.edu if (FullSystem) 885361Srstrong@cs.ucsd.edu thread = new SimpleThread(this, 0, params->system, params->itb, params->dtb, 895361Srstrong@cs.ucsd.edu params->isa[0]); 905361Srstrong@cs.ucsd.edu else 913671Sbinkertn@umich.edu thread = new SimpleThread(this, /* thread_num */ 0, params->system, 923671Sbinkertn@umich.edu params->workload[0], params->itb, 933671Sbinkertn@umich.edu params->dtb, params->isa[0]); 943671Sbinkertn@umich.edu 953671Sbinkertn@umich.edu thread->setStatus(ThreadContext::Halted); 963671Sbinkertn@umich.edu tc = thread->getTC(); 973671Sbinkertn@umich.edu threadContexts.push_back(tc); 983671Sbinkertn@umich.edu} 993671Sbinkertn@umich.edu 1003671Sbinkertn@umich.eduBaseKvmCPU::~BaseKvmCPU() 1013671Sbinkertn@umich.edu{ 1023671Sbinkertn@umich.edu if (_kvmRun) 1033671Sbinkertn@umich.edu munmap(_kvmRun, vcpuMMapSize); 1043671Sbinkertn@umich.edu close(vcpuFD); 1053671Sbinkertn@umich.edu} 1063671Sbinkertn@umich.edu 1073671Sbinkertn@umich.eduvoid 1083671Sbinkertn@umich.eduBaseKvmCPU::init() 1093671Sbinkertn@umich.edu{ 1104116Sgblack@eecs.umich.edu BaseCPU::init(); 1114116Sgblack@eecs.umich.edu 1123671Sbinkertn@umich.edu if (numThreads != 1) 1133671Sbinkertn@umich.edu fatal("KVM: Multithreading not supported"); 1143671Sbinkertn@umich.edu 1153671Sbinkertn@umich.edu tc->initMemProxies(tc); 1163671Sbinkertn@umich.edu 1173671Sbinkertn@umich.edu // initialize CPU, including PC 1183671Sbinkertn@umich.edu if (FullSystem && !switchedOut()) 1193671Sbinkertn@umich.edu TheISA::initCPU(tc, tc->contextId()); 1203671Sbinkertn@umich.edu} 1213671Sbinkertn@umich.edu 1223671Sbinkertn@umich.eduvoid 1233671Sbinkertn@umich.eduBaseKvmCPU::startup() 1243671Sbinkertn@umich.edu{ 1253671Sbinkertn@umich.edu const BaseKvmCPUParams * const p( 1263671Sbinkertn@umich.edu dynamic_cast<const BaseKvmCPUParams *>(params())); 1273671Sbinkertn@umich.edu 1283671Sbinkertn@umich.edu Kvm &kvm(*vm.kvm); 1293671Sbinkertn@umich.edu 1303671Sbinkertn@umich.edu BaseCPU::startup(); 1313671Sbinkertn@umich.edu 1323671Sbinkertn@umich.edu assert(vcpuFD == -1); 1333671Sbinkertn@umich.edu 1343671Sbinkertn@umich.edu // Tell the VM that a CPU is about to start. 1353671Sbinkertn@umich.edu vm.cpuStartup(); 1363671Sbinkertn@umich.edu 1374555Sbinkertn@umich.edu // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are 1383671Sbinkertn@umich.edu // not guaranteed that the parent KVM VM has initialized at that 1393671Sbinkertn@umich.edu // point. Initialize virtual CPUs here instead. 1403671Sbinkertn@umich.edu vcpuFD = vm.createVCPU(vcpuID); 1413671Sbinkertn@umich.edu 1423671Sbinkertn@umich.edu // Map the KVM run structure */ 1435378Ssaidi@eecs.umich.edu vcpuMMapSize = kvm.getVCPUMMapSize(); 1445378Ssaidi@eecs.umich.edu _kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize, 1453671Sbinkertn@umich.edu PROT_READ | PROT_WRITE, MAP_SHARED, 1463671Sbinkertn@umich.edu vcpuFD, 0); 1473671Sbinkertn@umich.edu if (_kvmRun == MAP_FAILED) 1483671Sbinkertn@umich.edu panic("KVM: Failed to map run data structure\n"); 1493671Sbinkertn@umich.edu 1503671Sbinkertn@umich.edu // Setup a pointer to the MMIO ring buffer if coalesced MMIO is 1513671Sbinkertn@umich.edu // available. The offset into the KVM's communication page is 1523671Sbinkertn@umich.edu // provided by the coalesced MMIO capability. 1533671Sbinkertn@umich.edu int mmioOffset(kvm.capCoalescedMMIO()); 1543671Sbinkertn@umich.edu if (!p->useCoalescedMMIO) { 1553671Sbinkertn@umich.edu inform("KVM: Coalesced MMIO disabled by config.\n"); 1565361Srstrong@cs.ucsd.edu } else if (mmioOffset) { 1573671Sbinkertn@umich.edu inform("KVM: Coalesced IO available\n"); 1588246Snate@binkert.org mmioRing = (struct kvm_coalesced_mmio_ring *)( 1593671Sbinkertn@umich.edu (char *)_kvmRun + (mmioOffset * pageSize)); 1603671Sbinkertn@umich.edu } else { 1613671Sbinkertn@umich.edu inform("KVM: Coalesced not supported by host OS\n"); 1623671Sbinkertn@umich.edu } 1633671Sbinkertn@umich.edu 1643671Sbinkertn@umich.edu thread->startup(); 1653671Sbinkertn@umich.edu 1663671Sbinkertn@umich.edu Event *startupEvent( 1673671Sbinkertn@umich.edu new EventWrapper<BaseKvmCPU, 1683671Sbinkertn@umich.edu &BaseKvmCPU::startupThread>(this, true)); 1693671Sbinkertn@umich.edu schedule(startupEvent, curTick()); 1703671Sbinkertn@umich.edu} 1713671Sbinkertn@umich.edu 1723671Sbinkertn@umich.eduBaseKvmCPU::Status 1733671Sbinkertn@umich.eduBaseKvmCPU::KVMCpuPort::nextIOState() const 1743671Sbinkertn@umich.edu{ 1753671Sbinkertn@umich.edu return (activeMMIOReqs || pendingMMIOPkts.size()) 1763671Sbinkertn@umich.edu ? RunningMMIOPending : RunningServiceCompletion; 1773671Sbinkertn@umich.edu} 1783671Sbinkertn@umich.edu 1793671Sbinkertn@umich.eduTick 1803671Sbinkertn@umich.eduBaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt) 1813671Sbinkertn@umich.edu{ 1823671Sbinkertn@umich.edu if (cpu->system->isAtomicMode()) { 1833671Sbinkertn@umich.edu Tick delay = sendAtomic(pkt); 1843671Sbinkertn@umich.edu delete pkt->req; 1853671Sbinkertn@umich.edu delete pkt; 1863671Sbinkertn@umich.edu return delay; 1875361Srstrong@cs.ucsd.edu } else { 1883671Sbinkertn@umich.edu if (pendingMMIOPkts.empty() && sendTimingReq(pkt)) { 1893671Sbinkertn@umich.edu activeMMIOReqs++; 1903671Sbinkertn@umich.edu } else { 1913671Sbinkertn@umich.edu pendingMMIOPkts.push(pkt); 1923671Sbinkertn@umich.edu } 1935361Srstrong@cs.ucsd.edu // Return value is irrelevant for timing-mode accesses. 1943671Sbinkertn@umich.edu return 0; 1953671Sbinkertn@umich.edu } 1963671Sbinkertn@umich.edu} 1973671Sbinkertn@umich.edu 1983671Sbinkertn@umich.edubool 1995361Srstrong@cs.ucsd.eduBaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt) 2003671Sbinkertn@umich.edu{ 2013671Sbinkertn@umich.edu DPRINTF(KvmIO, "KVM: Finished timing request\n"); 2023671Sbinkertn@umich.edu 2033671Sbinkertn@umich.edu delete pkt->req; 2043671Sbinkertn@umich.edu delete pkt; 2053671Sbinkertn@umich.edu activeMMIOReqs--; 2063671Sbinkertn@umich.edu 2073671Sbinkertn@umich.edu // We can switch back into KVM when all pending and in-flight MMIO 2083671Sbinkertn@umich.edu // operations have completed. 2093671Sbinkertn@umich.edu if (!(activeMMIOReqs || pendingMMIOPkts.size())) { 2103671Sbinkertn@umich.edu DPRINTF(KvmIO, "KVM: Finished all outstanding timing requests\n"); 2113671Sbinkertn@umich.edu cpu->finishMMIOPending(); 2123671Sbinkertn@umich.edu } 2133671Sbinkertn@umich.edu return true; 2143671Sbinkertn@umich.edu} 2153671Sbinkertn@umich.edu 2163671Sbinkertn@umich.eduvoid 2173671Sbinkertn@umich.eduBaseKvmCPU::KVMCpuPort::recvReqRetry() 2183671Sbinkertn@umich.edu{ 2193671Sbinkertn@umich.edu DPRINTF(KvmIO, "KVM: Retry for timing request\n"); 2203671Sbinkertn@umich.edu 2213671Sbinkertn@umich.edu assert(pendingMMIOPkts.size()); 2223671Sbinkertn@umich.edu 2233671Sbinkertn@umich.edu // Assuming that we can issue infinite requests this cycle is a bit 2243671Sbinkertn@umich.edu // unrealistic, but it's not worth modeling something more complex in 2253671Sbinkertn@umich.edu // KVM. 2263671Sbinkertn@umich.edu while (pendingMMIOPkts.size() && sendTimingReq(pendingMMIOPkts.front())) { 2273671Sbinkertn@umich.edu pendingMMIOPkts.pop(); 2283671Sbinkertn@umich.edu activeMMIOReqs++; 2293671Sbinkertn@umich.edu } 2303671Sbinkertn@umich.edu} 2313671Sbinkertn@umich.edu 2323671Sbinkertn@umich.eduvoid 2333671Sbinkertn@umich.eduBaseKvmCPU::finishMMIOPending() 2343671Sbinkertn@umich.edu{ 2353671Sbinkertn@umich.edu assert(_status = RunningMMIOPending); 2363671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 2373671Sbinkertn@umich.edu 2383671Sbinkertn@umich.edu _status = RunningServiceCompletion; 2393671Sbinkertn@umich.edu schedule(tickEvent, nextCycle()); 2403671Sbinkertn@umich.edu} 2413671Sbinkertn@umich.edu 2423671Sbinkertn@umich.eduvoid 2433671Sbinkertn@umich.eduBaseKvmCPU::startupThread() 2443671Sbinkertn@umich.edu{ 2453671Sbinkertn@umich.edu // Do thread-specific initialization. We need to setup signal 2463671Sbinkertn@umich.edu // delivery for counters and timers from within the thread that 2473671Sbinkertn@umich.edu // will execute the event queue to ensure that signals are 2483671Sbinkertn@umich.edu // delivered to the right threads. 2493671Sbinkertn@umich.edu const BaseKvmCPUParams * const p( 2503671Sbinkertn@umich.edu dynamic_cast<const BaseKvmCPUParams *>(params())); 2513671Sbinkertn@umich.edu 2525361Srstrong@cs.ucsd.edu vcpuThread = pthread_self(); 2533671Sbinkertn@umich.edu 2543671Sbinkertn@umich.edu // Setup signal handlers. This has to be done after the vCPU is 2553671Sbinkertn@umich.edu // created since it manipulates the vCPU signal mask. 2563671Sbinkertn@umich.edu setupSignalHandler(); 2573671Sbinkertn@umich.edu 2583671Sbinkertn@umich.edu setupCounters(); 2593671Sbinkertn@umich.edu 2603671Sbinkertn@umich.edu if (p->usePerfOverflow) 2613671Sbinkertn@umich.edu runTimer.reset(new PerfKvmTimer(hwCycles, 2623671Sbinkertn@umich.edu KVM_KICK_SIGNAL, 2633671Sbinkertn@umich.edu p->hostFactor, 2643671Sbinkertn@umich.edu p->hostFreq)); 2653671Sbinkertn@umich.edu else 2665361Srstrong@cs.ucsd.edu runTimer.reset(new PosixKvmTimer(KVM_KICK_SIGNAL, CLOCK_MONOTONIC, 2673671Sbinkertn@umich.edu p->hostFactor, 2683671Sbinkertn@umich.edu p->hostFreq)); 2693671Sbinkertn@umich.edu 2703671Sbinkertn@umich.edu} 2713671Sbinkertn@umich.edu 2725361Srstrong@cs.ucsd.eduvoid 2733671Sbinkertn@umich.eduBaseKvmCPU::regStats() 2743671Sbinkertn@umich.edu{ 2753671Sbinkertn@umich.edu using namespace Stats; 2763671Sbinkertn@umich.edu 2773671Sbinkertn@umich.edu BaseCPU::regStats(); 2783671Sbinkertn@umich.edu 2793671Sbinkertn@umich.edu numInsts 2805361Srstrong@cs.ucsd.edu .name(name() + ".committedInsts") 2813671Sbinkertn@umich.edu .desc("Number of instructions committed") 2823671Sbinkertn@umich.edu ; 2833671Sbinkertn@umich.edu 2843671Sbinkertn@umich.edu numVMExits 2853671Sbinkertn@umich.edu .name(name() + ".numVMExits") 2865361Srstrong@cs.ucsd.edu .desc("total number of KVM exits") 2873671Sbinkertn@umich.edu ; 2883671Sbinkertn@umich.edu 2893671Sbinkertn@umich.edu numVMHalfEntries 2903671Sbinkertn@umich.edu .name(name() + ".numVMHalfEntries") 2913671Sbinkertn@umich.edu .desc("number of KVM entries to finalize pending operations") 2925361Srstrong@cs.ucsd.edu ; 2933671Sbinkertn@umich.edu 2943671Sbinkertn@umich.edu numExitSignal 2953671Sbinkertn@umich.edu .name(name() + ".numExitSignal") 2963671Sbinkertn@umich.edu .desc("exits due to signal delivery") 2973671Sbinkertn@umich.edu ; 2985361Srstrong@cs.ucsd.edu 2993671Sbinkertn@umich.edu numMMIO 3003671Sbinkertn@umich.edu .name(name() + ".numMMIO") 3013671Sbinkertn@umich.edu .desc("number of VM exits due to memory mapped IO") 3023671Sbinkertn@umich.edu ; 3033671Sbinkertn@umich.edu 3043671Sbinkertn@umich.edu numCoalescedMMIO 3053671Sbinkertn@umich.edu .name(name() + ".numCoalescedMMIO") 3063671Sbinkertn@umich.edu .desc("number of coalesced memory mapped IO requests") 3073671Sbinkertn@umich.edu ; 3083671Sbinkertn@umich.edu 3093671Sbinkertn@umich.edu numIO 3103671Sbinkertn@umich.edu .name(name() + ".numIO") 3113671Sbinkertn@umich.edu .desc("number of VM exits due to legacy IO") 3123671Sbinkertn@umich.edu ; 3133671Sbinkertn@umich.edu 3143671Sbinkertn@umich.edu numHalt 3153671Sbinkertn@umich.edu .name(name() + ".numHalt") 3163671Sbinkertn@umich.edu .desc("number of VM exits due to wait for interrupt instructions") 3173671Sbinkertn@umich.edu ; 3185361Srstrong@cs.ucsd.edu 3193671Sbinkertn@umich.edu numInterrupts 3203671Sbinkertn@umich.edu .name(name() + ".numInterrupts") 3213671Sbinkertn@umich.edu .desc("number of interrupts delivered") 3223671Sbinkertn@umich.edu ; 3233671Sbinkertn@umich.edu 3243671Sbinkertn@umich.edu numHypercalls 3253671Sbinkertn@umich.edu .name(name() + ".numHypercalls") 3263671Sbinkertn@umich.edu .desc("number of hypercalls") 3275361Srstrong@cs.ucsd.edu ; 3283671Sbinkertn@umich.edu} 3293671Sbinkertn@umich.edu 3303671Sbinkertn@umich.eduvoid 3313671Sbinkertn@umich.eduBaseKvmCPU::serializeThread(CheckpointOut &cp, ThreadID tid) const 3323671Sbinkertn@umich.edu{ 3335361Srstrong@cs.ucsd.edu if (DTRACE(Checkpoint)) { 3343671Sbinkertn@umich.edu DPRINTF(Checkpoint, "KVM: Serializing thread %i:\n", tid); 3353671Sbinkertn@umich.edu dump(); 3363671Sbinkertn@umich.edu } 3373671Sbinkertn@umich.edu 3383671Sbinkertn@umich.edu assert(tid == 0); 3393671Sbinkertn@umich.edu assert(_status == Idle); 3403671Sbinkertn@umich.edu thread->serialize(cp); 3415361Srstrong@cs.ucsd.edu} 3423671Sbinkertn@umich.edu 3433671Sbinkertn@umich.eduvoid 3443671Sbinkertn@umich.eduBaseKvmCPU::unserializeThread(CheckpointIn &cp, ThreadID tid) 3453671Sbinkertn@umich.edu{ 3463671Sbinkertn@umich.edu DPRINTF(Checkpoint, "KVM: Unserialize thread %i:\n", tid); 3475361Srstrong@cs.ucsd.edu 3483671Sbinkertn@umich.edu assert(tid == 0); 3493671Sbinkertn@umich.edu assert(_status == Idle); 3503671Sbinkertn@umich.edu thread->unserialize(cp); 3513671Sbinkertn@umich.edu threadContextDirty = true; 3523671Sbinkertn@umich.edu} 3533671Sbinkertn@umich.edu 3543671Sbinkertn@umich.eduDrainState 3553671Sbinkertn@umich.eduBaseKvmCPU::drain() 3563671Sbinkertn@umich.edu{ 3573671Sbinkertn@umich.edu if (switchedOut()) 3583671Sbinkertn@umich.edu return DrainState::Drained; 3593671Sbinkertn@umich.edu 3603671Sbinkertn@umich.edu DPRINTF(Drain, "BaseKvmCPU::drain\n"); 3613671Sbinkertn@umich.edu switch (_status) { 3623671Sbinkertn@umich.edu case Running: 3633671Sbinkertn@umich.edu // The base KVM code is normally ready when it is in the 3645361Srstrong@cs.ucsd.edu // Running state, but the architecture specific code might be 3653671Sbinkertn@umich.edu // of a different opinion. This may happen when the CPU been 3663671Sbinkertn@umich.edu // notified of an event that hasn't been accepted by the vCPU 3673671Sbinkertn@umich.edu // yet. 3683671Sbinkertn@umich.edu if (!archIsDrained()) 3693671Sbinkertn@umich.edu return DrainState::Draining; 3703671Sbinkertn@umich.edu 3713671Sbinkertn@umich.edu // The state of the CPU is consistent, so we don't need to do 3725361Srstrong@cs.ucsd.edu // anything special to drain it. We simply de-schedule the 3733671Sbinkertn@umich.edu // tick event and enter the Idle state to prevent nasty things 3743671Sbinkertn@umich.edu // like MMIOs from happening. 3753671Sbinkertn@umich.edu if (tickEvent.scheduled()) 3763671Sbinkertn@umich.edu deschedule(tickEvent); 3773671Sbinkertn@umich.edu _status = Idle; 3783671Sbinkertn@umich.edu 3793671Sbinkertn@umich.edu /** FALLTHROUGH */ 3805361Srstrong@cs.ucsd.edu case Idle: 3813671Sbinkertn@umich.edu // Idle, no need to drain 3823671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 3833671Sbinkertn@umich.edu 3843671Sbinkertn@umich.edu // Sync the thread context here since we'll need it when we 3853671Sbinkertn@umich.edu // switch CPUs or checkpoint the CPU. 3863671Sbinkertn@umich.edu syncThreadContext(); 3873671Sbinkertn@umich.edu 3883671Sbinkertn@umich.edu return DrainState::Drained; 3893671Sbinkertn@umich.edu 3905361Srstrong@cs.ucsd.edu case RunningServiceCompletion: 3913671Sbinkertn@umich.edu // The CPU has just requested a service that was handled in 3923671Sbinkertn@umich.edu // the RunningService state, but the results have still not 3933671Sbinkertn@umich.edu // been reported to the CPU. Now, we /could/ probably just 3943671Sbinkertn@umich.edu // update the register state ourselves instead of letting KVM 3953671Sbinkertn@umich.edu // handle it, but that would be tricky. Instead, we enter KVM 3963671Sbinkertn@umich.edu // and let it do its stuff. 3973671Sbinkertn@umich.edu DPRINTF(Drain, "KVM CPU is waiting for service completion, " 3983671Sbinkertn@umich.edu "requesting drain.\n"); 3993671Sbinkertn@umich.edu return DrainState::Draining; 4003671Sbinkertn@umich.edu 4013671Sbinkertn@umich.edu case RunningMMIOPending: 4023671Sbinkertn@umich.edu // We need to drain since there are in-flight timing accesses 4033671Sbinkertn@umich.edu DPRINTF(Drain, "KVM CPU is waiting for timing accesses to complete, " 4043671Sbinkertn@umich.edu "requesting drain.\n"); 4053671Sbinkertn@umich.edu return DrainState::Draining; 4063671Sbinkertn@umich.edu 4073671Sbinkertn@umich.edu case RunningService: 4083671Sbinkertn@umich.edu // We need to drain since the CPU is waiting for service (e.g., MMIOs) 4093671Sbinkertn@umich.edu DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n"); 4103671Sbinkertn@umich.edu return DrainState::Draining; 4113671Sbinkertn@umich.edu 4123671Sbinkertn@umich.edu default: 4135361Srstrong@cs.ucsd.edu panic("KVM: Unhandled CPU state in drain()\n"); 4143671Sbinkertn@umich.edu return DrainState::Drained; 4153671Sbinkertn@umich.edu } 4163671Sbinkertn@umich.edu} 4173671Sbinkertn@umich.edu 4183671Sbinkertn@umich.eduvoid 4193671Sbinkertn@umich.eduBaseKvmCPU::drainResume() 4203671Sbinkertn@umich.edu{ 4213671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 4223671Sbinkertn@umich.edu 4233671Sbinkertn@umich.edu // We might have been switched out. In that case, we don't need to 4243671Sbinkertn@umich.edu // do anything. 4253671Sbinkertn@umich.edu if (switchedOut()) 4263671Sbinkertn@umich.edu return; 4273671Sbinkertn@umich.edu 4283671Sbinkertn@umich.edu DPRINTF(Kvm, "drainResume\n"); 4293671Sbinkertn@umich.edu verifyMemoryMode(); 4303671Sbinkertn@umich.edu 4315361Srstrong@cs.ucsd.edu // The tick event is de-scheduled as a part of the draining 4323671Sbinkertn@umich.edu // process. Re-schedule it if the thread context is active. 4333671Sbinkertn@umich.edu if (tc->status() == ThreadContext::Active) { 4343671Sbinkertn@umich.edu schedule(tickEvent, nextCycle()); 4353671Sbinkertn@umich.edu _status = Running; 4363671Sbinkertn@umich.edu } else { 4373671Sbinkertn@umich.edu _status = Idle; 4383671Sbinkertn@umich.edu } 4393671Sbinkertn@umich.edu} 4403671Sbinkertn@umich.edu 4413671Sbinkertn@umich.eduvoid 4423671Sbinkertn@umich.eduBaseKvmCPU::notifyFork() 4433671Sbinkertn@umich.edu{ 4443671Sbinkertn@umich.edu // We should have drained prior to forking, which means that the 4453671Sbinkertn@umich.edu // tick event shouldn't be scheduled and the CPU is idle. 4463671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 4473671Sbinkertn@umich.edu assert(_status == Idle); 4483671Sbinkertn@umich.edu 4493671Sbinkertn@umich.edu if (vcpuFD != -1) { 4503671Sbinkertn@umich.edu if (close(vcpuFD) == -1) 4513671Sbinkertn@umich.edu warn("kvm CPU: notifyFork failed to close vcpuFD\n"); 4523671Sbinkertn@umich.edu 4533671Sbinkertn@umich.edu if (_kvmRun) 4543671Sbinkertn@umich.edu munmap(_kvmRun, vcpuMMapSize); 4553671Sbinkertn@umich.edu 4563671Sbinkertn@umich.edu vcpuFD = -1; 4573671Sbinkertn@umich.edu _kvmRun = NULL; 4583671Sbinkertn@umich.edu 4593671Sbinkertn@umich.edu hwInstructions.detach(); 4603671Sbinkertn@umich.edu hwCycles.detach(); 4613671Sbinkertn@umich.edu } 4623671Sbinkertn@umich.edu} 4633671Sbinkertn@umich.edu 4645361Srstrong@cs.ucsd.eduvoid 4653671Sbinkertn@umich.eduBaseKvmCPU::switchOut() 4663671Sbinkertn@umich.edu{ 4673671Sbinkertn@umich.edu DPRINTF(Kvm, "switchOut\n"); 4683671Sbinkertn@umich.edu 4695361Srstrong@cs.ucsd.edu BaseCPU::switchOut(); 4703671Sbinkertn@umich.edu 4713671Sbinkertn@umich.edu // We should have drained prior to executing a switchOut, which 4723671Sbinkertn@umich.edu // means that the tick event shouldn't be scheduled and the CPU is 4733671Sbinkertn@umich.edu // idle. 4745361Srstrong@cs.ucsd.edu assert(!tickEvent.scheduled()); 4753671Sbinkertn@umich.edu assert(_status == Idle); 4763671Sbinkertn@umich.edu} 4773671Sbinkertn@umich.edu 4783671Sbinkertn@umich.eduvoid 4795361Srstrong@cs.ucsd.eduBaseKvmCPU::takeOverFrom(BaseCPU *cpu) 4803671Sbinkertn@umich.edu{ 4813671Sbinkertn@umich.edu DPRINTF(Kvm, "takeOverFrom\n"); 4823671Sbinkertn@umich.edu 4833671Sbinkertn@umich.edu BaseCPU::takeOverFrom(cpu); 4845361Srstrong@cs.ucsd.edu 4853671Sbinkertn@umich.edu // We should have drained prior to executing a switchOut, which 4863671Sbinkertn@umich.edu // means that the tick event shouldn't be scheduled and the CPU is 4873671Sbinkertn@umich.edu // idle. 4883671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 4893671Sbinkertn@umich.edu assert(_status == Idle); 4903671Sbinkertn@umich.edu assert(threadContexts.size() == 1); 4913671Sbinkertn@umich.edu 4923671Sbinkertn@umich.edu // Force an update of the KVM state here instead of flagging the 4933671Sbinkertn@umich.edu // TC as dirty. This is not ideal from a performance point of 4943671Sbinkertn@umich.edu // view, but it makes debugging easier as it allows meaningful KVM 4953671Sbinkertn@umich.edu // state to be dumped before and after a takeover. 4963671Sbinkertn@umich.edu updateKvmState(); 4973671Sbinkertn@umich.edu threadContextDirty = false; 4983671Sbinkertn@umich.edu} 4993671Sbinkertn@umich.edu 5005361Srstrong@cs.ucsd.eduvoid 5013671Sbinkertn@umich.eduBaseKvmCPU::verifyMemoryMode() const 5023671Sbinkertn@umich.edu{ 5033671Sbinkertn@umich.edu if (!(system->bypassCaches())) { 5043671Sbinkertn@umich.edu fatal("The KVM-based CPUs requires the memory system to be in the " 5053671Sbinkertn@umich.edu "'noncaching' mode.\n"); 5063671Sbinkertn@umich.edu } 5073671Sbinkertn@umich.edu} 5083671Sbinkertn@umich.edu 5093671Sbinkertn@umich.eduvoid 5103671Sbinkertn@umich.eduBaseKvmCPU::wakeup(ThreadID tid) 5115361Srstrong@cs.ucsd.edu{ 5123671Sbinkertn@umich.edu DPRINTF(Kvm, "wakeup()\n"); 5133671Sbinkertn@umich.edu // This method might have been called from another 5143671Sbinkertn@umich.edu // context. Migrate to this SimObject's event queue when 5153671Sbinkertn@umich.edu // delivering the wakeup signal. 5163671Sbinkertn@umich.edu EventQueue::ScopedMigration migrate(eventQueue()); 5173671Sbinkertn@umich.edu 5183671Sbinkertn@umich.edu // Kick the vCPU to get it to come out of KVM. 5193671Sbinkertn@umich.edu kick(); 5203671Sbinkertn@umich.edu 5213671Sbinkertn@umich.edu if (thread->status() != ThreadContext::Suspended) 5225361Srstrong@cs.ucsd.edu return; 5233671Sbinkertn@umich.edu 5243671Sbinkertn@umich.edu thread->activate(); 5253671Sbinkertn@umich.edu} 5263671Sbinkertn@umich.edu 5273671Sbinkertn@umich.eduvoid 5283671Sbinkertn@umich.eduBaseKvmCPU::activateContext(ThreadID thread_num) 5293671Sbinkertn@umich.edu{ 5303671Sbinkertn@umich.edu DPRINTF(Kvm, "ActivateContext %d\n", thread_num); 5313671Sbinkertn@umich.edu 5323671Sbinkertn@umich.edu assert(thread_num == 0); 5335361Srstrong@cs.ucsd.edu assert(thread); 5343671Sbinkertn@umich.edu 5353671Sbinkertn@umich.edu assert(_status == Idle); 5363671Sbinkertn@umich.edu assert(!tickEvent.scheduled()); 5373671Sbinkertn@umich.edu 5383671Sbinkertn@umich.edu numCycles += ticksToCycles(thread->lastActivate - thread->lastSuspend); 5393671Sbinkertn@umich.edu 5403671Sbinkertn@umich.edu schedule(tickEvent, clockEdge(Cycles(0))); 5413671Sbinkertn@umich.edu _status = Running; 5423671Sbinkertn@umich.edu} 5433671Sbinkertn@umich.edu 5445361Srstrong@cs.ucsd.edu 5453671Sbinkertn@umich.eduvoid 5463671Sbinkertn@umich.eduBaseKvmCPU::suspendContext(ThreadID thread_num) 5473671Sbinkertn@umich.edu{ 5483671Sbinkertn@umich.edu DPRINTF(Kvm, "SuspendContext %d\n", thread_num); 5493671Sbinkertn@umich.edu 5503671Sbinkertn@umich.edu assert(thread_num == 0); 5513671Sbinkertn@umich.edu assert(thread); 5523671Sbinkertn@umich.edu 5533671Sbinkertn@umich.edu if (_status == Idle) 5543671Sbinkertn@umich.edu return; 5553671Sbinkertn@umich.edu 5563671Sbinkertn@umich.edu assert(_status == Running || _status == RunningServiceCompletion); 5574271Sgblack@eecs.umich.edu 5585361Srstrong@cs.ucsd.edu // The tick event may no be scheduled if the quest has requested 5593671Sbinkertn@umich.edu // the monitor to wait for interrupts. The normal CPU models can 5603671Sbinkertn@umich.edu // get their tick events descheduled by quiesce instructions, but 5613671Sbinkertn@umich.edu // that can't happen here. 5623671Sbinkertn@umich.edu if (tickEvent.scheduled()) 5633671Sbinkertn@umich.edu deschedule(tickEvent); 5643671Sbinkertn@umich.edu 5655361Srstrong@cs.ucsd.edu _status = Idle; 5663671Sbinkertn@umich.edu} 5673671Sbinkertn@umich.edu 5683671Sbinkertn@umich.eduvoid 5693671Sbinkertn@umich.eduBaseKvmCPU::deallocateContext(ThreadID thread_num) 5703671Sbinkertn@umich.edu{ 5713671Sbinkertn@umich.edu // for now, these are equivalent 5723671Sbinkertn@umich.edu suspendContext(thread_num); 5733671Sbinkertn@umich.edu} 5743671Sbinkertn@umich.edu 5753671Sbinkertn@umich.eduvoid 5763671Sbinkertn@umich.eduBaseKvmCPU::haltContext(ThreadID thread_num) 5773671Sbinkertn@umich.edu{ 5785361Srstrong@cs.ucsd.edu // for now, these are equivalent 5793671Sbinkertn@umich.edu suspendContext(thread_num); 5803671Sbinkertn@umich.edu} 5813671Sbinkertn@umich.edu 5823671Sbinkertn@umich.eduThreadContext * 5833671Sbinkertn@umich.eduBaseKvmCPU::getContext(int tn) 5843671Sbinkertn@umich.edu{ 5853671Sbinkertn@umich.edu assert(tn == 0); 5863671Sbinkertn@umich.edu syncThreadContext(); 5873671Sbinkertn@umich.edu return tc; 5883671Sbinkertn@umich.edu} 5893671Sbinkertn@umich.edu 5903671Sbinkertn@umich.edu 5913671Sbinkertn@umich.eduCounter 5923671Sbinkertn@umich.eduBaseKvmCPU::totalInsts() const 5935361Srstrong@cs.ucsd.edu{ 5943671Sbinkertn@umich.edu return ctrInsts; 5953671Sbinkertn@umich.edu} 5963671Sbinkertn@umich.edu 5973671Sbinkertn@umich.eduCounter 5983671Sbinkertn@umich.eduBaseKvmCPU::totalOps() const 5993671Sbinkertn@umich.edu{ 6003671Sbinkertn@umich.edu hack_once("Pretending totalOps is equivalent to totalInsts()\n"); 6013671Sbinkertn@umich.edu return ctrInsts; 6023671Sbinkertn@umich.edu} 6033671Sbinkertn@umich.edu 6043671Sbinkertn@umich.eduvoid 6053671Sbinkertn@umich.eduBaseKvmCPU::dump() const 6063671Sbinkertn@umich.edu{ 6075361Srstrong@cs.ucsd.edu inform("State dumping not implemented."); 6083671Sbinkertn@umich.edu} 6093671Sbinkertn@umich.edu 6103671Sbinkertn@umich.eduvoid 6113671Sbinkertn@umich.eduBaseKvmCPU::tick() 6123671Sbinkertn@umich.edu{ 6133671Sbinkertn@umich.edu Tick delay(0); 6143671Sbinkertn@umich.edu assert(_status != Idle && _status != RunningMMIOPending); 6155361Srstrong@cs.ucsd.edu 6163671Sbinkertn@umich.edu switch (_status) { 6173671Sbinkertn@umich.edu case RunningService: 6183671Sbinkertn@umich.edu // handleKvmExit() will determine the next state of the CPU 6193671Sbinkertn@umich.edu delay = handleKvmExit(); 6203671Sbinkertn@umich.edu 6213671Sbinkertn@umich.edu if (tryDrain()) 6223671Sbinkertn@umich.edu _status = Idle; 6233671Sbinkertn@umich.edu break; 6243671Sbinkertn@umich.edu 6253671Sbinkertn@umich.edu case RunningServiceCompletion: 6263671Sbinkertn@umich.edu case Running: { 6273671Sbinkertn@umich.edu const uint64_t nextInstEvent( 6283671Sbinkertn@umich.edu !comInstEventQueue[0]->empty() ? 6293671Sbinkertn@umich.edu comInstEventQueue[0]->nextTick() : UINT64_MAX); 6303671Sbinkertn@umich.edu // Enter into KVM and complete pending IO instructions if we 6313671Sbinkertn@umich.edu // have an instruction event pending. 6323671Sbinkertn@umich.edu const Tick ticksToExecute( 6333671Sbinkertn@umich.edu nextInstEvent > ctrInsts ? 6343671Sbinkertn@umich.edu curEventQueue()->nextTick() - curTick() : 0); 6353671Sbinkertn@umich.edu 6363671Sbinkertn@umich.edu if (alwaysSyncTC) 6373671Sbinkertn@umich.edu threadContextDirty = true; 6383671Sbinkertn@umich.edu 6393671Sbinkertn@umich.edu // We might need to update the KVM state. 6403671Sbinkertn@umich.edu syncKvmState(); 6413671Sbinkertn@umich.edu 6423671Sbinkertn@umich.edu // Setup any pending instruction count breakpoints using 6433671Sbinkertn@umich.edu // PerfEvent if we are going to execute more than just an IO 6443671Sbinkertn@umich.edu // completion. 6453671Sbinkertn@umich.edu if (ticksToExecute > 0) 6463671Sbinkertn@umich.edu setupInstStop(); 6475361Srstrong@cs.ucsd.edu 6483671Sbinkertn@umich.edu DPRINTF(KvmRun, "Entering KVM...\n"); 6493671Sbinkertn@umich.edu if (drainState() == DrainState::Draining) { 6503671Sbinkertn@umich.edu // Force an immediate exit from KVM after completing 6513671Sbinkertn@umich.edu // pending operations. The architecture-specific code 6523671Sbinkertn@umich.edu // takes care to run until it is in a state where it can 6533671Sbinkertn@umich.edu // safely be drained. 6543671Sbinkertn@umich.edu delay = kvmRunDrain(); 6553671Sbinkertn@umich.edu } else { 6563671Sbinkertn@umich.edu delay = kvmRun(ticksToExecute); 6573671Sbinkertn@umich.edu } 6583671Sbinkertn@umich.edu 6593671Sbinkertn@umich.edu // The CPU might have been suspended before entering into 6603671Sbinkertn@umich.edu // KVM. Assume that the CPU was suspended /before/ entering 6613671Sbinkertn@umich.edu // into KVM and skip the exit handling. 6623671Sbinkertn@umich.edu if (_status == Idle) 6633671Sbinkertn@umich.edu break; 6643671Sbinkertn@umich.edu 6653671Sbinkertn@umich.edu // Entering into KVM implies that we'll have to reload the thread 6667416SAli.Saidi@ARM.com // context from KVM if we want to access it. Flag the KVM state as 6673671Sbinkertn@umich.edu // dirty with respect to the cached thread context. 6684116Sgblack@eecs.umich.edu kvmStateDirty = true; 6694116Sgblack@eecs.umich.edu 6703671Sbinkertn@umich.edu if (alwaysSyncTC) 6713671Sbinkertn@umich.edu syncThreadContext(); 6723671Sbinkertn@umich.edu 6733671Sbinkertn@umich.edu // Enter into the RunningService state unless the 6743671Sbinkertn@umich.edu // simulation was stopped by a timer. 6753671Sbinkertn@umich.edu if (_kvmRun->exit_reason != KVM_EXIT_INTR) { 6763671Sbinkertn@umich.edu _status = RunningService; 6773671Sbinkertn@umich.edu } else { 6783671Sbinkertn@umich.edu ++numExitSignal; 6793671Sbinkertn@umich.edu _status = Running; 6803671Sbinkertn@umich.edu } 6813671Sbinkertn@umich.edu 6823671Sbinkertn@umich.edu // Service any pending instruction events. The vCPU should 6833671Sbinkertn@umich.edu // have exited in time for the event using the instruction 6843671Sbinkertn@umich.edu // counter configured by setupInstStop(). 6853671Sbinkertn@umich.edu comInstEventQueue[0]->serviceEvents(ctrInsts); 6863671Sbinkertn@umich.edu system->instEventQueue.serviceEvents(system->totalNumInsts); 6873671Sbinkertn@umich.edu 6883671Sbinkertn@umich.edu if (tryDrain()) 6893671Sbinkertn@umich.edu _status = Idle; 6903671Sbinkertn@umich.edu } break; 6913671Sbinkertn@umich.edu 6923671Sbinkertn@umich.edu default: 6933671Sbinkertn@umich.edu panic("BaseKvmCPU entered tick() in an illegal state (%i)\n", 6943671Sbinkertn@umich.edu _status); 6953671Sbinkertn@umich.edu } 6963671Sbinkertn@umich.edu 6973671Sbinkertn@umich.edu // Schedule a new tick if we are still running 6983671Sbinkertn@umich.edu if (_status != Idle && _status != RunningMMIOPending) 6995361Srstrong@cs.ucsd.edu schedule(tickEvent, clockEdge(ticksToCycles(delay))); 7003671Sbinkertn@umich.edu} 7013671Sbinkertn@umich.edu 7023671Sbinkertn@umich.eduTick 7033671Sbinkertn@umich.eduBaseKvmCPU::kvmRunDrain() 7045361Srstrong@cs.ucsd.edu{ 7053671Sbinkertn@umich.edu // By default, the only thing we need to drain is a pending IO 7063671Sbinkertn@umich.edu // operation which assumes that we are in the 7073671Sbinkertn@umich.edu // RunningServiceCompletion or RunningMMIOPending state. 7083671Sbinkertn@umich.edu assert(_status == RunningServiceCompletion || 7093671Sbinkertn@umich.edu _status == RunningMMIOPending); 7105361Srstrong@cs.ucsd.edu 7113671Sbinkertn@umich.edu // Deliver the data from the pending IO operation and immediately 7123671Sbinkertn@umich.edu // exit. 7133671Sbinkertn@umich.edu return kvmRun(0); 7143671Sbinkertn@umich.edu} 7153671Sbinkertn@umich.edu 7163671Sbinkertn@umich.eduuint64_t 7173671Sbinkertn@umich.eduBaseKvmCPU::getHostCycles() const 7183671Sbinkertn@umich.edu{ 7193671Sbinkertn@umich.edu return hwCycles.read(); 7203671Sbinkertn@umich.edu} 7213671Sbinkertn@umich.edu 7223671Sbinkertn@umich.eduTick 7233671Sbinkertn@umich.eduBaseKvmCPU::kvmRun(Tick ticks) 7243671Sbinkertn@umich.edu{ 7253671Sbinkertn@umich.edu Tick ticksExecuted; 7263671Sbinkertn@umich.edu fatal_if(vcpuFD == -1, 7275361Srstrong@cs.ucsd.edu "Trying to run a KVM CPU in a forked child process. " 7283671Sbinkertn@umich.edu "This is not supported.\n"); 7293671Sbinkertn@umich.edu DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks); 7303671Sbinkertn@umich.edu 7313671Sbinkertn@umich.edu if (ticks == 0) { 7323671Sbinkertn@umich.edu // Settings ticks == 0 is a special case which causes an entry 7333671Sbinkertn@umich.edu // into KVM that finishes pending operations (e.g., IO) and 7343671Sbinkertn@umich.edu // then immediately exits. 7353671Sbinkertn@umich.edu DPRINTF(KvmRun, "KVM: Delivering IO without full guest entry\n"); 7363671Sbinkertn@umich.edu 7373671Sbinkertn@umich.edu ++numVMHalfEntries; 7383671Sbinkertn@umich.edu 7393671Sbinkertn@umich.edu // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this 7403671Sbinkertn@umich.edu // thread). The KVM control signal is masked while executing 7413671Sbinkertn@umich.edu // in gem5 and gets unmasked temporarily as when entering 7423671Sbinkertn@umich.edu // KVM. See setSignalMask() and setupSignalHandler(). 7433671Sbinkertn@umich.edu kick(); 7443671Sbinkertn@umich.edu 7453671Sbinkertn@umich.edu // Start the vCPU. KVM will check for signals after completing 7463671Sbinkertn@umich.edu // pending operations (IO). Since the KVM_KICK_SIGNAL is 7473671Sbinkertn@umich.edu // pending, this forces an immediate exit to gem5 again. We 7483671Sbinkertn@umich.edu // don't bother to setup timers since this shouldn't actually 7493671Sbinkertn@umich.edu // execute any code (other than completing half-executed IO 7503671Sbinkertn@umich.edu // instructions) in the guest. 7513671Sbinkertn@umich.edu ioctlRun(); 7523671Sbinkertn@umich.edu 7533671Sbinkertn@umich.edu // We always execute at least one cycle to prevent the 7543671Sbinkertn@umich.edu // BaseKvmCPU::tick() to be rescheduled on the same tick 755 // twice. 756 ticksExecuted = clockPeriod(); 757 } else { 758 // This method is executed as a result of a tick event. That 759 // means that the event queue will be locked when entering the 760 // method. We temporarily unlock the event queue to allow 761 // other threads to steal control of this thread to inject 762 // interrupts. They will typically lock the queue and then 763 // force an exit from KVM by kicking the vCPU. 764 EventQueue::ScopedRelease release(curEventQueue()); 765 766 if (ticks < runTimer->resolution()) { 767 DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n", 768 ticks, runTimer->resolution()); 769 ticks = runTimer->resolution(); 770 } 771 772 // Get hardware statistics after synchronizing contexts. The KVM 773 // state update might affect guest cycle counters. 774 uint64_t baseCycles(getHostCycles()); 775 uint64_t baseInstrs(hwInstructions.read()); 776 777 // Arm the run timer and start the cycle timer if it isn't 778 // controlled by the overflow timer. Starting/stopping the cycle 779 // timer automatically starts the other perf timers as they are in 780 // the same counter group. 781 runTimer->arm(ticks); 782 if (!perfControlledByTimer) 783 hwCycles.start(); 784 785 ioctlRun(); 786 787 runTimer->disarm(); 788 if (!perfControlledByTimer) 789 hwCycles.stop(); 790 791 // The control signal may have been delivered after we exited 792 // from KVM. It will be pending in that case since it is 793 // masked when we aren't executing in KVM. Discard it to make 794 // sure we don't deliver it immediately next time we try to 795 // enter into KVM. 796 discardPendingSignal(KVM_KICK_SIGNAL); 797 798 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles); 799 const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor); 800 const uint64_t instsExecuted(hwInstructions.read() - baseInstrs); 801 ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted); 802 803 /* Update statistics */ 804 numCycles += simCyclesExecuted;; 805 numInsts += instsExecuted; 806 ctrInsts += instsExecuted; 807 system->totalNumInsts += instsExecuted; 808 809 DPRINTF(KvmRun, 810 "KVM: Executed %i instructions in %i cycles " 811 "(%i ticks, sim cycles: %i).\n", 812 instsExecuted, hostCyclesExecuted, ticksExecuted, simCyclesExecuted); 813 } 814 815 ++numVMExits; 816 817 return ticksExecuted + flushCoalescedMMIO(); 818} 819 820void 821BaseKvmCPU::kvmNonMaskableInterrupt() 822{ 823 ++numInterrupts; 824 if (ioctl(KVM_NMI) == -1) 825 panic("KVM: Failed to deliver NMI to virtual CPU\n"); 826} 827 828void 829BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt &interrupt) 830{ 831 ++numInterrupts; 832 if (ioctl(KVM_INTERRUPT, (void *)&interrupt) == -1) 833 panic("KVM: Failed to deliver interrupt to virtual CPU\n"); 834} 835 836void 837BaseKvmCPU::getRegisters(struct kvm_regs ®s) const 838{ 839 if (ioctl(KVM_GET_REGS, ®s) == -1) 840 panic("KVM: Failed to get guest registers\n"); 841} 842 843void 844BaseKvmCPU::setRegisters(const struct kvm_regs ®s) 845{ 846 if (ioctl(KVM_SET_REGS, (void *)®s) == -1) 847 panic("KVM: Failed to set guest registers\n"); 848} 849 850void 851BaseKvmCPU::getSpecialRegisters(struct kvm_sregs ®s) const 852{ 853 if (ioctl(KVM_GET_SREGS, ®s) == -1) 854 panic("KVM: Failed to get guest special registers\n"); 855} 856 857void 858BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs ®s) 859{ 860 if (ioctl(KVM_SET_SREGS, (void *)®s) == -1) 861 panic("KVM: Failed to set guest special registers\n"); 862} 863 864void 865BaseKvmCPU::getFPUState(struct kvm_fpu &state) const 866{ 867 if (ioctl(KVM_GET_FPU, &state) == -1) 868 panic("KVM: Failed to get guest FPU state\n"); 869} 870 871void 872BaseKvmCPU::setFPUState(const struct kvm_fpu &state) 873{ 874 if (ioctl(KVM_SET_FPU, (void *)&state) == -1) 875 panic("KVM: Failed to set guest FPU state\n"); 876} 877 878 879void 880BaseKvmCPU::setOneReg(uint64_t id, const void *addr) 881{ 882#ifdef KVM_SET_ONE_REG 883 struct kvm_one_reg reg; 884 reg.id = id; 885 reg.addr = (uint64_t)addr; 886 887 if (ioctl(KVM_SET_ONE_REG, ®) == -1) { 888 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n", 889 id, errno); 890 } 891#else 892 panic("KVM_SET_ONE_REG is unsupported on this platform.\n"); 893#endif 894} 895 896void 897BaseKvmCPU::getOneReg(uint64_t id, void *addr) const 898{ 899#ifdef KVM_GET_ONE_REG 900 struct kvm_one_reg reg; 901 reg.id = id; 902 reg.addr = (uint64_t)addr; 903 904 if (ioctl(KVM_GET_ONE_REG, ®) == -1) { 905 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n", 906 id, errno); 907 } 908#else 909 panic("KVM_GET_ONE_REG is unsupported on this platform.\n"); 910#endif 911} 912 913std::string 914BaseKvmCPU::getAndFormatOneReg(uint64_t id) const 915{ 916#ifdef KVM_GET_ONE_REG 917 std::ostringstream ss; 918 919 ss.setf(std::ios::hex, std::ios::basefield); 920 ss.setf(std::ios::showbase); 921#define HANDLE_INTTYPE(len) \ 922 case KVM_REG_SIZE_U ## len: { \ 923 uint ## len ## _t value; \ 924 getOneReg(id, &value); \ 925 ss << value; \ 926 } break 927 928#define HANDLE_ARRAY(len) \ 929 case KVM_REG_SIZE_U ## len: { \ 930 uint8_t value[len / 8]; \ 931 getOneReg(id, value); \ 932 ccprintf(ss, "[0x%x", value[0]); \ 933 for (int i = 1; i < len / 8; ++i) \ 934 ccprintf(ss, ", 0x%x", value[i]); \ 935 ccprintf(ss, "]"); \ 936 } break 937 938 switch (id & KVM_REG_SIZE_MASK) { 939 HANDLE_INTTYPE(8); 940 HANDLE_INTTYPE(16); 941 HANDLE_INTTYPE(32); 942 HANDLE_INTTYPE(64); 943 HANDLE_ARRAY(128); 944 HANDLE_ARRAY(256); 945 HANDLE_ARRAY(512); 946 HANDLE_ARRAY(1024); 947 default: 948 ss << "??"; 949 } 950 951#undef HANDLE_INTTYPE 952#undef HANDLE_ARRAY 953 954 return ss.str(); 955#else 956 panic("KVM_GET_ONE_REG is unsupported on this platform.\n"); 957#endif 958} 959 960void 961BaseKvmCPU::syncThreadContext() 962{ 963 if (!kvmStateDirty) 964 return; 965 966 assert(!threadContextDirty); 967 968 updateThreadContext(); 969 kvmStateDirty = false; 970} 971 972void 973BaseKvmCPU::syncKvmState() 974{ 975 if (!threadContextDirty) 976 return; 977 978 assert(!kvmStateDirty); 979 980 updateKvmState(); 981 threadContextDirty = false; 982} 983 984Tick 985BaseKvmCPU::handleKvmExit() 986{ 987 DPRINTF(KvmRun, "handleKvmExit (exit_reason: %i)\n", _kvmRun->exit_reason); 988 assert(_status == RunningService); 989 990 // Switch into the running state by default. Individual handlers 991 // can override this. 992 _status = Running; 993 switch (_kvmRun->exit_reason) { 994 case KVM_EXIT_UNKNOWN: 995 return handleKvmExitUnknown(); 996 997 case KVM_EXIT_EXCEPTION: 998 return handleKvmExitException(); 999 1000 case KVM_EXIT_IO: 1001 { 1002 ++numIO; 1003 Tick ticks = handleKvmExitIO(); 1004 _status = dataPort.nextIOState(); 1005 return ticks; 1006 } 1007 1008 case KVM_EXIT_HYPERCALL: 1009 ++numHypercalls; 1010 return handleKvmExitHypercall(); 1011 1012 case KVM_EXIT_HLT: 1013 /* The guest has halted and is waiting for interrupts */ 1014 DPRINTF(Kvm, "handleKvmExitHalt\n"); 1015 ++numHalt; 1016 1017 // Suspend the thread until the next interrupt arrives 1018 thread->suspend(); 1019 1020 // This is actually ignored since the thread is suspended. 1021 return 0; 1022 1023 case KVM_EXIT_MMIO: 1024 { 1025 /* Service memory mapped IO requests */ 1026 DPRINTF(KvmIO, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n", 1027 _kvmRun->mmio.is_write, 1028 _kvmRun->mmio.phys_addr, _kvmRun->mmio.len); 1029 1030 ++numMMIO; 1031 Tick ticks = doMMIOAccess(_kvmRun->mmio.phys_addr, _kvmRun->mmio.data, 1032 _kvmRun->mmio.len, _kvmRun->mmio.is_write); 1033 // doMMIOAccess could have triggered a suspend, in which case we don't 1034 // want to overwrite the _status. 1035 if (_status != Idle) 1036 _status = dataPort.nextIOState(); 1037 return ticks; 1038 } 1039 1040 case KVM_EXIT_IRQ_WINDOW_OPEN: 1041 return handleKvmExitIRQWindowOpen(); 1042 1043 case KVM_EXIT_FAIL_ENTRY: 1044 return handleKvmExitFailEntry(); 1045 1046 case KVM_EXIT_INTR: 1047 /* KVM was interrupted by a signal, restart it in the next 1048 * tick. */ 1049 return 0; 1050 1051 case KVM_EXIT_INTERNAL_ERROR: 1052 panic("KVM: Internal error (suberror: %u)\n", 1053 _kvmRun->internal.suberror); 1054 1055 default: 1056 dump(); 1057 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun->exit_reason); 1058 } 1059} 1060 1061Tick 1062BaseKvmCPU::handleKvmExitIO() 1063{ 1064 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n", 1065 _kvmRun->io.direction, _kvmRun->io.size, 1066 _kvmRun->io.port, _kvmRun->io.count); 1067} 1068 1069Tick 1070BaseKvmCPU::handleKvmExitHypercall() 1071{ 1072 panic("KVM: Unhandled hypercall\n"); 1073} 1074 1075Tick 1076BaseKvmCPU::handleKvmExitIRQWindowOpen() 1077{ 1078 warn("KVM: Unhandled IRQ window.\n"); 1079 return 0; 1080} 1081 1082 1083Tick 1084BaseKvmCPU::handleKvmExitUnknown() 1085{ 1086 dump(); 1087 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n", 1088 _kvmRun->hw.hardware_exit_reason); 1089} 1090 1091Tick 1092BaseKvmCPU::handleKvmExitException() 1093{ 1094 dump(); 1095 panic("KVM: Got exception when starting vCPU " 1096 "(exception: %u, error_code: %u)\n", 1097 _kvmRun->ex.exception, _kvmRun->ex.error_code); 1098} 1099 1100Tick 1101BaseKvmCPU::handleKvmExitFailEntry() 1102{ 1103 dump(); 1104 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n", 1105 _kvmRun->fail_entry.hardware_entry_failure_reason); 1106} 1107 1108Tick 1109BaseKvmCPU::doMMIOAccess(Addr paddr, void *data, int size, bool write) 1110{ 1111 ThreadContext *tc(thread->getTC()); 1112 syncThreadContext(); 1113 1114 RequestPtr mmio_req = new Request(paddr, size, Request::UNCACHEABLE, 1115 dataMasterId()); 1116 mmio_req->setContext(tc->contextId()); 1117 // Some architectures do need to massage physical addresses a bit 1118 // before they are inserted into the memory system. This enables 1119 // APIC accesses on x86 and m5ops where supported through a MMIO 1120 // interface. 1121 BaseTLB::Mode tlb_mode(write ? BaseTLB::Write : BaseTLB::Read); 1122 Fault fault(tc->getDTBPtr()->finalizePhysical(mmio_req, tc, tlb_mode)); 1123 if (fault != NoFault) 1124 warn("Finalization of MMIO address failed: %s\n", fault->name()); 1125 1126 1127 const MemCmd cmd(write ? MemCmd::WriteReq : MemCmd::ReadReq); 1128 PacketPtr pkt = new Packet(mmio_req, cmd); 1129 pkt->dataStatic(data); 1130 1131 if (mmio_req->isMmappedIpr()) { 1132 // We currently assume that there is no need to migrate to a 1133 // different event queue when doing IPRs. Currently, IPRs are 1134 // only used for m5ops, so it should be a valid assumption. 1135 const Cycles ipr_delay(write ? 1136 TheISA::handleIprWrite(tc, pkt) : 1137 TheISA::handleIprRead(tc, pkt)); 1138 threadContextDirty = true; 1139 delete pkt->req; 1140 delete pkt; 1141 return clockPeriod() * ipr_delay; 1142 } else { 1143 // Temporarily lock and migrate to the event queue of the 1144 // VM. This queue is assumed to "own" all devices we need to 1145 // access if running in multi-core mode. 1146 EventQueue::ScopedMigration migrate(vm.eventQueue()); 1147 1148 return dataPort.submitIO(pkt); 1149 } 1150} 1151 1152void 1153BaseKvmCPU::setSignalMask(const sigset_t *mask) 1154{ 1155 std::unique_ptr<struct kvm_signal_mask> kvm_mask; 1156 1157 if (mask) { 1158 kvm_mask.reset((struct kvm_signal_mask *)operator new( 1159 sizeof(struct kvm_signal_mask) + sizeof(*mask))); 1160 // The kernel and the user-space headers have different ideas 1161 // about the size of sigset_t. This seems like a massive hack, 1162 // but is actually what qemu does. 1163 assert(sizeof(*mask) >= 8); 1164 kvm_mask->len = 8; 1165 memcpy(kvm_mask->sigset, mask, kvm_mask->len); 1166 } 1167 1168 if (ioctl(KVM_SET_SIGNAL_MASK, (void *)kvm_mask.get()) == -1) 1169 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n", 1170 errno); 1171} 1172 1173int 1174BaseKvmCPU::ioctl(int request, long p1) const 1175{ 1176 if (vcpuFD == -1) 1177 panic("KVM: CPU ioctl called before initialization\n"); 1178 1179 return ::ioctl(vcpuFD, request, p1); 1180} 1181 1182Tick 1183BaseKvmCPU::flushCoalescedMMIO() 1184{ 1185 if (!mmioRing) 1186 return 0; 1187 1188 DPRINTF(KvmIO, "KVM: Flushing the coalesced MMIO ring buffer\n"); 1189 1190 // TODO: We might need to do synchronization when we start to 1191 // support multiple CPUs 1192 Tick ticks(0); 1193 while (mmioRing->first != mmioRing->last) { 1194 struct kvm_coalesced_mmio &ent( 1195 mmioRing->coalesced_mmio[mmioRing->first]); 1196 1197 DPRINTF(KvmIO, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n", 1198 ent.phys_addr, ent.len); 1199 1200 ++numCoalescedMMIO; 1201 ticks += doMMIOAccess(ent.phys_addr, ent.data, ent.len, true); 1202 1203 mmioRing->first = (mmioRing->first + 1) % KVM_COALESCED_MMIO_MAX; 1204 } 1205 1206 return ticks; 1207} 1208 1209/** 1210 * Dummy handler for KVM kick signals. 1211 * 1212 * @note This function is usually not called since the kernel doesn't 1213 * seem to deliver signals when the signal is only unmasked when 1214 * running in KVM. This doesn't matter though since we are only 1215 * interested in getting KVM to exit, which happens as expected. See 1216 * setupSignalHandler() and kvmRun() for details about KVM signal 1217 * handling. 1218 */ 1219static void 1220onKickSignal(int signo, siginfo_t *si, void *data) 1221{ 1222} 1223 1224void 1225BaseKvmCPU::setupSignalHandler() 1226{ 1227 struct sigaction sa; 1228 1229 memset(&sa, 0, sizeof(sa)); 1230 sa.sa_sigaction = onKickSignal; 1231 sa.sa_flags = SA_SIGINFO | SA_RESTART; 1232 if (sigaction(KVM_KICK_SIGNAL, &sa, NULL) == -1) 1233 panic("KVM: Failed to setup vCPU timer signal handler\n"); 1234 1235 sigset_t sigset; 1236 if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1) 1237 panic("KVM: Failed get signal mask\n"); 1238 1239 // Request KVM to setup the same signal mask as we're currently 1240 // running with except for the KVM control signal. We'll sometimes 1241 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from 1242 // KVM after servicing IO requests. See kvmRun(). 1243 sigdelset(&sigset, KVM_KICK_SIGNAL); 1244 setSignalMask(&sigset); 1245 1246 // Mask our control signals so they aren't delivered unless we're 1247 // actually executing inside KVM. 1248 sigaddset(&sigset, KVM_KICK_SIGNAL); 1249 if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1) 1250 panic("KVM: Failed mask the KVM control signals\n"); 1251} 1252 1253bool 1254BaseKvmCPU::discardPendingSignal(int signum) const 1255{ 1256 int discardedSignal; 1257 1258 // Setting the timeout to zero causes sigtimedwait to return 1259 // immediately. 1260 struct timespec timeout; 1261 timeout.tv_sec = 0; 1262 timeout.tv_nsec = 0; 1263 1264 sigset_t sigset; 1265 sigemptyset(&sigset); 1266 sigaddset(&sigset, signum); 1267 1268 do { 1269 discardedSignal = sigtimedwait(&sigset, NULL, &timeout); 1270 } while (discardedSignal == -1 && errno == EINTR); 1271 1272 if (discardedSignal == signum) 1273 return true; 1274 else if (discardedSignal == -1 && errno == EAGAIN) 1275 return false; 1276 else 1277 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n", 1278 discardedSignal, errno); 1279} 1280 1281void 1282BaseKvmCPU::setupCounters() 1283{ 1284 DPRINTF(Kvm, "Attaching cycle counter...\n"); 1285 PerfKvmCounterConfig cfgCycles(PERF_TYPE_HARDWARE, 1286 PERF_COUNT_HW_CPU_CYCLES); 1287 cfgCycles.disabled(true) 1288 .pinned(true); 1289 1290 // Try to exclude the host. We set both exclude_hv and 1291 // exclude_host since different architectures use slightly 1292 // different APIs in the kernel. 1293 cfgCycles.exclude_hv(true) 1294 .exclude_host(true); 1295 1296 if (perfControlledByTimer) { 1297 // We need to configure the cycles counter to send overflows 1298 // since we are going to use it to trigger timer signals that 1299 // trap back into m5 from KVM. In practice, this means that we 1300 // need to set some non-zero sample period that gets 1301 // overridden when the timer is armed. 1302 cfgCycles.wakeupEvents(1) 1303 .samplePeriod(42); 1304 } 1305 1306 hwCycles.attach(cfgCycles, 1307 0); // TID (0 => currentThread) 1308 1309 setupInstCounter(); 1310} 1311 1312bool 1313BaseKvmCPU::tryDrain() 1314{ 1315 if (drainState() != DrainState::Draining) 1316 return false; 1317 1318 if (!archIsDrained()) { 1319 DPRINTF(Drain, "tryDrain: Architecture code is not ready.\n"); 1320 return false; 1321 } 1322 1323 if (_status == Idle || _status == Running) { 1324 DPRINTF(Drain, 1325 "tryDrain: CPU transitioned into the Idle state, drain done\n"); 1326 signalDrainDone(); 1327 return true; 1328 } else { 1329 DPRINTF(Drain, "tryDrain: CPU not ready.\n"); 1330 return false; 1331 } 1332} 1333 1334void 1335BaseKvmCPU::ioctlRun() 1336{ 1337 if (ioctl(KVM_RUN) == -1) { 1338 if (errno != EINTR) 1339 panic("KVM: Failed to start virtual CPU (errno: %i)\n", 1340 errno); 1341 } 1342} 1343 1344void 1345BaseKvmCPU::setupInstStop() 1346{ 1347 if (comInstEventQueue[0]->empty()) { 1348 setupInstCounter(0); 1349 } else { 1350 const uint64_t next(comInstEventQueue[0]->nextTick()); 1351 1352 assert(next > ctrInsts); 1353 setupInstCounter(next - ctrInsts); 1354 } 1355} 1356 1357void 1358BaseKvmCPU::setupInstCounter(uint64_t period) 1359{ 1360 // No need to do anything if we aren't attaching for the first 1361 // time or the period isn't changing. 1362 if (period == activeInstPeriod && hwInstructions.attached()) 1363 return; 1364 1365 PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE, 1366 PERF_COUNT_HW_INSTRUCTIONS); 1367 1368 // Try to exclude the host. We set both exclude_hv and 1369 // exclude_host since different architectures use slightly 1370 // different APIs in the kernel. 1371 cfgInstructions.exclude_hv(true) 1372 .exclude_host(true); 1373 1374 if (period) { 1375 // Setup a sampling counter if that has been requested. 1376 cfgInstructions.wakeupEvents(1) 1377 .samplePeriod(period); 1378 } 1379 1380 // We need to detach and re-attach the counter to reliably change 1381 // sampling settings. See PerfKvmCounter::period() for details. 1382 if (hwInstructions.attached()) 1383 hwInstructions.detach(); 1384 assert(hwCycles.attached()); 1385 hwInstructions.attach(cfgInstructions, 1386 0, // TID (0 => currentThread) 1387 hwCycles); 1388 1389 if (period) 1390 hwInstructions.enableSignals(KVM_KICK_SIGNAL); 1391 1392 activeInstPeriod = period; 1393} 1394