copy_engine.cc revision 12087
15794SN/A/* 28841SN/A * Copyright (c) 2012 ARM Limited 38841SN/A * All rights reserved 48841SN/A * 58841SN/A * The license below extends only to copyright in the software and shall 68841SN/A * not be construed as granting a license to any other intellectual 78841SN/A * property including but not limited to intellectual property relating 88841SN/A * to a hardware implementation of the functionality of the software 98841SN/A * licensed hereunder. You may use the software subject to the license 108841SN/A * terms below provided that you ensure that this notice is replicated 118841SN/A * unmodified and in its entirety in all distributions of the software, 128841SN/A * modified or unmodified, in source code or in binary form. 138841SN/A * 145794SN/A * Copyright (c) 2008 The Regents of The University of Michigan 155794SN/A * All rights reserved. 165794SN/A * 175794SN/A * Redistribution and use in source and binary forms, with or without 185794SN/A * modification, are permitted provided that the following conditions are 195794SN/A * met: redistributions of source code must retain the above copyright 205794SN/A * notice, this list of conditions and the following disclaimer; 215794SN/A * redistributions in binary form must reproduce the above copyright 225794SN/A * notice, this list of conditions and the following disclaimer in the 235794SN/A * documentation and/or other materials provided with the distribution; 245794SN/A * neither the name of the copyright holders nor the names of its 255794SN/A * contributors may be used to endorse or promote products derived from 265794SN/A * this software without specific prior written permission. 275794SN/A * 285794SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 295794SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 305794SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 315794SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 325794SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 335794SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 345794SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 355794SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 365794SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 375794SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 385794SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 395794SN/A * 405794SN/A * Authors: Ali Saidi 415794SN/A */ 425794SN/A 435794SN/A/* @file 445794SN/A * Device model for Intel's I/O AT DMA copy engine. 455794SN/A */ 465794SN/A 4711261Sandreas.sandberg@arm.com#include "dev/pci/copy_engine.hh" 4811261Sandreas.sandberg@arm.com 495794SN/A#include <algorithm> 505794SN/A 515954SN/A#include "base/cp_annotate.hh" 525794SN/A#include "base/trace.hh" 538232SN/A#include "debug/DMACopyEngine.hh" 549152SN/A#include "debug/Drain.hh" 555794SN/A#include "mem/packet.hh" 565794SN/A#include "mem/packet_access.hh" 575794SN/A#include "params/CopyEngine.hh" 585794SN/A#include "sim/stats.hh" 595794SN/A#include "sim/system.hh" 605794SN/A 615794SN/Ausing namespace CopyEngineReg; 625794SN/A 635794SN/ACopyEngine::CopyEngine(const Params *p) 649807SN/A : PciDevice(p) 655794SN/A{ 665794SN/A // All Reg regs are initialized to 0 by default 675794SN/A regs.chanCount = p->ChanCnt; 685794SN/A regs.xferCap = findMsbSet(p->XferCap); 695794SN/A regs.attnStatus = 0; 705794SN/A 715794SN/A if (regs.chanCount > 64) 725794SN/A fatal("CopyEngine interface doesn't support more than 64 DMA engines\n"); 735794SN/A 745794SN/A for (int x = 0; x < regs.chanCount; x++) { 755794SN/A CopyEngineChannel *ch = new CopyEngineChannel(this, x); 765794SN/A chan.push_back(ch); 775794SN/A } 785794SN/A} 795794SN/A 805794SN/A 815794SN/ACopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine *_ce, int cid) 829165SN/A : cePort(_ce, _ce->sys), 838851SN/A ce(_ce), channelId(cid), busy(false), underReset(false), 8412087Sspwilson2@wisc.edu refreshNext(false), latBeforeBegin(ce->params()->latBeforeBegin), 8512087Sspwilson2@wisc.edu latAfterCompletion(ce->params()->latAfterCompletion), 8612087Sspwilson2@wisc.edu completionDataReg(0), nextState(Idle), 8712087Sspwilson2@wisc.edu fetchCompleteEvent([this]{ fetchDescComplete(); }, name()), 8812087Sspwilson2@wisc.edu addrCompleteEvent([this]{ fetchAddrComplete(); }, name()), 8912087Sspwilson2@wisc.edu readCompleteEvent([this]{ readCopyBytesComplete(); }, name()), 9012087Sspwilson2@wisc.edu writeCompleteEvent([this]{ writeCopyBytesComplete(); }, name()), 9112087Sspwilson2@wisc.edu statusCompleteEvent([this]{ writeStatusComplete(); }, name()) 925794SN/A 935794SN/A{ 945794SN/A cr.status.dma_transfer_status(3); 955794SN/A cr.descChainAddr = 0; 965794SN/A cr.completionAddr = 0; 975794SN/A 985794SN/A curDmaDesc = new DmaDesc; 995794SN/A memset(curDmaDesc, 0, sizeof(DmaDesc)); 1005794SN/A copyBuffer = new uint8_t[ce->params()->XferCap]; 1015794SN/A} 1025794SN/A 1035794SN/ACopyEngine::~CopyEngine() 1045794SN/A{ 1055794SN/A for (int x = 0; x < chan.size(); x++) { 1065794SN/A delete chan[x]; 1075794SN/A } 1085794SN/A} 1095794SN/A 1105794SN/ACopyEngine::CopyEngineChannel::~CopyEngineChannel() 1115794SN/A{ 1125794SN/A delete curDmaDesc; 1135794SN/A delete [] copyBuffer; 1145794SN/A} 1155794SN/A 1169294SN/ABaseMasterPort & 1179294SN/ACopyEngine::getMasterPort(const std::string &if_name, PortID idx) 1185794SN/A{ 1198922SN/A if (if_name != "dma") { 1208922SN/A // pass it along to our super class 1219807SN/A return PciDevice::getMasterPort(if_name, idx); 1228922SN/A } else { 1238922SN/A if (idx >= static_cast<int>(chan.size())) { 1248922SN/A panic("CopyEngine::getMasterPort: unknown index %d\n", idx); 1258922SN/A } 1268922SN/A 1278922SN/A return chan[idx]->getMasterPort(); 1288841SN/A } 1295794SN/A} 1305794SN/A 1318841SN/A 1329294SN/ABaseMasterPort & 1338922SN/ACopyEngine::CopyEngineChannel::getMasterPort() 1345794SN/A{ 1358922SN/A return cePort; 1365794SN/A} 1375794SN/A 1385794SN/Avoid 1395794SN/ACopyEngine::CopyEngineChannel::recvCommand() 1405794SN/A{ 1415794SN/A if (cr.command.start_dma()) { 1425794SN/A assert(!busy); 1435794SN/A cr.status.dma_transfer_status(0); 1445794SN/A nextState = DescriptorFetch; 1455794SN/A fetchAddress = cr.descChainAddr; 14610913SN/A if (ce->drainState() == DrainState::Running) 1475794SN/A fetchDescriptor(cr.descChainAddr); 1485794SN/A } else if (cr.command.append_dma()) { 1495794SN/A if (!busy) { 1505794SN/A nextState = AddressFetch; 15110913SN/A if (ce->drainState() == DrainState::Running) 1525794SN/A fetchNextAddr(lastDescriptorAddr); 1535794SN/A } else 1545794SN/A refreshNext = true; 1555794SN/A } else if (cr.command.reset_dma()) { 1565794SN/A if (busy) 1575794SN/A underReset = true; 1585794SN/A else { 1595794SN/A cr.status.dma_transfer_status(3); 1605794SN/A nextState = Idle; 1615794SN/A } 1625794SN/A } else if (cr.command.resume_dma() || cr.command.abort_dma() || 1635794SN/A cr.command.suspend_dma()) 1645794SN/A panic("Resume, Abort, and Suspend are not supported\n"); 1655794SN/A cr.command(0); 1665794SN/A} 1675794SN/A 1685794SN/ATick 1695794SN/ACopyEngine::read(PacketPtr pkt) 1705794SN/A{ 1715794SN/A int bar; 1725794SN/A Addr daddr; 1735794SN/A 1745794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 1755794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 1765794SN/A 1775794SN/A // Only Memory register BAR is allowed 1785794SN/A assert(bar == 0); 1795794SN/A 1805794SN/A int size = pkt->getSize(); 1815794SN/A if (size != sizeof(uint64_t) && size != sizeof(uint32_t) && 1825794SN/A size != sizeof(uint16_t) && size != sizeof(uint8_t)) { 1835794SN/A panic("Unknown size for MMIO access: %d\n", pkt->getSize()); 1845794SN/A } 1855794SN/A 1865794SN/A DPRINTF(DMACopyEngine, "Read device register %#X size: %d\n", daddr, size); 1875794SN/A 1885794SN/A /// 1895794SN/A /// Handle read of register here 1905794SN/A /// 1915794SN/A 1925794SN/A if (daddr < 0x80) { 1935794SN/A switch (daddr) { 1945794SN/A case GEN_CHANCOUNT: 1955794SN/A assert(size == sizeof(regs.chanCount)); 1965794SN/A pkt->set<uint8_t>(regs.chanCount); 1975794SN/A break; 1985794SN/A case GEN_XFERCAP: 1995794SN/A assert(size == sizeof(regs.xferCap)); 2005794SN/A pkt->set<uint8_t>(regs.xferCap); 2015794SN/A break; 2025794SN/A case GEN_INTRCTRL: 2035794SN/A assert(size == sizeof(uint8_t)); 2045794SN/A pkt->set<uint8_t>(regs.intrctrl()); 2055794SN/A regs.intrctrl.master_int_enable(0); 2065794SN/A break; 2075794SN/A case GEN_ATTNSTATUS: 2085794SN/A assert(size == sizeof(regs.attnStatus)); 2095794SN/A pkt->set<uint32_t>(regs.attnStatus); 2105794SN/A regs.attnStatus = 0; 2115794SN/A break; 2125794SN/A default: 2135794SN/A panic("Read request to unknown register number: %#x\n", daddr); 2145794SN/A } 2155794SN/A pkt->makeAtomicResponse(); 2165794SN/A return pioDelay; 2175794SN/A } 2185794SN/A 2195794SN/A 2205794SN/A // Find which channel we're accessing 2215794SN/A int chanid = 0; 2225794SN/A daddr -= 0x80; 2235794SN/A while (daddr >= 0x80) { 2245794SN/A chanid++; 2255794SN/A daddr -= 0x80; 2265794SN/A } 2275794SN/A 2285794SN/A if (chanid >= regs.chanCount) 2295794SN/A panic("Access to channel %d (device only configured for %d channels)", 2305794SN/A chanid, regs.chanCount); 2315794SN/A 2325794SN/A /// 2335794SN/A /// Channel registers are handled here 2345794SN/A /// 2355794SN/A chan[chanid]->channelRead(pkt, daddr, size); 2365794SN/A 2375794SN/A pkt->makeAtomicResponse(); 2385794SN/A return pioDelay; 2395794SN/A} 2405794SN/A 2415794SN/Avoid 2425794SN/ACopyEngine::CopyEngineChannel::channelRead(Packet *pkt, Addr daddr, int size) 2435794SN/A{ 2445794SN/A switch (daddr) { 2455794SN/A case CHAN_CONTROL: 2465794SN/A assert(size == sizeof(uint16_t)); 2475794SN/A pkt->set<uint16_t>(cr.ctrl()); 2485794SN/A cr.ctrl.in_use(1); 2495794SN/A break; 2505794SN/A case CHAN_STATUS: 2515794SN/A assert(size == sizeof(uint64_t)); 2525794SN/A pkt->set<uint64_t>(cr.status() | ~busy); 2535794SN/A break; 2545794SN/A case CHAN_CHAINADDR: 2555794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2565794SN/A if (size == sizeof(uint64_t)) 2575794SN/A pkt->set<uint64_t>(cr.descChainAddr); 2585794SN/A else 2595794SN/A pkt->set<uint32_t>(bits(cr.descChainAddr,0,31)); 2605794SN/A break; 2615794SN/A case CHAN_CHAINADDR_HIGH: 2625794SN/A assert(size == sizeof(uint32_t)); 2635794SN/A pkt->set<uint32_t>(bits(cr.descChainAddr,32,63)); 2645794SN/A break; 2655794SN/A case CHAN_COMMAND: 2665794SN/A assert(size == sizeof(uint8_t)); 2675794SN/A pkt->set<uint32_t>(cr.command()); 2685794SN/A break; 2695794SN/A case CHAN_CMPLNADDR: 2705794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2715794SN/A if (size == sizeof(uint64_t)) 2725794SN/A pkt->set<uint64_t>(cr.completionAddr); 2735794SN/A else 2745794SN/A pkt->set<uint32_t>(bits(cr.completionAddr,0,31)); 2755794SN/A break; 2765794SN/A case CHAN_CMPLNADDR_HIGH: 2775794SN/A assert(size == sizeof(uint32_t)); 2785794SN/A pkt->set<uint32_t>(bits(cr.completionAddr,32,63)); 2795794SN/A break; 2805794SN/A case CHAN_ERROR: 2815794SN/A assert(size == sizeof(uint32_t)); 2825794SN/A pkt->set<uint32_t>(cr.error()); 2835794SN/A break; 2845794SN/A default: 2855794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 2865794SN/A channelId, daddr); 2875794SN/A } 2885794SN/A} 2895794SN/A 2905794SN/A 2915794SN/ATick 2925794SN/ACopyEngine::write(PacketPtr pkt) 2935794SN/A{ 2945794SN/A int bar; 2955794SN/A Addr daddr; 2965794SN/A 2975794SN/A 2985794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 2995794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 3005794SN/A 3015794SN/A // Only Memory register BAR is allowed 3025794SN/A assert(bar == 0); 3035794SN/A 3045794SN/A int size = pkt->getSize(); 3055794SN/A 3065794SN/A /// 3075794SN/A /// Handle write of register here 3085794SN/A /// 3095794SN/A 3105794SN/A if (size == sizeof(uint64_t)) { 3115794SN/A uint64_t val M5_VAR_USED = pkt->get<uint64_t>(); 3125794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3135794SN/A } else if (size == sizeof(uint32_t)) { 3145794SN/A uint32_t val M5_VAR_USED = pkt->get<uint32_t>(); 3155794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3165794SN/A } else if (size == sizeof(uint16_t)) { 3175794SN/A uint16_t val M5_VAR_USED = pkt->get<uint16_t>(); 3185794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3195794SN/A } else if (size == sizeof(uint8_t)) { 3205794SN/A uint8_t val M5_VAR_USED = pkt->get<uint8_t>(); 3215794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3225794SN/A } else { 3235794SN/A panic("Unknown size for MMIO access: %d\n", size); 3245794SN/A } 3255794SN/A 3265794SN/A if (daddr < 0x80) { 3275794SN/A switch (daddr) { 3285794SN/A case GEN_CHANCOUNT: 3295794SN/A case GEN_XFERCAP: 3305794SN/A case GEN_ATTNSTATUS: 3315794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3325794SN/A daddr); 3335794SN/A break; 3345794SN/A case GEN_INTRCTRL: 3355794SN/A regs.intrctrl.master_int_enable(bits(pkt->get<uint8_t>(),0,1)); 3365794SN/A break; 3375794SN/A default: 3385794SN/A panic("Read request to unknown register number: %#x\n", daddr); 3395794SN/A } 3405794SN/A pkt->makeAtomicResponse(); 3415794SN/A return pioDelay; 3425794SN/A } 3435794SN/A 3445794SN/A // Find which channel we're accessing 3455794SN/A int chanid = 0; 3465794SN/A daddr -= 0x80; 3475794SN/A while (daddr >= 0x80) { 3485794SN/A chanid++; 3495794SN/A daddr -= 0x80; 3505794SN/A } 3515794SN/A 3525794SN/A if (chanid >= regs.chanCount) 3535794SN/A panic("Access to channel %d (device only configured for %d channels)", 3545794SN/A chanid, regs.chanCount); 3555794SN/A 3565794SN/A /// 3575794SN/A /// Channel registers are handled here 3585794SN/A /// 3595794SN/A chan[chanid]->channelWrite(pkt, daddr, size); 3605794SN/A 3615794SN/A pkt->makeAtomicResponse(); 3625794SN/A return pioDelay; 3635794SN/A} 3645794SN/A 3655794SN/Avoid 3665794SN/ACopyEngine::CopyEngineChannel::channelWrite(Packet *pkt, Addr daddr, int size) 3675794SN/A{ 3685794SN/A switch (daddr) { 3695794SN/A case CHAN_CONTROL: 3705794SN/A assert(size == sizeof(uint16_t)); 3715794SN/A int old_int_disable; 3725794SN/A old_int_disable = cr.ctrl.interrupt_disable(); 3735794SN/A cr.ctrl(pkt->get<uint16_t>()); 3745794SN/A if (cr.ctrl.interrupt_disable()) 3755794SN/A cr.ctrl.interrupt_disable(0); 3765794SN/A else 3775794SN/A cr.ctrl.interrupt_disable(old_int_disable); 3785794SN/A break; 3795794SN/A case CHAN_STATUS: 3805794SN/A assert(size == sizeof(uint64_t)); 3815794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3825794SN/A daddr); 3835794SN/A break; 3845794SN/A case CHAN_CHAINADDR: 3855794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 3865794SN/A if (size == sizeof(uint64_t)) 3875794SN/A cr.descChainAddr = pkt->get<uint64_t>(); 3885794SN/A else 3895794SN/A cr.descChainAddr = (uint64_t)pkt->get<uint32_t>() | 3905794SN/A (cr.descChainAddr & ~mask(32)); 3915794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 3925794SN/A break; 3935794SN/A case CHAN_CHAINADDR_HIGH: 3945794SN/A assert(size == sizeof(uint32_t)); 3955794SN/A cr.descChainAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 3965794SN/A (cr.descChainAddr & mask(32)); 3975794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 3985794SN/A break; 3995794SN/A case CHAN_COMMAND: 4005794SN/A assert(size == sizeof(uint8_t)); 4015794SN/A cr.command(pkt->get<uint8_t>()); 4025794SN/A recvCommand(); 4035794SN/A break; 4045794SN/A case CHAN_CMPLNADDR: 4055794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 4065794SN/A if (size == sizeof(uint64_t)) 4075794SN/A cr.completionAddr = pkt->get<uint64_t>(); 4085794SN/A else 4095794SN/A cr.completionAddr = pkt->get<uint32_t>() | 4105794SN/A (cr.completionAddr & ~mask(32)); 4115794SN/A break; 4125794SN/A case CHAN_CMPLNADDR_HIGH: 4135794SN/A assert(size == sizeof(uint32_t)); 4145794SN/A cr.completionAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 4155794SN/A (cr.completionAddr & mask(32)); 4165794SN/A break; 4175794SN/A case CHAN_ERROR: 4185794SN/A assert(size == sizeof(uint32_t)); 4195794SN/A cr.error(~pkt->get<uint32_t>() & cr.error()); 4205794SN/A break; 4215794SN/A default: 4225794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 4235794SN/A channelId, daddr); 4245794SN/A } 4255794SN/A} 4265794SN/A 4275794SN/Avoid 4285794SN/ACopyEngine::regStats() 4295794SN/A{ 43011522Sstephan.diestelhorst@arm.com PciDevice::regStats(); 43111522Sstephan.diestelhorst@arm.com 4325794SN/A using namespace Stats; 4335794SN/A bytesCopied 4345794SN/A .init(regs.chanCount) 4355794SN/A .name(name() + ".bytes_copied") 4365794SN/A .desc("Number of bytes copied by each engine") 4375794SN/A .flags(total) 4385794SN/A ; 4395794SN/A copiesProcessed 4405794SN/A .init(regs.chanCount) 4415794SN/A .name(name() + ".copies_processed") 4425794SN/A .desc("Number of copies processed by each engine") 4435794SN/A .flags(total) 4445794SN/A ; 4455794SN/A} 4465794SN/A 4475794SN/Avoid 4485794SN/ACopyEngine::CopyEngineChannel::fetchDescriptor(Addr address) 4495794SN/A{ 4505954SN/A anDq(); 4515954SN/A anBegin("FetchDescriptor"); 4525794SN/A DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n", 45311202SN/A address, ce->pciToDma(address)); 4545794SN/A assert(address); 4555794SN/A busy = true; 4565794SN/A 4575794SN/A DPRINTF(DMACopyEngine, "dmaAction: %#x, %d bytes, to addr %#x\n", 45811202SN/A ce->pciToDma(address), sizeof(DmaDesc), curDmaDesc); 4595794SN/A 46011202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(address), 4618851SN/A sizeof(DmaDesc), &fetchCompleteEvent, 4628851SN/A (uint8_t*)curDmaDesc, latBeforeBegin); 4635794SN/A lastDescriptorAddr = address; 4645794SN/A} 4655794SN/A 4665794SN/Avoid 4675794SN/ACopyEngine::CopyEngineChannel::fetchDescComplete() 4685794SN/A{ 4695794SN/A DPRINTF(DMACopyEngine, "Read of descriptor complete\n"); 4705794SN/A 4715794SN/A if ((curDmaDesc->command & DESC_CTRL_NULL)) { 4725794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, skipping\n"); 4735794SN/A assert(!(curDmaDesc->command & DESC_CTRL_CP_STS)); 4745794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 4755794SN/A panic("Shouldn't be able to get here\n"); 4765794SN/A nextState = CompletionWrite; 4775794SN/A if (inDrain()) return; 4785794SN/A writeCompletionStatus(); 4795794SN/A } else { 4805954SN/A anBegin("Idle"); 4815954SN/A anWait(); 4825794SN/A busy = false; 4835794SN/A nextState = Idle; 4845794SN/A inDrain(); 4855794SN/A } 4865794SN/A return; 4875794SN/A } 4885794SN/A 4895794SN/A if (curDmaDesc->command & ~DESC_CTRL_CP_STS) 4905794SN/A panic("Descriptor has flag other that completion status set\n"); 4915794SN/A 4925794SN/A nextState = DMARead; 4935794SN/A if (inDrain()) return; 4945794SN/A readCopyBytes(); 4955794SN/A} 4965794SN/A 4975794SN/Avoid 4985794SN/ACopyEngine::CopyEngineChannel::readCopyBytes() 4995794SN/A{ 5005954SN/A anBegin("ReadCopyBytes"); 5015794SN/A DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n", 5025794SN/A curDmaDesc->len, curDmaDesc->dest, 50311202SN/A ce->pciToDma(curDmaDesc->src)); 50411202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(curDmaDesc->src), 5058851SN/A curDmaDesc->len, &readCompleteEvent, copyBuffer, 0); 5065794SN/A} 5075794SN/A 5085794SN/Avoid 5095794SN/ACopyEngine::CopyEngineChannel::readCopyBytesComplete() 5105794SN/A{ 5115794SN/A DPRINTF(DMACopyEngine, "Read of bytes to copy complete\n"); 5125794SN/A 5135794SN/A nextState = DMAWrite; 5145794SN/A if (inDrain()) return; 5155794SN/A writeCopyBytes(); 5165794SN/A} 5175794SN/A 5185794SN/Avoid 5195794SN/ACopyEngine::CopyEngineChannel::writeCopyBytes() 5205794SN/A{ 5215954SN/A anBegin("WriteCopyBytes"); 5225794SN/A DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n", 5235794SN/A curDmaDesc->len, curDmaDesc->dest, 52411202SN/A ce->pciToDma(curDmaDesc->dest)); 5255794SN/A 52611202SN/A cePort.dmaAction(MemCmd::WriteReq, ce->pciToDma(curDmaDesc->dest), 5278851SN/A curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0); 5285794SN/A 5295794SN/A ce->bytesCopied[channelId] += curDmaDesc->len; 5305794SN/A ce->copiesProcessed[channelId]++; 5315794SN/A} 5325794SN/A 5335794SN/Avoid 5345794SN/ACopyEngine::CopyEngineChannel::writeCopyBytesComplete() 5355794SN/A{ 5365794SN/A DPRINTF(DMACopyEngine, "Write of bytes to copy complete user1: %#x\n", 5375794SN/A curDmaDesc->user1); 5385794SN/A 5395794SN/A cr.status.compl_desc_addr(lastDescriptorAddr >> 6); 5405794SN/A completionDataReg = cr.status() | 1; 5415794SN/A 5425954SN/A anQ("DMAUsedDescQ", channelId, 1); 5435954SN/A anQ("AppRecvQ", curDmaDesc->user1, curDmaDesc->len); 5445794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 5455794SN/A nextState = CompletionWrite; 5465794SN/A if (inDrain()) return; 5475794SN/A writeCompletionStatus(); 5485794SN/A return; 5495794SN/A } 5505794SN/A 5515794SN/A continueProcessing(); 5525794SN/A} 5535794SN/A 5545794SN/Avoid 5555794SN/ACopyEngine::CopyEngineChannel::continueProcessing() 5565794SN/A{ 5575794SN/A busy = false; 5585794SN/A 5595794SN/A if (underReset) { 5605954SN/A anBegin("Reset"); 5615954SN/A anWait(); 5625794SN/A underReset = false; 5635794SN/A refreshNext = false; 5645794SN/A busy = false; 5655794SN/A nextState = Idle; 5665794SN/A return; 5675794SN/A } 5685794SN/A 5695794SN/A if (curDmaDesc->next) { 5705794SN/A nextState = DescriptorFetch; 5715794SN/A fetchAddress = curDmaDesc->next; 5725794SN/A if (inDrain()) return; 5735794SN/A fetchDescriptor(curDmaDesc->next); 5745794SN/A } else if (refreshNext) { 5755794SN/A nextState = AddressFetch; 5765794SN/A refreshNext = false; 5775794SN/A if (inDrain()) return; 5785794SN/A fetchNextAddr(lastDescriptorAddr); 5795794SN/A } else { 5805794SN/A inDrain(); 5815794SN/A nextState = Idle; 5825954SN/A anWait(); 5835954SN/A anBegin("Idle"); 5845794SN/A } 5855794SN/A} 5865794SN/A 5875794SN/Avoid 5885794SN/ACopyEngine::CopyEngineChannel::writeCompletionStatus() 5895794SN/A{ 5905954SN/A anBegin("WriteCompletionStatus"); 5915794SN/A DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n", 5925794SN/A completionDataReg, cr.completionAddr, 59311202SN/A ce->pciToDma(cr.completionAddr)); 5945794SN/A 5958851SN/A cePort.dmaAction(MemCmd::WriteReq, 59611202SN/A ce->pciToDma(cr.completionAddr), 5978851SN/A sizeof(completionDataReg), &statusCompleteEvent, 5988851SN/A (uint8_t*)&completionDataReg, latAfterCompletion); 5995794SN/A} 6005794SN/A 6015794SN/Avoid 6025794SN/ACopyEngine::CopyEngineChannel::writeStatusComplete() 6035794SN/A{ 6045794SN/A DPRINTF(DMACopyEngine, "Writing completion status complete\n"); 6055794SN/A continueProcessing(); 6065794SN/A} 6075794SN/A 6085794SN/Avoid 6095794SN/ACopyEngine::CopyEngineChannel::fetchNextAddr(Addr address) 6105794SN/A{ 6115954SN/A anBegin("FetchNextAddr"); 6125794SN/A DPRINTF(DMACopyEngine, "Fetching next address...\n"); 6135794SN/A busy = true; 6148851SN/A cePort.dmaAction(MemCmd::ReadReq, 61511202SN/A ce->pciToDma(address + offsetof(DmaDesc, next)), 6168851SN/A sizeof(Addr), &addrCompleteEvent, 6178851SN/A (uint8_t*)curDmaDesc + offsetof(DmaDesc, next), 0); 6185794SN/A} 6195794SN/A 6205794SN/Avoid 6215794SN/ACopyEngine::CopyEngineChannel::fetchAddrComplete() 6225794SN/A{ 6235794SN/A DPRINTF(DMACopyEngine, "Fetching next address complete: %#x\n", 6245794SN/A curDmaDesc->next); 6255794SN/A if (!curDmaDesc->next) { 6265794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n"); 6275794SN/A busy = false; 6285794SN/A nextState = Idle; 6295954SN/A anWait(); 6305954SN/A anBegin("Idle"); 6315794SN/A inDrain(); 6325794SN/A return; 6335794SN/A } 6345794SN/A nextState = DescriptorFetch; 6355794SN/A fetchAddress = curDmaDesc->next; 6365794SN/A if (inDrain()) return; 6375794SN/A fetchDescriptor(curDmaDesc->next); 6385794SN/A} 6395794SN/A 6405794SN/Abool 6415794SN/ACopyEngine::CopyEngineChannel::inDrain() 6425794SN/A{ 64310913SN/A if (drainState() == DrainState::Draining) { 6449152SN/A DPRINTF(Drain, "CopyEngine done draining, processing drain event\n"); 64510913SN/A signalDrainDone(); 6465794SN/A } 6475794SN/A 64810913SN/A return ce->drainState() != DrainState::Running; 6495794SN/A} 6505794SN/A 65110913SN/ADrainState 65210913SN/ACopyEngine::CopyEngineChannel::drain() 6535794SN/A{ 65410913SN/A if (nextState == Idle || ce->drainState() != DrainState::Running) { 65510913SN/A return DrainState::Drained; 65610913SN/A } else { 65710913SN/A DPRINTF(Drain, "CopyEngineChannel not drained\n"); 65810913SN/A return DrainState::Draining; 65910913SN/A } 6605794SN/A} 6615794SN/A 6625794SN/Avoid 66310905SN/ACopyEngine::serialize(CheckpointOut &cp) const 6645794SN/A{ 66510905SN/A PciDevice::serialize(cp); 66610905SN/A regs.serialize(cp); 66710905SN/A for (int x =0; x < chan.size(); x++) 66810905SN/A chan[x]->serializeSection(cp, csprintf("channel%d", x)); 6695794SN/A} 6705794SN/A 6715794SN/Avoid 67210905SN/ACopyEngine::unserialize(CheckpointIn &cp) 6735794SN/A{ 67410905SN/A PciDevice::unserialize(cp); 67510905SN/A regs.unserialize(cp); 6765794SN/A for (int x = 0; x < chan.size(); x++) 67710905SN/A chan[x]->unserializeSection(cp, csprintf("channel%d", x)); 6785794SN/A} 6795794SN/A 6805794SN/Avoid 68110905SN/ACopyEngine::CopyEngineChannel::serialize(CheckpointOut &cp) const 6825794SN/A{ 6835794SN/A SERIALIZE_SCALAR(channelId); 6845794SN/A SERIALIZE_SCALAR(busy); 6855794SN/A SERIALIZE_SCALAR(underReset); 6865794SN/A SERIALIZE_SCALAR(refreshNext); 6875794SN/A SERIALIZE_SCALAR(lastDescriptorAddr); 6885794SN/A SERIALIZE_SCALAR(completionDataReg); 6895794SN/A SERIALIZE_SCALAR(fetchAddress); 6905794SN/A int nextState = this->nextState; 6915794SN/A SERIALIZE_SCALAR(nextState); 69210905SN/A arrayParamOut(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 6935794SN/A SERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 69410905SN/A cr.serialize(cp); 6955794SN/A 6965794SN/A} 6975794SN/Avoid 69810905SN/ACopyEngine::CopyEngineChannel::unserialize(CheckpointIn &cp) 6995794SN/A{ 7005794SN/A UNSERIALIZE_SCALAR(channelId); 7015794SN/A UNSERIALIZE_SCALAR(busy); 7025794SN/A UNSERIALIZE_SCALAR(underReset); 7035794SN/A UNSERIALIZE_SCALAR(refreshNext); 7045794SN/A UNSERIALIZE_SCALAR(lastDescriptorAddr); 7055794SN/A UNSERIALIZE_SCALAR(completionDataReg); 7065794SN/A UNSERIALIZE_SCALAR(fetchAddress); 7075794SN/A int nextState; 7085794SN/A UNSERIALIZE_SCALAR(nextState); 7095794SN/A this->nextState = (ChannelState)nextState; 71010905SN/A arrayParamIn(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 7115794SN/A UNSERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 71210905SN/A cr.unserialize(cp); 7135794SN/A 7145794SN/A} 7155794SN/A 7165794SN/Avoid 7175794SN/ACopyEngine::CopyEngineChannel::restartStateMachine() 7185794SN/A{ 7195794SN/A switch(nextState) { 7205794SN/A case AddressFetch: 7215794SN/A fetchNextAddr(lastDescriptorAddr); 7225794SN/A break; 7235794SN/A case DescriptorFetch: 7245794SN/A fetchDescriptor(fetchAddress); 7255794SN/A break; 7265794SN/A case DMARead: 7275794SN/A readCopyBytes(); 7285794SN/A break; 7295794SN/A case DMAWrite: 7305794SN/A writeCopyBytes(); 7315794SN/A break; 7325794SN/A case CompletionWrite: 7335794SN/A writeCompletionStatus(); 7345794SN/A break; 7355794SN/A case Idle: 7365794SN/A break; 7375794SN/A default: 7385794SN/A panic("Unknown state for CopyEngineChannel\n"); 7395794SN/A } 7405794SN/A} 7415794SN/A 7425794SN/Avoid 7439342SN/ACopyEngine::CopyEngineChannel::drainResume() 7445794SN/A{ 7455794SN/A DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState); 7465794SN/A restartStateMachine(); 7475794SN/A} 7485794SN/A 7495794SN/ACopyEngine * 7505794SN/ACopyEngineParams::create() 7515794SN/A{ 7525794SN/A return new CopyEngine(this); 7535794SN/A} 754