15794SN/A/* 28841SN/A * Copyright (c) 2012 ARM Limited 38841SN/A * All rights reserved 48841SN/A * 58841SN/A * The license below extends only to copyright in the software and shall 68841SN/A * not be construed as granting a license to any other intellectual 78841SN/A * property including but not limited to intellectual property relating 88841SN/A * to a hardware implementation of the functionality of the software 98841SN/A * licensed hereunder. You may use the software subject to the license 108841SN/A * terms below provided that you ensure that this notice is replicated 118841SN/A * unmodified and in its entirety in all distributions of the software, 128841SN/A * modified or unmodified, in source code or in binary form. 138841SN/A * 145794SN/A * Copyright (c) 2008 The Regents of The University of Michigan 155794SN/A * All rights reserved. 165794SN/A * 175794SN/A * Redistribution and use in source and binary forms, with or without 185794SN/A * modification, are permitted provided that the following conditions are 195794SN/A * met: redistributions of source code must retain the above copyright 205794SN/A * notice, this list of conditions and the following disclaimer; 215794SN/A * redistributions in binary form must reproduce the above copyright 225794SN/A * notice, this list of conditions and the following disclaimer in the 235794SN/A * documentation and/or other materials provided with the distribution; 245794SN/A * neither the name of the copyright holders nor the names of its 255794SN/A * contributors may be used to endorse or promote products derived from 265794SN/A * this software without specific prior written permission. 275794SN/A * 285794SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 295794SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 305794SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 315794SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 325794SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 335794SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 345794SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 355794SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 365794SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 375794SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 385794SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 395794SN/A * 405794SN/A * Authors: Ali Saidi 415794SN/A */ 425794SN/A 435794SN/A/* @file 445794SN/A * Device model for Intel's I/O AT DMA copy engine. 455794SN/A */ 465794SN/A 4711261Sandreas.sandberg@arm.com#include "dev/pci/copy_engine.hh" 4811261Sandreas.sandberg@arm.com 495794SN/A#include <algorithm> 505794SN/A 515954SN/A#include "base/cp_annotate.hh" 525794SN/A#include "base/trace.hh" 538232SN/A#include "debug/DMACopyEngine.hh" 549152SN/A#include "debug/Drain.hh" 555794SN/A#include "mem/packet.hh" 565794SN/A#include "mem/packet_access.hh" 575794SN/A#include "params/CopyEngine.hh" 585794SN/A#include "sim/stats.hh" 595794SN/A#include "sim/system.hh" 605794SN/A 615794SN/Ausing namespace CopyEngineReg; 625794SN/A 635794SN/ACopyEngine::CopyEngine(const Params *p) 649807SN/A : PciDevice(p) 655794SN/A{ 665794SN/A // All Reg regs are initialized to 0 by default 675794SN/A regs.chanCount = p->ChanCnt; 685794SN/A regs.xferCap = findMsbSet(p->XferCap); 695794SN/A regs.attnStatus = 0; 705794SN/A 715794SN/A if (regs.chanCount > 64) 725794SN/A fatal("CopyEngine interface doesn't support more than 64 DMA engines\n"); 735794SN/A 745794SN/A for (int x = 0; x < regs.chanCount; x++) { 755794SN/A CopyEngineChannel *ch = new CopyEngineChannel(this, x); 765794SN/A chan.push_back(ch); 775794SN/A } 785794SN/A} 795794SN/A 805794SN/A 815794SN/ACopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine *_ce, int cid) 829165SN/A : cePort(_ce, _ce->sys), 838851SN/A ce(_ce), channelId(cid), busy(false), underReset(false), 8412087Sspwilson2@wisc.edu refreshNext(false), latBeforeBegin(ce->params()->latBeforeBegin), 8512087Sspwilson2@wisc.edu latAfterCompletion(ce->params()->latAfterCompletion), 8612087Sspwilson2@wisc.edu completionDataReg(0), nextState(Idle), 8712087Sspwilson2@wisc.edu fetchCompleteEvent([this]{ fetchDescComplete(); }, name()), 8812087Sspwilson2@wisc.edu addrCompleteEvent([this]{ fetchAddrComplete(); }, name()), 8912087Sspwilson2@wisc.edu readCompleteEvent([this]{ readCopyBytesComplete(); }, name()), 9012087Sspwilson2@wisc.edu writeCompleteEvent([this]{ writeCopyBytesComplete(); }, name()), 9112087Sspwilson2@wisc.edu statusCompleteEvent([this]{ writeStatusComplete(); }, name()) 925794SN/A 935794SN/A{ 945794SN/A cr.status.dma_transfer_status(3); 955794SN/A cr.descChainAddr = 0; 965794SN/A cr.completionAddr = 0; 975794SN/A 985794SN/A curDmaDesc = new DmaDesc; 995794SN/A memset(curDmaDesc, 0, sizeof(DmaDesc)); 1005794SN/A copyBuffer = new uint8_t[ce->params()->XferCap]; 1015794SN/A} 1025794SN/A 1035794SN/ACopyEngine::~CopyEngine() 1045794SN/A{ 1055794SN/A for (int x = 0; x < chan.size(); x++) { 1065794SN/A delete chan[x]; 1075794SN/A } 1085794SN/A} 1095794SN/A 1105794SN/ACopyEngine::CopyEngineChannel::~CopyEngineChannel() 1115794SN/A{ 1125794SN/A delete curDmaDesc; 1135794SN/A delete [] copyBuffer; 1145794SN/A} 1155794SN/A 11613784Sgabeblack@google.comPort & 11713784Sgabeblack@google.comCopyEngine::getPort(const std::string &if_name, PortID idx) 1185794SN/A{ 1198922SN/A if (if_name != "dma") { 1208922SN/A // pass it along to our super class 12113784Sgabeblack@google.com return PciDevice::getPort(if_name, idx); 1228922SN/A } else { 1238922SN/A if (idx >= static_cast<int>(chan.size())) { 12413784Sgabeblack@google.com panic("CopyEngine::getPort: unknown index %d\n", idx); 1258922SN/A } 1268922SN/A 12713784Sgabeblack@google.com return chan[idx]->getPort(); 1288841SN/A } 1295794SN/A} 1305794SN/A 1318841SN/A 13213784Sgabeblack@google.comPort & 13313784Sgabeblack@google.comCopyEngine::CopyEngineChannel::getPort() 1345794SN/A{ 1358922SN/A return cePort; 1365794SN/A} 1375794SN/A 1385794SN/Avoid 1395794SN/ACopyEngine::CopyEngineChannel::recvCommand() 1405794SN/A{ 1415794SN/A if (cr.command.start_dma()) { 1425794SN/A assert(!busy); 1435794SN/A cr.status.dma_transfer_status(0); 1445794SN/A nextState = DescriptorFetch; 1455794SN/A fetchAddress = cr.descChainAddr; 14610913SN/A if (ce->drainState() == DrainState::Running) 1475794SN/A fetchDescriptor(cr.descChainAddr); 1485794SN/A } else if (cr.command.append_dma()) { 1495794SN/A if (!busy) { 1505794SN/A nextState = AddressFetch; 15110913SN/A if (ce->drainState() == DrainState::Running) 1525794SN/A fetchNextAddr(lastDescriptorAddr); 1535794SN/A } else 1545794SN/A refreshNext = true; 1555794SN/A } else if (cr.command.reset_dma()) { 1565794SN/A if (busy) 1575794SN/A underReset = true; 1585794SN/A else { 1595794SN/A cr.status.dma_transfer_status(3); 1605794SN/A nextState = Idle; 1615794SN/A } 1625794SN/A } else if (cr.command.resume_dma() || cr.command.abort_dma() || 1635794SN/A cr.command.suspend_dma()) 1645794SN/A panic("Resume, Abort, and Suspend are not supported\n"); 1655794SN/A cr.command(0); 1665794SN/A} 1675794SN/A 1685794SN/ATick 1695794SN/ACopyEngine::read(PacketPtr pkt) 1705794SN/A{ 1715794SN/A int bar; 1725794SN/A Addr daddr; 1735794SN/A 1745794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 1755794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 1765794SN/A 1775794SN/A // Only Memory register BAR is allowed 1785794SN/A assert(bar == 0); 1795794SN/A 1805794SN/A int size = pkt->getSize(); 1815794SN/A if (size != sizeof(uint64_t) && size != sizeof(uint32_t) && 1825794SN/A size != sizeof(uint16_t) && size != sizeof(uint8_t)) { 1835794SN/A panic("Unknown size for MMIO access: %d\n", pkt->getSize()); 1845794SN/A } 1855794SN/A 1865794SN/A DPRINTF(DMACopyEngine, "Read device register %#X size: %d\n", daddr, size); 1875794SN/A 1885794SN/A /// 1895794SN/A /// Handle read of register here 1905794SN/A /// 1915794SN/A 1925794SN/A if (daddr < 0x80) { 1935794SN/A switch (daddr) { 1945794SN/A case GEN_CHANCOUNT: 1955794SN/A assert(size == sizeof(regs.chanCount)); 19613342Sgabeblack@google.com pkt->setLE<uint8_t>(regs.chanCount); 1975794SN/A break; 1985794SN/A case GEN_XFERCAP: 1995794SN/A assert(size == sizeof(regs.xferCap)); 20013342Sgabeblack@google.com pkt->setLE<uint8_t>(regs.xferCap); 2015794SN/A break; 2025794SN/A case GEN_INTRCTRL: 2035794SN/A assert(size == sizeof(uint8_t)); 20413342Sgabeblack@google.com pkt->setLE<uint8_t>(regs.intrctrl()); 2055794SN/A regs.intrctrl.master_int_enable(0); 2065794SN/A break; 2075794SN/A case GEN_ATTNSTATUS: 2085794SN/A assert(size == sizeof(regs.attnStatus)); 20913342Sgabeblack@google.com pkt->setLE<uint32_t>(regs.attnStatus); 2105794SN/A regs.attnStatus = 0; 2115794SN/A break; 2125794SN/A default: 2135794SN/A panic("Read request to unknown register number: %#x\n", daddr); 2145794SN/A } 2155794SN/A pkt->makeAtomicResponse(); 2165794SN/A return pioDelay; 2175794SN/A } 2185794SN/A 2195794SN/A 2205794SN/A // Find which channel we're accessing 2215794SN/A int chanid = 0; 2225794SN/A daddr -= 0x80; 2235794SN/A while (daddr >= 0x80) { 2245794SN/A chanid++; 2255794SN/A daddr -= 0x80; 2265794SN/A } 2275794SN/A 2285794SN/A if (chanid >= regs.chanCount) 2295794SN/A panic("Access to channel %d (device only configured for %d channels)", 2305794SN/A chanid, regs.chanCount); 2315794SN/A 2325794SN/A /// 2335794SN/A /// Channel registers are handled here 2345794SN/A /// 2355794SN/A chan[chanid]->channelRead(pkt, daddr, size); 2365794SN/A 2375794SN/A pkt->makeAtomicResponse(); 2385794SN/A return pioDelay; 2395794SN/A} 2405794SN/A 2415794SN/Avoid 2425794SN/ACopyEngine::CopyEngineChannel::channelRead(Packet *pkt, Addr daddr, int size) 2435794SN/A{ 2445794SN/A switch (daddr) { 2455794SN/A case CHAN_CONTROL: 2465794SN/A assert(size == sizeof(uint16_t)); 24713342Sgabeblack@google.com pkt->setLE<uint16_t>(cr.ctrl()); 2485794SN/A cr.ctrl.in_use(1); 2495794SN/A break; 2505794SN/A case CHAN_STATUS: 2515794SN/A assert(size == sizeof(uint64_t)); 25213342Sgabeblack@google.com pkt->setLE<uint64_t>(cr.status() | (busy ? 0 : 1)); 2535794SN/A break; 2545794SN/A case CHAN_CHAINADDR: 2555794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2565794SN/A if (size == sizeof(uint64_t)) 25713342Sgabeblack@google.com pkt->setLE<uint64_t>(cr.descChainAddr); 2585794SN/A else 25913342Sgabeblack@google.com pkt->setLE<uint32_t>(bits(cr.descChainAddr,0,31)); 2605794SN/A break; 2615794SN/A case CHAN_CHAINADDR_HIGH: 2625794SN/A assert(size == sizeof(uint32_t)); 26313342Sgabeblack@google.com pkt->setLE<uint32_t>(bits(cr.descChainAddr,32,63)); 2645794SN/A break; 2655794SN/A case CHAN_COMMAND: 2665794SN/A assert(size == sizeof(uint8_t)); 26713342Sgabeblack@google.com pkt->setLE<uint32_t>(cr.command()); 2685794SN/A break; 2695794SN/A case CHAN_CMPLNADDR: 2705794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2715794SN/A if (size == sizeof(uint64_t)) 27213342Sgabeblack@google.com pkt->setLE<uint64_t>(cr.completionAddr); 2735794SN/A else 27413342Sgabeblack@google.com pkt->setLE<uint32_t>(bits(cr.completionAddr,0,31)); 2755794SN/A break; 2765794SN/A case CHAN_CMPLNADDR_HIGH: 2775794SN/A assert(size == sizeof(uint32_t)); 27813342Sgabeblack@google.com pkt->setLE<uint32_t>(bits(cr.completionAddr,32,63)); 2795794SN/A break; 2805794SN/A case CHAN_ERROR: 2815794SN/A assert(size == sizeof(uint32_t)); 28213342Sgabeblack@google.com pkt->setLE<uint32_t>(cr.error()); 2835794SN/A break; 2845794SN/A default: 2855794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 2865794SN/A channelId, daddr); 2875794SN/A } 2885794SN/A} 2895794SN/A 2905794SN/A 2915794SN/ATick 2925794SN/ACopyEngine::write(PacketPtr pkt) 2935794SN/A{ 2945794SN/A int bar; 2955794SN/A Addr daddr; 2965794SN/A 2975794SN/A 2985794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 2995794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 3005794SN/A 3015794SN/A // Only Memory register BAR is allowed 3025794SN/A assert(bar == 0); 3035794SN/A 3045794SN/A int size = pkt->getSize(); 3055794SN/A 3065794SN/A /// 3075794SN/A /// Handle write of register here 3085794SN/A /// 3095794SN/A 3105794SN/A if (size == sizeof(uint64_t)) { 31113342Sgabeblack@google.com uint64_t val M5_VAR_USED = pkt->getLE<uint64_t>(); 31213342Sgabeblack@google.com DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", 31313342Sgabeblack@google.com daddr, val); 3145794SN/A } else if (size == sizeof(uint32_t)) { 31513342Sgabeblack@google.com uint32_t val M5_VAR_USED = pkt->getLE<uint32_t>(); 31613342Sgabeblack@google.com DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", 31713342Sgabeblack@google.com daddr, val); 3185794SN/A } else if (size == sizeof(uint16_t)) { 31913342Sgabeblack@google.com uint16_t val M5_VAR_USED = pkt->getLE<uint16_t>(); 32013342Sgabeblack@google.com DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", 32113342Sgabeblack@google.com daddr, val); 3225794SN/A } else if (size == sizeof(uint8_t)) { 32313342Sgabeblack@google.com uint8_t val M5_VAR_USED = pkt->getLE<uint8_t>(); 32413342Sgabeblack@google.com DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", 32513342Sgabeblack@google.com daddr, val); 3265794SN/A } else { 3275794SN/A panic("Unknown size for MMIO access: %d\n", size); 3285794SN/A } 3295794SN/A 3305794SN/A if (daddr < 0x80) { 3315794SN/A switch (daddr) { 3325794SN/A case GEN_CHANCOUNT: 3335794SN/A case GEN_XFERCAP: 3345794SN/A case GEN_ATTNSTATUS: 3355794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3365794SN/A daddr); 3375794SN/A break; 3385794SN/A case GEN_INTRCTRL: 33913342Sgabeblack@google.com regs.intrctrl.master_int_enable(bits(pkt->getLE<uint8_t>(), 0, 1)); 3405794SN/A break; 3415794SN/A default: 3425794SN/A panic("Read request to unknown register number: %#x\n", daddr); 3435794SN/A } 3445794SN/A pkt->makeAtomicResponse(); 3455794SN/A return pioDelay; 3465794SN/A } 3475794SN/A 3485794SN/A // Find which channel we're accessing 3495794SN/A int chanid = 0; 3505794SN/A daddr -= 0x80; 3515794SN/A while (daddr >= 0x80) { 3525794SN/A chanid++; 3535794SN/A daddr -= 0x80; 3545794SN/A } 3555794SN/A 3565794SN/A if (chanid >= regs.chanCount) 3575794SN/A panic("Access to channel %d (device only configured for %d channels)", 3585794SN/A chanid, regs.chanCount); 3595794SN/A 3605794SN/A /// 3615794SN/A /// Channel registers are handled here 3625794SN/A /// 3635794SN/A chan[chanid]->channelWrite(pkt, daddr, size); 3645794SN/A 3655794SN/A pkt->makeAtomicResponse(); 3665794SN/A return pioDelay; 3675794SN/A} 3685794SN/A 3695794SN/Avoid 3705794SN/ACopyEngine::CopyEngineChannel::channelWrite(Packet *pkt, Addr daddr, int size) 3715794SN/A{ 3725794SN/A switch (daddr) { 3735794SN/A case CHAN_CONTROL: 3745794SN/A assert(size == sizeof(uint16_t)); 3755794SN/A int old_int_disable; 3765794SN/A old_int_disable = cr.ctrl.interrupt_disable(); 37713342Sgabeblack@google.com cr.ctrl(pkt->getLE<uint16_t>()); 3785794SN/A if (cr.ctrl.interrupt_disable()) 3795794SN/A cr.ctrl.interrupt_disable(0); 3805794SN/A else 3815794SN/A cr.ctrl.interrupt_disable(old_int_disable); 3825794SN/A break; 3835794SN/A case CHAN_STATUS: 3845794SN/A assert(size == sizeof(uint64_t)); 3855794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3865794SN/A daddr); 3875794SN/A break; 3885794SN/A case CHAN_CHAINADDR: 3895794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 3905794SN/A if (size == sizeof(uint64_t)) 39113342Sgabeblack@google.com cr.descChainAddr = pkt->getLE<uint64_t>(); 3925794SN/A else 39313342Sgabeblack@google.com cr.descChainAddr = (uint64_t)pkt->getLE<uint32_t>() | 3945794SN/A (cr.descChainAddr & ~mask(32)); 3955794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 3965794SN/A break; 3975794SN/A case CHAN_CHAINADDR_HIGH: 3985794SN/A assert(size == sizeof(uint32_t)); 39913342Sgabeblack@google.com cr.descChainAddr = ((uint64_t)pkt->getLE<uint32_t>() << 32) | 4005794SN/A (cr.descChainAddr & mask(32)); 4015794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 4025794SN/A break; 4035794SN/A case CHAN_COMMAND: 4045794SN/A assert(size == sizeof(uint8_t)); 40513342Sgabeblack@google.com cr.command(pkt->getLE<uint8_t>()); 4065794SN/A recvCommand(); 4075794SN/A break; 4085794SN/A case CHAN_CMPLNADDR: 4095794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 4105794SN/A if (size == sizeof(uint64_t)) 41113342Sgabeblack@google.com cr.completionAddr = pkt->getLE<uint64_t>(); 4125794SN/A else 41313342Sgabeblack@google.com cr.completionAddr = pkt->getLE<uint32_t>() | 4145794SN/A (cr.completionAddr & ~mask(32)); 4155794SN/A break; 4165794SN/A case CHAN_CMPLNADDR_HIGH: 4175794SN/A assert(size == sizeof(uint32_t)); 41813342Sgabeblack@google.com cr.completionAddr = ((uint64_t)pkt->getLE<uint32_t>() <<32) | 4195794SN/A (cr.completionAddr & mask(32)); 4205794SN/A break; 4215794SN/A case CHAN_ERROR: 4225794SN/A assert(size == sizeof(uint32_t)); 42313342Sgabeblack@google.com cr.error(~pkt->getLE<uint32_t>() & cr.error()); 4245794SN/A break; 4255794SN/A default: 4265794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 4275794SN/A channelId, daddr); 4285794SN/A } 4295794SN/A} 4305794SN/A 4315794SN/Avoid 4325794SN/ACopyEngine::regStats() 4335794SN/A{ 43411522Sstephan.diestelhorst@arm.com PciDevice::regStats(); 43511522Sstephan.diestelhorst@arm.com 4365794SN/A using namespace Stats; 4375794SN/A bytesCopied 4385794SN/A .init(regs.chanCount) 4395794SN/A .name(name() + ".bytes_copied") 4405794SN/A .desc("Number of bytes copied by each engine") 4415794SN/A .flags(total) 4425794SN/A ; 4435794SN/A copiesProcessed 4445794SN/A .init(regs.chanCount) 4455794SN/A .name(name() + ".copies_processed") 4465794SN/A .desc("Number of copies processed by each engine") 4475794SN/A .flags(total) 4485794SN/A ; 4495794SN/A} 4505794SN/A 4515794SN/Avoid 4525794SN/ACopyEngine::CopyEngineChannel::fetchDescriptor(Addr address) 4535794SN/A{ 4545954SN/A anDq(); 4555954SN/A anBegin("FetchDescriptor"); 4565794SN/A DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n", 45711202SN/A address, ce->pciToDma(address)); 4585794SN/A assert(address); 4595794SN/A busy = true; 4605794SN/A 4615794SN/A DPRINTF(DMACopyEngine, "dmaAction: %#x, %d bytes, to addr %#x\n", 46211202SN/A ce->pciToDma(address), sizeof(DmaDesc), curDmaDesc); 4635794SN/A 46411202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(address), 4658851SN/A sizeof(DmaDesc), &fetchCompleteEvent, 4668851SN/A (uint8_t*)curDmaDesc, latBeforeBegin); 4675794SN/A lastDescriptorAddr = address; 4685794SN/A} 4695794SN/A 4705794SN/Avoid 4715794SN/ACopyEngine::CopyEngineChannel::fetchDescComplete() 4725794SN/A{ 4735794SN/A DPRINTF(DMACopyEngine, "Read of descriptor complete\n"); 4745794SN/A 4755794SN/A if ((curDmaDesc->command & DESC_CTRL_NULL)) { 4765794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, skipping\n"); 4775794SN/A assert(!(curDmaDesc->command & DESC_CTRL_CP_STS)); 4785794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 4795794SN/A panic("Shouldn't be able to get here\n"); 4805794SN/A nextState = CompletionWrite; 4815794SN/A if (inDrain()) return; 4825794SN/A writeCompletionStatus(); 4835794SN/A } else { 4845954SN/A anBegin("Idle"); 4855954SN/A anWait(); 4865794SN/A busy = false; 4875794SN/A nextState = Idle; 4885794SN/A inDrain(); 4895794SN/A } 4905794SN/A return; 4915794SN/A } 4925794SN/A 4935794SN/A if (curDmaDesc->command & ~DESC_CTRL_CP_STS) 4945794SN/A panic("Descriptor has flag other that completion status set\n"); 4955794SN/A 4965794SN/A nextState = DMARead; 4975794SN/A if (inDrain()) return; 4985794SN/A readCopyBytes(); 4995794SN/A} 5005794SN/A 5015794SN/Avoid 5025794SN/ACopyEngine::CopyEngineChannel::readCopyBytes() 5035794SN/A{ 5045954SN/A anBegin("ReadCopyBytes"); 5055794SN/A DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n", 5065794SN/A curDmaDesc->len, curDmaDesc->dest, 50711202SN/A ce->pciToDma(curDmaDesc->src)); 50811202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(curDmaDesc->src), 5098851SN/A curDmaDesc->len, &readCompleteEvent, copyBuffer, 0); 5105794SN/A} 5115794SN/A 5125794SN/Avoid 5135794SN/ACopyEngine::CopyEngineChannel::readCopyBytesComplete() 5145794SN/A{ 5155794SN/A DPRINTF(DMACopyEngine, "Read of bytes to copy complete\n"); 5165794SN/A 5175794SN/A nextState = DMAWrite; 5185794SN/A if (inDrain()) return; 5195794SN/A writeCopyBytes(); 5205794SN/A} 5215794SN/A 5225794SN/Avoid 5235794SN/ACopyEngine::CopyEngineChannel::writeCopyBytes() 5245794SN/A{ 5255954SN/A anBegin("WriteCopyBytes"); 5265794SN/A DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n", 5275794SN/A curDmaDesc->len, curDmaDesc->dest, 52811202SN/A ce->pciToDma(curDmaDesc->dest)); 5295794SN/A 53011202SN/A cePort.dmaAction(MemCmd::WriteReq, ce->pciToDma(curDmaDesc->dest), 5318851SN/A curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0); 5325794SN/A 5335794SN/A ce->bytesCopied[channelId] += curDmaDesc->len; 5345794SN/A ce->copiesProcessed[channelId]++; 5355794SN/A} 5365794SN/A 5375794SN/Avoid 5385794SN/ACopyEngine::CopyEngineChannel::writeCopyBytesComplete() 5395794SN/A{ 5405794SN/A DPRINTF(DMACopyEngine, "Write of bytes to copy complete user1: %#x\n", 5415794SN/A curDmaDesc->user1); 5425794SN/A 5435794SN/A cr.status.compl_desc_addr(lastDescriptorAddr >> 6); 5445794SN/A completionDataReg = cr.status() | 1; 5455794SN/A 5465954SN/A anQ("DMAUsedDescQ", channelId, 1); 5475954SN/A anQ("AppRecvQ", curDmaDesc->user1, curDmaDesc->len); 5485794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 5495794SN/A nextState = CompletionWrite; 5505794SN/A if (inDrain()) return; 5515794SN/A writeCompletionStatus(); 5525794SN/A return; 5535794SN/A } 5545794SN/A 5555794SN/A continueProcessing(); 5565794SN/A} 5575794SN/A 5585794SN/Avoid 5595794SN/ACopyEngine::CopyEngineChannel::continueProcessing() 5605794SN/A{ 5615794SN/A busy = false; 5625794SN/A 5635794SN/A if (underReset) { 5645954SN/A anBegin("Reset"); 5655954SN/A anWait(); 5665794SN/A underReset = false; 5675794SN/A refreshNext = false; 5685794SN/A busy = false; 5695794SN/A nextState = Idle; 5705794SN/A return; 5715794SN/A } 5725794SN/A 5735794SN/A if (curDmaDesc->next) { 5745794SN/A nextState = DescriptorFetch; 5755794SN/A fetchAddress = curDmaDesc->next; 5765794SN/A if (inDrain()) return; 5775794SN/A fetchDescriptor(curDmaDesc->next); 5785794SN/A } else if (refreshNext) { 5795794SN/A nextState = AddressFetch; 5805794SN/A refreshNext = false; 5815794SN/A if (inDrain()) return; 5825794SN/A fetchNextAddr(lastDescriptorAddr); 5835794SN/A } else { 5845794SN/A inDrain(); 5855794SN/A nextState = Idle; 5865954SN/A anWait(); 5875954SN/A anBegin("Idle"); 5885794SN/A } 5895794SN/A} 5905794SN/A 5915794SN/Avoid 5925794SN/ACopyEngine::CopyEngineChannel::writeCompletionStatus() 5935794SN/A{ 5945954SN/A anBegin("WriteCompletionStatus"); 5955794SN/A DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n", 5965794SN/A completionDataReg, cr.completionAddr, 59711202SN/A ce->pciToDma(cr.completionAddr)); 5985794SN/A 5998851SN/A cePort.dmaAction(MemCmd::WriteReq, 60011202SN/A ce->pciToDma(cr.completionAddr), 6018851SN/A sizeof(completionDataReg), &statusCompleteEvent, 6028851SN/A (uint8_t*)&completionDataReg, latAfterCompletion); 6035794SN/A} 6045794SN/A 6055794SN/Avoid 6065794SN/ACopyEngine::CopyEngineChannel::writeStatusComplete() 6075794SN/A{ 6085794SN/A DPRINTF(DMACopyEngine, "Writing completion status complete\n"); 6095794SN/A continueProcessing(); 6105794SN/A} 6115794SN/A 6125794SN/Avoid 6135794SN/ACopyEngine::CopyEngineChannel::fetchNextAddr(Addr address) 6145794SN/A{ 6155954SN/A anBegin("FetchNextAddr"); 6165794SN/A DPRINTF(DMACopyEngine, "Fetching next address...\n"); 6175794SN/A busy = true; 6188851SN/A cePort.dmaAction(MemCmd::ReadReq, 61911202SN/A ce->pciToDma(address + offsetof(DmaDesc, next)), 6208851SN/A sizeof(Addr), &addrCompleteEvent, 6218851SN/A (uint8_t*)curDmaDesc + offsetof(DmaDesc, next), 0); 6225794SN/A} 6235794SN/A 6245794SN/Avoid 6255794SN/ACopyEngine::CopyEngineChannel::fetchAddrComplete() 6265794SN/A{ 6275794SN/A DPRINTF(DMACopyEngine, "Fetching next address complete: %#x\n", 6285794SN/A curDmaDesc->next); 6295794SN/A if (!curDmaDesc->next) { 6305794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n"); 6315794SN/A busy = false; 6325794SN/A nextState = Idle; 6335954SN/A anWait(); 6345954SN/A anBegin("Idle"); 6355794SN/A inDrain(); 6365794SN/A return; 6375794SN/A } 6385794SN/A nextState = DescriptorFetch; 6395794SN/A fetchAddress = curDmaDesc->next; 6405794SN/A if (inDrain()) return; 6415794SN/A fetchDescriptor(curDmaDesc->next); 6425794SN/A} 6435794SN/A 6445794SN/Abool 6455794SN/ACopyEngine::CopyEngineChannel::inDrain() 6465794SN/A{ 64710913SN/A if (drainState() == DrainState::Draining) { 6489152SN/A DPRINTF(Drain, "CopyEngine done draining, processing drain event\n"); 64910913SN/A signalDrainDone(); 6505794SN/A } 6515794SN/A 65210913SN/A return ce->drainState() != DrainState::Running; 6535794SN/A} 6545794SN/A 65510913SN/ADrainState 65610913SN/ACopyEngine::CopyEngineChannel::drain() 6575794SN/A{ 65810913SN/A if (nextState == Idle || ce->drainState() != DrainState::Running) { 65910913SN/A return DrainState::Drained; 66010913SN/A } else { 66110913SN/A DPRINTF(Drain, "CopyEngineChannel not drained\n"); 66210913SN/A return DrainState::Draining; 66310913SN/A } 6645794SN/A} 6655794SN/A 6665794SN/Avoid 66710905SN/ACopyEngine::serialize(CheckpointOut &cp) const 6685794SN/A{ 66910905SN/A PciDevice::serialize(cp); 67010905SN/A regs.serialize(cp); 67110905SN/A for (int x =0; x < chan.size(); x++) 67210905SN/A chan[x]->serializeSection(cp, csprintf("channel%d", x)); 6735794SN/A} 6745794SN/A 6755794SN/Avoid 67610905SN/ACopyEngine::unserialize(CheckpointIn &cp) 6775794SN/A{ 67810905SN/A PciDevice::unserialize(cp); 67910905SN/A regs.unserialize(cp); 6805794SN/A for (int x = 0; x < chan.size(); x++) 68110905SN/A chan[x]->unserializeSection(cp, csprintf("channel%d", x)); 6825794SN/A} 6835794SN/A 6845794SN/Avoid 68510905SN/ACopyEngine::CopyEngineChannel::serialize(CheckpointOut &cp) const 6865794SN/A{ 6875794SN/A SERIALIZE_SCALAR(channelId); 6885794SN/A SERIALIZE_SCALAR(busy); 6895794SN/A SERIALIZE_SCALAR(underReset); 6905794SN/A SERIALIZE_SCALAR(refreshNext); 6915794SN/A SERIALIZE_SCALAR(lastDescriptorAddr); 6925794SN/A SERIALIZE_SCALAR(completionDataReg); 6935794SN/A SERIALIZE_SCALAR(fetchAddress); 6945794SN/A int nextState = this->nextState; 6955794SN/A SERIALIZE_SCALAR(nextState); 69610905SN/A arrayParamOut(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 6975794SN/A SERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 69810905SN/A cr.serialize(cp); 6995794SN/A 7005794SN/A} 7015794SN/Avoid 70210905SN/ACopyEngine::CopyEngineChannel::unserialize(CheckpointIn &cp) 7035794SN/A{ 7045794SN/A UNSERIALIZE_SCALAR(channelId); 7055794SN/A UNSERIALIZE_SCALAR(busy); 7065794SN/A UNSERIALIZE_SCALAR(underReset); 7075794SN/A UNSERIALIZE_SCALAR(refreshNext); 7085794SN/A UNSERIALIZE_SCALAR(lastDescriptorAddr); 7095794SN/A UNSERIALIZE_SCALAR(completionDataReg); 7105794SN/A UNSERIALIZE_SCALAR(fetchAddress); 7115794SN/A int nextState; 7125794SN/A UNSERIALIZE_SCALAR(nextState); 7135794SN/A this->nextState = (ChannelState)nextState; 71410905SN/A arrayParamIn(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 7155794SN/A UNSERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 71610905SN/A cr.unserialize(cp); 7175794SN/A 7185794SN/A} 7195794SN/A 7205794SN/Avoid 7215794SN/ACopyEngine::CopyEngineChannel::restartStateMachine() 7225794SN/A{ 7235794SN/A switch(nextState) { 7245794SN/A case AddressFetch: 7255794SN/A fetchNextAddr(lastDescriptorAddr); 7265794SN/A break; 7275794SN/A case DescriptorFetch: 7285794SN/A fetchDescriptor(fetchAddress); 7295794SN/A break; 7305794SN/A case DMARead: 7315794SN/A readCopyBytes(); 7325794SN/A break; 7335794SN/A case DMAWrite: 7345794SN/A writeCopyBytes(); 7355794SN/A break; 7365794SN/A case CompletionWrite: 7375794SN/A writeCompletionStatus(); 7385794SN/A break; 7395794SN/A case Idle: 7405794SN/A break; 7415794SN/A default: 7425794SN/A panic("Unknown state for CopyEngineChannel\n"); 7435794SN/A } 7445794SN/A} 7455794SN/A 7465794SN/Avoid 7479342SN/ACopyEngine::CopyEngineChannel::drainResume() 7485794SN/A{ 7495794SN/A DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState); 7505794SN/A restartStateMachine(); 7515794SN/A} 7525794SN/A 7535794SN/ACopyEngine * 7545794SN/ACopyEngineParams::create() 7555794SN/A{ 7565794SN/A return new CopyEngine(this); 7575794SN/A} 758