copy_engine.cc revision 11522
15794SN/A/* 28841SN/A * Copyright (c) 2012 ARM Limited 38841SN/A * All rights reserved 48841SN/A * 58841SN/A * The license below extends only to copyright in the software and shall 68841SN/A * not be construed as granting a license to any other intellectual 78841SN/A * property including but not limited to intellectual property relating 88841SN/A * to a hardware implementation of the functionality of the software 98841SN/A * licensed hereunder. You may use the software subject to the license 108841SN/A * terms below provided that you ensure that this notice is replicated 118841SN/A * unmodified and in its entirety in all distributions of the software, 128841SN/A * modified or unmodified, in source code or in binary form. 138841SN/A * 145794SN/A * Copyright (c) 2008 The Regents of The University of Michigan 155794SN/A * All rights reserved. 165794SN/A * 175794SN/A * Redistribution and use in source and binary forms, with or without 185794SN/A * modification, are permitted provided that the following conditions are 195794SN/A * met: redistributions of source code must retain the above copyright 205794SN/A * notice, this list of conditions and the following disclaimer; 215794SN/A * redistributions in binary form must reproduce the above copyright 225794SN/A * notice, this list of conditions and the following disclaimer in the 235794SN/A * documentation and/or other materials provided with the distribution; 245794SN/A * neither the name of the copyright holders nor the names of its 255794SN/A * contributors may be used to endorse or promote products derived from 265794SN/A * this software without specific prior written permission. 275794SN/A * 285794SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 295794SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 305794SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 315794SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 325794SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 335794SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 345794SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 355794SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 365794SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 375794SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 385794SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 395794SN/A * 405794SN/A * Authors: Ali Saidi 415794SN/A */ 425794SN/A 435794SN/A/* @file 445794SN/A * Device model for Intel's I/O AT DMA copy engine. 455794SN/A */ 465794SN/A 4711261Sandreas.sandberg@arm.com#include "dev/pci/copy_engine.hh" 4811261Sandreas.sandberg@arm.com 495794SN/A#include <algorithm> 505794SN/A 515954SN/A#include "base/cp_annotate.hh" 525794SN/A#include "base/trace.hh" 538232SN/A#include "debug/DMACopyEngine.hh" 549152SN/A#include "debug/Drain.hh" 555794SN/A#include "mem/packet.hh" 565794SN/A#include "mem/packet_access.hh" 575794SN/A#include "params/CopyEngine.hh" 585794SN/A#include "sim/stats.hh" 595794SN/A#include "sim/system.hh" 605794SN/A 615794SN/Ausing namespace CopyEngineReg; 625794SN/A 635794SN/ACopyEngine::CopyEngine(const Params *p) 649807SN/A : PciDevice(p) 655794SN/A{ 665794SN/A // All Reg regs are initialized to 0 by default 675794SN/A regs.chanCount = p->ChanCnt; 685794SN/A regs.xferCap = findMsbSet(p->XferCap); 695794SN/A regs.attnStatus = 0; 705794SN/A 715794SN/A if (regs.chanCount > 64) 725794SN/A fatal("CopyEngine interface doesn't support more than 64 DMA engines\n"); 735794SN/A 745794SN/A for (int x = 0; x < regs.chanCount; x++) { 755794SN/A CopyEngineChannel *ch = new CopyEngineChannel(this, x); 765794SN/A chan.push_back(ch); 775794SN/A } 785794SN/A} 795794SN/A 805794SN/A 815794SN/ACopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine *_ce, int cid) 829165SN/A : cePort(_ce, _ce->sys), 838851SN/A ce(_ce), channelId(cid), busy(false), underReset(false), 845794SN/A refreshNext(false), latBeforeBegin(ce->params()->latBeforeBegin), 855794SN/A latAfterCompletion(ce->params()->latAfterCompletion), 8610913SN/A completionDataReg(0), nextState(Idle), 875794SN/A fetchCompleteEvent(this), addrCompleteEvent(this), 885794SN/A readCompleteEvent(this), writeCompleteEvent(this), 895794SN/A statusCompleteEvent(this) 905794SN/A 915794SN/A{ 925794SN/A cr.status.dma_transfer_status(3); 935794SN/A cr.descChainAddr = 0; 945794SN/A cr.completionAddr = 0; 955794SN/A 965794SN/A curDmaDesc = new DmaDesc; 975794SN/A memset(curDmaDesc, 0, sizeof(DmaDesc)); 985794SN/A copyBuffer = new uint8_t[ce->params()->XferCap]; 995794SN/A} 1005794SN/A 1015794SN/ACopyEngine::~CopyEngine() 1025794SN/A{ 1035794SN/A for (int x = 0; x < chan.size(); x++) { 1045794SN/A delete chan[x]; 1055794SN/A } 1065794SN/A} 1075794SN/A 1085794SN/ACopyEngine::CopyEngineChannel::~CopyEngineChannel() 1095794SN/A{ 1105794SN/A delete curDmaDesc; 1115794SN/A delete [] copyBuffer; 1125794SN/A} 1135794SN/A 1149294SN/ABaseMasterPort & 1159294SN/ACopyEngine::getMasterPort(const std::string &if_name, PortID idx) 1165794SN/A{ 1178922SN/A if (if_name != "dma") { 1188922SN/A // pass it along to our super class 1199807SN/A return PciDevice::getMasterPort(if_name, idx); 1208922SN/A } else { 1218922SN/A if (idx >= static_cast<int>(chan.size())) { 1228922SN/A panic("CopyEngine::getMasterPort: unknown index %d\n", idx); 1238922SN/A } 1248922SN/A 1258922SN/A return chan[idx]->getMasterPort(); 1268841SN/A } 1275794SN/A} 1285794SN/A 1298841SN/A 1309294SN/ABaseMasterPort & 1318922SN/ACopyEngine::CopyEngineChannel::getMasterPort() 1325794SN/A{ 1338922SN/A return cePort; 1345794SN/A} 1355794SN/A 1365794SN/Avoid 1375794SN/ACopyEngine::CopyEngineChannel::recvCommand() 1385794SN/A{ 1395794SN/A if (cr.command.start_dma()) { 1405794SN/A assert(!busy); 1415794SN/A cr.status.dma_transfer_status(0); 1425794SN/A nextState = DescriptorFetch; 1435794SN/A fetchAddress = cr.descChainAddr; 14410913SN/A if (ce->drainState() == DrainState::Running) 1455794SN/A fetchDescriptor(cr.descChainAddr); 1465794SN/A } else if (cr.command.append_dma()) { 1475794SN/A if (!busy) { 1485794SN/A nextState = AddressFetch; 14910913SN/A if (ce->drainState() == DrainState::Running) 1505794SN/A fetchNextAddr(lastDescriptorAddr); 1515794SN/A } else 1525794SN/A refreshNext = true; 1535794SN/A } else if (cr.command.reset_dma()) { 1545794SN/A if (busy) 1555794SN/A underReset = true; 1565794SN/A else { 1575794SN/A cr.status.dma_transfer_status(3); 1585794SN/A nextState = Idle; 1595794SN/A } 1605794SN/A } else if (cr.command.resume_dma() || cr.command.abort_dma() || 1615794SN/A cr.command.suspend_dma()) 1625794SN/A panic("Resume, Abort, and Suspend are not supported\n"); 1635794SN/A cr.command(0); 1645794SN/A} 1655794SN/A 1665794SN/ATick 1675794SN/ACopyEngine::read(PacketPtr pkt) 1685794SN/A{ 1695794SN/A int bar; 1705794SN/A Addr daddr; 1715794SN/A 1725794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 1735794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 1745794SN/A 1755794SN/A // Only Memory register BAR is allowed 1765794SN/A assert(bar == 0); 1775794SN/A 1785794SN/A int size = pkt->getSize(); 1795794SN/A if (size != sizeof(uint64_t) && size != sizeof(uint32_t) && 1805794SN/A size != sizeof(uint16_t) && size != sizeof(uint8_t)) { 1815794SN/A panic("Unknown size for MMIO access: %d\n", pkt->getSize()); 1825794SN/A } 1835794SN/A 1845794SN/A DPRINTF(DMACopyEngine, "Read device register %#X size: %d\n", daddr, size); 1855794SN/A 1865794SN/A /// 1875794SN/A /// Handle read of register here 1885794SN/A /// 1895794SN/A 1905794SN/A if (daddr < 0x80) { 1915794SN/A switch (daddr) { 1925794SN/A case GEN_CHANCOUNT: 1935794SN/A assert(size == sizeof(regs.chanCount)); 1945794SN/A pkt->set<uint8_t>(regs.chanCount); 1955794SN/A break; 1965794SN/A case GEN_XFERCAP: 1975794SN/A assert(size == sizeof(regs.xferCap)); 1985794SN/A pkt->set<uint8_t>(regs.xferCap); 1995794SN/A break; 2005794SN/A case GEN_INTRCTRL: 2015794SN/A assert(size == sizeof(uint8_t)); 2025794SN/A pkt->set<uint8_t>(regs.intrctrl()); 2035794SN/A regs.intrctrl.master_int_enable(0); 2045794SN/A break; 2055794SN/A case GEN_ATTNSTATUS: 2065794SN/A assert(size == sizeof(regs.attnStatus)); 2075794SN/A pkt->set<uint32_t>(regs.attnStatus); 2085794SN/A regs.attnStatus = 0; 2095794SN/A break; 2105794SN/A default: 2115794SN/A panic("Read request to unknown register number: %#x\n", daddr); 2125794SN/A } 2135794SN/A pkt->makeAtomicResponse(); 2145794SN/A return pioDelay; 2155794SN/A } 2165794SN/A 2175794SN/A 2185794SN/A // Find which channel we're accessing 2195794SN/A int chanid = 0; 2205794SN/A daddr -= 0x80; 2215794SN/A while (daddr >= 0x80) { 2225794SN/A chanid++; 2235794SN/A daddr -= 0x80; 2245794SN/A } 2255794SN/A 2265794SN/A if (chanid >= regs.chanCount) 2275794SN/A panic("Access to channel %d (device only configured for %d channels)", 2285794SN/A chanid, regs.chanCount); 2295794SN/A 2305794SN/A /// 2315794SN/A /// Channel registers are handled here 2325794SN/A /// 2335794SN/A chan[chanid]->channelRead(pkt, daddr, size); 2345794SN/A 2355794SN/A pkt->makeAtomicResponse(); 2365794SN/A return pioDelay; 2375794SN/A} 2385794SN/A 2395794SN/Avoid 2405794SN/ACopyEngine::CopyEngineChannel::channelRead(Packet *pkt, Addr daddr, int size) 2415794SN/A{ 2425794SN/A switch (daddr) { 2435794SN/A case CHAN_CONTROL: 2445794SN/A assert(size == sizeof(uint16_t)); 2455794SN/A pkt->set<uint16_t>(cr.ctrl()); 2465794SN/A cr.ctrl.in_use(1); 2475794SN/A break; 2485794SN/A case CHAN_STATUS: 2495794SN/A assert(size == sizeof(uint64_t)); 2505794SN/A pkt->set<uint64_t>(cr.status() | ~busy); 2515794SN/A break; 2525794SN/A case CHAN_CHAINADDR: 2535794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2545794SN/A if (size == sizeof(uint64_t)) 2555794SN/A pkt->set<uint64_t>(cr.descChainAddr); 2565794SN/A else 2575794SN/A pkt->set<uint32_t>(bits(cr.descChainAddr,0,31)); 2585794SN/A break; 2595794SN/A case CHAN_CHAINADDR_HIGH: 2605794SN/A assert(size == sizeof(uint32_t)); 2615794SN/A pkt->set<uint32_t>(bits(cr.descChainAddr,32,63)); 2625794SN/A break; 2635794SN/A case CHAN_COMMAND: 2645794SN/A assert(size == sizeof(uint8_t)); 2655794SN/A pkt->set<uint32_t>(cr.command()); 2665794SN/A break; 2675794SN/A case CHAN_CMPLNADDR: 2685794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2695794SN/A if (size == sizeof(uint64_t)) 2705794SN/A pkt->set<uint64_t>(cr.completionAddr); 2715794SN/A else 2725794SN/A pkt->set<uint32_t>(bits(cr.completionAddr,0,31)); 2735794SN/A break; 2745794SN/A case CHAN_CMPLNADDR_HIGH: 2755794SN/A assert(size == sizeof(uint32_t)); 2765794SN/A pkt->set<uint32_t>(bits(cr.completionAddr,32,63)); 2775794SN/A break; 2785794SN/A case CHAN_ERROR: 2795794SN/A assert(size == sizeof(uint32_t)); 2805794SN/A pkt->set<uint32_t>(cr.error()); 2815794SN/A break; 2825794SN/A default: 2835794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 2845794SN/A channelId, daddr); 2855794SN/A } 2865794SN/A} 2875794SN/A 2885794SN/A 2895794SN/ATick 2905794SN/ACopyEngine::write(PacketPtr pkt) 2915794SN/A{ 2925794SN/A int bar; 2935794SN/A Addr daddr; 2945794SN/A 2955794SN/A 2965794SN/A if (!getBAR(pkt->getAddr(), bar, daddr)) 2975794SN/A panic("Invalid PCI memory access to unmapped memory.\n"); 2985794SN/A 2995794SN/A // Only Memory register BAR is allowed 3005794SN/A assert(bar == 0); 3015794SN/A 3025794SN/A int size = pkt->getSize(); 3035794SN/A 3045794SN/A /// 3055794SN/A /// Handle write of register here 3065794SN/A /// 3075794SN/A 3085794SN/A if (size == sizeof(uint64_t)) { 3095794SN/A uint64_t val M5_VAR_USED = pkt->get<uint64_t>(); 3105794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3115794SN/A } else if (size == sizeof(uint32_t)) { 3125794SN/A uint32_t val M5_VAR_USED = pkt->get<uint32_t>(); 3135794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3145794SN/A } else if (size == sizeof(uint16_t)) { 3155794SN/A uint16_t val M5_VAR_USED = pkt->get<uint16_t>(); 3165794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3175794SN/A } else if (size == sizeof(uint8_t)) { 3185794SN/A uint8_t val M5_VAR_USED = pkt->get<uint8_t>(); 3195794SN/A DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 3205794SN/A } else { 3215794SN/A panic("Unknown size for MMIO access: %d\n", size); 3225794SN/A } 3235794SN/A 3245794SN/A if (daddr < 0x80) { 3255794SN/A switch (daddr) { 3265794SN/A case GEN_CHANCOUNT: 3275794SN/A case GEN_XFERCAP: 3285794SN/A case GEN_ATTNSTATUS: 3295794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3305794SN/A daddr); 3315794SN/A break; 3325794SN/A case GEN_INTRCTRL: 3335794SN/A regs.intrctrl.master_int_enable(bits(pkt->get<uint8_t>(),0,1)); 3345794SN/A break; 3355794SN/A default: 3365794SN/A panic("Read request to unknown register number: %#x\n", daddr); 3375794SN/A } 3385794SN/A pkt->makeAtomicResponse(); 3395794SN/A return pioDelay; 3405794SN/A } 3415794SN/A 3425794SN/A // Find which channel we're accessing 3435794SN/A int chanid = 0; 3445794SN/A daddr -= 0x80; 3455794SN/A while (daddr >= 0x80) { 3465794SN/A chanid++; 3475794SN/A daddr -= 0x80; 3485794SN/A } 3495794SN/A 3505794SN/A if (chanid >= regs.chanCount) 3515794SN/A panic("Access to channel %d (device only configured for %d channels)", 3525794SN/A chanid, regs.chanCount); 3535794SN/A 3545794SN/A /// 3555794SN/A /// Channel registers are handled here 3565794SN/A /// 3575794SN/A chan[chanid]->channelWrite(pkt, daddr, size); 3585794SN/A 3595794SN/A pkt->makeAtomicResponse(); 3605794SN/A return pioDelay; 3615794SN/A} 3625794SN/A 3635794SN/Avoid 3645794SN/ACopyEngine::CopyEngineChannel::channelWrite(Packet *pkt, Addr daddr, int size) 3655794SN/A{ 3665794SN/A switch (daddr) { 3675794SN/A case CHAN_CONTROL: 3685794SN/A assert(size == sizeof(uint16_t)); 3695794SN/A int old_int_disable; 3705794SN/A old_int_disable = cr.ctrl.interrupt_disable(); 3715794SN/A cr.ctrl(pkt->get<uint16_t>()); 3725794SN/A if (cr.ctrl.interrupt_disable()) 3735794SN/A cr.ctrl.interrupt_disable(0); 3745794SN/A else 3755794SN/A cr.ctrl.interrupt_disable(old_int_disable); 3765794SN/A break; 3775794SN/A case CHAN_STATUS: 3785794SN/A assert(size == sizeof(uint64_t)); 3795794SN/A DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 3805794SN/A daddr); 3815794SN/A break; 3825794SN/A case CHAN_CHAINADDR: 3835794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 3845794SN/A if (size == sizeof(uint64_t)) 3855794SN/A cr.descChainAddr = pkt->get<uint64_t>(); 3865794SN/A else 3875794SN/A cr.descChainAddr = (uint64_t)pkt->get<uint32_t>() | 3885794SN/A (cr.descChainAddr & ~mask(32)); 3895794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 3905794SN/A break; 3915794SN/A case CHAN_CHAINADDR_HIGH: 3925794SN/A assert(size == sizeof(uint32_t)); 3935794SN/A cr.descChainAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 3945794SN/A (cr.descChainAddr & mask(32)); 3955794SN/A DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 3965794SN/A break; 3975794SN/A case CHAN_COMMAND: 3985794SN/A assert(size == sizeof(uint8_t)); 3995794SN/A cr.command(pkt->get<uint8_t>()); 4005794SN/A recvCommand(); 4015794SN/A break; 4025794SN/A case CHAN_CMPLNADDR: 4035794SN/A assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 4045794SN/A if (size == sizeof(uint64_t)) 4055794SN/A cr.completionAddr = pkt->get<uint64_t>(); 4065794SN/A else 4075794SN/A cr.completionAddr = pkt->get<uint32_t>() | 4085794SN/A (cr.completionAddr & ~mask(32)); 4095794SN/A break; 4105794SN/A case CHAN_CMPLNADDR_HIGH: 4115794SN/A assert(size == sizeof(uint32_t)); 4125794SN/A cr.completionAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 4135794SN/A (cr.completionAddr & mask(32)); 4145794SN/A break; 4155794SN/A case CHAN_ERROR: 4165794SN/A assert(size == sizeof(uint32_t)); 4175794SN/A cr.error(~pkt->get<uint32_t>() & cr.error()); 4185794SN/A break; 4195794SN/A default: 4205794SN/A panic("Read request to unknown channel register number: (%d)%#x\n", 4215794SN/A channelId, daddr); 4225794SN/A } 4235794SN/A} 4245794SN/A 4255794SN/Avoid 4265794SN/ACopyEngine::regStats() 4275794SN/A{ 42811522Sstephan.diestelhorst@arm.com PciDevice::regStats(); 42911522Sstephan.diestelhorst@arm.com 4305794SN/A using namespace Stats; 4315794SN/A bytesCopied 4325794SN/A .init(regs.chanCount) 4335794SN/A .name(name() + ".bytes_copied") 4345794SN/A .desc("Number of bytes copied by each engine") 4355794SN/A .flags(total) 4365794SN/A ; 4375794SN/A copiesProcessed 4385794SN/A .init(regs.chanCount) 4395794SN/A .name(name() + ".copies_processed") 4405794SN/A .desc("Number of copies processed by each engine") 4415794SN/A .flags(total) 4425794SN/A ; 4435794SN/A} 4445794SN/A 4455794SN/Avoid 4465794SN/ACopyEngine::CopyEngineChannel::fetchDescriptor(Addr address) 4475794SN/A{ 4485954SN/A anDq(); 4495954SN/A anBegin("FetchDescriptor"); 4505794SN/A DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n", 45111202SN/A address, ce->pciToDma(address)); 4525794SN/A assert(address); 4535794SN/A busy = true; 4545794SN/A 4555794SN/A DPRINTF(DMACopyEngine, "dmaAction: %#x, %d bytes, to addr %#x\n", 45611202SN/A ce->pciToDma(address), sizeof(DmaDesc), curDmaDesc); 4575794SN/A 45811202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(address), 4598851SN/A sizeof(DmaDesc), &fetchCompleteEvent, 4608851SN/A (uint8_t*)curDmaDesc, latBeforeBegin); 4615794SN/A lastDescriptorAddr = address; 4625794SN/A} 4635794SN/A 4645794SN/Avoid 4655794SN/ACopyEngine::CopyEngineChannel::fetchDescComplete() 4665794SN/A{ 4675794SN/A DPRINTF(DMACopyEngine, "Read of descriptor complete\n"); 4685794SN/A 4695794SN/A if ((curDmaDesc->command & DESC_CTRL_NULL)) { 4705794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, skipping\n"); 4715794SN/A assert(!(curDmaDesc->command & DESC_CTRL_CP_STS)); 4725794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 4735794SN/A panic("Shouldn't be able to get here\n"); 4745794SN/A nextState = CompletionWrite; 4755794SN/A if (inDrain()) return; 4765794SN/A writeCompletionStatus(); 4775794SN/A } else { 4785954SN/A anBegin("Idle"); 4795954SN/A anWait(); 4805794SN/A busy = false; 4815794SN/A nextState = Idle; 4825794SN/A inDrain(); 4835794SN/A } 4845794SN/A return; 4855794SN/A } 4865794SN/A 4875794SN/A if (curDmaDesc->command & ~DESC_CTRL_CP_STS) 4885794SN/A panic("Descriptor has flag other that completion status set\n"); 4895794SN/A 4905794SN/A nextState = DMARead; 4915794SN/A if (inDrain()) return; 4925794SN/A readCopyBytes(); 4935794SN/A} 4945794SN/A 4955794SN/Avoid 4965794SN/ACopyEngine::CopyEngineChannel::readCopyBytes() 4975794SN/A{ 4985954SN/A anBegin("ReadCopyBytes"); 4995794SN/A DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n", 5005794SN/A curDmaDesc->len, curDmaDesc->dest, 50111202SN/A ce->pciToDma(curDmaDesc->src)); 50211202SN/A cePort.dmaAction(MemCmd::ReadReq, ce->pciToDma(curDmaDesc->src), 5038851SN/A curDmaDesc->len, &readCompleteEvent, copyBuffer, 0); 5045794SN/A} 5055794SN/A 5065794SN/Avoid 5075794SN/ACopyEngine::CopyEngineChannel::readCopyBytesComplete() 5085794SN/A{ 5095794SN/A DPRINTF(DMACopyEngine, "Read of bytes to copy complete\n"); 5105794SN/A 5115794SN/A nextState = DMAWrite; 5125794SN/A if (inDrain()) return; 5135794SN/A writeCopyBytes(); 5145794SN/A} 5155794SN/A 5165794SN/Avoid 5175794SN/ACopyEngine::CopyEngineChannel::writeCopyBytes() 5185794SN/A{ 5195954SN/A anBegin("WriteCopyBytes"); 5205794SN/A DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n", 5215794SN/A curDmaDesc->len, curDmaDesc->dest, 52211202SN/A ce->pciToDma(curDmaDesc->dest)); 5235794SN/A 52411202SN/A cePort.dmaAction(MemCmd::WriteReq, ce->pciToDma(curDmaDesc->dest), 5258851SN/A curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0); 5265794SN/A 5275794SN/A ce->bytesCopied[channelId] += curDmaDesc->len; 5285794SN/A ce->copiesProcessed[channelId]++; 5295794SN/A} 5305794SN/A 5315794SN/Avoid 5325794SN/ACopyEngine::CopyEngineChannel::writeCopyBytesComplete() 5335794SN/A{ 5345794SN/A DPRINTF(DMACopyEngine, "Write of bytes to copy complete user1: %#x\n", 5355794SN/A curDmaDesc->user1); 5365794SN/A 5375794SN/A cr.status.compl_desc_addr(lastDescriptorAddr >> 6); 5385794SN/A completionDataReg = cr.status() | 1; 5395794SN/A 5405954SN/A anQ("DMAUsedDescQ", channelId, 1); 5415954SN/A anQ("AppRecvQ", curDmaDesc->user1, curDmaDesc->len); 5425794SN/A if (curDmaDesc->command & DESC_CTRL_CP_STS) { 5435794SN/A nextState = CompletionWrite; 5445794SN/A if (inDrain()) return; 5455794SN/A writeCompletionStatus(); 5465794SN/A return; 5475794SN/A } 5485794SN/A 5495794SN/A continueProcessing(); 5505794SN/A} 5515794SN/A 5525794SN/Avoid 5535794SN/ACopyEngine::CopyEngineChannel::continueProcessing() 5545794SN/A{ 5555794SN/A busy = false; 5565794SN/A 5575794SN/A if (underReset) { 5585954SN/A anBegin("Reset"); 5595954SN/A anWait(); 5605794SN/A underReset = false; 5615794SN/A refreshNext = false; 5625794SN/A busy = false; 5635794SN/A nextState = Idle; 5645794SN/A return; 5655794SN/A } 5665794SN/A 5675794SN/A if (curDmaDesc->next) { 5685794SN/A nextState = DescriptorFetch; 5695794SN/A fetchAddress = curDmaDesc->next; 5705794SN/A if (inDrain()) return; 5715794SN/A fetchDescriptor(curDmaDesc->next); 5725794SN/A } else if (refreshNext) { 5735794SN/A nextState = AddressFetch; 5745794SN/A refreshNext = false; 5755794SN/A if (inDrain()) return; 5765794SN/A fetchNextAddr(lastDescriptorAddr); 5775794SN/A } else { 5785794SN/A inDrain(); 5795794SN/A nextState = Idle; 5805954SN/A anWait(); 5815954SN/A anBegin("Idle"); 5825794SN/A } 5835794SN/A} 5845794SN/A 5855794SN/Avoid 5865794SN/ACopyEngine::CopyEngineChannel::writeCompletionStatus() 5875794SN/A{ 5885954SN/A anBegin("WriteCompletionStatus"); 5895794SN/A DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n", 5905794SN/A completionDataReg, cr.completionAddr, 59111202SN/A ce->pciToDma(cr.completionAddr)); 5925794SN/A 5938851SN/A cePort.dmaAction(MemCmd::WriteReq, 59411202SN/A ce->pciToDma(cr.completionAddr), 5958851SN/A sizeof(completionDataReg), &statusCompleteEvent, 5968851SN/A (uint8_t*)&completionDataReg, latAfterCompletion); 5975794SN/A} 5985794SN/A 5995794SN/Avoid 6005794SN/ACopyEngine::CopyEngineChannel::writeStatusComplete() 6015794SN/A{ 6025794SN/A DPRINTF(DMACopyEngine, "Writing completion status complete\n"); 6035794SN/A continueProcessing(); 6045794SN/A} 6055794SN/A 6065794SN/Avoid 6075794SN/ACopyEngine::CopyEngineChannel::fetchNextAddr(Addr address) 6085794SN/A{ 6095954SN/A anBegin("FetchNextAddr"); 6105794SN/A DPRINTF(DMACopyEngine, "Fetching next address...\n"); 6115794SN/A busy = true; 6128851SN/A cePort.dmaAction(MemCmd::ReadReq, 61311202SN/A ce->pciToDma(address + offsetof(DmaDesc, next)), 6148851SN/A sizeof(Addr), &addrCompleteEvent, 6158851SN/A (uint8_t*)curDmaDesc + offsetof(DmaDesc, next), 0); 6165794SN/A} 6175794SN/A 6185794SN/Avoid 6195794SN/ACopyEngine::CopyEngineChannel::fetchAddrComplete() 6205794SN/A{ 6215794SN/A DPRINTF(DMACopyEngine, "Fetching next address complete: %#x\n", 6225794SN/A curDmaDesc->next); 6235794SN/A if (!curDmaDesc->next) { 6245794SN/A DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n"); 6255794SN/A busy = false; 6265794SN/A nextState = Idle; 6275954SN/A anWait(); 6285954SN/A anBegin("Idle"); 6295794SN/A inDrain(); 6305794SN/A return; 6315794SN/A } 6325794SN/A nextState = DescriptorFetch; 6335794SN/A fetchAddress = curDmaDesc->next; 6345794SN/A if (inDrain()) return; 6355794SN/A fetchDescriptor(curDmaDesc->next); 6365794SN/A} 6375794SN/A 6385794SN/Abool 6395794SN/ACopyEngine::CopyEngineChannel::inDrain() 6405794SN/A{ 64110913SN/A if (drainState() == DrainState::Draining) { 6429152SN/A DPRINTF(Drain, "CopyEngine done draining, processing drain event\n"); 64310913SN/A signalDrainDone(); 6445794SN/A } 6455794SN/A 64610913SN/A return ce->drainState() != DrainState::Running; 6475794SN/A} 6485794SN/A 64910913SN/ADrainState 65010913SN/ACopyEngine::CopyEngineChannel::drain() 6515794SN/A{ 65210913SN/A if (nextState == Idle || ce->drainState() != DrainState::Running) { 65310913SN/A return DrainState::Drained; 65410913SN/A } else { 65510913SN/A DPRINTF(Drain, "CopyEngineChannel not drained\n"); 65610913SN/A return DrainState::Draining; 65710913SN/A } 6585794SN/A} 6595794SN/A 6605794SN/Avoid 66110905SN/ACopyEngine::serialize(CheckpointOut &cp) const 6625794SN/A{ 66310905SN/A PciDevice::serialize(cp); 66410905SN/A regs.serialize(cp); 66510905SN/A for (int x =0; x < chan.size(); x++) 66610905SN/A chan[x]->serializeSection(cp, csprintf("channel%d", x)); 6675794SN/A} 6685794SN/A 6695794SN/Avoid 67010905SN/ACopyEngine::unserialize(CheckpointIn &cp) 6715794SN/A{ 67210905SN/A PciDevice::unserialize(cp); 67310905SN/A regs.unserialize(cp); 6745794SN/A for (int x = 0; x < chan.size(); x++) 67510905SN/A chan[x]->unserializeSection(cp, csprintf("channel%d", x)); 6765794SN/A} 6775794SN/A 6785794SN/Avoid 67910905SN/ACopyEngine::CopyEngineChannel::serialize(CheckpointOut &cp) const 6805794SN/A{ 6815794SN/A SERIALIZE_SCALAR(channelId); 6825794SN/A SERIALIZE_SCALAR(busy); 6835794SN/A SERIALIZE_SCALAR(underReset); 6845794SN/A SERIALIZE_SCALAR(refreshNext); 6855794SN/A SERIALIZE_SCALAR(lastDescriptorAddr); 6865794SN/A SERIALIZE_SCALAR(completionDataReg); 6875794SN/A SERIALIZE_SCALAR(fetchAddress); 6885794SN/A int nextState = this->nextState; 6895794SN/A SERIALIZE_SCALAR(nextState); 69010905SN/A arrayParamOut(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 6915794SN/A SERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 69210905SN/A cr.serialize(cp); 6935794SN/A 6945794SN/A} 6955794SN/Avoid 69610905SN/ACopyEngine::CopyEngineChannel::unserialize(CheckpointIn &cp) 6975794SN/A{ 6985794SN/A UNSERIALIZE_SCALAR(channelId); 6995794SN/A UNSERIALIZE_SCALAR(busy); 7005794SN/A UNSERIALIZE_SCALAR(underReset); 7015794SN/A UNSERIALIZE_SCALAR(refreshNext); 7025794SN/A UNSERIALIZE_SCALAR(lastDescriptorAddr); 7035794SN/A UNSERIALIZE_SCALAR(completionDataReg); 7045794SN/A UNSERIALIZE_SCALAR(fetchAddress); 7055794SN/A int nextState; 7065794SN/A UNSERIALIZE_SCALAR(nextState); 7075794SN/A this->nextState = (ChannelState)nextState; 70810905SN/A arrayParamIn(cp, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 7095794SN/A UNSERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 71010905SN/A cr.unserialize(cp); 7115794SN/A 7125794SN/A} 7135794SN/A 7145794SN/Avoid 7155794SN/ACopyEngine::CopyEngineChannel::restartStateMachine() 7165794SN/A{ 7175794SN/A switch(nextState) { 7185794SN/A case AddressFetch: 7195794SN/A fetchNextAddr(lastDescriptorAddr); 7205794SN/A break; 7215794SN/A case DescriptorFetch: 7225794SN/A fetchDescriptor(fetchAddress); 7235794SN/A break; 7245794SN/A case DMARead: 7255794SN/A readCopyBytes(); 7265794SN/A break; 7275794SN/A case DMAWrite: 7285794SN/A writeCopyBytes(); 7295794SN/A break; 7305794SN/A case CompletionWrite: 7315794SN/A writeCompletionStatus(); 7325794SN/A break; 7335794SN/A case Idle: 7345794SN/A break; 7355794SN/A default: 7365794SN/A panic("Unknown state for CopyEngineChannel\n"); 7375794SN/A } 7385794SN/A} 7395794SN/A 7405794SN/Avoid 7419342SN/ACopyEngine::CopyEngineChannel::drainResume() 7425794SN/A{ 7435794SN/A DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState); 7445794SN/A restartStateMachine(); 7455794SN/A} 7465794SN/A 7475794SN/ACopyEngine * 7485794SN/ACopyEngineParams::create() 7495794SN/A{ 7505794SN/A return new CopyEngine(this); 7515794SN/A} 752