copy_engine.cc revision 9165
12115Shsul@eecs.umich.edu/* 21881Sbinkertn@umich.edu * Copyright (c) 2012 ARM Limited 31881Sbinkertn@umich.edu * All rights reserved 41881Sbinkertn@umich.edu * 51881Sbinkertn@umich.edu * The license below extends only to copyright in the software and shall 61881Sbinkertn@umich.edu * not be construed as granting a license to any other intellectual 71881Sbinkertn@umich.edu * property including but not limited to intellectual property relating 81881Sbinkertn@umich.edu * to a hardware implementation of the functionality of the software 91881Sbinkertn@umich.edu * licensed hereunder. You may use the software subject to the license 101881Sbinkertn@umich.edu * terms below provided that you ensure that this notice is replicated 111881Sbinkertn@umich.edu * unmodified and in its entirety in all distributions of the software, 121881Sbinkertn@umich.edu * modified or unmodified, in source code or in binary form. 131881Sbinkertn@umich.edu * 141881Sbinkertn@umich.edu * Copyright (c) 2008 The Regents of The University of Michigan 151881Sbinkertn@umich.edu * All rights reserved. 161881Sbinkertn@umich.edu * 171881Sbinkertn@umich.edu * Redistribution and use in source and binary forms, with or without 181881Sbinkertn@umich.edu * modification, are permitted provided that the following conditions are 191881Sbinkertn@umich.edu * met: redistributions of source code must retain the above copyright 201881Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer; 211881Sbinkertn@umich.edu * redistributions in binary form must reproduce the above copyright 221881Sbinkertn@umich.edu * notice, this list of conditions and the following disclaimer in the 231881Sbinkertn@umich.edu * documentation and/or other materials provided with the distribution; 241881Sbinkertn@umich.edu * neither the name of the copyright holders nor the names of its 251881Sbinkertn@umich.edu * contributors may be used to endorse or promote products derived from 261881Sbinkertn@umich.edu * this software without specific prior written permission. 271881Sbinkertn@umich.edu * 281881Sbinkertn@umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 291881Sbinkertn@umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 301881Sbinkertn@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312006Sbinkertn@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 321881Sbinkertn@umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 331881Sbinkertn@umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 341881Sbinkertn@umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 351881Sbinkertn@umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 361881Sbinkertn@umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372006Sbinkertn@umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 381881Sbinkertn@umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392006Sbinkertn@umich.edu * 402006Sbinkertn@umich.edu * Authors: Ali Saidi 412006Sbinkertn@umich.edu */ 421881Sbinkertn@umich.edu 431881Sbinkertn@umich.edu/* @file 441881Sbinkertn@umich.edu * Device model for Intel's I/O AT DMA copy engine. 451881Sbinkertn@umich.edu */ 461881Sbinkertn@umich.edu 471881Sbinkertn@umich.edu#include <algorithm> 481881Sbinkertn@umich.edu 491881Sbinkertn@umich.edu#include "base/cp_annotate.hh" 501881Sbinkertn@umich.edu#include "base/trace.hh" 511881Sbinkertn@umich.edu#include "debug/DMACopyEngine.hh" 521881Sbinkertn@umich.edu#include "debug/Drain.hh" 531881Sbinkertn@umich.edu#include "dev/copy_engine.hh" 541881Sbinkertn@umich.edu#include "mem/packet.hh" 551881Sbinkertn@umich.edu#include "mem/packet_access.hh" 561881Sbinkertn@umich.edu#include "params/CopyEngine.hh" 571881Sbinkertn@umich.edu#include "sim/stats.hh" 581881Sbinkertn@umich.edu#include "sim/system.hh" 591881Sbinkertn@umich.edu 601881Sbinkertn@umich.eduusing namespace CopyEngineReg; 611881Sbinkertn@umich.edu 621881Sbinkertn@umich.eduCopyEngine::CopyEngine(const Params *p) 631881Sbinkertn@umich.edu : PciDev(p) 641881Sbinkertn@umich.edu{ 651881Sbinkertn@umich.edu // All Reg regs are initialized to 0 by default 661881Sbinkertn@umich.edu regs.chanCount = p->ChanCnt; 671881Sbinkertn@umich.edu regs.xferCap = findMsbSet(p->XferCap); 681881Sbinkertn@umich.edu regs.attnStatus = 0; 691881Sbinkertn@umich.edu 701881Sbinkertn@umich.edu if (regs.chanCount > 64) 711881Sbinkertn@umich.edu fatal("CopyEngine interface doesn't support more than 64 DMA engines\n"); 721881Sbinkertn@umich.edu 731881Sbinkertn@umich.edu for (int x = 0; x < regs.chanCount; x++) { 741881Sbinkertn@umich.edu CopyEngineChannel *ch = new CopyEngineChannel(this, x); 751881Sbinkertn@umich.edu chan.push_back(ch); 761881Sbinkertn@umich.edu } 771881Sbinkertn@umich.edu} 781881Sbinkertn@umich.edu 792115Shsul@eecs.umich.edu 801881Sbinkertn@umich.eduCopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine *_ce, int cid) 811881Sbinkertn@umich.edu : cePort(_ce, _ce->sys), 821881Sbinkertn@umich.edu ce(_ce), channelId(cid), busy(false), underReset(false), 831881Sbinkertn@umich.edu refreshNext(false), latBeforeBegin(ce->params()->latBeforeBegin), 841881Sbinkertn@umich.edu latAfterCompletion(ce->params()->latAfterCompletion), 851881Sbinkertn@umich.edu completionDataReg(0), nextState(Idle), drainEvent(NULL), 862115Shsul@eecs.umich.edu fetchCompleteEvent(this), addrCompleteEvent(this), 871881Sbinkertn@umich.edu readCompleteEvent(this), writeCompleteEvent(this), 881881Sbinkertn@umich.edu statusCompleteEvent(this) 891881Sbinkertn@umich.edu 901881Sbinkertn@umich.edu{ 911881Sbinkertn@umich.edu cr.status.dma_transfer_status(3); 921881Sbinkertn@umich.edu cr.descChainAddr = 0; 931881Sbinkertn@umich.edu cr.completionAddr = 0; 941881Sbinkertn@umich.edu 951881Sbinkertn@umich.edu curDmaDesc = new DmaDesc; 961881Sbinkertn@umich.edu memset(curDmaDesc, 0, sizeof(DmaDesc)); 971881Sbinkertn@umich.edu copyBuffer = new uint8_t[ce->params()->XferCap]; 981881Sbinkertn@umich.edu} 991881Sbinkertn@umich.edu 1001881Sbinkertn@umich.eduCopyEngine::~CopyEngine() 1011881Sbinkertn@umich.edu{ 1021881Sbinkertn@umich.edu for (int x = 0; x < chan.size(); x++) { 1031881Sbinkertn@umich.edu delete chan[x]; 1041881Sbinkertn@umich.edu } 1051881Sbinkertn@umich.edu} 1061881Sbinkertn@umich.edu 1071881Sbinkertn@umich.eduCopyEngine::CopyEngineChannel::~CopyEngineChannel() 1081881Sbinkertn@umich.edu{ 1091881Sbinkertn@umich.edu delete curDmaDesc; 1101881Sbinkertn@umich.edu delete [] copyBuffer; 1111881Sbinkertn@umich.edu} 1121881Sbinkertn@umich.edu 1131881Sbinkertn@umich.eduMasterPort & 1141881Sbinkertn@umich.eduCopyEngine::getMasterPort(const std::string &if_name, int idx) 1151881Sbinkertn@umich.edu{ 1161881Sbinkertn@umich.edu if (if_name != "dma") { 1171881Sbinkertn@umich.edu // pass it along to our super class 1181881Sbinkertn@umich.edu return PciDev::getMasterPort(if_name, idx); 1191881Sbinkertn@umich.edu } else { 1201881Sbinkertn@umich.edu if (idx >= static_cast<int>(chan.size())) { 1211881Sbinkertn@umich.edu panic("CopyEngine::getMasterPort: unknown index %d\n", idx); 1221881Sbinkertn@umich.edu } 1231881Sbinkertn@umich.edu 1241881Sbinkertn@umich.edu return chan[idx]->getMasterPort(); 1252115Shsul@eecs.umich.edu } 1261881Sbinkertn@umich.edu} 1271881Sbinkertn@umich.edu 1281881Sbinkertn@umich.edu 1292115Shsul@eecs.umich.eduMasterPort & 1301881Sbinkertn@umich.eduCopyEngine::CopyEngineChannel::getMasterPort() 1311881Sbinkertn@umich.edu{ 1321881Sbinkertn@umich.edu return cePort; 1332115Shsul@eecs.umich.edu} 1341881Sbinkertn@umich.edu 1351881Sbinkertn@umich.eduvoid 1361881Sbinkertn@umich.eduCopyEngine::CopyEngineChannel::recvCommand() 1371881Sbinkertn@umich.edu{ 1381881Sbinkertn@umich.edu if (cr.command.start_dma()) { 1391881Sbinkertn@umich.edu assert(!busy); 1401881Sbinkertn@umich.edu cr.status.dma_transfer_status(0); 1411881Sbinkertn@umich.edu nextState = DescriptorFetch; 1421881Sbinkertn@umich.edu fetchAddress = cr.descChainAddr; 1431881Sbinkertn@umich.edu if (ce->getState() == SimObject::Running) 1441881Sbinkertn@umich.edu fetchDescriptor(cr.descChainAddr); 1452115Shsul@eecs.umich.edu } else if (cr.command.append_dma()) { 1462115Shsul@eecs.umich.edu if (!busy) { 1472115Shsul@eecs.umich.edu nextState = AddressFetch; 1482115Shsul@eecs.umich.edu if (ce->getState() == SimObject::Running) 1492115Shsul@eecs.umich.edu fetchNextAddr(lastDescriptorAddr); 1502115Shsul@eecs.umich.edu } else 1512115Shsul@eecs.umich.edu refreshNext = true; 1522115Shsul@eecs.umich.edu } else if (cr.command.reset_dma()) { 1532115Shsul@eecs.umich.edu if (busy) 1542115Shsul@eecs.umich.edu underReset = true; 1552115Shsul@eecs.umich.edu else { 1562115Shsul@eecs.umich.edu cr.status.dma_transfer_status(3); 1572115Shsul@eecs.umich.edu nextState = Idle; 1582115Shsul@eecs.umich.edu } 1592115Shsul@eecs.umich.edu } else if (cr.command.resume_dma() || cr.command.abort_dma() || 1602115Shsul@eecs.umich.edu cr.command.suspend_dma()) 1612115Shsul@eecs.umich.edu panic("Resume, Abort, and Suspend are not supported\n"); 1622115Shsul@eecs.umich.edu cr.command(0); 1632115Shsul@eecs.umich.edu} 1642115Shsul@eecs.umich.edu 1652115Shsul@eecs.umich.eduTick 1661881Sbinkertn@umich.eduCopyEngine::read(PacketPtr pkt) 1671881Sbinkertn@umich.edu{ 1681881Sbinkertn@umich.edu int bar; 1691881Sbinkertn@umich.edu Addr daddr; 1701881Sbinkertn@umich.edu 1711881Sbinkertn@umich.edu if (!getBAR(pkt->getAddr(), bar, daddr)) 1721881Sbinkertn@umich.edu panic("Invalid PCI memory access to unmapped memory.\n"); 1731881Sbinkertn@umich.edu 1741881Sbinkertn@umich.edu // Only Memory register BAR is allowed 1751881Sbinkertn@umich.edu assert(bar == 0); 1762006Sbinkertn@umich.edu 1771881Sbinkertn@umich.edu int size = pkt->getSize(); 1781881Sbinkertn@umich.edu if (size != sizeof(uint64_t) && size != sizeof(uint32_t) && 1791881Sbinkertn@umich.edu size != sizeof(uint16_t) && size != sizeof(uint8_t)) { 1801881Sbinkertn@umich.edu panic("Unknown size for MMIO access: %d\n", pkt->getSize()); 1811881Sbinkertn@umich.edu } 1821881Sbinkertn@umich.edu 1832115Shsul@eecs.umich.edu DPRINTF(DMACopyEngine, "Read device register %#X size: %d\n", daddr, size); 1842115Shsul@eecs.umich.edu 1851881Sbinkertn@umich.edu pkt->allocate(); 1861881Sbinkertn@umich.edu 1871881Sbinkertn@umich.edu /// 1881881Sbinkertn@umich.edu /// Handle read of register here 1891881Sbinkertn@umich.edu /// 1902115Shsul@eecs.umich.edu 1911881Sbinkertn@umich.edu if (daddr < 0x80) { 1921881Sbinkertn@umich.edu switch (daddr) { 1932115Shsul@eecs.umich.edu case GEN_CHANCOUNT: 1941881Sbinkertn@umich.edu assert(size == sizeof(regs.chanCount)); 1951881Sbinkertn@umich.edu pkt->set<uint8_t>(regs.chanCount); 1961881Sbinkertn@umich.edu break; 1971881Sbinkertn@umich.edu case GEN_XFERCAP: 1981881Sbinkertn@umich.edu assert(size == sizeof(regs.xferCap)); 1992115Shsul@eecs.umich.edu pkt->set<uint8_t>(regs.xferCap); 2002115Shsul@eecs.umich.edu break; 2012006Sbinkertn@umich.edu case GEN_INTRCTRL: 2022115Shsul@eecs.umich.edu assert(size == sizeof(uint8_t)); 2031881Sbinkertn@umich.edu pkt->set<uint8_t>(regs.intrctrl()); 2041881Sbinkertn@umich.edu regs.intrctrl.master_int_enable(0); 2052115Shsul@eecs.umich.edu break; 2062115Shsul@eecs.umich.edu case GEN_ATTNSTATUS: 2072115Shsul@eecs.umich.edu assert(size == sizeof(regs.attnStatus)); 2082115Shsul@eecs.umich.edu pkt->set<uint32_t>(regs.attnStatus); 2092115Shsul@eecs.umich.edu regs.attnStatus = 0; 2102115Shsul@eecs.umich.edu break; 2111881Sbinkertn@umich.edu default: 2121881Sbinkertn@umich.edu panic("Read request to unknown register number: %#x\n", daddr); 2131881Sbinkertn@umich.edu } 2141881Sbinkertn@umich.edu pkt->makeAtomicResponse(); 2151881Sbinkertn@umich.edu return pioDelay; 2161881Sbinkertn@umich.edu } 2171881Sbinkertn@umich.edu 2181881Sbinkertn@umich.edu 2191881Sbinkertn@umich.edu // Find which channel we're accessing 2202006Sbinkertn@umich.edu int chanid = 0; 2212006Sbinkertn@umich.edu daddr -= 0x80; 2221881Sbinkertn@umich.edu while (daddr >= 0x80) { 2231881Sbinkertn@umich.edu chanid++; 2241881Sbinkertn@umich.edu daddr -= 0x80; 2251881Sbinkertn@umich.edu } 2261881Sbinkertn@umich.edu 2271881Sbinkertn@umich.edu if (chanid >= regs.chanCount) 2281881Sbinkertn@umich.edu panic("Access to channel %d (device only configured for %d channels)", 2292006Sbinkertn@umich.edu chanid, regs.chanCount); 2302006Sbinkertn@umich.edu 2312006Sbinkertn@umich.edu /// 2322006Sbinkertn@umich.edu /// Channel registers are handled here 2332006Sbinkertn@umich.edu /// 2342006Sbinkertn@umich.edu chan[chanid]->channelRead(pkt, daddr, size); 2352006Sbinkertn@umich.edu 2362006Sbinkertn@umich.edu pkt->makeAtomicResponse(); 2372006Sbinkertn@umich.edu return pioDelay; 2382006Sbinkertn@umich.edu} 2392006Sbinkertn@umich.edu 2402006Sbinkertn@umich.eduvoid 2412006Sbinkertn@umich.eduCopyEngine::CopyEngineChannel::channelRead(Packet *pkt, Addr daddr, int size) 2422006Sbinkertn@umich.edu{ 2432006Sbinkertn@umich.edu switch (daddr) { 2442006Sbinkertn@umich.edu case CHAN_CONTROL: 2452006Sbinkertn@umich.edu assert(size == sizeof(uint16_t)); 2462006Sbinkertn@umich.edu pkt->set<uint16_t>(cr.ctrl()); 2472006Sbinkertn@umich.edu cr.ctrl.in_use(1); 2482006Sbinkertn@umich.edu break; 2492006Sbinkertn@umich.edu case CHAN_STATUS: 2502006Sbinkertn@umich.edu assert(size == sizeof(uint64_t)); 2512006Sbinkertn@umich.edu pkt->set<uint64_t>(cr.status() | ~busy); 2521881Sbinkertn@umich.edu break; 2532006Sbinkertn@umich.edu case CHAN_CHAINADDR: 2541881Sbinkertn@umich.edu assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2551881Sbinkertn@umich.edu if (size == sizeof(uint64_t)) 2561881Sbinkertn@umich.edu pkt->set<uint64_t>(cr.descChainAddr); 2571881Sbinkertn@umich.edu else 2581881Sbinkertn@umich.edu pkt->set<uint32_t>(bits(cr.descChainAddr,0,31)); 2591881Sbinkertn@umich.edu break; 2601881Sbinkertn@umich.edu case CHAN_CHAINADDR_HIGH: 2611881Sbinkertn@umich.edu assert(size == sizeof(uint32_t)); 2621881Sbinkertn@umich.edu pkt->set<uint32_t>(bits(cr.descChainAddr,32,63)); 2631881Sbinkertn@umich.edu break; 2641881Sbinkertn@umich.edu case CHAN_COMMAND: 2651881Sbinkertn@umich.edu assert(size == sizeof(uint8_t)); 2661881Sbinkertn@umich.edu pkt->set<uint32_t>(cr.command()); 2671881Sbinkertn@umich.edu break; 2681881Sbinkertn@umich.edu case CHAN_CMPLNADDR: 2691881Sbinkertn@umich.edu assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 2701881Sbinkertn@umich.edu if (size == sizeof(uint64_t)) 2711881Sbinkertn@umich.edu pkt->set<uint64_t>(cr.completionAddr); 2721881Sbinkertn@umich.edu else 2731881Sbinkertn@umich.edu pkt->set<uint32_t>(bits(cr.completionAddr,0,31)); 2741881Sbinkertn@umich.edu break; 2751881Sbinkertn@umich.edu case CHAN_CMPLNADDR_HIGH: 2761881Sbinkertn@umich.edu assert(size == sizeof(uint32_t)); 2771881Sbinkertn@umich.edu pkt->set<uint32_t>(bits(cr.completionAddr,32,63)); 2781881Sbinkertn@umich.edu break; 2791881Sbinkertn@umich.edu case CHAN_ERROR: 2801881Sbinkertn@umich.edu assert(size == sizeof(uint32_t)); 2811881Sbinkertn@umich.edu pkt->set<uint32_t>(cr.error()); 2821881Sbinkertn@umich.edu break; 2832115Shsul@eecs.umich.edu default: 2842117Shsul@eecs.umich.edu panic("Read request to unknown channel register number: (%d)%#x\n", 2852117Shsul@eecs.umich.edu channelId, daddr); 2861881Sbinkertn@umich.edu } 2872006Sbinkertn@umich.edu} 2882006Sbinkertn@umich.edu 2892006Sbinkertn@umich.edu 2902006Sbinkertn@umich.eduTick 2911881Sbinkertn@umich.eduCopyEngine::write(PacketPtr pkt) 2921881Sbinkertn@umich.edu{ 2931881Sbinkertn@umich.edu int bar; 2941881Sbinkertn@umich.edu Addr daddr; 2951881Sbinkertn@umich.edu 2961881Sbinkertn@umich.edu 2972006Sbinkertn@umich.edu if (!getBAR(pkt->getAddr(), bar, daddr)) 2982006Sbinkertn@umich.edu panic("Invalid PCI memory access to unmapped memory.\n"); 2991881Sbinkertn@umich.edu 3002115Shsul@eecs.umich.edu // Only Memory register BAR is allowed 301 assert(bar == 0); 302 303 int size = pkt->getSize(); 304 305 /// 306 /// Handle write of register here 307 /// 308 309 if (size == sizeof(uint64_t)) { 310 uint64_t val M5_VAR_USED = pkt->get<uint64_t>(); 311 DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 312 } else if (size == sizeof(uint32_t)) { 313 uint32_t val M5_VAR_USED = pkt->get<uint32_t>(); 314 DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 315 } else if (size == sizeof(uint16_t)) { 316 uint16_t val M5_VAR_USED = pkt->get<uint16_t>(); 317 DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 318 } else if (size == sizeof(uint8_t)) { 319 uint8_t val M5_VAR_USED = pkt->get<uint8_t>(); 320 DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val); 321 } else { 322 panic("Unknown size for MMIO access: %d\n", size); 323 } 324 325 if (daddr < 0x80) { 326 switch (daddr) { 327 case GEN_CHANCOUNT: 328 case GEN_XFERCAP: 329 case GEN_ATTNSTATUS: 330 DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 331 daddr); 332 break; 333 case GEN_INTRCTRL: 334 regs.intrctrl.master_int_enable(bits(pkt->get<uint8_t>(),0,1)); 335 break; 336 default: 337 panic("Read request to unknown register number: %#x\n", daddr); 338 } 339 pkt->makeAtomicResponse(); 340 return pioDelay; 341 } 342 343 // Find which channel we're accessing 344 int chanid = 0; 345 daddr -= 0x80; 346 while (daddr >= 0x80) { 347 chanid++; 348 daddr -= 0x80; 349 } 350 351 if (chanid >= regs.chanCount) 352 panic("Access to channel %d (device only configured for %d channels)", 353 chanid, regs.chanCount); 354 355 /// 356 /// Channel registers are handled here 357 /// 358 chan[chanid]->channelWrite(pkt, daddr, size); 359 360 pkt->makeAtomicResponse(); 361 return pioDelay; 362} 363 364void 365CopyEngine::CopyEngineChannel::channelWrite(Packet *pkt, Addr daddr, int size) 366{ 367 switch (daddr) { 368 case CHAN_CONTROL: 369 assert(size == sizeof(uint16_t)); 370 int old_int_disable; 371 old_int_disable = cr.ctrl.interrupt_disable(); 372 cr.ctrl(pkt->get<uint16_t>()); 373 if (cr.ctrl.interrupt_disable()) 374 cr.ctrl.interrupt_disable(0); 375 else 376 cr.ctrl.interrupt_disable(old_int_disable); 377 break; 378 case CHAN_STATUS: 379 assert(size == sizeof(uint64_t)); 380 DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n", 381 daddr); 382 break; 383 case CHAN_CHAINADDR: 384 assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 385 if (size == sizeof(uint64_t)) 386 cr.descChainAddr = pkt->get<uint64_t>(); 387 else 388 cr.descChainAddr = (uint64_t)pkt->get<uint32_t>() | 389 (cr.descChainAddr & ~mask(32)); 390 DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 391 break; 392 case CHAN_CHAINADDR_HIGH: 393 assert(size == sizeof(uint32_t)); 394 cr.descChainAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 395 (cr.descChainAddr & mask(32)); 396 DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr); 397 break; 398 case CHAN_COMMAND: 399 assert(size == sizeof(uint8_t)); 400 cr.command(pkt->get<uint8_t>()); 401 recvCommand(); 402 break; 403 case CHAN_CMPLNADDR: 404 assert(size == sizeof(uint64_t) || size == sizeof(uint32_t)); 405 if (size == sizeof(uint64_t)) 406 cr.completionAddr = pkt->get<uint64_t>(); 407 else 408 cr.completionAddr = pkt->get<uint32_t>() | 409 (cr.completionAddr & ~mask(32)); 410 break; 411 case CHAN_CMPLNADDR_HIGH: 412 assert(size == sizeof(uint32_t)); 413 cr.completionAddr = ((uint64_t)pkt->get<uint32_t>() <<32) | 414 (cr.completionAddr & mask(32)); 415 break; 416 case CHAN_ERROR: 417 assert(size == sizeof(uint32_t)); 418 cr.error(~pkt->get<uint32_t>() & cr.error()); 419 break; 420 default: 421 panic("Read request to unknown channel register number: (%d)%#x\n", 422 channelId, daddr); 423 } 424} 425 426void 427CopyEngine::regStats() 428{ 429 using namespace Stats; 430 bytesCopied 431 .init(regs.chanCount) 432 .name(name() + ".bytes_copied") 433 .desc("Number of bytes copied by each engine") 434 .flags(total) 435 ; 436 copiesProcessed 437 .init(regs.chanCount) 438 .name(name() + ".copies_processed") 439 .desc("Number of copies processed by each engine") 440 .flags(total) 441 ; 442} 443 444void 445CopyEngine::CopyEngineChannel::fetchDescriptor(Addr address) 446{ 447 anDq(); 448 anBegin("FetchDescriptor"); 449 DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n", 450 address, ce->platform->pciToDma(address)); 451 assert(address); 452 busy = true; 453 454 DPRINTF(DMACopyEngine, "dmaAction: %#x, %d bytes, to addr %#x\n", 455 ce->platform->pciToDma(address), sizeof(DmaDesc), curDmaDesc); 456 457 cePort.dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(address), 458 sizeof(DmaDesc), &fetchCompleteEvent, 459 (uint8_t*)curDmaDesc, latBeforeBegin); 460 lastDescriptorAddr = address; 461} 462 463void 464CopyEngine::CopyEngineChannel::fetchDescComplete() 465{ 466 DPRINTF(DMACopyEngine, "Read of descriptor complete\n"); 467 468 if ((curDmaDesc->command & DESC_CTRL_NULL)) { 469 DPRINTF(DMACopyEngine, "Got NULL descriptor, skipping\n"); 470 assert(!(curDmaDesc->command & DESC_CTRL_CP_STS)); 471 if (curDmaDesc->command & DESC_CTRL_CP_STS) { 472 panic("Shouldn't be able to get here\n"); 473 nextState = CompletionWrite; 474 if (inDrain()) return; 475 writeCompletionStatus(); 476 } else { 477 anBegin("Idle"); 478 anWait(); 479 busy = false; 480 nextState = Idle; 481 inDrain(); 482 } 483 return; 484 } 485 486 if (curDmaDesc->command & ~DESC_CTRL_CP_STS) 487 panic("Descriptor has flag other that completion status set\n"); 488 489 nextState = DMARead; 490 if (inDrain()) return; 491 readCopyBytes(); 492} 493 494void 495CopyEngine::CopyEngineChannel::readCopyBytes() 496{ 497 anBegin("ReadCopyBytes"); 498 DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n", 499 curDmaDesc->len, curDmaDesc->dest, 500 ce->platform->pciToDma(curDmaDesc->src)); 501 cePort.dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(curDmaDesc->src), 502 curDmaDesc->len, &readCompleteEvent, copyBuffer, 0); 503} 504 505void 506CopyEngine::CopyEngineChannel::readCopyBytesComplete() 507{ 508 DPRINTF(DMACopyEngine, "Read of bytes to copy complete\n"); 509 510 nextState = DMAWrite; 511 if (inDrain()) return; 512 writeCopyBytes(); 513} 514 515void 516CopyEngine::CopyEngineChannel::writeCopyBytes() 517{ 518 anBegin("WriteCopyBytes"); 519 DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n", 520 curDmaDesc->len, curDmaDesc->dest, 521 ce->platform->pciToDma(curDmaDesc->dest)); 522 523 cePort.dmaAction(MemCmd::WriteReq, ce->platform->pciToDma(curDmaDesc->dest), 524 curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0); 525 526 ce->bytesCopied[channelId] += curDmaDesc->len; 527 ce->copiesProcessed[channelId]++; 528} 529 530void 531CopyEngine::CopyEngineChannel::writeCopyBytesComplete() 532{ 533 DPRINTF(DMACopyEngine, "Write of bytes to copy complete user1: %#x\n", 534 curDmaDesc->user1); 535 536 cr.status.compl_desc_addr(lastDescriptorAddr >> 6); 537 completionDataReg = cr.status() | 1; 538 539 anQ("DMAUsedDescQ", channelId, 1); 540 anQ("AppRecvQ", curDmaDesc->user1, curDmaDesc->len); 541 if (curDmaDesc->command & DESC_CTRL_CP_STS) { 542 nextState = CompletionWrite; 543 if (inDrain()) return; 544 writeCompletionStatus(); 545 return; 546 } 547 548 continueProcessing(); 549} 550 551void 552CopyEngine::CopyEngineChannel::continueProcessing() 553{ 554 busy = false; 555 556 if (underReset) { 557 anBegin("Reset"); 558 anWait(); 559 underReset = false; 560 refreshNext = false; 561 busy = false; 562 nextState = Idle; 563 return; 564 } 565 566 if (curDmaDesc->next) { 567 nextState = DescriptorFetch; 568 fetchAddress = curDmaDesc->next; 569 if (inDrain()) return; 570 fetchDescriptor(curDmaDesc->next); 571 } else if (refreshNext) { 572 nextState = AddressFetch; 573 refreshNext = false; 574 if (inDrain()) return; 575 fetchNextAddr(lastDescriptorAddr); 576 } else { 577 inDrain(); 578 nextState = Idle; 579 anWait(); 580 anBegin("Idle"); 581 } 582} 583 584void 585CopyEngine::CopyEngineChannel::writeCompletionStatus() 586{ 587 anBegin("WriteCompletionStatus"); 588 DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n", 589 completionDataReg, cr.completionAddr, 590 ce->platform->pciToDma(cr.completionAddr)); 591 592 cePort.dmaAction(MemCmd::WriteReq, 593 ce->platform->pciToDma(cr.completionAddr), 594 sizeof(completionDataReg), &statusCompleteEvent, 595 (uint8_t*)&completionDataReg, latAfterCompletion); 596} 597 598void 599CopyEngine::CopyEngineChannel::writeStatusComplete() 600{ 601 DPRINTF(DMACopyEngine, "Writing completion status complete\n"); 602 continueProcessing(); 603} 604 605void 606CopyEngine::CopyEngineChannel::fetchNextAddr(Addr address) 607{ 608 anBegin("FetchNextAddr"); 609 DPRINTF(DMACopyEngine, "Fetching next address...\n"); 610 busy = true; 611 cePort.dmaAction(MemCmd::ReadReq, 612 ce->platform->pciToDma(address + offsetof(DmaDesc, next)), 613 sizeof(Addr), &addrCompleteEvent, 614 (uint8_t*)curDmaDesc + offsetof(DmaDesc, next), 0); 615} 616 617void 618CopyEngine::CopyEngineChannel::fetchAddrComplete() 619{ 620 DPRINTF(DMACopyEngine, "Fetching next address complete: %#x\n", 621 curDmaDesc->next); 622 if (!curDmaDesc->next) { 623 DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n"); 624 busy = false; 625 nextState = Idle; 626 anWait(); 627 anBegin("Idle"); 628 inDrain(); 629 return; 630 } 631 nextState = DescriptorFetch; 632 fetchAddress = curDmaDesc->next; 633 if (inDrain()) return; 634 fetchDescriptor(curDmaDesc->next); 635} 636 637bool 638CopyEngine::CopyEngineChannel::inDrain() 639{ 640 if (ce->getState() == SimObject::Draining) { 641 DPRINTF(Drain, "CopyEngine done draining, processing drain event\n"); 642 assert(drainEvent); 643 drainEvent->process(); 644 drainEvent = NULL; 645 } 646 647 return ce->getState() != SimObject::Running; 648} 649 650unsigned int 651CopyEngine::CopyEngineChannel::drain(Event *de) 652{ 653 if (nextState == Idle || ce->getState() != SimObject::Running) 654 return 0; 655 unsigned int count = 1; 656 count += cePort.drain(de); 657 658 DPRINTF(Drain, "CopyEngineChannel not drained\n"); 659 drainEvent = de; 660 return count; 661} 662 663unsigned int 664CopyEngine::drain(Event *de) 665{ 666 unsigned int count; 667 count = pioPort.drain(de) + dmaPort.drain(de) + configPort.drain(de); 668 for (int x = 0;x < chan.size(); x++) 669 count += chan[x]->drain(de); 670 671 if (count) 672 changeState(Draining); 673 else 674 changeState(Drained); 675 676 DPRINTF(Drain, "CopyEngine not drained\n"); 677 return count; 678} 679 680void 681CopyEngine::serialize(std::ostream &os) 682{ 683 PciDev::serialize(os); 684 regs.serialize(os); 685 for (int x =0; x < chan.size(); x++) { 686 nameOut(os, csprintf("%s.channel%d", name(), x)); 687 chan[x]->serialize(os); 688 } 689} 690 691void 692CopyEngine::unserialize(Checkpoint *cp, const std::string §ion) 693{ 694 PciDev::unserialize(cp, section); 695 regs.unserialize(cp, section); 696 for (int x = 0; x < chan.size(); x++) 697 chan[x]->unserialize(cp, csprintf("%s.channel%d", section, x)); 698} 699 700void 701CopyEngine::CopyEngineChannel::serialize(std::ostream &os) 702{ 703 SERIALIZE_SCALAR(channelId); 704 SERIALIZE_SCALAR(busy); 705 SERIALIZE_SCALAR(underReset); 706 SERIALIZE_SCALAR(refreshNext); 707 SERIALIZE_SCALAR(lastDescriptorAddr); 708 SERIALIZE_SCALAR(completionDataReg); 709 SERIALIZE_SCALAR(fetchAddress); 710 int nextState = this->nextState; 711 SERIALIZE_SCALAR(nextState); 712 arrayParamOut(os, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 713 SERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 714 cr.serialize(os); 715 716} 717void 718CopyEngine::CopyEngineChannel::unserialize(Checkpoint *cp, const std::string §ion) 719{ 720 UNSERIALIZE_SCALAR(channelId); 721 UNSERIALIZE_SCALAR(busy); 722 UNSERIALIZE_SCALAR(underReset); 723 UNSERIALIZE_SCALAR(refreshNext); 724 UNSERIALIZE_SCALAR(lastDescriptorAddr); 725 UNSERIALIZE_SCALAR(completionDataReg); 726 UNSERIALIZE_SCALAR(fetchAddress); 727 int nextState; 728 UNSERIALIZE_SCALAR(nextState); 729 this->nextState = (ChannelState)nextState; 730 arrayParamIn(cp, section, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc)); 731 UNSERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap); 732 cr.unserialize(cp, section); 733 734} 735 736void 737CopyEngine::CopyEngineChannel::restartStateMachine() 738{ 739 switch(nextState) { 740 case AddressFetch: 741 fetchNextAddr(lastDescriptorAddr); 742 break; 743 case DescriptorFetch: 744 fetchDescriptor(fetchAddress); 745 break; 746 case DMARead: 747 readCopyBytes(); 748 break; 749 case DMAWrite: 750 writeCopyBytes(); 751 break; 752 case CompletionWrite: 753 writeCompletionStatus(); 754 break; 755 case Idle: 756 break; 757 default: 758 panic("Unknown state for CopyEngineChannel\n"); 759 } 760} 761 762void 763CopyEngine::resume() 764{ 765 SimObject::resume(); 766 for (int x = 0;x < chan.size(); x++) 767 chan[x]->resume(); 768} 769 770 771void 772CopyEngine::CopyEngineChannel::resume() 773{ 774 DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState); 775 restartStateMachine(); 776} 777 778CopyEngine * 779CopyEngineParams::create() 780{ 781 return new CopyEngine(this); 782} 783