1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Nathan Binkert
29 */
30
31#include "dev/net/sinic.hh"
32
33#include <deque>
34#include <limits>
35#include <string>
36
37#ifdef SINIC_VTOPHYS
38#include "arch/vtophys.hh"
39
40#endif
41#include "base/compiler.hh"
42#include "base/debug.hh"
43#include "base/inet.hh"
44#include "base/types.hh"
45#include "config/the_isa.hh"
46#include "debug/EthernetAll.hh"
47#include "dev/net/etherlink.hh"
48#include "mem/packet.hh"
49#include "mem/packet_access.hh"
50#include "sim/eventq.hh"
51#include "sim/stats.hh"
52
53using namespace std;
54using namespace Net;
55using namespace TheISA;
56
57namespace Sinic {
58
59const char *RxStateStrings[] =
60{
61 "rxIdle",
62 "rxFifoBlock",
63 "rxBeginCopy",
64 "rxCopy",
65 "rxCopyDone"
66};
67
68const char *TxStateStrings[] =
69{
70 "txIdle",
71 "txFifoBlock",
72 "txBeginCopy",
73 "txCopy",
74 "txCopyDone"
75};
76
77
78///////////////////////////////////////////////////////////////////////
79//
80// Sinic PCI Device
81//
82Base::Base(const Params *p)
83 : EtherDevBase(p), rxEnable(false), txEnable(false),
84 intrDelay(p->intr_delay), intrTick(0), cpuIntrEnable(false),
85 cpuPendingIntr(false), intrEvent(0), interface(NULL)
86{
87}
88
89Device::Device(const Params *p)
90 : Base(p), rxUnique(0), txUnique(0),
91 virtualRegs(p->virtual_count < 1 ? 1 : p->virtual_count),
92 rxFifo(p->rx_fifo_size), txFifo(p->tx_fifo_size),
93 rxKickTick(0), txKickTick(0),
94 txEvent([this]{ txEventTransmit(); }, name()),
95 rxDmaEvent([this]{ rxDmaDone(); }, name()),
96 txDmaEvent([this]{ txDmaDone(); }, name()),
97 dmaReadDelay(p->dma_read_delay), dmaReadFactor(p->dma_read_factor),
98 dmaWriteDelay(p->dma_write_delay), dmaWriteFactor(p->dma_write_factor)
99{
100 interface = new Interface(name() + ".int0", this);
101 reset();
102
103}
104
105Device::~Device()
106{}
107
108void
109Device::regStats()
110{
111 Base::regStats();
112
113 _maxVnicDistance = 0;
114
115 maxVnicDistance
116 .name(name() + ".maxVnicDistance")
117 .desc("maximum vnic distance")
118 ;
119
120 totalVnicDistance
121 .name(name() + ".totalVnicDistance")
122 .desc("total vnic distance")
123 ;
124 numVnicDistance
125 .name(name() + ".numVnicDistance")
126 .desc("number of vnic distance measurements")
127 ;
128
129 avgVnicDistance
130 .name(name() + ".avgVnicDistance")
131 .desc("average vnic distance")
132 ;
133
134 avgVnicDistance = totalVnicDistance / numVnicDistance;
135}
136
137void
138Device::resetStats()
139{
140 Base::resetStats();
141
142 _maxVnicDistance = 0;
143}
144
145EtherInt*
146Device::getEthPort(const std::string &if_name, int idx)
147{
148 if (if_name == "interface") {
149 if (interface->getPeer())
150 panic("interface already connected to\n");
151
152 return interface;
153 }
154 return NULL;
155}
156
157
158void
159Device::prepareIO(ContextID cpu, int index)
160{
161 int size = virtualRegs.size();
162 if (index > size)
163 panic("Trying to access a vnic that doesn't exist %d > %d\n",
164 index, size);
165}
166
167//add stats for head of line blocking
168//add stats for average fifo length
169//add stats for average number of vnics busy
170
171void
172Device::prepareRead(ContextID cpu, int index)
173{
174 using namespace Regs;
175 prepareIO(cpu, index);
176
177 VirtualReg &vnic = virtualRegs[index];
178
179 // update rx registers
180 uint64_t rxdone = vnic.RxDone;
181 rxdone = set_RxDone_Packets(rxdone, rxFifo.countPacketsAfter(rxFifoPtr));
182 rxdone = set_RxDone_Empty(rxdone, rxFifo.empty());
183 rxdone = set_RxDone_High(rxdone, rxFifo.size() > regs.RxFifoHigh);
184 rxdone = set_RxDone_NotHigh(rxdone, rxLow);
185 regs.RxData = vnic.RxData;
186 regs.RxDone = rxdone;
187 regs.RxWait = rxdone;
188
189 // update tx regsiters
190 uint64_t txdone = vnic.TxDone;
191 txdone = set_TxDone_Packets(txdone, txFifo.packets());
192 txdone = set_TxDone_Full(txdone, txFifo.avail() < regs.TxMaxCopy);
193 txdone = set_TxDone_Low(txdone, txFifo.size() < regs.TxFifoLow);
194 regs.TxData = vnic.TxData;
195 regs.TxDone = txdone;
196 regs.TxWait = txdone;
197
198 int head = 0xffff;
199
200 if (!rxFifo.empty()) {
201 int vnic = rxFifo.begin()->priv;
202 if (vnic != -1 && virtualRegs[vnic].rxPacketOffset > 0)
203 head = vnic;
204 }
205
206 regs.RxStatus = set_RxStatus_Head(regs.RxStatus, head);
207 regs.RxStatus = set_RxStatus_Busy(regs.RxStatus, rxBusyCount);
208 regs.RxStatus = set_RxStatus_Mapped(regs.RxStatus, rxMappedCount);
209 regs.RxStatus = set_RxStatus_Dirty(regs.RxStatus, rxDirtyCount);
210}
211
212void
213Device::prepareWrite(ContextID cpu, int index)
214{
215 prepareIO(cpu, index);
216}
217
218/**
219 * I/O read of device register
220 */
221Tick
222Device::read(PacketPtr pkt)
223{
224 assert(config.command & PCI_CMD_MSE);
225 assert(pkt->getAddr() >= BARAddrs[0] && pkt->getSize() < BARSize[0]);
226
227 ContextID cpu = pkt->req->contextId();
228 Addr daddr = pkt->getAddr() - BARAddrs[0];
229 Addr index = daddr >> Regs::VirtualShift;
230 Addr raddr = daddr & Regs::VirtualMask;
231
232 if (!regValid(raddr))
233 panic("invalid register: cpu=%d vnic=%d da=%#x pa=%#x size=%d",
234 cpu, index, daddr, pkt->getAddr(), pkt->getSize());
235
236 const Regs::Info &info = regInfo(raddr);
237 if (!info.read)
238 panic("read %s (write only): "
239 "cpu=%d vnic=%d da=%#x pa=%#x size=%d",
240 info.name, cpu, index, daddr, pkt->getAddr(), pkt->getSize());
241
242 panic("read %s (invalid size): "
243 "cpu=%d vnic=%d da=%#x pa=%#x size=%d",
244 info.name, cpu, index, daddr, pkt->getAddr(), pkt->getSize());
245
246 prepareRead(cpu, index);
247
248 uint64_t value M5_VAR_USED = 0;
249 if (pkt->getSize() == 4) {
250 uint32_t reg = regData32(raddr);
|
409 panic("vtophys won't work here in newmem.\n");
410#ifdef SINIC_VTOPHYS
411 Addr vaddr = Regs::get_TxData_Addr(reg64);
412 Addr paddr = vtophys(req->xc, vaddr);
413 DPRINTF(EthernetPIO, "write TxData vnic %d (txunique %d): "
414 "vaddr=%#x, paddr=%#x\n",
415 index, vnic.txUnique, vaddr, paddr);
416
417 vnic.TxData = Regs::set_TxData_Addr(vnic.TxData, paddr);
418#endif
419 } else {
420 DPRINTF(EthernetPIO, "write TxData vnic %d (txunique %d)\n",
421 index, vnic.txUnique);
422 }
423
424 if (txList.empty() || txList.front() != index)
425 txList.push_back(index);
426 if (txEnable && txState == txIdle && txList.front() == index) {
427 txState = txFifoBlock;
428 txKick();
429 }
430 break;
431 }
432
433 return pioDelay;
434}
435
436void
437Device::devIntrPost(uint32_t interrupts)
438{
439 if ((interrupts & Regs::Intr_Res))
440 panic("Cannot set a reserved interrupt");
441
442 regs.IntrStatus |= interrupts;
443
444 DPRINTF(EthernetIntr,
445 "interrupt written to intStatus: intr=%#x status=%#x mask=%#x\n",
446 interrupts, regs.IntrStatus, regs.IntrMask);
447
448 interrupts = regs.IntrStatus & regs.IntrMask;
449
450 // Intr_RxHigh is special, we only signal it if we've emptied the fifo
451 // and then filled it above the high watermark
452 if (rxEmpty)
453 rxEmpty = false;
454 else
455 interrupts &= ~Regs::Intr_RxHigh;
456
457 // Intr_TxLow is special, we only signal it if we've filled up the fifo
458 // and then dropped below the low watermark
459 if (txFull)
460 txFull = false;
461 else
462 interrupts &= ~Regs::Intr_TxLow;
463
464 if (interrupts) {
465 Tick when = curTick();
466 if ((interrupts & Regs::Intr_NoDelay) == 0)
467 when += intrDelay;
468 cpuIntrPost(when);
469 }
470}
471
472void
473Device::devIntrClear(uint32_t interrupts)
474{
475 if ((interrupts & Regs::Intr_Res))
476 panic("Cannot clear a reserved interrupt");
477
478 regs.IntrStatus &= ~interrupts;
479
480 DPRINTF(EthernetIntr,
481 "interrupt cleared from intStatus: intr=%x status=%x mask=%x\n",
482 interrupts, regs.IntrStatus, regs.IntrMask);
483
484 if (!(regs.IntrStatus & regs.IntrMask))
485 cpuIntrClear();
486}
487
488void
489Device::devIntrChangeMask(uint32_t newmask)
490{
491 if (regs.IntrMask == newmask)
492 return;
493
494 regs.IntrMask = newmask;
495
496 DPRINTF(EthernetIntr,
497 "interrupt mask changed: intStatus=%x intMask=%x masked=%x\n",
498 regs.IntrStatus, regs.IntrMask, regs.IntrStatus & regs.IntrMask);
499
500 if (regs.IntrStatus & regs.IntrMask)
501 cpuIntrPost(curTick());
502 else
503 cpuIntrClear();
504}
505
506void
507Base::cpuIntrPost(Tick when)
508{
509 // If the interrupt you want to post is later than an interrupt
510 // already scheduled, just let it post in the coming one and don't
511 // schedule another.
512 // HOWEVER, must be sure that the scheduled intrTick is in the
513 // future (this was formerly the source of a bug)
514 /**
515 * @todo this warning should be removed and the intrTick code should
516 * be fixed.
517 */
518 assert(when >= curTick());
519 assert(intrTick >= curTick() || intrTick == 0);
520 if (!cpuIntrEnable) {
521 DPRINTF(EthernetIntr, "interrupts not enabled.\n",
522 intrTick);
523 return;
524 }
525
526 if (when > intrTick && intrTick != 0) {
527 DPRINTF(EthernetIntr, "don't need to schedule event...intrTick=%d\n",
528 intrTick);
529 return;
530 }
531
532 intrTick = when;
533 if (intrTick < curTick()) {
534 intrTick = curTick();
535 }
536
537 DPRINTF(EthernetIntr, "going to schedule an interrupt for intrTick=%d\n",
538 intrTick);
539
540 if (intrEvent)
541 intrEvent->squash();
542
543 intrEvent = new EventFunctionWrapper([this]{ cpuInterrupt(); },
544 name(), true);
545 schedule(intrEvent, intrTick);
546}
547
548void
549Base::cpuInterrupt()
550{
551 assert(intrTick == curTick());
552
553 // Whether or not there's a pending interrupt, we don't care about
554 // it anymore
555 intrEvent = 0;
556 intrTick = 0;
557
558 // Don't send an interrupt if there's already one
559 if (cpuPendingIntr) {
560 DPRINTF(EthernetIntr,
561 "would send an interrupt now, but there's already pending\n");
562 } else {
563 // Send interrupt
564 cpuPendingIntr = true;
565
566 DPRINTF(EthernetIntr, "posting interrupt\n");
567 intrPost();
568 }
569}
570
571void
572Base::cpuIntrClear()
573{
574 if (!cpuPendingIntr)
575 return;
576
577 if (intrEvent) {
578 intrEvent->squash();
579 intrEvent = 0;
580 }
581
582 intrTick = 0;
583
584 cpuPendingIntr = false;
585
586 DPRINTF(EthernetIntr, "clearing cchip interrupt\n");
587 intrClear();
588}
589
590bool
591Base::cpuIntrPending() const
592{ return cpuPendingIntr; }
593
594void
595Device::changeConfig(uint32_t newconf)
596{
597 uint32_t changed = regs.Config ^ newconf;
598 if (!changed)
599 return;
600
601 regs.Config = newconf;
602
603 if ((changed & Regs::Config_IntEn)) {
604 cpuIntrEnable = regs.Config & Regs::Config_IntEn;
605 if (cpuIntrEnable) {
606 if (regs.IntrStatus & regs.IntrMask)
607 cpuIntrPost(curTick());
608 } else {
609 cpuIntrClear();
610 }
611 }
612
613 if ((changed & Regs::Config_TxEn)) {
614 txEnable = regs.Config & Regs::Config_TxEn;
615 if (txEnable)
616 txKick();
617 }
618
619 if ((changed & Regs::Config_RxEn)) {
620 rxEnable = regs.Config & Regs::Config_RxEn;
621 if (rxEnable)
622 rxKick();
623 }
624}
625
626void
627Device::command(uint32_t command)
628{
629 if (command & Regs::Command_Intr)
630 devIntrPost(Regs::Intr_Soft);
631
632 if (command & Regs::Command_Reset)
633 reset();
634}
635
636void
637Device::reset()
638{
639 using namespace Regs;
640
641 memset(®s, 0, sizeof(regs));
642
643 regs.Config = 0;
644 if (params()->rx_thread)
645 regs.Config |= Config_RxThread;
646 if (params()->tx_thread)
647 regs.Config |= Config_TxThread;
648 if (params()->rss)
649 regs.Config |= Config_RSS;
650 if (params()->zero_copy)
651 regs.Config |= Config_ZeroCopy;
652 if (params()->delay_copy)
653 regs.Config |= Config_DelayCopy;
654 if (params()->virtual_addr)
655 regs.Config |= Config_Vaddr;
656
657 if (params()->delay_copy && params()->zero_copy)
658 panic("Can't delay copy and zero copy");
659
660 regs.IntrMask = Intr_Soft | Intr_RxHigh | Intr_RxPacket | Intr_TxLow;
661 regs.RxMaxCopy = params()->rx_max_copy;
662 regs.TxMaxCopy = params()->tx_max_copy;
663 regs.ZeroCopySize = params()->zero_copy_size;
664 regs.ZeroCopyMark = params()->zero_copy_threshold;
665 regs.VirtualCount = params()->virtual_count;
666 regs.RxMaxIntr = params()->rx_max_intr;
667 regs.RxFifoSize = params()->rx_fifo_size;
668 regs.TxFifoSize = params()->tx_fifo_size;
669 regs.RxFifoLow = params()->rx_fifo_low_mark;
670 regs.TxFifoLow = params()->tx_fifo_threshold;
671 regs.RxFifoHigh = params()->rx_fifo_threshold;
672 regs.TxFifoHigh = params()->tx_fifo_high_mark;
673 regs.HwAddr = params()->hardware_address;
674
675 if (regs.RxMaxCopy < regs.ZeroCopyMark)
676 panic("Must be able to copy at least as many bytes as the threshold");
677
678 if (regs.ZeroCopySize >= regs.ZeroCopyMark)
679 panic("The number of bytes to copy must be less than the threshold");
680
681 rxList.clear();
682 rxBusy.clear();
683 rxActive = -1;
684 txList.clear();
685 rxBusyCount = 0;
686 rxDirtyCount = 0;
687 rxMappedCount = 0;
688
689 rxState = rxIdle;
690 txState = txIdle;
691
692 rxFifo.clear();
693 rxFifoPtr = rxFifo.end();
694 txFifo.clear();
695 rxEmpty = false;
696 rxLow = true;
697 txFull = false;
698
699 int size = virtualRegs.size();
700 virtualRegs.clear();
701 virtualRegs.resize(size);
702 for (int i = 0; i < size; ++i)
703 virtualRegs[i].rxIndex = rxFifo.end();
704}
705
706void
707Device::rxDmaDone()
708{
709 assert(rxState == rxCopy);
710 rxState = rxCopyDone;
711 DPRINTF(EthernetDMA, "end rx dma write paddr=%#x len=%d\n",
712 rxDmaAddr, rxDmaLen);
713 DDUMP(EthernetData, rxDmaData, rxDmaLen);
714
715 // If the transmit state machine has a pending DMA, let it go first
716 if (txState == txBeginCopy)
717 txKick();
718
719 rxKick();
720}
721
722void
723Device::rxKick()
724{
725 VirtualReg *vnic = NULL;
726
727 DPRINTF(EthernetSM, "rxKick: rxState=%s (rxFifo.size=%d)\n",
728 RxStateStrings[rxState], rxFifo.size());
729
730 if (rxKickTick > curTick()) {
731 DPRINTF(EthernetSM, "rxKick: exiting, can't run till %d\n",
732 rxKickTick);
733 return;
734 }
735
736 next:
737 rxFifo.check();
738 if (rxState == rxIdle)
739 goto exit;
740
741 if (rxActive == -1) {
742 if (rxState != rxFifoBlock)
743 panic("no active vnic while in state %s", RxStateStrings[rxState]);
744
745 DPRINTF(EthernetSM, "processing rxState=%s\n",
746 RxStateStrings[rxState]);
747 } else {
748 vnic = &virtualRegs[rxActive];
749 DPRINTF(EthernetSM,
750 "processing rxState=%s for vnic %d (rxunique %d)\n",
751 RxStateStrings[rxState], rxActive, vnic->rxUnique);
752 }
753
754 switch (rxState) {
755 case rxFifoBlock:
756 if (DTRACE(EthernetSM)) {
757 PacketFifo::iterator end = rxFifo.end();
758 int size = virtualRegs.size();
759 for (int i = 0; i < size; ++i) {
760 VirtualReg *vn = &virtualRegs[i];
761 bool busy = Regs::get_RxDone_Busy(vn->RxDone);
762 if (vn->rxIndex != end) {
763#ifndef NDEBUG
764 bool dirty = vn->rxPacketOffset > 0;
765 const char *status;
766
767 if (busy && dirty)
768 status = "busy,dirty";
769 else if (busy)
770 status = "busy";
771 else if (dirty)
772 status = "dirty";
773 else
774 status = "mapped";
775
776 DPRINTF(EthernetSM,
777 "vnic %d %s (rxunique %d), packet %d, slack %d\n",
778 i, status, vn->rxUnique,
779 rxFifo.countPacketsBefore(vn->rxIndex),
780 vn->rxIndex->slack);
781#endif
782 } else if (busy) {
783 DPRINTF(EthernetSM, "vnic %d unmapped (rxunique %d)\n",
784 i, vn->rxUnique);
785 }
786 }
787 }
788
789 if (!rxBusy.empty()) {
790 rxActive = rxBusy.front();
791 rxBusy.pop_front();
792 vnic = &virtualRegs[rxActive];
793
794 if (vnic->rxIndex == rxFifo.end())
795 panic("continuing vnic without packet\n");
796
797 DPRINTF(EthernetSM,
798 "continue processing for vnic %d (rxunique %d)\n",
799 rxActive, vnic->rxUnique);
800
801 rxState = rxBeginCopy;
802
803 int vnic_distance = rxFifo.countPacketsBefore(vnic->rxIndex);
804 totalVnicDistance += vnic_distance;
805 numVnicDistance += 1;
806 if (vnic_distance > _maxVnicDistance) {
807 maxVnicDistance = vnic_distance;
808 _maxVnicDistance = vnic_distance;
809 }
810
811 break;
812 }
813
814 if (rxFifoPtr == rxFifo.end()) {
815 DPRINTF(EthernetSM, "receive waiting for data. Nothing to do.\n");
816 goto exit;
817 }
818
819 if (rxList.empty())
820 panic("Not idle, but nothing to do!");
821
822 assert(!rxFifo.empty());
823
824 rxActive = rxList.front();
825 rxList.pop_front();
826 vnic = &virtualRegs[rxActive];
827
828 DPRINTF(EthernetSM,
829 "processing new packet for vnic %d (rxunique %d)\n",
830 rxActive, vnic->rxUnique);
831
832 // Grab a new packet from the fifo.
833 vnic->rxIndex = rxFifoPtr++;
834 vnic->rxIndex->priv = rxActive;
835 vnic->rxPacketOffset = 0;
836 vnic->rxPacketBytes = vnic->rxIndex->packet->length;
837 assert(vnic->rxPacketBytes);
838 rxMappedCount++;
839
840 vnic->rxDoneData = 0;
841 /* scope for variables */ {
842 IpPtr ip(vnic->rxIndex->packet);
843 if (ip) {
844 DPRINTF(Ethernet, "ID is %d\n", ip->id());
845 vnic->rxDoneData |= Regs::RxDone_IpPacket;
846 rxIpChecksums++;
847 if (cksum(ip) != 0) {
848 DPRINTF(EthernetCksum, "Rx IP Checksum Error\n");
849 vnic->rxDoneData |= Regs::RxDone_IpError;
850 }
851 TcpPtr tcp(ip);
852 UdpPtr udp(ip);
853 if (tcp) {
854 DPRINTF(Ethernet,
855 "Src Port=%d, Dest Port=%d, Seq=%d, Ack=%d\n",
856 tcp->sport(), tcp->dport(), tcp->seq(),
857 tcp->ack());
858 vnic->rxDoneData |= Regs::RxDone_TcpPacket;
859 rxTcpChecksums++;
860 if (cksum(tcp) != 0) {
861 DPRINTF(EthernetCksum, "Rx TCP Checksum Error\n");
862 vnic->rxDoneData |= Regs::RxDone_TcpError;
863 }
864 } else if (udp) {
865 vnic->rxDoneData |= Regs::RxDone_UdpPacket;
866 rxUdpChecksums++;
867 if (cksum(udp) != 0) {
868 DPRINTF(EthernetCksum, "Rx UDP Checksum Error\n");
869 vnic->rxDoneData |= Regs::RxDone_UdpError;
870 }
871 }
872 }
873 }
874 rxState = rxBeginCopy;
875 break;
876
877 case rxBeginCopy:
878 if (dmaPending() || drainState() != DrainState::Running)
879 goto exit;
880
881 rxDmaAddr = pciToDma(Regs::get_RxData_Addr(vnic->RxData));
882 rxDmaLen = min<unsigned>(Regs::get_RxData_Len(vnic->RxData),
883 vnic->rxPacketBytes);
884
885 /*
886 * if we're doing zero/delay copy and we're below the fifo
887 * threshold, see if we should try to do the zero/defer copy
888 */
889 if ((Regs::get_Config_ZeroCopy(regs.Config) ||
890 Regs::get_Config_DelayCopy(regs.Config)) &&
891 !Regs::get_RxData_NoDelay(vnic->RxData) && rxLow) {
892 if (rxDmaLen > regs.ZeroCopyMark)
893 rxDmaLen = regs.ZeroCopySize;
894 }
895 rxDmaData = vnic->rxIndex->packet->data + vnic->rxPacketOffset;
896 rxState = rxCopy;
897 if (rxDmaAddr == 1LL) {
898 rxState = rxCopyDone;
899 break;
900 }
901
902 dmaWrite(rxDmaAddr, rxDmaLen, &rxDmaEvent, rxDmaData);
903 break;
904
905 case rxCopy:
906 DPRINTF(EthernetSM, "receive machine still copying\n");
907 goto exit;
908
909 case rxCopyDone:
910 vnic->RxDone = vnic->rxDoneData;
911 vnic->RxDone |= Regs::RxDone_Complete;
912 rxBusyCount--;
913
914 if (vnic->rxPacketBytes == rxDmaLen) {
915 if (vnic->rxPacketOffset)
916 rxDirtyCount--;
917
918 // Packet is complete. Indicate how many bytes were copied
919 vnic->RxDone = Regs::set_RxDone_CopyLen(vnic->RxDone, rxDmaLen);
920
921 DPRINTF(EthernetSM,
922 "rxKick: packet complete on vnic %d (rxunique %d)\n",
923 rxActive, vnic->rxUnique);
924 rxFifo.remove(vnic->rxIndex);
925 vnic->rxIndex = rxFifo.end();
926 rxMappedCount--;
927 } else {
928 if (!vnic->rxPacketOffset)
929 rxDirtyCount++;
930
931 vnic->rxPacketBytes -= rxDmaLen;
932 vnic->rxPacketOffset += rxDmaLen;
933 vnic->RxDone |= Regs::RxDone_More;
934 vnic->RxDone = Regs::set_RxDone_CopyLen(vnic->RxDone,
935 vnic->rxPacketBytes);
936 DPRINTF(EthernetSM,
937 "rxKick: packet not complete on vnic %d (rxunique %d): "
938 "%d bytes left\n",
939 rxActive, vnic->rxUnique, vnic->rxPacketBytes);
940 }
941
942 rxActive = -1;
943 rxState = rxBusy.empty() && rxList.empty() ? rxIdle : rxFifoBlock;
944
945 if (rxFifo.empty()) {
946 devIntrPost(Regs::Intr_RxEmpty);
947 rxEmpty = true;
948 }
949
950 if (rxFifo.size() < regs.RxFifoLow)
951 rxLow = true;
952
953 if (rxFifo.size() > regs.RxFifoHigh)
954 rxLow = false;
955
956 devIntrPost(Regs::Intr_RxDMA);
957 break;
958
959 default:
960 panic("Invalid rxState!");
961 }
962
963 DPRINTF(EthernetSM, "entering next rxState=%s\n",
964 RxStateStrings[rxState]);
965
966 goto next;
967
968 exit:
969 /**
970 * @todo do we want to schedule a future kick?
971 */
972 DPRINTF(EthernetSM, "rx state machine exited rxState=%s\n",
973 RxStateStrings[rxState]);
974}
975
976void
977Device::txDmaDone()
978{
979 assert(txState == txCopy);
980 txState = txCopyDone;
981 DPRINTF(EthernetDMA, "tx dma read paddr=%#x len=%d\n",
982 txDmaAddr, txDmaLen);
983 DDUMP(EthernetData, txDmaData, txDmaLen);
984
985 // If the receive state machine has a pending DMA, let it go first
986 if (rxState == rxBeginCopy)
987 rxKick();
988
989 txKick();
990}
991
992void
993Device::transmit()
994{
995 if (txFifo.empty()) {
996 DPRINTF(Ethernet, "nothing to transmit\n");
997 return;
998 }
999
1000 uint32_t interrupts;
1001 EthPacketPtr packet = txFifo.front();
1002 if (!interface->sendPacket(packet)) {
1003 DPRINTF(Ethernet, "Packet Transmit: failed txFifo available %d\n",
1004 txFifo.avail());
1005 return;
1006 }
1007
1008 txFifo.pop();
1009#if TRACING_ON
1010 if (DTRACE(Ethernet)) {
1011 IpPtr ip(packet);
1012 if (ip) {
1013 DPRINTF(Ethernet, "ID is %d\n", ip->id());
1014 TcpPtr tcp(ip);
1015 if (tcp) {
1016 DPRINTF(Ethernet,
1017 "Src Port=%d, Dest Port=%d, Seq=%d, Ack=%d\n",
1018 tcp->sport(), tcp->dport(), tcp->seq(),
1019 tcp->ack());
1020 }
1021 }
1022 }
1023#endif
1024
1025 DDUMP(EthernetData, packet->data, packet->length);
1026 txBytes += packet->length;
1027 txPackets++;
1028
1029 DPRINTF(Ethernet, "Packet Transmit: successful txFifo Available %d\n",
1030 txFifo.avail());
1031
1032 interrupts = Regs::Intr_TxPacket;
1033 if (txFifo.size() < regs.TxFifoLow)
1034 interrupts |= Regs::Intr_TxLow;
1035 devIntrPost(interrupts);
1036}
1037
1038void
1039Device::txKick()
1040{
1041 VirtualReg *vnic;
1042 DPRINTF(EthernetSM, "txKick: txState=%s (txFifo.size=%d)\n",
1043 TxStateStrings[txState], txFifo.size());
1044
1045 if (txKickTick > curTick()) {
1046 DPRINTF(EthernetSM, "txKick: exiting, can't run till %d\n",
1047 txKickTick);
1048 return;
1049 }
1050
1051 next:
1052 if (txState == txIdle)
1053 goto exit;
1054
1055 assert(!txList.empty());
1056 vnic = &virtualRegs[txList.front()];
1057
1058 switch (txState) {
1059 case txFifoBlock:
1060 assert(Regs::get_TxDone_Busy(vnic->TxDone));
1061 if (!txPacket) {
1062 // Grab a new packet from the fifo.
1063 txPacket = make_shared<EthPacketData>(16384);
1064 txPacketOffset = 0;
1065 }
1066
1067 if (txFifo.avail() - txPacket->length <
1068 Regs::get_TxData_Len(vnic->TxData)) {
1069 DPRINTF(EthernetSM, "transmit fifo full. Nothing to do.\n");
1070 goto exit;
1071 }
1072
1073 txState = txBeginCopy;
1074 break;
1075
1076 case txBeginCopy:
1077 if (dmaPending() || drainState() != DrainState::Running)
1078 goto exit;
1079
1080 txDmaAddr = pciToDma(Regs::get_TxData_Addr(vnic->TxData));
1081 txDmaLen = Regs::get_TxData_Len(vnic->TxData);
1082 txDmaData = txPacket->data + txPacketOffset;
1083 txState = txCopy;
1084
1085 dmaRead(txDmaAddr, txDmaLen, &txDmaEvent, txDmaData);
1086 break;
1087
1088 case txCopy:
1089 DPRINTF(EthernetSM, "transmit machine still copying\n");
1090 goto exit;
1091
1092 case txCopyDone:
1093 vnic->TxDone = txDmaLen | Regs::TxDone_Complete;
1094 txPacket->simLength += txDmaLen;
1095 txPacket->length += txDmaLen;
1096 if ((vnic->TxData & Regs::TxData_More)) {
1097 txPacketOffset += txDmaLen;
1098 txState = txIdle;
1099 devIntrPost(Regs::Intr_TxDMA);
1100 break;
1101 }
1102
1103 assert(txPacket->length <= txFifo.avail());
1104 if ((vnic->TxData & Regs::TxData_Checksum)) {
1105 IpPtr ip(txPacket);
1106 if (ip) {
1107 TcpPtr tcp(ip);
1108 if (tcp) {
1109 tcp->sum(0);
1110 tcp->sum(cksum(tcp));
1111 txTcpChecksums++;
1112 }
1113
1114 UdpPtr udp(ip);
1115 if (udp) {
1116 udp->sum(0);
1117 udp->sum(cksum(udp));
1118 txUdpChecksums++;
1119 }
1120
1121 ip->sum(0);
1122 ip->sum(cksum(ip));
1123 txIpChecksums++;
1124 }
1125 }
1126
1127 txFifo.push(txPacket);
1128 if (txFifo.avail() < regs.TxMaxCopy) {
1129 devIntrPost(Regs::Intr_TxFull);
1130 txFull = true;
1131 }
1132 txPacket = 0;
1133 transmit();
1134 txList.pop_front();
1135 txState = txList.empty() ? txIdle : txFifoBlock;
1136 devIntrPost(Regs::Intr_TxDMA);
1137 break;
1138
1139 default:
1140 panic("Invalid txState!");
1141 }
1142
1143 DPRINTF(EthernetSM, "entering next txState=%s\n",
1144 TxStateStrings[txState]);
1145
1146 goto next;
1147
1148 exit:
1149 /**
1150 * @todo do we want to schedule a future kick?
1151 */
1152 DPRINTF(EthernetSM, "tx state machine exited txState=%s\n",
1153 TxStateStrings[txState]);
1154}
1155
1156void
1157Device::transferDone()
1158{
1159 if (txFifo.empty()) {
1160 DPRINTF(Ethernet, "transfer complete: txFifo empty...nothing to do\n");
1161 return;
1162 }
1163
1164 DPRINTF(Ethernet, "transfer complete: data in txFifo...schedule xmit\n");
1165
1166 reschedule(txEvent, clockEdge(Cycles(1)), true);
1167}
1168
1169bool
1170Device::rxFilter(const EthPacketPtr &packet)
1171{
1172 if (!Regs::get_Config_Filter(regs.Config))
1173 return false;
1174
1175 panic("receive filter not implemented\n");
1176 bool drop = true;
1177
1178#if 0
1179 string type;
1180
1181 EthHdr *eth = packet->eth();
1182 if (eth->unicast()) {
1183 // If we're accepting all unicast addresses
1184 if (acceptUnicast)
1185 drop = false;
1186
1187 // If we make a perfect match
1188 if (acceptPerfect && params->eaddr == eth.dst())
1189 drop = false;
1190
1191 if (acceptArp && eth->type() == ETH_TYPE_ARP)
1192 drop = false;
1193
1194 } else if (eth->broadcast()) {
1195 // if we're accepting broadcasts
1196 if (acceptBroadcast)
1197 drop = false;
1198
1199 } else if (eth->multicast()) {
1200 // if we're accepting all multicasts
1201 if (acceptMulticast)
1202 drop = false;
1203
1204 }
1205
1206 if (drop) {
1207 DPRINTF(Ethernet, "rxFilter drop\n");
1208 DDUMP(EthernetData, packet->data, packet->length);
1209 }
1210#endif
1211 return drop;
1212}
1213
1214bool
1215Device::recvPacket(EthPacketPtr packet)
1216{
1217 rxBytes += packet->length;
1218 rxPackets++;
1219
1220 DPRINTF(Ethernet, "Receiving packet from wire, rxFifo Available is %d\n",
1221 rxFifo.avail());
1222
1223 if (!rxEnable) {
1224 DPRINTF(Ethernet, "receive disabled...packet dropped\n");
1225 return true;
1226 }
1227
1228 if (rxFilter(packet)) {
1229 DPRINTF(Ethernet, "packet filtered...dropped\n");
1230 return true;
1231 }
1232
1233 if (rxFifo.size() >= regs.RxFifoHigh)
1234 devIntrPost(Regs::Intr_RxHigh);
1235
1236 if (!rxFifo.push(packet)) {
1237 DPRINTF(Ethernet,
1238 "packet will not fit in receive buffer...packet dropped\n");
1239 return false;
1240 }
1241
1242 // If we were at the last element, back up one ot go to the new
1243 // last element of the list.
1244 if (rxFifoPtr == rxFifo.end())
1245 --rxFifoPtr;
1246
1247 devIntrPost(Regs::Intr_RxPacket);
1248 rxKick();
1249 return true;
1250}
1251
1252void
1253Device::drainResume()
1254{
1255 Drainable::drainResume();
1256
1257 // During drain we could have left the state machines in a waiting state and
1258 // they wouldn't get out until some other event occured to kick them.
1259 // This way they'll get out immediately
1260 txKick();
1261 rxKick();
1262}
1263
1264//=====================================================================
1265//
1266//
1267void
1268Base::serialize(CheckpointOut &cp) const
1269{
1270 // Serialize the PciDevice base class
1271 PciDevice::serialize(cp);
1272
1273 SERIALIZE_SCALAR(rxEnable);
1274 SERIALIZE_SCALAR(txEnable);
1275 SERIALIZE_SCALAR(cpuIntrEnable);
1276
1277 /*
1278 * Keep track of pending interrupt status.
1279 */
1280 SERIALIZE_SCALAR(intrTick);
1281 SERIALIZE_SCALAR(cpuPendingIntr);
1282 Tick intrEventTick = 0;
1283 if (intrEvent)
1284 intrEventTick = intrEvent->when();
1285 SERIALIZE_SCALAR(intrEventTick);
1286}
1287
1288void
1289Base::unserialize(CheckpointIn &cp)
1290{
1291 // Unserialize the PciDevice base class
1292 PciDevice::unserialize(cp);
1293
1294 UNSERIALIZE_SCALAR(rxEnable);
1295 UNSERIALIZE_SCALAR(txEnable);
1296 UNSERIALIZE_SCALAR(cpuIntrEnable);
1297
1298 /*
1299 * Keep track of pending interrupt status.
1300 */
1301 UNSERIALIZE_SCALAR(intrTick);
1302 UNSERIALIZE_SCALAR(cpuPendingIntr);
1303 Tick intrEventTick;
1304 UNSERIALIZE_SCALAR(intrEventTick);
1305 if (intrEventTick) {
1306 intrEvent = new EventFunctionWrapper([this]{ cpuInterrupt(); },
1307 name(), true);
1308 schedule(intrEvent, intrEventTick);
1309 }
1310}
1311
1312void
1313Device::serialize(CheckpointOut &cp) const
1314{
1315 int count;
1316
1317 // Serialize the PciDevice base class
1318 Base::serialize(cp);
1319
1320 if (rxState == rxCopy)
1321 panic("can't serialize with an in flight dma request rxState=%s",
1322 RxStateStrings[rxState]);
1323
1324 if (txState == txCopy)
1325 panic("can't serialize with an in flight dma request txState=%s",
1326 TxStateStrings[txState]);
1327
1328 /*
1329 * Serialize the device registers that could be modified by the OS.
1330 */
1331 SERIALIZE_SCALAR(regs.Config);
1332 SERIALIZE_SCALAR(regs.IntrStatus);
1333 SERIALIZE_SCALAR(regs.IntrMask);
1334 SERIALIZE_SCALAR(regs.RxData);
1335 SERIALIZE_SCALAR(regs.TxData);
1336
1337 /*
1338 * Serialize the virtual nic state
1339 */
1340 int virtualRegsSize = virtualRegs.size();
1341 SERIALIZE_SCALAR(virtualRegsSize);
1342 for (int i = 0; i < virtualRegsSize; ++i) {
1343 const VirtualReg *vnic = &virtualRegs[i];
1344
1345 std::string reg = csprintf("vnic%d", i);
1346 paramOut(cp, reg + ".RxData", vnic->RxData);
1347 paramOut(cp, reg + ".RxDone", vnic->RxDone);
1348 paramOut(cp, reg + ".TxData", vnic->TxData);
1349 paramOut(cp, reg + ".TxDone", vnic->TxDone);
1350
1351 bool rxPacketExists = vnic->rxIndex != rxFifo.end();
1352 paramOut(cp, reg + ".rxPacketExists", rxPacketExists);
1353 if (rxPacketExists) {
1354 int rxPacket = 0;
1355 auto i = rxFifo.begin();
1356 while (i != vnic->rxIndex) {
1357 assert(i != rxFifo.end());
1358 ++i;
1359 ++rxPacket;
1360 }
1361
1362 paramOut(cp, reg + ".rxPacket", rxPacket);
1363 paramOut(cp, reg + ".rxPacketOffset", vnic->rxPacketOffset);
1364 paramOut(cp, reg + ".rxPacketBytes", vnic->rxPacketBytes);
1365 }
1366 paramOut(cp, reg + ".rxDoneData", vnic->rxDoneData);
1367 }
1368
1369 int rxFifoPtr = -1;
1370 if (this->rxFifoPtr != rxFifo.end())
1371 rxFifoPtr = rxFifo.countPacketsBefore(this->rxFifoPtr);
1372 SERIALIZE_SCALAR(rxFifoPtr);
1373
1374 SERIALIZE_SCALAR(rxActive);
1375 SERIALIZE_SCALAR(rxBusyCount);
1376 SERIALIZE_SCALAR(rxDirtyCount);
1377 SERIALIZE_SCALAR(rxMappedCount);
1378
1379 VirtualList::const_iterator i, end;
1380 for (count = 0, i = rxList.begin(), end = rxList.end(); i != end; ++i)
1381 paramOut(cp, csprintf("rxList%d", count++), *i);
1382 int rxListSize = count;
1383 SERIALIZE_SCALAR(rxListSize);
1384
1385 for (count = 0, i = rxBusy.begin(), end = rxBusy.end(); i != end; ++i)
1386 paramOut(cp, csprintf("rxBusy%d", count++), *i);
1387 int rxBusySize = count;
1388 SERIALIZE_SCALAR(rxBusySize);
1389
1390 for (count = 0, i = txList.begin(), end = txList.end(); i != end; ++i)
1391 paramOut(cp, csprintf("txList%d", count++), *i);
1392 int txListSize = count;
1393 SERIALIZE_SCALAR(txListSize);
1394
1395 /*
1396 * Serialize rx state machine
1397 */
1398 int rxState = this->rxState;
1399 SERIALIZE_SCALAR(rxState);
1400 SERIALIZE_SCALAR(rxEmpty);
1401 SERIALIZE_SCALAR(rxLow);
1402 rxFifo.serialize("rxFifo", cp);
1403
1404 /*
1405 * Serialize tx state machine
1406 */
1407 int txState = this->txState;
1408 SERIALIZE_SCALAR(txState);
1409 SERIALIZE_SCALAR(txFull);
1410 txFifo.serialize("txFifo", cp);
1411 bool txPacketExists = txPacket != nullptr;
1412 SERIALIZE_SCALAR(txPacketExists);
1413 if (txPacketExists) {
1414 txPacket->serialize("txPacket", cp);
1415 SERIALIZE_SCALAR(txPacketOffset);
1416 SERIALIZE_SCALAR(txPacketBytes);
1417 }
1418
1419 /*
1420 * If there's a pending transmit, store the time so we can
1421 * reschedule it later
1422 */
1423 Tick transmitTick = txEvent.scheduled() ? txEvent.when() - curTick() : 0;
1424 SERIALIZE_SCALAR(transmitTick);
1425}
1426
1427void
1428Device::unserialize(CheckpointIn &cp)
1429{
1430 // Unserialize the PciDevice base class
1431 Base::unserialize(cp);
1432
1433 /*
1434 * Unserialize the device registers that may have been written by the OS.
1435 */
1436 UNSERIALIZE_SCALAR(regs.Config);
1437 UNSERIALIZE_SCALAR(regs.IntrStatus);
1438 UNSERIALIZE_SCALAR(regs.IntrMask);
1439 UNSERIALIZE_SCALAR(regs.RxData);
1440 UNSERIALIZE_SCALAR(regs.TxData);
1441
1442 UNSERIALIZE_SCALAR(rxActive);
1443 UNSERIALIZE_SCALAR(rxBusyCount);
1444 UNSERIALIZE_SCALAR(rxDirtyCount);
1445 UNSERIALIZE_SCALAR(rxMappedCount);
1446
1447 int rxListSize;
1448 UNSERIALIZE_SCALAR(rxListSize);
1449 rxList.clear();
1450 for (int i = 0; i < rxListSize; ++i) {
1451 int value;
1452 paramIn(cp, csprintf("rxList%d", i), value);
1453 rxList.push_back(value);
1454 }
1455
1456 int rxBusySize;
1457 UNSERIALIZE_SCALAR(rxBusySize);
1458 rxBusy.clear();
1459 for (int i = 0; i < rxBusySize; ++i) {
1460 int value;
1461 paramIn(cp, csprintf("rxBusy%d", i), value);
1462 rxBusy.push_back(value);
1463 }
1464
1465 int txListSize;
1466 UNSERIALIZE_SCALAR(txListSize);
1467 txList.clear();
1468 for (int i = 0; i < txListSize; ++i) {
1469 int value;
1470 paramIn(cp, csprintf("txList%d", i), value);
1471 txList.push_back(value);
1472 }
1473
1474 /*
1475 * Unserialize rx state machine
1476 */
1477 int rxState;
1478 UNSERIALIZE_SCALAR(rxState);
1479 UNSERIALIZE_SCALAR(rxEmpty);
1480 UNSERIALIZE_SCALAR(rxLow);
1481 this->rxState = (RxState) rxState;
1482 rxFifo.unserialize("rxFifo", cp);
1483
1484 int rxFifoPtr;
1485 UNSERIALIZE_SCALAR(rxFifoPtr);
1486 if (rxFifoPtr >= 0) {
1487 this->rxFifoPtr = rxFifo.begin();
1488 for (int i = 0; i < rxFifoPtr; ++i)
1489 ++this->rxFifoPtr;
1490 } else {
1491 this->rxFifoPtr = rxFifo.end();
1492 }
1493
1494 /*
1495 * Unserialize tx state machine
1496 */
1497 int txState;
1498 UNSERIALIZE_SCALAR(txState);
1499 UNSERIALIZE_SCALAR(txFull);
1500 this->txState = (TxState) txState;
1501 txFifo.unserialize("txFifo", cp);
1502 bool txPacketExists;
1503 UNSERIALIZE_SCALAR(txPacketExists);
1504 txPacket = 0;
1505 if (txPacketExists) {
1506 txPacket = make_shared<EthPacketData>(16384);
1507 txPacket->unserialize("txPacket", cp);
1508 UNSERIALIZE_SCALAR(txPacketOffset);
1509 UNSERIALIZE_SCALAR(txPacketBytes);
1510 }
1511
1512 /*
1513 * unserialize the virtual nic registers/state
1514 *
1515 * this must be done after the unserialization of the rxFifo
1516 * because the packet iterators depend on the fifo being populated
1517 */
1518 int virtualRegsSize;
1519 UNSERIALIZE_SCALAR(virtualRegsSize);
1520 virtualRegs.clear();
1521 virtualRegs.resize(virtualRegsSize);
1522 for (int i = 0; i < virtualRegsSize; ++i) {
1523 VirtualReg *vnic = &virtualRegs[i];
1524 std::string reg = csprintf("vnic%d", i);
1525
1526 paramIn(cp, reg + ".RxData", vnic->RxData);
1527 paramIn(cp, reg + ".RxDone", vnic->RxDone);
1528 paramIn(cp, reg + ".TxData", vnic->TxData);
1529 paramIn(cp, reg + ".TxDone", vnic->TxDone);
1530
1531 vnic->rxUnique = rxUnique++;
1532 vnic->txUnique = txUnique++;
1533
1534 bool rxPacketExists;
1535 paramIn(cp, reg + ".rxPacketExists", rxPacketExists);
1536 if (rxPacketExists) {
1537 int rxPacket;
1538 paramIn(cp, reg + ".rxPacket", rxPacket);
1539 vnic->rxIndex = rxFifo.begin();
1540 while (rxPacket--)
1541 ++vnic->rxIndex;
1542
1543 paramIn(cp, reg + ".rxPacketOffset",
1544 vnic->rxPacketOffset);
1545 paramIn(cp, reg + ".rxPacketBytes", vnic->rxPacketBytes);
1546 } else {
1547 vnic->rxIndex = rxFifo.end();
1548 }
1549 paramIn(cp, reg + ".rxDoneData", vnic->rxDoneData);
1550 }
1551
1552 /*
1553 * If there's a pending transmit, reschedule it now
1554 */
1555 Tick transmitTick;
1556 UNSERIALIZE_SCALAR(transmitTick);
1557 if (transmitTick)
1558 schedule(txEvent, curTick() + transmitTick);
1559
1560 pioPort.sendRangeChange();
1561
1562}
1563
1564} // namespace Sinic
1565
1566Sinic::Device *
1567SinicParams::create()
1568{
1569 return new Sinic::Device(this);
1570}
|