gic_v3_its.cc revision 14231 
1/*
2 * Copyright (c) 2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Giacomo Travaglini
38 */
39
40#include "dev/arm/gic_v3_its.hh"
41
42#include "debug/AddrRanges.hh"
43#include "debug/Drain.hh"
44#include "debug/GIC.hh"
45#include "debug/ITS.hh"
46#include "dev/arm/gic_v3.hh"
47#include "dev/arm/gic_v3_distributor.hh"
48#include "dev/arm/gic_v3_redistributor.hh"
49#include "mem/packet_access.hh"
50
51#define COMMAND(x, method) { x, DispatchEntry(#x, method) }
52
53const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0138);
54
55const uint32_t Gicv3Its::CTLR_QUIESCENT = 0x80000000;
56
57ItsProcess::ItsProcess(Gicv3Its &_its)
58  : its(_its), coroutine(nullptr)
59{
60}
61
62ItsProcess::~ItsProcess()
63{
64}
65
66void
67ItsProcess::reinit()
68{
69    coroutine.reset(new Coroutine(
70        std::bind(&ItsProcess::main, this, std::placeholders::_1)));
71}
72
73const std::string
74ItsProcess::name() const
75{
76    return its.name();
77}
78
79ItsAction
80ItsProcess::run(PacketPtr pkt)
81{
82    assert(coroutine != nullptr);
83    assert(*coroutine);
84    return (*coroutine)(pkt).get();
85}
86
87void
88ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)
89{
90    ItsAction a;
91    a.type = ItsActionType::SEND_REQ;
92
93    RequestPtr req = std::make_shared<Request>(
94        addr, size, 0, its.masterId);
95
96    req->taskId(ContextSwitchTaskId::DMA);
97
98    a.pkt = new Packet(req, MemCmd::ReadReq);
99    a.pkt->dataStatic(ptr);
100
101    a.delay = 0;
102
103    PacketPtr pkt = yield(a).get();
104
105    assert(pkt);
106    assert(pkt->getSize() >= size);
107
108    delete pkt;
109}
110
111void
112ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)
113{
114    ItsAction a;
115    a.type = ItsActionType::SEND_REQ;
116
117    RequestPtr req = std::make_shared<Request>(
118        addr, size, 0, its.masterId);
119
120    req->taskId(ContextSwitchTaskId::DMA);
121
122    a.pkt = new Packet(req, MemCmd::WriteReq);
123    a.pkt->dataStatic(ptr);
124
125    a.delay = 0;
126
127    PacketPtr pkt = yield(a).get();
128
129    assert(pkt);
130    assert(pkt->getSize() >= size);
131
132    delete pkt;
133}
134
135void
136ItsProcess::terminate(Yield &yield)
137{
138    ItsAction a;
139    a.type = ItsActionType::TERMINATE;
140    a.pkt = NULL;
141    a.delay = 0;
142    yield(a);
143}
144
145void
146ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)
147{
148    const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
149    const Addr address = base + (device_id * sizeof(dte));
150
151    DPRINTF(ITS, "Writing DTE at address %#x: %#x\n", address, dte);
152
153    doWrite(yield, address, &dte, sizeof(dte));
154}
155
156void
157ItsProcess::writeIrqTranslationTable(
158    Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)
159{
160    const Addr address = itt_base + (event_id * sizeof(itte));
161
162    doWrite(yield, address, &itte, sizeof(itte));
163
164    DPRINTF(ITS, "Writing ITTE at address %#x: %#x\n", address, itte);
165}
166
167void
168ItsProcess::writeIrqCollectionTable(
169    Yield &yield, uint32_t collection_id, CTE cte)
170{
171    const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
172    const Addr address = base + (collection_id * sizeof(cte));
173
174    doWrite(yield, address, &cte, sizeof(cte));
175
176    DPRINTF(ITS, "Writing CTE at address %#x: %#x\n", address, cte);
177}
178
179uint64_t
180ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)
181{
182    uint64_t dte;
183    const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
184    const Addr address = base + (device_id * sizeof(dte));
185
186    doRead(yield, address, &dte, sizeof(dte));
187
188    DPRINTF(ITS, "Reading DTE at address %#x: %#x\n", address, dte);
189    return dte;
190}
191
192uint64_t
193ItsProcess::readIrqTranslationTable(
194    Yield &yield, const Addr itt_base, uint32_t event_id)
195{
196    uint64_t itte;
197    const Addr address = itt_base + (event_id * sizeof(itte));
198
199    doRead(yield, address, &itte, sizeof(itte));
200
201    DPRINTF(ITS, "Reading ITTE at address %#x: %#x\n", address, itte);
202    return itte;
203}
204
205uint64_t
206ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)
207{
208    uint64_t cte;
209    const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
210    const Addr address = base + (collection_id * sizeof(cte));
211
212    doRead(yield, address, &cte, sizeof(cte));
213
214    DPRINTF(ITS, "Reading CTE at address %#x: %#x\n", address, cte);
215    return cte;
216}
217
218ItsTranslation::ItsTranslation(Gicv3Its &_its)
219  : ItsProcess(_its)
220{
221    reinit();
222    its.pendingTranslations++;
223    its.gitsControl.quiescent = 0;
224}
225
226ItsTranslation::~ItsTranslation()
227{
228    assert(its.pendingTranslations >= 1);
229    its.pendingTranslations--;
230    if (!its.pendingTranslations && !its.pendingCommands)
231        its.gitsControl.quiescent = 1;
232}
233
234void
235ItsTranslation::main(Yield &yield)
236{
237    PacketPtr pkt = yield.get();
238
239    const uint32_t device_id = pkt->req->streamId();
240    const uint32_t event_id = pkt->getLE<uint32_t>();
241
242    auto result = translateLPI(yield, device_id, event_id);
243
244    uint32_t intid = result.first;
245    Gicv3Redistributor *redist = result.second;
246
247    // Set the LPI in the redistributor
248    redist->setClrLPI(intid, true);
249
250    // Update the value in GITS_TRANSLATER only once we know
251    // there was no error in the tranlation process (before
252    // terminating the translation
253    its.gitsTranslater = event_id;
254
255    terminate(yield);
256}
257
258std::pair<uint32_t, Gicv3Redistributor *>
259ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,
260                             uint32_t event_id)
261{
262    if (its.deviceOutOfRange(device_id)) {
263        terminate(yield);
264    }
265
266    DTE dte = readDeviceTable(yield, device_id);
267
268    if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {
269        terminate(yield);
270    }
271
272    ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);
273    const auto collection_id = itte.icid;
274
275    if (!itte.valid || its.collectionOutOfRange(collection_id)) {
276        terminate(yield);
277    }
278
279    CTE cte = readIrqCollectionTable(yield, collection_id);
280
281    if (!cte.valid) {
282        terminate(yield);
283    }
284
285    // Returning the INTID and the target Redistributor
286    return std::make_pair(itte.intNum, its.getRedistributor(cte));
287}
288
289ItsCommand::DispatchTable ItsCommand::cmdDispatcher =
290{
291    COMMAND(CLEAR, &ItsCommand::clear),
292    COMMAND(DISCARD, &ItsCommand::discard),
293    COMMAND(INT, &ItsCommand::doInt),
294    COMMAND(INV, &ItsCommand::inv),
295    COMMAND(INVALL, &ItsCommand::invall),
296    COMMAND(MAPC, &ItsCommand::mapc),
297    COMMAND(MAPD, &ItsCommand::mapd),
298    COMMAND(MAPI, &ItsCommand::mapi),
299    COMMAND(MAPTI, &ItsCommand::mapti),
300    COMMAND(MOVALL, &ItsCommand::movall),
301    COMMAND(MOVI, &ItsCommand::movi),
302    COMMAND(SYNC, &ItsCommand::sync),
303    COMMAND(VINVALL, &ItsCommand::vinvall),
304    COMMAND(VMAPI, &ItsCommand::vmapi),
305    COMMAND(VMAPP, &ItsCommand::vmapp),
306    COMMAND(VMAPTI, &ItsCommand::vmapti),
307    COMMAND(VMOVI, &ItsCommand::vmovi),
308    COMMAND(VMOVP, &ItsCommand::vmovp),
309    COMMAND(VSYNC, &ItsCommand::vsync),
310};
311
312ItsCommand::ItsCommand(Gicv3Its &_its)
313  : ItsProcess(_its)
314{
315    reinit();
316    its.pendingCommands = true;
317
318    its.gitsControl.quiescent = 0;
319}
320
321ItsCommand::~ItsCommand()
322{
323    its.pendingCommands = false;
324
325    if (!its.pendingTranslations)
326        its.gitsControl.quiescent = 1;
327}
328
329std::string
330ItsCommand::commandName(uint32_t cmd)
331{
332    const auto entry = cmdDispatcher.find(cmd);
333    return entry != cmdDispatcher.end() ? entry->second.name : "INVALID";
334}
335
336void
337ItsCommand::main(Yield &yield)
338{
339    ItsAction a;
340    a.type = ItsActionType::INITIAL_NOP;
341    a.pkt = nullptr;
342    a.delay = 0;
343    yield(a);
344
345    while (its.gitsCwriter.offset != its.gitsCreadr.offset) {
346        CommandEntry command;
347
348        // Reading the command from CMDQ
349        readCommand(yield, command);
350
351        processCommand(yield, command);
352
353        its.incrementReadPointer();
354    }
355
356    terminate(yield);
357}
358
359void
360ItsCommand::readCommand(Yield &yield, CommandEntry &command)
361{
362    // read the command pointed by GITS_CREADR
363    const Addr cmd_addr =
364        (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);
365
366    doRead(yield, cmd_addr, &command, sizeof(command));
367
368    DPRINTF(ITS, "Command %s read from queue at address: %#x\n",
369            commandName(command.type), cmd_addr);
370    DPRINTF(ITS, "dw0: %#x dw1: %#x dw2: %#x dw3: %#x\n",
371            command.raw[0], command.raw[1], command.raw[2], command.raw[3]);
372}
373
374void
375ItsCommand::processCommand(Yield &yield, CommandEntry &command)
376{
377    const auto entry = cmdDispatcher.find(command.type);
378
379    if (entry != cmdDispatcher.end()) {
380        // Execute the command
381        entry->second.exec(this, yield, command);
382    } else {
383        panic("Unrecognized command type: %u", command.type);
384    }
385}
386
387void
388ItsCommand::clear(Yield &yield, CommandEntry &command)
389{
390    if (deviceOutOfRange(command)) {
391        its.incrementReadPointer();
392        terminate(yield);
393    }
394
395    DTE dte = readDeviceTable(yield, command.deviceId);
396
397    if (!dte.valid || idOutOfRange(command, dte)) {
398        its.incrementReadPointer();
399        terminate(yield);
400    }
401
402    ITTE itte = readIrqTranslationTable(
403        yield, dte.ittAddress, command.eventId);
404
405    if (!itte.valid) {
406        its.incrementReadPointer();
407        terminate(yield);
408    }
409
410    const auto collection_id = itte.icid;
411    CTE cte = readIrqCollectionTable(yield, collection_id);
412
413    if (!cte.valid) {
414        its.incrementReadPointer();
415        terminate(yield);
416    }
417
418    // Clear the LPI in the redistributor
419    its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
420}
421
422void
423ItsCommand::discard(Yield &yield, CommandEntry &command)
424{
425    if (deviceOutOfRange(command)) {
426        its.incrementReadPointer();
427        terminate(yield);
428    }
429
430    DTE dte = readDeviceTable(yield, command.deviceId);
431
432    if (!dte.valid || idOutOfRange(command, dte)) {
433        its.incrementReadPointer();
434        terminate(yield);
435    }
436
437    ITTE itte = readIrqTranslationTable(
438        yield, dte.ittAddress, command.eventId);
439
440    if (!itte.valid) {
441        its.incrementReadPointer();
442        terminate(yield);
443    }
444
445    const auto collection_id = itte.icid;
446    Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);
447
448    if (!cte.valid) {
449        its.incrementReadPointer();
450        terminate(yield);
451    }
452
453    its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
454
455    // Then removes the mapping from the ITT (invalidating)
456    itte.valid = 0;
457    writeIrqTranslationTable(
458        yield, dte.ittAddress, command.eventId, itte);
459}
460
461void
462ItsCommand::doInt(Yield &yield, CommandEntry &command)
463{
464    if (deviceOutOfRange(command)) {
465        its.incrementReadPointer();
466        terminate(yield);
467    }
468
469    DTE dte = readDeviceTable(yield, command.deviceId);
470
471    if (!dte.valid || idOutOfRange(command, dte)) {
472        its.incrementReadPointer();
473        terminate(yield);
474    }
475
476    ITTE itte = readIrqTranslationTable(
477        yield, dte.ittAddress, command.eventId);
478
479    if (!itte.valid) {
480        its.incrementReadPointer();
481        terminate(yield);
482    }
483
484    const auto collection_id = itte.icid;
485    CTE cte = readIrqCollectionTable(yield, collection_id);
486
487    if (!cte.valid) {
488        its.incrementReadPointer();
489        terminate(yield);
490    }
491
492    // Set the LPI in the redistributor
493    its.getRedistributor(cte)->setClrLPI(itte.intNum, true);
494}
495
496void
497ItsCommand::inv(Yield &yield, CommandEntry &command)
498{
499    if (deviceOutOfRange(command)) {
500        its.incrementReadPointer();
501        terminate(yield);
502    }
503
504    DTE dte = readDeviceTable(yield, command.deviceId);
505
506    if (!dte.valid || idOutOfRange(command, dte)) {
507        its.incrementReadPointer();
508        terminate(yield);
509    }
510
511    ITTE itte = readIrqTranslationTable(
512        yield, dte.ittAddress, command.eventId);
513
514    if (!itte.valid) {
515        its.incrementReadPointer();
516        terminate(yield);
517    }
518
519    const auto collection_id = itte.icid;
520    CTE cte = readIrqCollectionTable(yield, collection_id);
521
522    if (!cte.valid) {
523        its.incrementReadPointer();
524        terminate(yield);
525    }
526    // Do nothing since caching is currently not supported in
527    // Redistributor
528}
529
530void
531ItsCommand::invall(Yield &yield, CommandEntry &command)
532{
533    if (collectionOutOfRange(command)) {
534        its.incrementReadPointer();
535        terminate(yield);
536    }
537
538    const auto icid = bits(command.raw[2], 15, 0);
539
540    CTE cte = readIrqCollectionTable(yield, icid);
541
542    if (!cte.valid) {
543        its.incrementReadPointer();
544        terminate(yield);
545    }
546    // Do nothing since caching is currently not supported in
547    // Redistributor
548}
549
550void
551ItsCommand::mapc(Yield &yield, CommandEntry &command)
552{
553    if (collectionOutOfRange(command)) {
554        its.incrementReadPointer();
555        terminate(yield);
556    }
557
558    CTE cte = 0;
559    cte.valid = bits(command.raw[2], 63);
560    cte.rdBase = bits(command.raw[2], 50, 16);
561
562    const auto icid = bits(command.raw[2], 15, 0);
563
564    writeIrqCollectionTable(yield, icid, cte);
565}
566
567void
568ItsCommand::mapd(Yield &yield, CommandEntry &command)
569{
570    if (deviceOutOfRange(command) || sizeOutOfRange(command)) {
571        its.incrementReadPointer();
572        terminate(yield);
573    }
574
575    DTE dte = 0;
576    dte.valid = bits(command.raw[2], 63);
577    dte.ittAddress = mbits(command.raw[2], 51, 8);
578    dte.ittRange = bits(command.raw[1], 4, 0);
579
580    writeDeviceTable(yield, command.deviceId, dte);
581}
582
583void
584ItsCommand::mapi(Yield &yield, CommandEntry &command)
585{
586    if (deviceOutOfRange(command)) {
587        its.incrementReadPointer();
588        terminate(yield);
589    }
590
591    if (collectionOutOfRange(command)) {
592        its.incrementReadPointer();
593        terminate(yield);
594    }
595
596    DTE dte = readDeviceTable(yield, command.deviceId);
597
598    if (!dte.valid || idOutOfRange(command, dte) ||
599        its.lpiOutOfRange(command.eventId)) {
600
601        its.incrementReadPointer();
602        terminate(yield);
603    }
604
605    Gicv3Its::ITTE itte = readIrqTranslationTable(
606        yield, dte.ittAddress, command.eventId);
607
608    itte.valid = 1;
609    itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
610    itte.intNum = command.eventId;
611    itte.icid = bits(command.raw[2], 15, 0);
612
613    writeIrqTranslationTable(
614        yield, dte.ittAddress, command.eventId, itte);
615}
616
617void
618ItsCommand::mapti(Yield &yield, CommandEntry &command)
619{
620    if (deviceOutOfRange(command)) {
621        its.incrementReadPointer();
622        terminate(yield);
623    }
624
625    if (collectionOutOfRange(command)) {
626        its.incrementReadPointer();
627        terminate(yield);
628    }
629
630    DTE dte = readDeviceTable(yield, command.deviceId);
631
632    const auto pintid = bits(command.raw[1], 63, 32);
633
634    if (!dte.valid || idOutOfRange(command, dte) ||
635        its.lpiOutOfRange(pintid)) {
636
637        its.incrementReadPointer();
638        terminate(yield);
639    }
640
641    ITTE itte = readIrqTranslationTable(
642        yield, dte.ittAddress, command.eventId);
643
644    itte.valid = 1;
645    itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
646    itte.intNum = pintid;
647    itte.icid = bits(command.raw[2], 15, 0);
648
649    writeIrqTranslationTable(
650        yield, dte.ittAddress, command.eventId, itte);
651}
652
653void
654ItsCommand::movall(Yield &yield, CommandEntry &command)
655{
656    const uint64_t rd1 = bits(command.raw[2], 50, 16);
657    const uint64_t rd2 = bits(command.raw[3], 50, 16);
658
659    if (rd1 != rd2) {
660        Gicv3Redistributor * redist1 = its.getRedistributor(rd1);
661        Gicv3Redistributor * redist2 = its.getRedistributor(rd2);
662
663        its.moveAllPendingState(redist1, redist2);
664    }
665}
666
667void
668ItsCommand::movi(Yield &yield, CommandEntry &command)
669{
670    if (deviceOutOfRange(command)) {
671        its.incrementReadPointer();
672        terminate(yield);
673    }
674
675    if (collectionOutOfRange(command)) {
676        its.incrementReadPointer();
677        terminate(yield);
678    }
679
680    DTE dte = readDeviceTable(yield, command.deviceId);
681
682    if (!dte.valid || idOutOfRange(command, dte)) {
683        its.incrementReadPointer();
684        terminate(yield);
685    }
686
687    ITTE itte = readIrqTranslationTable(
688        yield, dte.ittAddress, command.eventId);
689
690    if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {
691        its.incrementReadPointer();
692        terminate(yield);
693    }
694
695    const auto collection_id1 = itte.icid;
696    CTE cte1 = readIrqCollectionTable(yield, collection_id1);
697
698    if (!cte1.valid) {
699        its.incrementReadPointer();
700        terminate(yield);
701    }
702
703    const auto collection_id2 = bits(command.raw[2], 15, 0);
704    CTE cte2 = readIrqCollectionTable(yield, collection_id2);
705
706    if (!cte2.valid) {
707        its.incrementReadPointer();
708        terminate(yield);
709    }
710
711    Gicv3Redistributor *first_redist = its.getRedistributor(cte1);
712    Gicv3Redistributor *second_redist = its.getRedistributor(cte2);
713
714    if (second_redist != first_redist) {
715        // move pending state of the interrupt from one redistributor
716        // to the other.
717        if (first_redist->isPendingLPI(itte.intNum)) {
718            first_redist->setClrLPI(itte.intNum, false);
719            second_redist->setClrLPI(itte.intNum, true);
720        }
721    }
722
723    itte.icid = collection_id2;
724    writeIrqTranslationTable(
725        yield, dte.ittAddress, command.eventId, itte);
726}
727
728void
729ItsCommand::sync(Yield &yield, CommandEntry &command)
730{
731    warn("ITS %s command unimplemented", __func__);
732}
733
734void
735ItsCommand::vinvall(Yield &yield, CommandEntry &command)
736{
737    panic("ITS %s command unimplemented", __func__);
738}
739
740void
741ItsCommand::vmapi(Yield &yield, CommandEntry &command)
742{
743    panic("ITS %s command unimplemented", __func__);
744}
745
746void
747ItsCommand::vmapp(Yield &yield, CommandEntry &command)
748{
749    panic("ITS %s command unimplemented", __func__);
750}
751
752void
753ItsCommand::vmapti(Yield &yield, CommandEntry &command)
754{
755    panic("ITS %s command unimplemented", __func__);
756}
757
758void
759ItsCommand::vmovi(Yield &yield, CommandEntry &command)
760{
761    panic("ITS %s command unimplemented", __func__);
762}
763
764void
765ItsCommand::vmovp(Yield &yield, CommandEntry &command)
766{
767    panic("ITS %s command unimplemented", __func__);
768}
769
770void
771ItsCommand::vsync(Yield &yield, CommandEntry &command)
772{
773    panic("ITS %s command unimplemented", __func__);
774}
775
776Gicv3Its::Gicv3Its(const Gicv3ItsParams *params)
777 : BasicPioDevice(params, params->pio_size),
778   dmaPort(name() + ".dma", *this),
779   gitsControl(CTLR_QUIESCENT),
780   gitsTyper(params->gits_typer),
781   gitsCbaser(0), gitsCreadr(0),
782   gitsCwriter(0), gitsIidr(0),
783   tableBases(NUM_BASER_REGS, 0),
784   masterId(params->system->getMasterId(this)),
785   gic(nullptr),
786   commandEvent([this] { checkCommandQueue(); }, name()),
787   pendingCommands(false),
788   pendingTranslations(0)
789{
790    BASER device_baser = 0;
791    device_baser.type = DEVICE_TABLE;
792    device_baser.entrySize = sizeof(uint64_t) - 1;
793    tableBases[0] = device_baser;
794
795    BASER icollect_baser = 0;
796    icollect_baser.type = COLLECTION_TABLE;
797    icollect_baser.entrySize = sizeof(uint64_t) - 1;
798    tableBases[1] = icollect_baser;
799}
800
801void
802Gicv3Its::setGIC(Gicv3 *_gic)
803{
804    assert(!gic);
805    gic = _gic;
806}
807
808AddrRangeList
809Gicv3Its::getAddrRanges() const
810{
811    assert(pioSize != 0);
812    AddrRangeList ranges;
813    DPRINTF(AddrRanges, "registering range: %#x-%#x\n", pioAddr, pioSize);
814    ranges.push_back(RangeSize(pioAddr, pioSize));
815    return ranges;
816}
817
818Tick
819Gicv3Its::read(PacketPtr pkt)
820{
821    const Addr addr = pkt->getAddr() - pioAddr;
822    uint64_t value = 0;
823
824    DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
825
826    switch (addr) {
827      case GITS_CTLR:
828        value = gitsControl;
829        break;
830
831      case GITS_IIDR:
832        value = gitsIidr;
833        break;
834
835      case GITS_TYPER:
836        value = gitsTyper;
837        break;
838
839      case GITS_CBASER:
840        value = gitsCbaser;
841        break;
842
843      case GITS_CBASER + 4:
844        value = gitsCbaser.high;
845        break;
846
847      case GITS_CWRITER:
848        value = gitsCwriter;
849        break;
850
851      case GITS_CWRITER + 4:
852        value = gitsCwriter.high;
853        break;
854
855      case GITS_CREADR:
856        value = gitsCreadr;
857        break;
858
859      case GITS_CREADR + 4:
860        value = gitsCreadr.high;
861        break;
862
863      case GITS_PIDR2:
864        value = gic->getDistributor()->gicdPidr2;
865        break;
866
867      case GITS_TRANSLATER:
868        value = gitsTranslater;
869        break;
870
871      default:
872        if (GITS_BASER.contains(addr)) {
873            auto relative_addr = addr - GITS_BASER.start();
874            auto baser_index = relative_addr / sizeof(uint64_t);
875
876            value = tableBases[baser_index];
877            break;
878        } else {
879            panic("Unrecognized register access\n");
880        }
881    }
882
883    pkt->setUintX(value, LittleEndianByteOrder);
884    pkt->makeAtomicResponse();
885    return pioDelay;
886}
887
888Tick
889Gicv3Its::write(PacketPtr pkt)
890{
891    Addr addr = pkt->getAddr() - pioAddr;
892
893    DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
894
895    switch (addr) {
896      case GITS_CTLR:
897        assert(pkt->getSize() == sizeof(uint32_t));
898        gitsControl = (pkt->getLE<uint32_t>() & ~CTLR_QUIESCENT);
899        // We should check here if the ITS has been disabled, and if
900        // that's the case, flush GICv3 caches to external memory.
901        // This is not happening now, since LPI caching is not
902        // currently implemented in gem5.
903        break;
904
905      case GITS_IIDR:
906        panic("GITS_IIDR is Read Only\n");
907
908      case GITS_TYPER:
909        panic("GITS_TYPER is Read Only\n");
910
911      case GITS_CBASER:
912        if (pkt->getSize() == sizeof(uint32_t)) {
913            gitsCbaser.low = pkt->getLE<uint32_t>();
914        } else {
915            assert(pkt->getSize() == sizeof(uint64_t));
916            gitsCbaser = pkt->getLE<uint64_t>();
917        }
918
919        gitsCreadr = 0; // Cleared when CBASER gets written
920
921        checkCommandQueue();
922        break;
923
924      case GITS_CBASER + 4:
925        assert(pkt->getSize() == sizeof(uint32_t));
926        gitsCbaser.high = pkt->getLE<uint32_t>();
927
928        gitsCreadr = 0; // Cleared when CBASER gets written
929
930        checkCommandQueue();
931        break;
932
933      case GITS_CWRITER:
934        if (pkt->getSize() == sizeof(uint32_t)) {
935            gitsCwriter.low = pkt->getLE<uint32_t>();
936        } else {
937            assert(pkt->getSize() == sizeof(uint64_t));
938            gitsCwriter = pkt->getLE<uint64_t>();
939        }
940
941        checkCommandQueue();
942        break;
943
944      case GITS_CWRITER + 4:
945        assert(pkt->getSize() == sizeof(uint32_t));
946        gitsCwriter.high = pkt->getLE<uint32_t>();
947
948        checkCommandQueue();
949        break;
950
951      case GITS_CREADR:
952        panic("GITS_READR is Read Only\n");
953
954      case GITS_TRANSLATER:
955        if (gitsControl.enabled) {
956            translate(pkt);
957        }
958        break;
959
960      default:
961        if (GITS_BASER.contains(addr)) {
962            auto relative_addr = addr - GITS_BASER.start();
963            auto baser_index = relative_addr / sizeof(uint64_t);
964
965            const uint64_t table_base = tableBases[baser_index];
966            const uint64_t w_mask = tableBases[baser_index].type ?
967                BASER_WMASK : BASER_WMASK_UNIMPL;
968            const uint64_t val = pkt->getLE<uint64_t>() & w_mask;
969
970            tableBases[baser_index] = table_base | val;
971            break;
972        } else {
973            panic("Unrecognized register access\n");
974        }
975    }
976
977    pkt->makeAtomicResponse();
978    return pioDelay;
979}
980
981bool
982Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const
983{
984    const uint32_t id_bits = gitsTyper.idBits;
985    return event_id >= (1ULL << (id_bits + 1)) ||
986        event_id >= ((1ULL << itt_range) + 1);
987}
988
989bool
990Gicv3Its::deviceOutOfRange(uint32_t device_id) const
991{
992    return device_id >= (1ULL << (gitsTyper.devBits + 1));
993}
994
995bool
996Gicv3Its::sizeOutOfRange(uint32_t size) const
997{
998    return size > gitsTyper.idBits;
999}
1000
1001bool
1002Gicv3Its::collectionOutOfRange(uint32_t collection_id) const
1003{
1004    // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID
1005    // Otherwise, #bits is specified by GITS_TYPER.CIDbits
1006    const auto cid_bits = gitsTyper.cil == 0 ?
1007        16 : gitsTyper.cidBits + 1;
1008
1009    return collection_id >= (1ULL << cid_bits);
1010}
1011
1012bool
1013Gicv3Its::lpiOutOfRange(uint32_t intid) const
1014{
1015    return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||
1016           (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&
1017            intid != Gicv3::INTID_SPURIOUS);
1018}
1019
1020DrainState
1021Gicv3Its::drain()
1022{
1023    if (!pendingCommands && !pendingTranslations) {
1024        return DrainState::Drained;
1025    } else {
1026        DPRINTF(Drain, "GICv3 ITS not drained\n");
1027        return DrainState::Draining;
1028    }
1029}
1030
1031void
1032Gicv3Its::serialize(CheckpointOut & cp) const
1033{
1034    SERIALIZE_SCALAR(gitsControl);
1035    SERIALIZE_SCALAR(gitsTyper);
1036    SERIALIZE_SCALAR(gitsCbaser);
1037    SERIALIZE_SCALAR(gitsCreadr);
1038    SERIALIZE_SCALAR(gitsCwriter);
1039    SERIALIZE_SCALAR(gitsIidr);
1040
1041    SERIALIZE_CONTAINER(tableBases);
1042}
1043
1044void
1045Gicv3Its::unserialize(CheckpointIn & cp)
1046{
1047    UNSERIALIZE_SCALAR(gitsControl);
1048    UNSERIALIZE_SCALAR(gitsTyper);
1049    UNSERIALIZE_SCALAR(gitsCbaser);
1050    UNSERIALIZE_SCALAR(gitsCreadr);
1051    UNSERIALIZE_SCALAR(gitsCwriter);
1052    UNSERIALIZE_SCALAR(gitsIidr);
1053
1054    UNSERIALIZE_CONTAINER(tableBases);
1055}
1056
1057void
1058Gicv3Its::incrementReadPointer()
1059{
1060    // Make the reader point to the next element
1061    gitsCreadr.offset = gitsCreadr.offset + 1;
1062
1063    // Check for wrapping
1064    auto queue_end = (4096 * (gitsCbaser.size + 1));
1065
1066    if (gitsCreadr.offset == queue_end) {
1067        gitsCreadr.offset = 0;
1068    }
1069}
1070
1071void
1072Gicv3Its::checkCommandQueue()
1073{
1074    if (!gitsControl.enabled || !gitsCbaser.valid)
1075        return;
1076
1077    if (gitsCwriter.offset != gitsCreadr.offset) {
1078        // writer and reader pointing to different command
1079        // entries: queue not empty.
1080        DPRINTF(ITS, "Reading command from queue\n");
1081
1082        if (!pendingCommands) {
1083            auto *cmd_proc = new ItsCommand(*this);
1084
1085            runProcess(cmd_proc, nullptr);
1086        } else {
1087            DPRINTF(ITS, "Waiting for pending command to finish\n");
1088        }
1089    }
1090}
1091
1092Port &
1093Gicv3Its::getPort(const std::string &if_name, PortID idx)
1094{
1095    if (if_name == "dma") {
1096        return dmaPort;
1097    }
1098    return BasicPioDevice::getPort(if_name, idx);
1099}
1100
1101void
1102Gicv3Its::recvReqRetry()
1103{
1104    assert(!packetsToRetry.empty());
1105
1106    while (!packetsToRetry.empty()) {
1107        ItsAction a = packetsToRetry.front();
1108
1109        assert(a.type == ItsActionType::SEND_REQ);
1110
1111        if (!dmaPort.sendTimingReq(a.pkt))
1112            break;
1113
1114        packetsToRetry.pop();
1115    }
1116}
1117
1118bool
1119Gicv3Its::recvTimingResp(PacketPtr pkt)
1120{
1121    // @todo: We need to pay for this and not just zero it out
1122    pkt->headerDelay = pkt->payloadDelay = 0;
1123
1124    ItsProcess *proc =
1125        safe_cast<ItsProcess *>(pkt->popSenderState());
1126
1127    runProcessTiming(proc, pkt);
1128
1129    return true;
1130}
1131
1132ItsAction
1133Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)
1134{
1135    if (sys->isAtomicMode()) {
1136        return runProcessAtomic(proc, pkt);
1137    } else if (sys->isTimingMode()) {
1138        return runProcessTiming(proc, pkt);
1139    } else {
1140        panic("Not in timing or atomic mode\n");
1141    }
1142}
1143
1144ItsAction
1145Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)
1146{
1147    ItsAction action = proc->run(pkt);
1148
1149    switch (action.type) {
1150      case ItsActionType::SEND_REQ:
1151        action.pkt->pushSenderState(proc);
1152
1153        if (packetsToRetry.empty() &&
1154            dmaPort.sendTimingReq(action.pkt)) {
1155
1156        } else {
1157            packetsToRetry.push(action);
1158        }
1159        break;
1160
1161      case ItsActionType::TERMINATE:
1162        delete proc;
1163        if (!pendingCommands && !commandEvent.scheduled()) {
1164            schedule(commandEvent, clockEdge());
1165        }
1166        break;
1167
1168      default:
1169        panic("Unknown action\n");
1170    }
1171
1172    return action;
1173}
1174
1175ItsAction
1176Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)
1177{
1178    ItsAction action;
1179    Tick delay = 0;
1180    bool terminate = false;
1181
1182    do {
1183        action = proc->run(pkt);
1184
1185        switch (action.type) {
1186          case ItsActionType::SEND_REQ:
1187            delay += dmaPort.sendAtomic(action.pkt);
1188            pkt = action.pkt;
1189            break;
1190
1191          case ItsActionType::TERMINATE:
1192            delete proc;
1193            terminate = true;
1194            break;
1195
1196          default:
1197            panic("Unknown action\n");
1198        }
1199
1200    } while (!terminate);
1201
1202    action.delay = delay;
1203
1204    return action;
1205}
1206
1207void
1208Gicv3Its::translate(PacketPtr pkt)
1209{
1210    DPRINTF(ITS, "Starting Translation Request\n");
1211
1212    auto *proc = new ItsTranslation(*this);
1213    runProcess(proc, pkt);
1214}
1215
1216Gicv3Redistributor*
1217Gicv3Its::getRedistributor(uint64_t rd_base)
1218{
1219    if (gitsTyper.pta == 1) {
1220        // RDBase is a redistributor address
1221        return gic->getRedistributorByAddr(rd_base << 16);
1222    } else {
1223        // RDBase is a redistributor number
1224        return gic->getRedistributor(rd_base);
1225    }
1226}
1227
1228Addr
1229Gicv3Its::pageAddress(Gicv3Its::ItsTables table)
1230{
1231    auto base_it = std::find_if(
1232        tableBases.begin(), tableBases.end(),
1233        [table] (const BASER &b) { return b.type == table; }
1234    );
1235
1236    panic_if(base_it == tableBases.end(),
1237        "ITS Table not recognised\n");
1238
1239    const BASER base = *base_it;
1240
1241    // real address depends on page size
1242    switch (base.pageSize) {
1243      case SIZE_4K:
1244      case SIZE_16K:
1245        return mbits(base, 47, 12);
1246      case SIZE_64K:
1247        return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);
1248      default:
1249        panic("Unsupported page size\n");
1250    }
1251}
1252
1253void
1254Gicv3Its::moveAllPendingState(
1255    Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)
1256{
1257    const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);
1258    uint8_t lpi_pending_table[largest_lpi_id / 8];
1259
1260    // Copying the pending table from redistributor 1 to redistributor 2
1261    rd1->memProxy->readBlob(
1262        rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1263        sizeof(lpi_pending_table));
1264
1265    rd2->memProxy->writeBlob(
1266        rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1267        sizeof(lpi_pending_table));
1268
1269    // Clearing pending table in redistributor 2
1270    rd1->memProxy->memsetBlob(
1271        rd1->lpiPendingTablePtr,
1272        0, sizeof(lpi_pending_table));
1273
1274    rd2->updateDistributor();
1275}
1276
1277Gicv3Its *
1278Gicv3ItsParams::create()
1279{
1280    return new Gicv3Its(this);
1281}
1282