1/*
2 * Copyright (c) 2017-2018 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Matteo Andreozzi
38 */
39
40#include "mem_ctrl.hh"
41
42#include "turnaround_policy.hh"
43
44namespace QoS {
45
46MemCtrl::MemCtrl(const QoSMemCtrlParams * p)
47  : AbstractMemory(p),
48    policy(p->qos_policy),
49    turnPolicy(p->qos_turnaround_policy),
50    queuePolicy(QueuePolicy::create(p)),
51    _numPriorities(p->qos_priorities),
52    qosPriorityEscalation(p->qos_priority_escalation),
53    qosSyncroScheduler(p->qos_syncro_scheduler),
54    totalReadQueueSize(0), totalWriteQueueSize(0),
55    busState(READ), busStateNext(READ)
56{
57    // Set the priority policy
58    if (policy) {
59        policy->setMemCtrl(this);
60    }
61
62    // Set the queue priority policy
63    if (queuePolicy) {
64        queuePolicy->setMemCtrl(this);
65    }
66
67    // Set the bus turnaround policy
68    if (turnPolicy) {
69        turnPolicy->setMemCtrl(this);
70    }
71
72    readQueueSizes.resize(_numPriorities);
73    writeQueueSizes.resize(_numPriorities);
74    serviceTick.resize(_numPriorities);
75}
76
77MemCtrl::~MemCtrl()
78{}
79
80void
81MemCtrl::init()
82{
83    AbstractMemory::init();
84}
85
86void
87MemCtrl::logRequest(BusState dir, MasterID m_id, uint8_t qos,
88                    Addr addr, uint64_t entries)
89{
90    // If needed, initialize all counters and statistics
91    // for this master
92    addMaster(m_id);
93
94    DPRINTF(QOS,
95            "QoSMemCtrl::logRequest MASTER %s [id %d] address %d"
96            " prio %d this master q packets %d"
97            " - queue size %d - requested entries %d\n",
98            masters[m_id], m_id, addr, qos, packetPriorities[m_id][qos],
99            (dir == READ) ? readQueueSizes[qos]: writeQueueSizes[qos],
100            entries);
101
102    if (dir == READ) {
103        readQueueSizes[qos] += entries;
104        totalReadQueueSize += entries;
105    } else if (dir == WRITE) {
106        writeQueueSizes[qos] += entries;
107        totalWriteQueueSize += entries;
108    }
109
110    packetPriorities[m_id][qos] += entries;
111    for (auto j = 0; j < entries; ++j) {
112        requestTimes[m_id][addr].push_back(curTick());
113    }
114
115    // Record statistics
116    avgPriority[m_id].sample(qos);
117
118    // Compute avg priority distance
119
120    for (uint8_t i = 0; i < packetPriorities[m_id].size(); ++i) {
121        uint8_t distance =
122            (abs(int(qos) - int(i))) * packetPriorities[m_id][i];
123
124        if (distance > 0) {
125            avgPriorityDistance[m_id].sample(distance);
126            DPRINTF(QOS,
127                    "QoSMemCtrl::logRequest MASTER %s [id %d]"
128                    " registering priority distance %d for priority %d"
129                    " (packets %d)\n",
130                    masters[m_id], m_id, distance, i,
131                    packetPriorities[m_id][i]);
132        }
133    }
134
135    DPRINTF(QOS,
136            "QoSMemCtrl::logRequest MASTER %s [id %d] prio %d "
137            "this master q packets %d - new queue size %d\n",
138            masters[m_id], m_id, qos, packetPriorities[m_id][qos],
139            (dir == READ) ? readQueueSizes[qos]: writeQueueSizes[qos]);
140
141}
142
143void
144MemCtrl::logResponse(BusState dir, MasterID m_id, uint8_t qos,
145                     Addr addr, uint64_t entries, double delay)
146{
147    panic_if(!hasMaster(m_id),
148        "Logging response with invalid master\n");
149
150    DPRINTF(QOS,
151            "QoSMemCtrl::logResponse MASTER %s [id %d] address %d prio"
152            " %d this master q packets %d"
153            " - queue size %d - requested entries %d\n",
154            masters[m_id], m_id, addr, qos, packetPriorities[m_id][qos],
155            (dir == READ) ? readQueueSizes[qos]: writeQueueSizes[qos],
156            entries);
157
158    if (dir == READ) {
159        readQueueSizes[qos] -= entries;
160        totalReadQueueSize -= entries;
161    } else if (dir == WRITE) {
162        writeQueueSizes[qos] -= entries;
163        totalWriteQueueSize -= entries;
164    }
165
166    panic_if(packetPriorities[m_id][qos] == 0,
167             "QoSMemCtrl::logResponse master %s negative packets for priority"
168             " %d", masters[m_id], qos);
169
170    packetPriorities[m_id][qos] -= entries;
171
172    for (auto j = 0; j < entries; ++j) {
173        auto it = requestTimes[m_id].find(addr);
174        panic_if(it == requestTimes[m_id].end(),
175                 "QoSMemCtrl::logResponse master %s unmatched response for"
176                 " address %d received", masters[m_id], addr);
177
178        // Load request time
179        uint64_t requestTime = it->second.front();
180
181        // Remove request entry
182        it->second.pop_front();
183
184        // Remove whole address entry if last one
185        if (it->second.empty()) {
186            requestTimes[m_id].erase(it);
187        }
188        // Compute latency
189        double latency = (double) (curTick() + delay - requestTime)
190                / SimClock::Float::s;
191
192        if (latency > 0) {
193            // Record per-priority latency stats
194            if (priorityMaxLatency[qos].value() < latency) {
195                priorityMaxLatency[qos] = latency;
196            }
197
198            if (priorityMinLatency[qos].value() > latency
199                    || priorityMinLatency[qos].value() == 0) {
200                priorityMinLatency[qos] = latency;
201            }
202        }
203    }
204
205    DPRINTF(QOS,
206            "QoSMemCtrl::logResponse MASTER %s [id %d] prio %d "
207            "this master q packets %d - new queue size %d\n",
208            masters[m_id], m_id, qos, packetPriorities[m_id][qos],
209            (dir == READ) ? readQueueSizes[qos]: writeQueueSizes[qos]);
210}
211
212uint8_t
213MemCtrl::schedule(MasterID m_id, uint64_t data)
214{
215    if (policy) {
216        return policy->schedule(m_id, data);
217    } else {
218        DPRINTF(QOS,
219                "QoSScheduler::schedule master ID [%d] "
220                "data received [%d], but QoS scheduler not initialized\n",
221                m_id,data);
222        return 0;
223    }
224}
225
226uint8_t
227MemCtrl::schedule(const PacketPtr pkt)
228{
229    assert(pkt->req);
230
231    if (policy) {
232        return schedule(pkt->req->masterId(), pkt->getSize());
233    } else {
234        DPRINTF(QOS, "QoSScheduler::schedule Packet received [Qv %d], "
235                "but QoS scheduler not initialized\n",
236                pkt->qosValue());
237        return pkt->qosValue();
238    }
239}
240
241MemCtrl::BusState
242MemCtrl::selectNextBusState()
243{
244    auto bus_state = getBusState();
245
246    if (turnPolicy) {
247        DPRINTF(QOS,
248                "QoSMemoryTurnaround::selectBusState running policy %s\n",
249                turnPolicy->name());
250
251        bus_state = turnPolicy->selectBusState();
252    } else {
253        DPRINTF(QOS,
254                "QoSMemoryTurnaround::selectBusState running "
255                "default bus direction selection policy\n");
256
257        if ((!getTotalReadQueueSize() && bus_state == MemCtrl::READ) ||
258            (!getTotalWriteQueueSize() && bus_state == MemCtrl::WRITE)) {
259            // READ/WRITE turnaround
260            bus_state = (bus_state == MemCtrl::READ) ? MemCtrl::WRITE :
261                                                       MemCtrl::READ;
262
263        }
264    }
265
266    return bus_state;
267}
268
269void
270MemCtrl::addMaster(MasterID m_id)
271{
272    if (!hasMaster(m_id)) {
273        masters.emplace(m_id, _system->getMasterName(m_id));
274        packetPriorities[m_id].resize(numPriorities(), 0);
275
276        DPRINTF(QOS,
277                "QoSMemCtrl::addMaster registering"
278                " Master %s [id %d]\n",
279                masters[m_id], m_id);
280    }
281}
282
283void
284MemCtrl::regStats()
285{
286    AbstractMemory::regStats();
287
288    using namespace Stats;
289
290    // Initializes per master statistics
291    avgPriority.init(_system->maxMasters()).name(name() + ".avgPriority")
292        .desc("Average QoS priority value for accepted requests")
293        .flags(nozero | nonan).precision(2);
294
295    avgPriorityDistance.init(_system->maxMasters())
296        .name(name() + ".avgPriorityDistance")
297        .desc("Average QoS priority distance between assigned and "
298        "queued values").flags(nozero | nonan);
299
300    priorityMinLatency.init(numPriorities())
301        .name(name() + ".priorityMinLatency")
302        .desc("per QoS priority minimum request to response latency (s)")
303        .precision(12);
304
305    priorityMaxLatency.init(numPriorities())
306        .name(name() + ".priorityMaxLatency")
307        .desc("per QoS priority maximum request to response latency (s)")
308        .precision(12);
309
310    numReadWriteTurnArounds.name(name() + ".numReadWriteTurnArounds")
311        .desc("Number of turnarounds from READ to WRITE");
312
313    numWriteReadTurnArounds.name(name() + ".numWriteReadTurnArounds")
314        .desc("Number of turnarounds from WRITE to READ");
315
316    numStayReadState.name(name() + ".numStayReadState")
317        .desc("Number of times bus staying in READ state");
318
319    numStayWriteState.name(name() + ".numStayWriteState")
320        .desc("Number of times bus staying in WRITE state");
321
322    for (int i = 0; i < _system->maxMasters(); i++) {
323        const std::string master = _system->getMasterName(i);
324        avgPriority.subname(i, master);
325        avgPriorityDistance.subname(i, master);
326    }
327
328    for (int j = 0; j < numPriorities(); ++j) {
329        priorityMinLatency.subname(j, std::to_string(j));
330        priorityMaxLatency.subname(j, std::to_string(j));
331    }
332}
333
334void
335MemCtrl::recordTurnaroundStats()
336{
337    if (busStateNext != busState) {
338        if (busState == READ) {
339            numWriteReadTurnArounds++;
340        } else if (busState == WRITE) {
341            numReadWriteTurnArounds++;
342        }
343    } else {
344        if (busState == READ) {
345            numStayReadState++;
346        } else if (busState == WRITE) {
347            numStayWriteState++;
348        }
349    }
350}
351
352} // namespace QoS
353