112966SMatteo.Andreozzi@arm.com/* 212966SMatteo.Andreozzi@arm.com * Copyright (c) 2018 ARM Limited 312966SMatteo.Andreozzi@arm.com * All rights reserved 412966SMatteo.Andreozzi@arm.com * 512966SMatteo.Andreozzi@arm.com * The license below extends only to copyright in the software and shall 612966SMatteo.Andreozzi@arm.com * not be construed as granting a license to any other intellectual 712966SMatteo.Andreozzi@arm.com * property including but not limited to intellectual property relating 812966SMatteo.Andreozzi@arm.com * to a hardware implementation of the functionality of the software 912966SMatteo.Andreozzi@arm.com * licensed hereunder. You may use the software subject to the license 1012966SMatteo.Andreozzi@arm.com * terms below provided that you ensure that this notice is replicated 1112966SMatteo.Andreozzi@arm.com * unmodified and in its entirety in all distributions of the software, 1212966SMatteo.Andreozzi@arm.com * modified or unmodified, in source code or in binary form. 1312966SMatteo.Andreozzi@arm.com * 1412966SMatteo.Andreozzi@arm.com * Redistribution and use in source and binary forms, with or without 1512966SMatteo.Andreozzi@arm.com * modification, are permitted provided that the following conditions are 1612966SMatteo.Andreozzi@arm.com * met: redistributions of source code must retain the above copyright 1712966SMatteo.Andreozzi@arm.com * notice, this list of conditions and the following disclaimer; 1812966SMatteo.Andreozzi@arm.com * redistributions in binary form must reproduce the above copyright 1912966SMatteo.Andreozzi@arm.com * notice, this list of conditions and the following disclaimer in the 2012966SMatteo.Andreozzi@arm.com * documentation and/or other materials provided with the distribution; 2112966SMatteo.Andreozzi@arm.com * neither the name of the copyright holders nor the names of its 2212966SMatteo.Andreozzi@arm.com * contributors may be used to endorse or promote products derived from 2312966SMatteo.Andreozzi@arm.com * this software without specific prior written permission. 2412966SMatteo.Andreozzi@arm.com * 2512966SMatteo.Andreozzi@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2612966SMatteo.Andreozzi@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2712966SMatteo.Andreozzi@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2812966SMatteo.Andreozzi@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2912966SMatteo.Andreozzi@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3012966SMatteo.Andreozzi@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3112966SMatteo.Andreozzi@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3212966SMatteo.Andreozzi@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3312966SMatteo.Andreozzi@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3412966SMatteo.Andreozzi@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3512966SMatteo.Andreozzi@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3612966SMatteo.Andreozzi@arm.com * 3712966SMatteo.Andreozzi@arm.com * Authors: Matteo Andreozzi 3812966SMatteo.Andreozzi@arm.com */ 3912966SMatteo.Andreozzi@arm.com 4012966SMatteo.Andreozzi@arm.com#include "debug/QOS.hh" 4112966SMatteo.Andreozzi@arm.com#include "mem/abstract_mem.hh" 4212966SMatteo.Andreozzi@arm.com#include "mem/qos/q_policy.hh" 4312966SMatteo.Andreozzi@arm.com#include "mem/qos/policy.hh" 4412966SMatteo.Andreozzi@arm.com#include "params/QoSMemCtrl.hh" 4512966SMatteo.Andreozzi@arm.com#include "sim/system.hh" 4612966SMatteo.Andreozzi@arm.com 4712966SMatteo.Andreozzi@arm.com#include <unordered_map> 4812966SMatteo.Andreozzi@arm.com#include <vector> 4912966SMatteo.Andreozzi@arm.com#include <deque> 5012966SMatteo.Andreozzi@arm.com 5112966SMatteo.Andreozzi@arm.com#ifndef __MEM_QOS_MEM_CTRL_HH__ 5212966SMatteo.Andreozzi@arm.com#define __MEM_QOS_MEM_CTRL_HH__ 5312966SMatteo.Andreozzi@arm.com 5412966SMatteo.Andreozzi@arm.comnamespace QoS { 5512966SMatteo.Andreozzi@arm.com 5612966SMatteo.Andreozzi@arm.com/** 5712966SMatteo.Andreozzi@arm.com * The QoS::MemCtrl is a base class for Memory objects 5812966SMatteo.Andreozzi@arm.com * which support QoS - it provides access to a set of QoS 5912966SMatteo.Andreozzi@arm.com * scheduling policies 6012966SMatteo.Andreozzi@arm.com */ 6112966SMatteo.Andreozzi@arm.comclass MemCtrl: public AbstractMemory 6212966SMatteo.Andreozzi@arm.com{ 6312966SMatteo.Andreozzi@arm.com public: 6412966SMatteo.Andreozzi@arm.com /** Bus Direction */ 6512966SMatteo.Andreozzi@arm.com enum BusState { READ, WRITE }; 6612966SMatteo.Andreozzi@arm.com 6712966SMatteo.Andreozzi@arm.com protected: 6812966SMatteo.Andreozzi@arm.com /** QoS Policy, assigns QoS priority to the incoming packets */ 6912966SMatteo.Andreozzi@arm.com const std::unique_ptr<Policy> policy; 7012966SMatteo.Andreozzi@arm.com 7112966SMatteo.Andreozzi@arm.com /** QoS Bus Turnaround Policy: selects the bus direction (READ/WRITE) */ 7212966SMatteo.Andreozzi@arm.com const std::unique_ptr<TurnaroundPolicy> turnPolicy; 7312966SMatteo.Andreozzi@arm.com 7412966SMatteo.Andreozzi@arm.com /** QoS Queue Policy: selects packet among same-priority queue */ 7512966SMatteo.Andreozzi@arm.com const std::unique_ptr<QueuePolicy> queuePolicy; 7612966SMatteo.Andreozzi@arm.com 7712966SMatteo.Andreozzi@arm.com /** Number of configured QoS priorities */ 7812966SMatteo.Andreozzi@arm.com const uint8_t _numPriorities; 7912966SMatteo.Andreozzi@arm.com 8012966SMatteo.Andreozzi@arm.com /** Enables QoS priority escalation */ 8112966SMatteo.Andreozzi@arm.com const bool qosPriorityEscalation; 8212966SMatteo.Andreozzi@arm.com 8312966SMatteo.Andreozzi@arm.com /** 8412966SMatteo.Andreozzi@arm.com * Enables QoS synchronized scheduling invokes the QoS scheduler 8512966SMatteo.Andreozzi@arm.com * on all masters, at every packet arrival. 8612966SMatteo.Andreozzi@arm.com */ 8712966SMatteo.Andreozzi@arm.com const bool qosSyncroScheduler; 8812966SMatteo.Andreozzi@arm.com 8912966SMatteo.Andreozzi@arm.com /** Hash of master ID - master name */ 9012966SMatteo.Andreozzi@arm.com std::unordered_map<MasterID, const std::string> masters; 9112966SMatteo.Andreozzi@arm.com 9212966SMatteo.Andreozzi@arm.com /** Hash of masters - number of packets queued per priority */ 9312966SMatteo.Andreozzi@arm.com std::unordered_map<MasterID, std::vector<uint64_t> > packetPriorities; 9412966SMatteo.Andreozzi@arm.com 9512966SMatteo.Andreozzi@arm.com /** Hash of masters - address of request - queue of times of request */ 9612966SMatteo.Andreozzi@arm.com std::unordered_map<MasterID, 9712966SMatteo.Andreozzi@arm.com std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes; 9812966SMatteo.Andreozzi@arm.com 9912966SMatteo.Andreozzi@arm.com /** 10012966SMatteo.Andreozzi@arm.com * Vector of QoS priorities/last service time. Refreshed at every 10112966SMatteo.Andreozzi@arm.com * qosSchedule call. 10212966SMatteo.Andreozzi@arm.com */ 10312966SMatteo.Andreozzi@arm.com std::vector<Tick> serviceTick; 10412966SMatteo.Andreozzi@arm.com 10512966SMatteo.Andreozzi@arm.com /** Read request packets queue length in #packets, per QoS priority */ 10612966SMatteo.Andreozzi@arm.com std::vector<uint64_t> readQueueSizes; 10712966SMatteo.Andreozzi@arm.com 10812966SMatteo.Andreozzi@arm.com /** Write request packets queue length in #packets, per QoS priority */ 10912966SMatteo.Andreozzi@arm.com std::vector<uint64_t> writeQueueSizes; 11012966SMatteo.Andreozzi@arm.com 11112966SMatteo.Andreozzi@arm.com /** Total read request packets queue length in #packets */ 11212966SMatteo.Andreozzi@arm.com uint64_t totalReadQueueSize; 11312966SMatteo.Andreozzi@arm.com 11412966SMatteo.Andreozzi@arm.com /** Total write request packets queue length in #packets */ 11512966SMatteo.Andreozzi@arm.com uint64_t totalWriteQueueSize; 11612966SMatteo.Andreozzi@arm.com 11712966SMatteo.Andreozzi@arm.com /** 11812966SMatteo.Andreozzi@arm.com * Bus state used to control the read/write switching and drive 11912966SMatteo.Andreozzi@arm.com * the scheduling of the next request. 12012966SMatteo.Andreozzi@arm.com */ 12112966SMatteo.Andreozzi@arm.com BusState busState; 12212966SMatteo.Andreozzi@arm.com 12312966SMatteo.Andreozzi@arm.com /** bus state for next request event triggered */ 12412966SMatteo.Andreozzi@arm.com BusState busStateNext; 12512966SMatteo.Andreozzi@arm.com 12612966SMatteo.Andreozzi@arm.com /** per-master average QoS priority */ 12712966SMatteo.Andreozzi@arm.com Stats::VectorStandardDeviation avgPriority; 12812966SMatteo.Andreozzi@arm.com /** per-master average QoS distance between assigned and queued values */ 12912966SMatteo.Andreozzi@arm.com Stats::VectorStandardDeviation avgPriorityDistance; 13012966SMatteo.Andreozzi@arm.com 13112966SMatteo.Andreozzi@arm.com /** per-priority minimum latency */ 13212966SMatteo.Andreozzi@arm.com Stats::Vector priorityMinLatency; 13312966SMatteo.Andreozzi@arm.com /** per-priority maximum latency */ 13412966SMatteo.Andreozzi@arm.com Stats::Vector priorityMaxLatency; 13512966SMatteo.Andreozzi@arm.com /** Count the number of turnarounds READ to WRITE */ 13612966SMatteo.Andreozzi@arm.com Stats::Scalar numReadWriteTurnArounds; 13712966SMatteo.Andreozzi@arm.com /** Count the number of turnarounds WRITE to READ */ 13812966SMatteo.Andreozzi@arm.com Stats::Scalar numWriteReadTurnArounds; 13912966SMatteo.Andreozzi@arm.com /** Count the number of times bus staying in READ state */ 14012966SMatteo.Andreozzi@arm.com Stats::Scalar numStayReadState; 14112966SMatteo.Andreozzi@arm.com /** Count the number of times bus staying in WRITE state */ 14212966SMatteo.Andreozzi@arm.com Stats::Scalar numStayWriteState; 14312966SMatteo.Andreozzi@arm.com 14412966SMatteo.Andreozzi@arm.com /** registers statistics */ 14512966SMatteo.Andreozzi@arm.com void regStats() override; 14612966SMatteo.Andreozzi@arm.com 14712966SMatteo.Andreozzi@arm.com /** 14812966SMatteo.Andreozzi@arm.com * Initializes dynamically counters and 14912966SMatteo.Andreozzi@arm.com * statistics for a given Master 15012966SMatteo.Andreozzi@arm.com * 15112966SMatteo.Andreozzi@arm.com * @param m_id the master ID 15212966SMatteo.Andreozzi@arm.com */ 15312966SMatteo.Andreozzi@arm.com void addMaster(const MasterID m_id); 15412966SMatteo.Andreozzi@arm.com 15512966SMatteo.Andreozzi@arm.com /** 15612966SMatteo.Andreozzi@arm.com * Called upon receiving a request or 15712966SMatteo.Andreozzi@arm.com * updates statistics and updates queues status 15812966SMatteo.Andreozzi@arm.com * 15912966SMatteo.Andreozzi@arm.com * @param dir request direction 16012966SMatteo.Andreozzi@arm.com * @param m_id master id 16112966SMatteo.Andreozzi@arm.com * @param qos packet qos value 16212966SMatteo.Andreozzi@arm.com * @param addr packet address 16312966SMatteo.Andreozzi@arm.com * @param entries number of entries to record 16412966SMatteo.Andreozzi@arm.com */ 16512966SMatteo.Andreozzi@arm.com void logRequest(BusState dir, MasterID m_id, uint8_t qos, 16612966SMatteo.Andreozzi@arm.com Addr addr, uint64_t entries); 16712966SMatteo.Andreozzi@arm.com 16812966SMatteo.Andreozzi@arm.com /** 16912966SMatteo.Andreozzi@arm.com * Called upon receiving a response, 17012966SMatteo.Andreozzi@arm.com * updates statistics and updates queues status 17112966SMatteo.Andreozzi@arm.com * 17212966SMatteo.Andreozzi@arm.com * @param dir response direction 17312966SMatteo.Andreozzi@arm.com * @param m_id master id 17412966SMatteo.Andreozzi@arm.com * @param qos packet qos value 17512966SMatteo.Andreozzi@arm.com * @param addr packet address 17612966SMatteo.Andreozzi@arm.com * @param entries number of entries to record 17712966SMatteo.Andreozzi@arm.com * @param delay response delay 17812966SMatteo.Andreozzi@arm.com */ 17912966SMatteo.Andreozzi@arm.com void logResponse(BusState dir, MasterID m_id, uint8_t qos, 18012966SMatteo.Andreozzi@arm.com Addr addr, uint64_t entries, double delay); 18112966SMatteo.Andreozzi@arm.com 18212966SMatteo.Andreozzi@arm.com /** 18312966SMatteo.Andreozzi@arm.com * Assign priority to a packet by executing 18412966SMatteo.Andreozzi@arm.com * the configured QoS policy. 18512966SMatteo.Andreozzi@arm.com * 18612966SMatteo.Andreozzi@arm.com * @param queues_ptr list of pointers to packet queues 18712966SMatteo.Andreozzi@arm.com * @param queue_entry_size size in bytes per each packet in the queue 18812966SMatteo.Andreozzi@arm.com * @param pkt pointer to the Packet 18912966SMatteo.Andreozzi@arm.com * @return a QoS priority value 19012966SMatteo.Andreozzi@arm.com */ 19112966SMatteo.Andreozzi@arm.com template<typename Queues> 19212966SMatteo.Andreozzi@arm.com uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr, 19312966SMatteo.Andreozzi@arm.com uint64_t queue_entry_size, const PacketPtr pkt); 19412966SMatteo.Andreozzi@arm.com 19512966SMatteo.Andreozzi@arm.com using SimObject::schedule; 19612966SMatteo.Andreozzi@arm.com uint8_t schedule(MasterID m_id, uint64_t data); 19712966SMatteo.Andreozzi@arm.com uint8_t schedule(const PacketPtr pkt); 19812966SMatteo.Andreozzi@arm.com 19912966SMatteo.Andreozzi@arm.com /** 20012966SMatteo.Andreozzi@arm.com * Returns next bus direction (READ or WRITE) 20112966SMatteo.Andreozzi@arm.com * based on configured policy. 20212966SMatteo.Andreozzi@arm.com */ 20312966SMatteo.Andreozzi@arm.com BusState selectNextBusState(); 20412966SMatteo.Andreozzi@arm.com 20512966SMatteo.Andreozzi@arm.com /** 20612966SMatteo.Andreozzi@arm.com * Set current bus direction (READ or WRITE) 20712966SMatteo.Andreozzi@arm.com * from next selected one 20812966SMatteo.Andreozzi@arm.com */ 20912966SMatteo.Andreozzi@arm.com void setCurrentBusState() { busState = busStateNext; } 21012966SMatteo.Andreozzi@arm.com 21112966SMatteo.Andreozzi@arm.com /** 21212966SMatteo.Andreozzi@arm.com * Record statistics on turnarounds based on 21312966SMatteo.Andreozzi@arm.com * busStateNext and busState values 21412966SMatteo.Andreozzi@arm.com */ 21512966SMatteo.Andreozzi@arm.com void recordTurnaroundStats(); 21612966SMatteo.Andreozzi@arm.com 21712966SMatteo.Andreozzi@arm.com /** 21812966SMatteo.Andreozzi@arm.com * Escalates/demotes priority of all packets 21912966SMatteo.Andreozzi@arm.com * belonging to the passed master to given 22012966SMatteo.Andreozzi@arm.com * priority value 22112966SMatteo.Andreozzi@arm.com * 22212966SMatteo.Andreozzi@arm.com * @param queues list of pointers to packet queues 22312966SMatteo.Andreozzi@arm.com * @param queue_entry_size size of an entry in the queue 22412966SMatteo.Andreozzi@arm.com * @param m_id master whose packets priority will change 22512966SMatteo.Andreozzi@arm.com * @param tgt_prio target priority value 22612966SMatteo.Andreozzi@arm.com */ 22712966SMatteo.Andreozzi@arm.com template<typename Queues> 22812966SMatteo.Andreozzi@arm.com void escalate(std::initializer_list<Queues*> queues, 22912966SMatteo.Andreozzi@arm.com uint64_t queue_entry_size, 23012966SMatteo.Andreozzi@arm.com MasterID m_id, uint8_t tgt_prio); 23112966SMatteo.Andreozzi@arm.com 23212966SMatteo.Andreozzi@arm.com /** 23312966SMatteo.Andreozzi@arm.com * Escalates/demotes priority of all packets 23412966SMatteo.Andreozzi@arm.com * belonging to the passed master to given 23512966SMatteo.Andreozzi@arm.com * priority value in a specified cluster of queues 23612966SMatteo.Andreozzi@arm.com * (e.g. read queues or write queues) which is passed 23712966SMatteo.Andreozzi@arm.com * as an argument to the function. 23812966SMatteo.Andreozzi@arm.com * The curr_prio/tgt_prio parameters are queue selectors in the 23912966SMatteo.Andreozzi@arm.com * queue cluster. 24012966SMatteo.Andreozzi@arm.com * 24112966SMatteo.Andreozzi@arm.com * @param queues reference to packet queues 24212966SMatteo.Andreozzi@arm.com * @param queue_entry_size size of an entry in the queue 24312966SMatteo.Andreozzi@arm.com * @param m_id master whose packets priority will change 24412966SMatteo.Andreozzi@arm.com * @param curr_prio source queue priority value 24512966SMatteo.Andreozzi@arm.com * @param tgt_prio target queue priority value 24612966SMatteo.Andreozzi@arm.com */ 24712966SMatteo.Andreozzi@arm.com template<typename Queues> 24812966SMatteo.Andreozzi@arm.com void escalateQueues(Queues& queues, uint64_t queue_entry_size, 24912966SMatteo.Andreozzi@arm.com MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio); 25012966SMatteo.Andreozzi@arm.com 25112966SMatteo.Andreozzi@arm.com public: 25212966SMatteo.Andreozzi@arm.com /** 25312966SMatteo.Andreozzi@arm.com * QoS Memory base class 25412966SMatteo.Andreozzi@arm.com * 25512966SMatteo.Andreozzi@arm.com * @param p pointer to QoSMemCtrl parameters 25612966SMatteo.Andreozzi@arm.com */ 25712966SMatteo.Andreozzi@arm.com MemCtrl(const QoSMemCtrlParams*); 25812966SMatteo.Andreozzi@arm.com 25912966SMatteo.Andreozzi@arm.com virtual ~MemCtrl(); 26012966SMatteo.Andreozzi@arm.com 26112966SMatteo.Andreozzi@arm.com /** 26212966SMatteo.Andreozzi@arm.com * Initializes this object 26312966SMatteo.Andreozzi@arm.com */ 26412966SMatteo.Andreozzi@arm.com void init() override; 26512966SMatteo.Andreozzi@arm.com 26612966SMatteo.Andreozzi@arm.com /** 26712966SMatteo.Andreozzi@arm.com * Gets the current bus state 26812966SMatteo.Andreozzi@arm.com * 26912966SMatteo.Andreozzi@arm.com * @return current bus state 27012966SMatteo.Andreozzi@arm.com */ 27112966SMatteo.Andreozzi@arm.com BusState getBusState() const { return busState; } 27212966SMatteo.Andreozzi@arm.com 27312966SMatteo.Andreozzi@arm.com /** 27412966SMatteo.Andreozzi@arm.com * Gets the next bus state 27512966SMatteo.Andreozzi@arm.com * 27612966SMatteo.Andreozzi@arm.com * @return next bus state 27712966SMatteo.Andreozzi@arm.com */ 27812966SMatteo.Andreozzi@arm.com BusState getBusStateNext() const { return busStateNext; } 27912966SMatteo.Andreozzi@arm.com 28012966SMatteo.Andreozzi@arm.com /** 28112966SMatteo.Andreozzi@arm.com * hasMaster returns true if the selected master(ID) has 28212966SMatteo.Andreozzi@arm.com * been registered in the memory controller, which happens if 28312966SMatteo.Andreozzi@arm.com * the memory controller has received at least a packet from 28412966SMatteo.Andreozzi@arm.com * that master. 28512966SMatteo.Andreozzi@arm.com * 28612966SMatteo.Andreozzi@arm.com * @param m_id master id to lookup 28712966SMatteo.Andreozzi@arm.com * @return true if the memory controller has received a packet 28812966SMatteo.Andreozzi@arm.com * from the master, false otherwise. 28912966SMatteo.Andreozzi@arm.com */ 29012966SMatteo.Andreozzi@arm.com bool hasMaster(MasterID m_id) const 29112966SMatteo.Andreozzi@arm.com { 29212966SMatteo.Andreozzi@arm.com return masters.find(m_id) != masters.end(); 29312966SMatteo.Andreozzi@arm.com } 29412966SMatteo.Andreozzi@arm.com 29512966SMatteo.Andreozzi@arm.com /** 29612966SMatteo.Andreozzi@arm.com * Gets a READ queue size 29712966SMatteo.Andreozzi@arm.com * 29812966SMatteo.Andreozzi@arm.com * @param prio QoS Priority of the queue 29912966SMatteo.Andreozzi@arm.com * @return queue size in packets 30012966SMatteo.Andreozzi@arm.com */ 30112966SMatteo.Andreozzi@arm.com uint64_t getReadQueueSize(const uint8_t prio) const 30212966SMatteo.Andreozzi@arm.com { return readQueueSizes[prio]; } 30312966SMatteo.Andreozzi@arm.com 30412966SMatteo.Andreozzi@arm.com /** 30512966SMatteo.Andreozzi@arm.com * Gets a WRITE queue size 30612966SMatteo.Andreozzi@arm.com * 30712966SMatteo.Andreozzi@arm.com * @param prio QoS Priority of the queue 30812966SMatteo.Andreozzi@arm.com * @return queue size in packets 30912966SMatteo.Andreozzi@arm.com */ 31012966SMatteo.Andreozzi@arm.com uint64_t getWriteQueueSize(const uint8_t prio) const 31112966SMatteo.Andreozzi@arm.com { return writeQueueSizes[prio]; } 31212966SMatteo.Andreozzi@arm.com 31312966SMatteo.Andreozzi@arm.com /** 31412966SMatteo.Andreozzi@arm.com * Gets the total combined READ queues size 31512966SMatteo.Andreozzi@arm.com * 31612966SMatteo.Andreozzi@arm.com * @return total queues size in packets 31712966SMatteo.Andreozzi@arm.com */ 31812966SMatteo.Andreozzi@arm.com uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; } 31912966SMatteo.Andreozzi@arm.com 32012966SMatteo.Andreozzi@arm.com /** 32112966SMatteo.Andreozzi@arm.com * Gets the total combined WRITE queues size 32212966SMatteo.Andreozzi@arm.com * 32312966SMatteo.Andreozzi@arm.com * @return total queues size in packets 32412966SMatteo.Andreozzi@arm.com */ 32512966SMatteo.Andreozzi@arm.com uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; } 32612966SMatteo.Andreozzi@arm.com 32712966SMatteo.Andreozzi@arm.com /** 32812966SMatteo.Andreozzi@arm.com * Gets the last service tick related to a QoS Priority 32912966SMatteo.Andreozzi@arm.com * 33012966SMatteo.Andreozzi@arm.com * @param prio QoS Priority 33112966SMatteo.Andreozzi@arm.com * @return tick 33212966SMatteo.Andreozzi@arm.com */ 33312966SMatteo.Andreozzi@arm.com Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; } 33412966SMatteo.Andreozzi@arm.com 33512966SMatteo.Andreozzi@arm.com /** 33612966SMatteo.Andreozzi@arm.com * Gets the total number of priority levels in the 33712966SMatteo.Andreozzi@arm.com * QoS memory controller. 33812966SMatteo.Andreozzi@arm.com * 33912966SMatteo.Andreozzi@arm.com * @return total number of priority levels 34012966SMatteo.Andreozzi@arm.com */ 34112966SMatteo.Andreozzi@arm.com uint8_t numPriorities() const { return _numPriorities; } 34212966SMatteo.Andreozzi@arm.com}; 34312966SMatteo.Andreozzi@arm.com 34412966SMatteo.Andreozzi@arm.comtemplate<typename Queues> 34512966SMatteo.Andreozzi@arm.comvoid 34612966SMatteo.Andreozzi@arm.comMemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size, 34712966SMatteo.Andreozzi@arm.com MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio) 34812966SMatteo.Andreozzi@arm.com{ 34912966SMatteo.Andreozzi@arm.com auto it = queues[curr_prio].begin(); 35012966SMatteo.Andreozzi@arm.com while (it != queues[curr_prio].end()) { 35112966SMatteo.Andreozzi@arm.com // No packets left to move 35212966SMatteo.Andreozzi@arm.com if (packetPriorities[m_id][curr_prio] == 0) 35312966SMatteo.Andreozzi@arm.com break; 35412966SMatteo.Andreozzi@arm.com 35512966SMatteo.Andreozzi@arm.com auto pkt = *it; 35612966SMatteo.Andreozzi@arm.com 35712966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 35812966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate checking priority %d packet " 35912966SMatteo.Andreozzi@arm.com "m_id %d address %d\n", curr_prio, 36012966SMatteo.Andreozzi@arm.com pkt->masterId(), pkt->getAddr()); 36112966SMatteo.Andreozzi@arm.com 36212966SMatteo.Andreozzi@arm.com // Found a packet to move 36312966SMatteo.Andreozzi@arm.com if (pkt->masterId() == m_id) { 36412966SMatteo.Andreozzi@arm.com 36512966SMatteo.Andreozzi@arm.com uint64_t moved_entries = divCeil(pkt->getSize(), 36612966SMatteo.Andreozzi@arm.com queue_entry_size); 36712966SMatteo.Andreozzi@arm.com 36812966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 36912966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate Master %s [id %d] moving " 37012966SMatteo.Andreozzi@arm.com "packet addr %d size %d (p size %d) from priority %d " 37112966SMatteo.Andreozzi@arm.com "to priority %d - " 37212966SMatteo.Andreozzi@arm.com "this master packets %d (entries to move %d)\n", 37312966SMatteo.Andreozzi@arm.com masters[m_id], m_id, pkt->getAddr(), 37412966SMatteo.Andreozzi@arm.com pkt->getSize(), 37512966SMatteo.Andreozzi@arm.com queue_entry_size, curr_prio, tgt_prio, 37612966SMatteo.Andreozzi@arm.com packetPriorities[m_id][curr_prio], moved_entries); 37712966SMatteo.Andreozzi@arm.com 37812966SMatteo.Andreozzi@arm.com 37912966SMatteo.Andreozzi@arm.com if (pkt->isRead()) { 38012966SMatteo.Andreozzi@arm.com panic_if(readQueueSizes[curr_prio] < moved_entries, 38112966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate master %s negative READ " 38212966SMatteo.Andreozzi@arm.com "packets for priority %d", 38312966SMatteo.Andreozzi@arm.com masters[m_id], tgt_prio); 38412966SMatteo.Andreozzi@arm.com readQueueSizes[curr_prio] -= moved_entries; 38512966SMatteo.Andreozzi@arm.com readQueueSizes[tgt_prio] += moved_entries; 38612966SMatteo.Andreozzi@arm.com } else if (pkt->isWrite()) { 38712966SMatteo.Andreozzi@arm.com panic_if(writeQueueSizes[curr_prio] < moved_entries, 38812966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate master %s negative WRITE " 38912966SMatteo.Andreozzi@arm.com "packets for priority %d", 39012966SMatteo.Andreozzi@arm.com masters[m_id], tgt_prio); 39112966SMatteo.Andreozzi@arm.com writeQueueSizes[curr_prio] -= moved_entries; 39212966SMatteo.Andreozzi@arm.com writeQueueSizes[tgt_prio] += moved_entries; 39312966SMatteo.Andreozzi@arm.com } 39412966SMatteo.Andreozzi@arm.com 39512966SMatteo.Andreozzi@arm.com // Change QoS priority and move packet 39612966SMatteo.Andreozzi@arm.com pkt->qosValue(tgt_prio); 39712966SMatteo.Andreozzi@arm.com queues[tgt_prio].push_back(pkt); 39812966SMatteo.Andreozzi@arm.com 39912966SMatteo.Andreozzi@arm.com // Erase element from source packet queue, this will 40012966SMatteo.Andreozzi@arm.com // increment the iterator 40112966SMatteo.Andreozzi@arm.com it = queues[curr_prio].erase(it); 40212966SMatteo.Andreozzi@arm.com panic_if(packetPriorities[m_id][curr_prio] < moved_entries, 40312966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate master %s negative packets " 40412966SMatteo.Andreozzi@arm.com "for priority %d", 40512966SMatteo.Andreozzi@arm.com masters[m_id], tgt_prio); 40612966SMatteo.Andreozzi@arm.com 40712966SMatteo.Andreozzi@arm.com packetPriorities[m_id][curr_prio] -= moved_entries; 40812966SMatteo.Andreozzi@arm.com packetPriorities[m_id][tgt_prio] += moved_entries; 40912966SMatteo.Andreozzi@arm.com } else { 41012966SMatteo.Andreozzi@arm.com // Increment iterator to next location in the queue 41112966SMatteo.Andreozzi@arm.com it++; 41212966SMatteo.Andreozzi@arm.com } 41312966SMatteo.Andreozzi@arm.com } 41412966SMatteo.Andreozzi@arm.com} 41512966SMatteo.Andreozzi@arm.com 41612966SMatteo.Andreozzi@arm.comtemplate<typename Queues> 41712966SMatteo.Andreozzi@arm.comvoid 41812966SMatteo.Andreozzi@arm.comMemCtrl::escalate(std::initializer_list<Queues*> queues, 41912966SMatteo.Andreozzi@arm.com uint64_t queue_entry_size, 42012966SMatteo.Andreozzi@arm.com MasterID m_id, uint8_t tgt_prio) 42112966SMatteo.Andreozzi@arm.com{ 42212966SMatteo.Andreozzi@arm.com // If needed, initialize all counters and statistics 42312966SMatteo.Andreozzi@arm.com // for this master 42412966SMatteo.Andreozzi@arm.com addMaster(m_id); 42512966SMatteo.Andreozzi@arm.com 42612966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 42712966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate Master %s [id %d] to priority " 42812966SMatteo.Andreozzi@arm.com "%d (currently %d packets)\n",masters[m_id], m_id, tgt_prio, 42912966SMatteo.Andreozzi@arm.com packetPriorities[m_id][tgt_prio]); 43012966SMatteo.Andreozzi@arm.com 43112966SMatteo.Andreozzi@arm.com for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) { 43212966SMatteo.Andreozzi@arm.com // Skip target priority 43312966SMatteo.Andreozzi@arm.com if (curr_prio == tgt_prio) 43412966SMatteo.Andreozzi@arm.com continue; 43512966SMatteo.Andreozzi@arm.com 43612966SMatteo.Andreozzi@arm.com // Process other priority packet 43712966SMatteo.Andreozzi@arm.com while (packetPriorities[m_id][curr_prio] > 0) { 43812966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 43912966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate MID %d checking priority %d " 44012966SMatteo.Andreozzi@arm.com "(packets %d)- current packets in prio %d: %d\n" 44112966SMatteo.Andreozzi@arm.com "\t(source read %d source write %d target read %d, " 44212966SMatteo.Andreozzi@arm.com "target write %d)\n", 44312966SMatteo.Andreozzi@arm.com m_id, curr_prio, packetPriorities[m_id][curr_prio], 44412966SMatteo.Andreozzi@arm.com tgt_prio, packetPriorities[m_id][tgt_prio], 44512966SMatteo.Andreozzi@arm.com readQueueSizes[curr_prio], 44612966SMatteo.Andreozzi@arm.com writeQueueSizes[curr_prio], readQueueSizes[tgt_prio], 44712966SMatteo.Andreozzi@arm.com writeQueueSizes[tgt_prio]); 44812966SMatteo.Andreozzi@arm.com 44912966SMatteo.Andreozzi@arm.com // Check both read and write queue 45012966SMatteo.Andreozzi@arm.com for (auto q : queues) { 45112966SMatteo.Andreozzi@arm.com escalateQueues(*q, queue_entry_size, m_id, 45212966SMatteo.Andreozzi@arm.com curr_prio, tgt_prio); 45312966SMatteo.Andreozzi@arm.com } 45412966SMatteo.Andreozzi@arm.com } 45512966SMatteo.Andreozzi@arm.com } 45612966SMatteo.Andreozzi@arm.com 45712966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 45812966SMatteo.Andreozzi@arm.com "QoSMemCtrl::escalate Completed master %s [id %d] to priority %d " 45912966SMatteo.Andreozzi@arm.com "(now %d packets)\n\t(total read %d, total write %d)\n", 46012966SMatteo.Andreozzi@arm.com masters[m_id], m_id, tgt_prio, packetPriorities[m_id][tgt_prio], 46112966SMatteo.Andreozzi@arm.com readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]); 46212966SMatteo.Andreozzi@arm.com} 46312966SMatteo.Andreozzi@arm.com 46412966SMatteo.Andreozzi@arm.comtemplate<typename Queues> 46512966SMatteo.Andreozzi@arm.comuint8_t 46612966SMatteo.Andreozzi@arm.comMemCtrl::qosSchedule(std::initializer_list<Queues*> queues, 46712966SMatteo.Andreozzi@arm.com const uint64_t queue_entry_size, 46812966SMatteo.Andreozzi@arm.com const PacketPtr pkt) 46912966SMatteo.Andreozzi@arm.com{ 47012966SMatteo.Andreozzi@arm.com // Schedule packet. 47112966SMatteo.Andreozzi@arm.com uint8_t pkt_priority = schedule(pkt); 47212966SMatteo.Andreozzi@arm.com 47312966SMatteo.Andreozzi@arm.com assert(pkt_priority < numPriorities()); 47412966SMatteo.Andreozzi@arm.com 47512966SMatteo.Andreozzi@arm.com pkt->qosValue(pkt_priority); 47612966SMatteo.Andreozzi@arm.com 47712966SMatteo.Andreozzi@arm.com if (qosSyncroScheduler) { 47812966SMatteo.Andreozzi@arm.com // Call the scheduling function on all other masters. 47912966SMatteo.Andreozzi@arm.com for (const auto& m : masters) { 48012966SMatteo.Andreozzi@arm.com 48112966SMatteo.Andreozzi@arm.com if (m.first == pkt->masterId()) 48212966SMatteo.Andreozzi@arm.com continue; 48312966SMatteo.Andreozzi@arm.com 48412966SMatteo.Andreozzi@arm.com uint8_t prio = schedule(m.first, 0); 48512966SMatteo.Andreozzi@arm.com 48612966SMatteo.Andreozzi@arm.com if (qosPriorityEscalation) { 48712966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 48812966SMatteo.Andreozzi@arm.com "QoSMemCtrl::qosSchedule: (syncro) escalating " 48912966SMatteo.Andreozzi@arm.com "MASTER %s to assigned priority %d\n", 49012966SMatteo.Andreozzi@arm.com _system->getMasterName(m.first), 49112966SMatteo.Andreozzi@arm.com prio); 49212966SMatteo.Andreozzi@arm.com escalate(queues, queue_entry_size, m.first, prio); 49312966SMatteo.Andreozzi@arm.com } 49412966SMatteo.Andreozzi@arm.com } 49512966SMatteo.Andreozzi@arm.com } 49612966SMatteo.Andreozzi@arm.com 49712966SMatteo.Andreozzi@arm.com if (qosPriorityEscalation) { 49812966SMatteo.Andreozzi@arm.com DPRINTF(QOS, 49912966SMatteo.Andreozzi@arm.com "QoSMemCtrl::qosSchedule: escalating " 50012966SMatteo.Andreozzi@arm.com "MASTER %s to assigned priority %d\n", 50112966SMatteo.Andreozzi@arm.com _system->getMasterName(pkt->masterId()), 50212966SMatteo.Andreozzi@arm.com pkt_priority); 50312966SMatteo.Andreozzi@arm.com escalate(queues, queue_entry_size, pkt->masterId(), pkt_priority); 50412966SMatteo.Andreozzi@arm.com } 50512966SMatteo.Andreozzi@arm.com 50612966SMatteo.Andreozzi@arm.com // Update last service tick for selected priority 50712966SMatteo.Andreozzi@arm.com serviceTick[pkt_priority] = curTick(); 50812966SMatteo.Andreozzi@arm.com 50912966SMatteo.Andreozzi@arm.com return pkt_priority; 51012966SMatteo.Andreozzi@arm.com} 51112966SMatteo.Andreozzi@arm.com 51212966SMatteo.Andreozzi@arm.com} // namespace QoS 51312966SMatteo.Andreozzi@arm.com 51412966SMatteo.Andreozzi@arm.com#endif /* __MEM_QOS_MEM_CTRL_HH__ */ 515