1/* 2 * Copyright (c) 2010-2016 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 27 unchanged lines hidden (view full) --- 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Andreas Hansson 41 * Ani Udipi 42 * Neha Agarwal 43 * Omar Naji |
44 * Wendy Elsasser |
45 */ 46 47#include "base/bitfield.hh" 48#include "base/trace.hh" 49#include "debug/DRAM.hh" 50#include "debug/DRAMPower.hh" 51#include "debug/DRAMState.hh" 52#include "debug/Drain.hh" 53#include "mem/dram_ctrl.hh" 54#include "sim/system.hh" 55 56using namespace std; 57using namespace Data; 58 59DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : 60 AbstractMemory(p), 61 port(name() + ".port", *this), isTimingMode(false), 62 retryRdReq(false), retryWrReq(false), 63 busState(READ), |
64 busStateNext(READ), |
65 nextReqEvent(this), respondEvent(this), 66 deviceSize(p->device_size), 67 deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), 68 deviceRowBufferSize(p->device_rowbuffer_size), 69 devicesPerRank(p->devices_per_rank), 70 burstSize((devicesPerRank * burstLength * deviceBusWidth) / 8), 71 rowBufferSize(devicesPerRank * deviceRowBufferSize), 72 columnsPerRowBuffer(rowBufferSize / burstSize), --- 405 unchanged lines hidden (view full) --- 478 479 assert(!readQueueFull(1)); 480 rdQLenPdf[readQueue.size() + respQueue.size()]++; 481 482 DPRINTF(DRAM, "Adding to read queue\n"); 483 484 readQueue.push_back(dram_pkt); 485 |
486 // increment read entries of the rank 487 ++dram_pkt->rankRef.readEntries; 488 |
489 // Update stats 490 avgRdQLen = readQueue.size() + respQueue.size(); 491 } 492 493 // Starting address of next dram pkt (aligend to burstSize boundary) 494 addr = (addr | (burstSize - 1)) + 1; 495 } 496 --- 47 unchanged lines hidden (view full) --- 544 DPRINTF(DRAM, "Adding to write queue\n"); 545 546 writeQueue.push_back(dram_pkt); 547 isInWriteQueue.insert(burstAlign(addr)); 548 assert(writeQueue.size() == isInWriteQueue.size()); 549 550 // Update stats 551 avgWrQLen = writeQueue.size(); |
552 553 // increment write entries of the rank 554 ++dram_pkt->rankRef.writeEntries; |
555 } else { 556 DPRINTF(DRAM, "Merging write burst with existing queue entry\n"); 557 558 // keep track of the fact that this burst effectively 559 // disappeared as it was merged with an existing one 560 mergedWrBursts++; 561 } 562 --- 96 unchanged lines hidden (view full) --- 659void 660DRAMCtrl::processRespondEvent() 661{ 662 DPRINTF(DRAM, 663 "processRespondEvent(): Some req has reached its readyTime\n"); 664 665 DRAMPacket* dram_pkt = respQueue.front(); 666 |
667 // if a read has reached its ready-time, decrement the number of reads 668 // At this point the packet has been handled and there is a possibility 669 // to switch to low-power mode if no other packet is available 670 --dram_pkt->rankRef.readEntries; 671 DPRINTF(DRAM, "number of read entries for rank %d is %d\n", 672 dram_pkt->rank, dram_pkt->rankRef.readEntries); 673 674 // counter should at least indicate one outstanding request 675 // for this read 676 assert(dram_pkt->rankRef.outstandingEvents > 0); 677 // read response received, decrement count 678 --dram_pkt->rankRef.outstandingEvents; 679 680 // at this moment should be either ACT or IDLE depending on 681 // if PRE has occurred to close all banks 682 assert((dram_pkt->rankRef.pwrState == PWR_ACT) || 683 (dram_pkt->rankRef.pwrState == PWR_IDLE)); 684 685 // track if this is the last packet before idling 686 // and that there are no outstanding commands to this rank 687 if (dram_pkt->rankRef.lowPowerEntryReady()) { 688 // verify that there are no events scheduled 689 assert(!dram_pkt->rankRef.activateEvent.scheduled()); 690 assert(!dram_pkt->rankRef.prechargeEvent.scheduled()); 691 assert(dram_pkt->rankRef.refreshState == REF_IDLE); 692 693 // if coming from active state, schedule power event to 694 // active power-down else go to precharge power-down 695 DPRINTF(DRAMState, "Rank %d sleep at tick %d; current power state is " 696 "%d\n", dram_pkt->rank, curTick(), dram_pkt->rankRef.pwrState); 697 698 // default to ACT power-down unless already in IDLE state 699 // could be in IDLE if PRE issued before data returned 700 PowerState next_pwr_state = PWR_ACT_PDN; 701 if (dram_pkt->rankRef.pwrState == PWR_IDLE) { 702 next_pwr_state = PWR_PRE_PDN; 703 } 704 705 dram_pkt->rankRef.powerDownSleep(next_pwr_state, curTick()); 706 } 707 |
708 if (dram_pkt->burstHelper) { 709 // it is a split packet 710 dram_pkt->burstHelper->burstsServiced++; 711 if (dram_pkt->burstHelper->burstsServiced == 712 dram_pkt->burstHelper->burstCount) { 713 // we have now serviced all children packets of a system packet 714 // so we can now respond to the requester 715 // @todo we probably want to have a different front end and back --- 340 unchanged lines hidden (view full) --- 1056 timeStampOffset, bank.bank, rank_ref.rank); 1057 } 1058 // if we look at the current number of active banks we might be 1059 // tempted to think the DRAM is now idle, however this can be 1060 // undone by an activate that is scheduled to happen before we 1061 // would have reached the idle state, so schedule an event and 1062 // rather check once we actually make it to the point in time when 1063 // the (last) precharge takes place |
1064 if (!rank_ref.prechargeEvent.scheduled()) { |
1065 schedule(rank_ref.prechargeEvent, pre_done_at); |
1066 // New event, increment count 1067 ++rank_ref.outstandingEvents; 1068 } else if (rank_ref.prechargeEvent.when() < pre_done_at) { |
1069 reschedule(rank_ref.prechargeEvent, pre_done_at); |
1070 } |
1071} 1072 1073void 1074DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) 1075{ 1076 DPRINTF(DRAM, "Timing access to addr %lld, rank/bank/row %d %d %d\n", 1077 dram_pkt->addr, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); 1078 1079 // get the rank 1080 Rank& rank = dram_pkt->rankRef; 1081 |
1082 // are we in or transitioning to a low-power state and have not scheduled 1083 // a power-up event? 1084 // if so, wake up from power down to issue RD/WR burst 1085 if (rank.inLowPowerState) { 1086 assert(rank.pwrState != PWR_SREF); 1087 rank.scheduleWakeUpEvent(tXP); 1088 } 1089 |
1090 // get the bank 1091 Bank& bank = dram_pkt->bankRef; 1092 1093 // for the state we need to track if it is a row hit or not 1094 bool row_hit = true; 1095 1096 // respect any constraints on the command (e.g. tRCD or tCCD) 1097 Tick cmd_at = std::max(bank.colAllowedAt, curTick()); --- 186 unchanged lines hidden (view full) --- 1284} 1285 1286void 1287DRAMCtrl::processNextReqEvent() 1288{ 1289 int busyRanks = 0; 1290 for (auto r : ranks) { 1291 if (!r->isAvailable()) { |
1292 if (r->pwrState != PWR_SREF) { 1293 // rank is busy refreshing 1294 DPRINTF(DRAMState, "Rank %d is not available\n", r->rank); 1295 busyRanks++; |
1296 |
1297 // let the rank know that if it was waiting to drain, it 1298 // is now done and ready to proceed 1299 r->checkDrainDone(); 1300 } 1301 1302 // check if we were in self-refresh and haven't started 1303 // to transition out 1304 if ((r->pwrState == PWR_SREF) && r->inLowPowerState) { 1305 DPRINTF(DRAMState, "Rank %d is in self-refresh\n", r->rank); 1306 // if we have commands queued to this rank and we don't have 1307 // a minimum number of active commands enqueued, 1308 // exit self-refresh 1309 if (r->forceSelfRefreshExit()) { 1310 DPRINTF(DRAMState, "rank %d was in self refresh and" 1311 " should wake up\n", r->rank); 1312 //wake up from self-refresh 1313 r->scheduleWakeUpEvent(tXS); 1314 // things are brought back into action once a refresh is 1315 // performed after self-refresh 1316 // continue with selection for other ranks 1317 } 1318 } |
1319 } 1320 } 1321 1322 if (busyRanks == ranksPerChannel) { 1323 // if all ranks are refreshing wait for them to finish 1324 // and stall this state machine without taking any further 1325 // action, and do not schedule a new nextReqEvent 1326 return; 1327 } 1328 |
1329 // pre-emptively set to false. Overwrite if in transitioning to 1330 // a new state |
1331 bool switched_cmd_type = false; |
1332 if (busState != busStateNext) { 1333 if (busState == READ) { 1334 DPRINTF(DRAM, "Switching to writes after %d reads with %d reads " 1335 "waiting\n", readsThisTime, readQueue.size()); |
1336 |
1337 // sample and reset the read-related stats as we are now 1338 // transitioning to writes, and all reads are done 1339 rdPerTurnAround.sample(readsThisTime); 1340 readsThisTime = 0; |
1341 |
1342 // now proceed to do the actual writes 1343 switched_cmd_type = true; 1344 } else { 1345 DPRINTF(DRAM, "Switching to reads after %d writes with %d writes " 1346 "waiting\n", writesThisTime, writeQueue.size()); |
1347 |
1348 wrPerTurnAround.sample(writesThisTime); 1349 writesThisTime = 0; |
1350 |
1351 switched_cmd_type = true; 1352 } 1353 // update busState to match next state until next transition 1354 busState = busStateNext; |
1355 } 1356 1357 // when we get here it is either a read or a write 1358 if (busState == READ) { 1359 1360 // track if we should switch or not 1361 bool switch_to_writes = false; 1362 --- 38 unchanged lines hidden (view full) --- 1401 // which are above the required threshold. However, to 1402 // avoid adding more complexity to the code, return and wait 1403 // for a refresh event to kick things into action again. 1404 if (!found_read) 1405 return; 1406 1407 DRAMPacket* dram_pkt = readQueue.front(); 1408 assert(dram_pkt->rankRef.isAvailable()); |
1409 |
1410 // here we get a bit creative and shift the bus busy time not 1411 // just the tWTR, but also a CAS latency to capture the fact 1412 // that we are allowed to prepare a new bank, but not issue a 1413 // read command until after tWTR, in essence we capture a 1414 // bubble on the data bus that is tWTR + tCL 1415 if (switched_cmd_type && dram_pkt->rank == activeRank) { 1416 busBusyUntil += tWTR + tCL; 1417 } 1418 1419 doDRAMAccess(dram_pkt); 1420 1421 // At this point we're done dealing with the request 1422 readQueue.pop_front(); 1423 |
1424 // Every respQueue which will generate an event, increment count 1425 ++dram_pkt->rankRef.outstandingEvents; 1426 |
1427 // sanity check 1428 assert(dram_pkt->size <= burstSize); 1429 assert(dram_pkt->readyTime >= curTick()); 1430 1431 // Insert into response queue. It will be sent back to the 1432 // requestor at its readyTime 1433 if (respQueue.empty()) { 1434 assert(!respondEvent.scheduled()); --- 11 unchanged lines hidden (view full) --- 1446 } 1447 } 1448 1449 // switching to writes, either because the read queue is empty 1450 // and the writes have passed the low threshold (or we are 1451 // draining), or because the writes hit the hight threshold 1452 if (switch_to_writes) { 1453 // transition to writing |
1454 busStateNext = WRITE; |
1455 } 1456 } else { 1457 // bool to check if write to free rank is found 1458 bool found_write = false; 1459 1460 // If we are changing command type, incorporate the minimum 1461 // bus turnaround delay 1462 found_write = chooseNext(writeQueue, --- 17 unchanged lines hidden (view full) --- 1480 // applied to colAllowedAt 1481 if (switched_cmd_type && dram_pkt->rank == activeRank) { 1482 busBusyUntil += tRTW; 1483 } 1484 1485 doDRAMAccess(dram_pkt); 1486 1487 writeQueue.pop_front(); |
1488 1489 // removed write from queue, decrement count 1490 --dram_pkt->rankRef.writeEntries; 1491 1492 // Schedule write done event to decrement event count 1493 // after the readyTime has been reached 1494 // Only schedule latest write event to minimize events 1495 // required; only need to ensure that final event scheduled covers 1496 // the time that writes are outstanding and bus is active 1497 // to holdoff power-down entry events 1498 if (!dram_pkt->rankRef.writeDoneEvent.scheduled()) { 1499 schedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime); 1500 // New event, increment count 1501 ++dram_pkt->rankRef.outstandingEvents; 1502 1503 } else if (dram_pkt->rankRef.writeDoneEvent.when() < 1504 dram_pkt-> readyTime) { 1505 reschedule(dram_pkt->rankRef.writeDoneEvent, dram_pkt->readyTime); 1506 } 1507 |
1508 isInWriteQueue.erase(burstAlign(dram_pkt->addr)); 1509 delete dram_pkt; 1510 1511 // If we emptied the write queue, or got sufficiently below the 1512 // threshold (using the minWritesPerSwitch as the hysteresis) and 1513 // are not draining, or we have reads waiting and have done enough 1514 // writes, then switch to reads. 1515 if (writeQueue.empty() || 1516 (writeQueue.size() + minWritesPerSwitch < writeLowThreshold && 1517 drainState() != DrainState::Draining) || 1518 (!readQueue.empty() && writesThisTime >= minWritesPerSwitch)) { 1519 // turn the bus back around for reads again |
1520 busStateNext = READ; |
1521 1522 // note that the we switch back to reads also in the idle 1523 // case, which eventually will check for any draining and 1524 // also pause any further scheduling if there is really 1525 // nothing to do 1526 } 1527 } 1528 // It is possible that a refresh to another rank kicks things back into --- 91 unchanged lines hidden (view full) --- 1620 } 1621 } 1622 1623 return make_pair(bank_mask, hidden_bank_prep); 1624} 1625 1626DRAMCtrl::Rank::Rank(DRAMCtrl& _memory, const DRAMCtrlParams* _p) 1627 : EventManager(&_memory), memory(_memory), |
1628 pwrStateTrans(PWR_IDLE), pwrStatePostRefresh(PWR_IDLE), 1629 pwrStateTick(0), refreshDueAt(0), pwrState(PWR_IDLE), 1630 refreshState(REF_IDLE), inLowPowerState(false), rank(0), 1631 readEntries(0), writeEntries(0), outstandingEvents(0), 1632 wakeUpAllowedAt(0), power(_p, false), numBanksActive(0), 1633 writeDoneEvent(*this), activateEvent(*this), prechargeEvent(*this), 1634 refreshEvent(*this), powerEvent(*this), wakeUpEvent(*this) |
1635{ } 1636 1637void 1638DRAMCtrl::Rank::startup(Tick ref_tick) 1639{ 1640 assert(ref_tick > curTick()); 1641 1642 pwrStateTick = curTick(); --- 5 unchanged lines hidden (view full) --- 1648 1649void 1650DRAMCtrl::Rank::suspend() 1651{ 1652 deschedule(refreshEvent); 1653 1654 // Update the stats 1655 updatePowerStats(); |
1656 1657 // don't automatically transition back to LP state after next REF 1658 pwrStatePostRefresh = PWR_IDLE; |
1659} 1660 |
1661bool 1662DRAMCtrl::Rank::lowPowerEntryReady() const 1663{ 1664 bool no_queued_cmds = ((memory.busStateNext == READ) && (readEntries == 0)) 1665 || ((memory.busStateNext == WRITE) && 1666 (writeEntries == 0)); 1667 1668 if (refreshState == REF_RUN) { 1669 // have not decremented outstandingEvents for refresh command 1670 // still check if there are no commands queued to force PD 1671 // entry after refresh completes 1672 return no_queued_cmds; 1673 } else { 1674 // ensure no commands in Q and no commands scheduled 1675 return (no_queued_cmds && (outstandingEvents == 0)); 1676 } 1677} 1678 |
1679void 1680DRAMCtrl::Rank::checkDrainDone() 1681{ 1682 // if this rank was waiting to drain it is now able to proceed to 1683 // precharge 1684 if (refreshState == REF_DRAIN) { 1685 DPRINTF(DRAM, "Refresh drain done, now precharging\n"); 1686 |
1687 refreshState = REF_PD_EXIT; |
1688 1689 // hand control back to the refresh event loop 1690 schedule(refreshEvent, curTick()); 1691 } 1692} 1693 1694void 1695DRAMCtrl::Rank::flushCmdList() --- 31 unchanged lines hidden (view full) --- 1727 // note that at this point numBanksActive could be back at 1728 // zero again due to a precharge scheduled in the future 1729 schedulePowerEvent(PWR_ACT, curTick()); 1730} 1731 1732void 1733DRAMCtrl::Rank::processPrechargeEvent() 1734{ |
1735 // counter should at least indicate one outstanding request 1736 // for this precharge 1737 assert(outstandingEvents > 0); 1738 // precharge complete, decrement count 1739 --outstandingEvents; 1740 |
1741 // if we reached zero, then special conditions apply as we track 1742 // if all banks are precharged for the power models 1743 if (numBanksActive == 0) { |
1744 // no reads to this rank in the Q and no pending 1745 // RD/WR or refresh commands 1746 if (lowPowerEntryReady()) { 1747 // should still be in ACT state since bank still open 1748 assert(pwrState == PWR_ACT); 1749 1750 // All banks closed - switch to precharge power down state. 1751 DPRINTF(DRAMState, "Rank %d sleep at tick %d\n", 1752 rank, curTick()); 1753 powerDownSleep(PWR_PRE_PDN, curTick()); 1754 } else { 1755 // we should transition to the idle state when the last bank 1756 // is precharged 1757 schedulePowerEvent(PWR_IDLE, curTick()); 1758 } |
1759 } 1760} 1761 1762void |
1763DRAMCtrl::Rank::processWriteDoneEvent() 1764{ 1765 // counter should at least indicate one outstanding request 1766 // for this write 1767 assert(outstandingEvents > 0); 1768 // Write transfer on bus has completed 1769 // decrement per rank counter 1770 --outstandingEvents; 1771} 1772 1773void |
1774DRAMCtrl::Rank::processRefreshEvent() 1775{ 1776 // when first preparing the refresh, remember when it was due |
1777 if ((refreshState == REF_IDLE) || (refreshState == REF_SREF_EXIT)) { |
1778 // remember when the refresh is due 1779 refreshDueAt = curTick(); 1780 1781 // proceed to drain 1782 refreshState = REF_DRAIN; 1783 |
1784 // make nonzero while refresh is pending to ensure 1785 // power down and self-refresh are not entered 1786 ++outstandingEvents; 1787 |
1788 DPRINTF(DRAM, "Refresh due\n"); 1789 } 1790 1791 // let any scheduled read or write to the same rank go ahead, 1792 // after which it will 1793 // hand control back to this event loop 1794 if (refreshState == REF_DRAIN) { 1795 // if a request is at the moment being handled and this request is 1796 // accessing the current rank then wait for it to finish 1797 if ((rank == memory.activeRank) 1798 && (memory.nextReqEvent.scheduled())) { 1799 // hand control over to the request loop until it is 1800 // evaluated next 1801 DPRINTF(DRAM, "Refresh awaiting draining\n"); 1802 1803 return; 1804 } else { |
1805 refreshState = REF_PD_EXIT; 1806 } 1807 } 1808 1809 // at this point, ensure that rank is not in a power-down state 1810 if (refreshState == REF_PD_EXIT) { 1811 // if rank was sleeping and we have't started exit process, 1812 // wake-up for refresh 1813 if (inLowPowerState) { 1814 DPRINTF(DRAM, "Wake Up for refresh\n"); 1815 // save state and return after refresh completes 1816 scheduleWakeUpEvent(memory.tXP); 1817 return; 1818 } else { |
1819 refreshState = REF_PRE; 1820 } 1821 } 1822 1823 // at this point, ensure that all banks are precharged 1824 if (refreshState == REF_PRE) { |
1825 // precharge any active bank 1826 if (numBanksActive != 0) { |
1827 // at the moment, we use a precharge all even if there is 1828 // only a single bank open 1829 DPRINTF(DRAM, "Precharging all\n"); 1830 1831 // first determine when we can precharge 1832 Tick pre_at = curTick(); 1833 1834 for (auto &b : banks) { --- 17 unchanged lines hidden (view full) --- 1852 } 1853 1854 // precharge all banks in rank 1855 cmdList.push_back(Command(MemCommand::PREA, 0, pre_at)); 1856 1857 DPRINTF(DRAMPower, "%llu,PREA,0,%d\n", 1858 divCeil(pre_at, memory.tCK) - 1859 memory.timeStampOffset, rank); |
1860 } else if ((pwrState == PWR_IDLE) && (outstandingEvents == 1)) { 1861 // Banks are closed, have transitioned to IDLE state, and 1862 // no outstanding ACT,RD/WR,Auto-PRE sequence scheduled |
1863 DPRINTF(DRAM, "All banks already precharged, starting refresh\n"); 1864 |
1865 // go ahead and kick the power state machine into gear since |
1866 // we are already idle 1867 schedulePowerEvent(PWR_REF, curTick()); |
1868 } else { 1869 // banks state is closed but haven't transitioned pwrState to IDLE 1870 // or have outstanding ACT,RD/WR,Auto-PRE sequence scheduled 1871 // should have outstanding precharge event in this case 1872 assert(prechargeEvent.scheduled()); 1873 // will start refresh when pwrState transitions to IDLE |
1874 } 1875 |
1876 assert(numBanksActive == 0); 1877 1878 // wait for all banks to be precharged, at which point the 1879 // power state machine will transition to the idle state, and 1880 // automatically move to a refresh, at that point it will also 1881 // call this method to get the refresh event loop going again 1882 return; 1883 } 1884 1885 // last but not least we perform the actual refresh |
1886 if (refreshState == REF_START) { |
1887 // should never get here with any banks active 1888 assert(numBanksActive == 0); 1889 assert(pwrState == PWR_REF); 1890 1891 Tick ref_done_at = curTick() + memory.tRFC; 1892 1893 for (auto &b : banks) { 1894 b.actAllowedAt = ref_done_at; 1895 } 1896 1897 // at the moment this affects all ranks 1898 cmdList.push_back(Command(MemCommand::REF, 0, curTick())); 1899 1900 // Update the stats 1901 updatePowerStats(); 1902 1903 DPRINTF(DRAMPower, "%llu,REF,0,%d\n", divCeil(curTick(), memory.tCK) - 1904 memory.timeStampOffset, rank); 1905 |
1906 // Update for next refresh 1907 refreshDueAt += memory.tREFI; 1908 |
1909 // make sure we did not wait so long that we cannot make up 1910 // for it |
1911 if (refreshDueAt < ref_done_at) { |
1912 fatal("Refresh was delayed so long we cannot catch up\n"); 1913 } 1914 |
1915 // Run the refresh and schedule event to transition power states 1916 // when refresh completes 1917 refreshState = REF_RUN; 1918 schedule(refreshEvent, ref_done_at); 1919 return; 1920 } |
1921 |
1922 if (refreshState == REF_RUN) { 1923 // should never get here with any banks active 1924 assert(numBanksActive == 0); 1925 assert(pwrState == PWR_REF); 1926 |
1927 assert(!powerEvent.scheduled()); 1928 |
1929 if ((memory.drainState() == DrainState::Draining) || 1930 (memory.drainState() == DrainState::Drained)) { 1931 // if draining, do not re-enter low-power mode. 1932 // simply go to IDLE and wait 1933 schedulePowerEvent(PWR_IDLE, curTick()); 1934 } else { 1935 // At the moment, we sleep when the refresh ends and wait to be 1936 // woken up again if previously in a low-power state. 1937 if (pwrStatePostRefresh != PWR_IDLE) { 1938 // power State should be power Refresh 1939 assert(pwrState == PWR_REF); 1940 DPRINTF(DRAMState, "Rank %d sleeping after refresh and was in " 1941 "power state %d before refreshing\n", rank, 1942 pwrStatePostRefresh); 1943 powerDownSleep(pwrState, curTick()); |
1944 |
1945 // Force PRE power-down if there are no outstanding commands 1946 // in Q after refresh. 1947 } else if (lowPowerEntryReady()) { 1948 DPRINTF(DRAMState, "Rank %d sleeping after refresh but was NOT" 1949 " in a low power state before refreshing\n", rank); 1950 powerDownSleep(PWR_PRE_PDN, curTick()); 1951 1952 } else { 1953 // move to the idle power state once the refresh is done, this 1954 // will also move the refresh state machine to the refresh 1955 // idle state 1956 schedulePowerEvent(PWR_IDLE, curTick()); 1957 } 1958 } 1959 1960 // if transitioning to self refresh do not schedule a new refresh; 1961 // when waking from self refresh, a refresh is scheduled again. 1962 if (pwrStateTrans != PWR_SREF) { 1963 // compensate for the delay in actually performing the refresh 1964 // when scheduling the next one 1965 schedule(refreshEvent, refreshDueAt - memory.tRP); 1966 1967 DPRINTF(DRAMState, "Refresh done at %llu and next refresh" 1968 " at %llu\n", curTick(), refreshDueAt); 1969 } |
1970 } 1971} 1972 1973void 1974DRAMCtrl::Rank::schedulePowerEvent(PowerState pwr_state, Tick tick) 1975{ 1976 // respect causality 1977 assert(tick >= curTick()); --- 9 unchanged lines hidden (view full) --- 1987 } else { 1988 panic("Scheduled power event at %llu to state %d, " 1989 "with scheduled event at %llu to %d\n", tick, pwr_state, 1990 powerEvent.when(), pwrStateTrans); 1991 } 1992} 1993 1994void |
1995DRAMCtrl::Rank::powerDownSleep(PowerState pwr_state, Tick tick) 1996{ 1997 // if low power state is active low, schedule to active low power state. 1998 // in reality tCKE is needed to enter active low power. This is neglected 1999 // here and could be added in the future. 2000 if (pwr_state == PWR_ACT_PDN) { 2001 schedulePowerEvent(pwr_state, tick); 2002 // push command to DRAMPower 2003 cmdList.push_back(Command(MemCommand::PDN_F_ACT, 0, tick)); 2004 DPRINTF(DRAMPower, "%llu,PDN_F_ACT,0,%d\n", divCeil(tick, 2005 memory.tCK) - memory.timeStampOffset, rank); 2006 } else if (pwr_state == PWR_PRE_PDN) { 2007 // if low power state is precharge low, schedule to precharge low 2008 // power state. In reality tCKE is needed to enter active low power. 2009 // This is neglected here. 2010 schedulePowerEvent(pwr_state, tick); 2011 //push Command to DRAMPower 2012 cmdList.push_back(Command(MemCommand::PDN_F_PRE, 0, tick)); 2013 DPRINTF(DRAMPower, "%llu,PDN_F_PRE,0,%d\n", divCeil(tick, 2014 memory.tCK) - memory.timeStampOffset, rank); 2015 } else if (pwr_state == PWR_REF) { 2016 // if a refresh just occured 2017 // transition to PRE_PDN now that all banks are closed 2018 // do not transition to SREF if commands are in Q; stay in PRE_PDN 2019 if (pwrStatePostRefresh == PWR_ACT_PDN || !lowPowerEntryReady()) { 2020 // prechage power down requires tCKE to enter. For simplicity 2021 // this is not considered. 2022 schedulePowerEvent(PWR_PRE_PDN, tick); 2023 //push Command to DRAMPower 2024 cmdList.push_back(Command(MemCommand::PDN_F_PRE, 0, tick)); 2025 DPRINTF(DRAMPower, "%llu,PDN_F_PRE,0,%d\n", divCeil(tick, 2026 memory.tCK) - memory.timeStampOffset, rank); 2027 } else { 2028 // last low power State was power precharge 2029 assert(pwrStatePostRefresh == PWR_PRE_PDN); 2030 // self refresh requires time tCKESR to enter. For simplicity, 2031 // this is not considered. 2032 schedulePowerEvent(PWR_SREF, tick); 2033 // push Command to DRAMPower 2034 cmdList.push_back(Command(MemCommand::SREN, 0, tick)); 2035 DPRINTF(DRAMPower, "%llu,SREN,0,%d\n", divCeil(tick, 2036 memory.tCK) - memory.timeStampOffset, rank); 2037 } 2038 } 2039 // Ensure that we don't power-down and back up in same tick 2040 // Once we commit to PD entry, do it and wait for at least 1tCK 2041 // This could be replaced with tCKE if/when that is added to the model 2042 wakeUpAllowedAt = tick + memory.tCK; 2043 2044 // Transitioning to a low power state, set flag 2045 inLowPowerState = true; 2046} 2047 2048void 2049DRAMCtrl::Rank::scheduleWakeUpEvent(Tick exit_delay) 2050{ 2051 Tick wake_up_tick = std::max(curTick(), wakeUpAllowedAt); 2052 2053 DPRINTF(DRAMState, "Scheduling wake-up for rank %d at tick %d\n", 2054 rank, wake_up_tick); 2055 2056 // if waking for refresh, hold previous state 2057 // else reset state back to IDLE 2058 if (refreshState == REF_PD_EXIT) { 2059 pwrStatePostRefresh = pwrState; 2060 } else { 2061 // don't automatically transition back to LP state after next REF 2062 pwrStatePostRefresh = PWR_IDLE; 2063 } 2064 2065 // schedule wake-up with event to ensure entry has completed before 2066 // we try to wake-up 2067 schedule(wakeUpEvent, wake_up_tick); 2068 2069 for (auto &b : banks) { 2070 // respect both causality and any existing bank 2071 // constraints, some banks could already have a 2072 // (auto) precharge scheduled 2073 b.colAllowedAt = std::max(wake_up_tick + exit_delay, b.colAllowedAt); 2074 b.preAllowedAt = std::max(wake_up_tick + exit_delay, b.preAllowedAt); 2075 b.actAllowedAt = std::max(wake_up_tick + exit_delay, b.actAllowedAt); 2076 } 2077 // Transitioning out of low power state, clear flag 2078 inLowPowerState = false; 2079 2080 // push to DRAMPower 2081 // use pwrStateTrans for cases where we have a power event scheduled 2082 // to enter low power that has not yet been processed 2083 if (pwrStateTrans == PWR_ACT_PDN) { 2084 cmdList.push_back(Command(MemCommand::PUP_ACT, 0, wake_up_tick)); 2085 DPRINTF(DRAMPower, "%llu,PUP_ACT,0,%d\n", divCeil(wake_up_tick, 2086 memory.tCK) - memory.timeStampOffset, rank); 2087 2088 } else if (pwrStateTrans == PWR_PRE_PDN) { 2089 cmdList.push_back(Command(MemCommand::PUP_PRE, 0, wake_up_tick)); 2090 DPRINTF(DRAMPower, "%llu,PUP_PRE,0,%d\n", divCeil(wake_up_tick, 2091 memory.tCK) - memory.timeStampOffset, rank); 2092 } else if (pwrStateTrans == PWR_SREF) { 2093 cmdList.push_back(Command(MemCommand::SREX, 0, wake_up_tick)); 2094 DPRINTF(DRAMPower, "%llu,SREX,0,%d\n", divCeil(wake_up_tick, 2095 memory.tCK) - memory.timeStampOffset, rank); 2096 } 2097} 2098 2099void 2100DRAMCtrl::Rank::processWakeUpEvent() 2101{ 2102 // Should be in a power-down or self-refresh state 2103 assert((pwrState == PWR_ACT_PDN) || (pwrState == PWR_PRE_PDN) || 2104 (pwrState == PWR_SREF)); 2105 2106 // Check current state to determine transition state 2107 if (pwrState == PWR_ACT_PDN) { 2108 // banks still open, transition to PWR_ACT 2109 schedulePowerEvent(PWR_ACT, curTick()); 2110 } else { 2111 // transitioning from a precharge power-down or self-refresh state 2112 // banks are closed - transition to PWR_IDLE 2113 schedulePowerEvent(PWR_IDLE, curTick()); 2114 } 2115} 2116 2117void |
2118DRAMCtrl::Rank::processPowerEvent() 2119{ |
2120 assert(curTick() >= pwrStateTick); |
2121 // remember where we were, and for how long 2122 Tick duration = curTick() - pwrStateTick; 2123 PowerState prev_state = pwrState; 2124 2125 // update the accounting 2126 pwrStateTime[prev_state] += duration; 2127 |
2128 // track to total idle time 2129 if ((prev_state == PWR_PRE_PDN) || (prev_state == PWR_ACT_PDN) || 2130 (prev_state == PWR_SREF)) { 2131 totalIdleTime += duration; 2132 } 2133 |
2134 pwrState = pwrStateTrans; 2135 pwrStateTick = curTick(); 2136 |
2137 // if rank was refreshing, make sure to start scheduling requests again 2138 if (prev_state == PWR_REF) { 2139 // bus IDLED prior to REF 2140 // counter should be one for refresh command only 2141 assert(outstandingEvents == 1); 2142 // REF complete, decrement count 2143 --outstandingEvents; |
2144 |
2145 DPRINTF(DRAMState, "Was refreshing for %llu ticks\n", duration); 2146 // if sleeping after refresh 2147 if (pwrState != PWR_IDLE) { 2148 assert((pwrState == PWR_PRE_PDN) || (pwrState == PWR_SREF)); 2149 DPRINTF(DRAMState, "Switching to power down state after refreshing" 2150 " rank %d at %llu tick\n", rank, curTick()); 2151 } 2152 if (pwrState != PWR_SREF) { 2153 // rank is not available in SREF 2154 // don't transition to IDLE in this case |
2155 refreshState = REF_IDLE; |
2156 } 2157 // a request event could be already scheduled by the state 2158 // machine of the other rank 2159 if (!memory.nextReqEvent.scheduled()) { 2160 DPRINTF(DRAM, "Scheduling next request after refreshing rank %d\n", 2161 rank); 2162 schedule(memory.nextReqEvent, curTick()); 2163 } 2164 } else if (pwrState == PWR_ACT) { 2165 if (refreshState == REF_PD_EXIT) { 2166 // kick the refresh event loop into action again 2167 assert(prev_state == PWR_ACT_PDN); |
2168 |
2169 // go back to REF event and close banks 2170 refreshState = REF_PRE; 2171 schedule(refreshEvent, curTick()); 2172 } 2173 } else if (pwrState == PWR_IDLE) { 2174 DPRINTF(DRAMState, "All banks precharged\n"); 2175 if (prev_state == PWR_SREF) { 2176 // set refresh state to REF_SREF_EXIT, ensuring isAvailable 2177 // continues to return false during tXS after SREF exit 2178 // Schedule a refresh which kicks things back into action 2179 // when it finishes 2180 refreshState = REF_SREF_EXIT; 2181 schedule(refreshEvent, curTick() + memory.tXS); 2182 } else { |
2183 // if we have a pending refresh, and are now moving to |
2184 // the idle state, directly transition to a refresh 2185 if ((refreshState == REF_PRE) || (refreshState == REF_PD_EXIT)) { 2186 // ensure refresh is restarted only after final PRE command. 2187 // do not restart refresh if controller is in an intermediate 2188 // state, after PRE_PDN exit, when banks are IDLE but an 2189 // ACT is scheduled. 2190 if (!activateEvent.scheduled()) { 2191 // there should be nothing waiting at this point 2192 assert(!powerEvent.scheduled()); 2193 // update the state in zero time and proceed below 2194 pwrState = PWR_REF; 2195 } else { 2196 // must have PRE scheduled to transition back to IDLE 2197 // and re-kick off refresh 2198 assert(prechargeEvent.scheduled()); 2199 } |
2200 } |
2201 } |
2202 } 2203 2204 // we transition to the refresh state, let the refresh state 2205 // machine know of this state update and let it deal with the 2206 // scheduling of the next power state transition as well as the 2207 // following refresh 2208 if (pwrState == PWR_REF) { |
2209 assert(refreshState == REF_PRE || refreshState == REF_PD_EXIT); |
2210 DPRINTF(DRAMState, "Refreshing\n"); |
2211 |
2212 // kick the refresh event loop into action again, and that 2213 // in turn will schedule a transition to the idle power 2214 // state once the refresh is done |
2215 if (refreshState == REF_PD_EXIT) { 2216 // Wait for PD exit timing to complete before issuing REF 2217 schedule(refreshEvent, curTick() + memory.tXP); 2218 } else { 2219 schedule(refreshEvent, curTick()); 2220 } 2221 // Banks transitioned to IDLE, start REF 2222 refreshState = REF_START; |
2223 } 2224} 2225 2226void 2227DRAMCtrl::Rank::updatePowerStats() 2228{ 2229 // All commands up to refresh have completed 2230 // flush cmdList to DRAMPower --- 20 unchanged lines hidden (view full) --- 2251 2252 actEnergy = energy.act_energy * memory.devicesPerRank; 2253 preEnergy = energy.pre_energy * memory.devicesPerRank; 2254 readEnergy = energy.read_energy * memory.devicesPerRank; 2255 writeEnergy = energy.write_energy * memory.devicesPerRank; 2256 refreshEnergy = energy.ref_energy * memory.devicesPerRank; 2257 actBackEnergy = energy.act_stdby_energy * memory.devicesPerRank; 2258 preBackEnergy = energy.pre_stdby_energy * memory.devicesPerRank; |
2259 actPowerDownEnergy = energy.f_act_pd_energy * memory.devicesPerRank; 2260 prePowerDownEnergy = energy.f_pre_pd_energy * memory.devicesPerRank; 2261 selfRefreshEnergy = energy.sref_energy * memory.devicesPerRank; |
2262 totalEnergy = energy.total_energy * memory.devicesPerRank; 2263 averagePower = rank_power.average_power * memory.devicesPerRank; 2264} 2265 2266void 2267DRAMCtrl::Rank::computeStats() 2268{ 2269 DPRINTF(DRAM,"Computing final stats\n"); --- 12 unchanged lines hidden (view full) --- 2282} 2283 2284void 2285DRAMCtrl::Rank::regStats() 2286{ 2287 using namespace Stats; 2288 2289 pwrStateTime |
2290 .init(6) |
2291 .name(name() + ".memoryStateTime") 2292 .desc("Time in different power states"); 2293 pwrStateTime.subname(0, "IDLE"); 2294 pwrStateTime.subname(1, "REF"); |
2295 pwrStateTime.subname(2, "SREF"); 2296 pwrStateTime.subname(3, "PRE_PDN"); 2297 pwrStateTime.subname(4, "ACT"); 2298 pwrStateTime.subname(5, "ACT_PDN"); |
2299 2300 actEnergy 2301 .name(name() + ".actEnergy") 2302 .desc("Energy for activate commands per rank (pJ)"); 2303 2304 preEnergy 2305 .name(name() + ".preEnergy") 2306 .desc("Energy for precharge commands per rank (pJ)"); --- 13 unchanged lines hidden (view full) --- 2320 actBackEnergy 2321 .name(name() + ".actBackEnergy") 2322 .desc("Energy for active background per rank (pJ)"); 2323 2324 preBackEnergy 2325 .name(name() + ".preBackEnergy") 2326 .desc("Energy for precharge background per rank (pJ)"); 2327 |
2328 actPowerDownEnergy 2329 .name(name() + ".actPowerDownEnergy") 2330 .desc("Energy for active power-down per rank (pJ)"); 2331 2332 prePowerDownEnergy 2333 .name(name() + ".prePowerDownEnergy") 2334 .desc("Energy for precharge power-down per rank (pJ)"); 2335 2336 selfRefreshEnergy 2337 .name(name() + ".selfRefreshEnergy") 2338 .desc("Energy for self refresh per rank (pJ)"); 2339 |
2340 totalEnergy 2341 .name(name() + ".totalEnergy") 2342 .desc("Total energy per rank (pJ)"); 2343 2344 averagePower 2345 .name(name() + ".averagePower") 2346 .desc("Core power per rank (mW)"); 2347 |
2348 totalIdleTime 2349 .name(name() + ".totalIdleTime") 2350 .desc("Total Idle time Per DRAM Rank"); 2351 |
2352 registerDumpCallback(new RankDumpCallback(this)); 2353} 2354void 2355DRAMCtrl::regStats() 2356{ 2357 using namespace Stats; 2358 2359 AbstractMemory::regStats(); --- 274 unchanged lines hidden (view full) --- 2634 // of that as well 2635 if (!(writeQueue.empty() && readQueue.empty() && respQueue.empty() && 2636 allRanksDrained())) { 2637 2638 DPRINTF(Drain, "DRAM controller not drained, write: %d, read: %d," 2639 " resp: %d\n", writeQueue.size(), readQueue.size(), 2640 respQueue.size()); 2641 |
2642 // the only queue that is not drained automatically over time |
2643 // is the write queue, thus kick things into action if needed 2644 if (!writeQueue.empty() && !nextReqEvent.scheduled()) { 2645 schedule(nextReqEvent, curTick()); 2646 } |
2647 2648 // also need to kick off events to exit self-refresh 2649 for (auto r : ranks) { 2650 // force self-refresh exit, which in turn will issue auto-refresh 2651 if (r->pwrState == PWR_SREF) { 2652 DPRINTF(DRAM,"Rank%d: Forcing self-refresh wakeup in drain\n", 2653 r->rank); 2654 r->scheduleWakeUpEvent(tXS); 2655 } 2656 } 2657 |
2658 return DrainState::Draining; 2659 } else { 2660 return DrainState::Drained; 2661 } 2662} 2663 2664bool 2665DRAMCtrl::allRanksDrained() const --- 76 unchanged lines hidden --- |