dram_ctrl.cc (10209:ac71c857e1e1) dram_ctrl.cc (10210:793e5ff26e0b)
1/*
2 * Copyright (c) 2010-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 61 unchanged lines hidden (view full) ---

70 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0),
71 readBufferSize(p->read_buffer_size),
72 writeBufferSize(p->write_buffer_size),
73 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0),
74 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0),
75 minWritesPerSwitch(p->min_writes_per_switch),
76 writesThisTime(0), readsThisTime(0),
77 tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST),
1/*
2 * Copyright (c) 2010-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 61 unchanged lines hidden (view full) ---

70 banksPerRank(p->banks_per_rank), channels(p->channels), rowsPerBank(0),
71 readBufferSize(p->read_buffer_size),
72 writeBufferSize(p->write_buffer_size),
73 writeHighThreshold(writeBufferSize * p->write_high_thresh_perc / 100.0),
74 writeLowThreshold(writeBufferSize * p->write_low_thresh_perc / 100.0),
75 minWritesPerSwitch(p->min_writes_per_switch),
76 writesThisTime(0), readsThisTime(0),
77 tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST),
78 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS),
78 tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR),
79 tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD),
80 tXAW(p->tXAW), activationLimit(p->activation_limit),
81 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping),
82 pageMgmt(p->page_policy),
83 maxAccessesPerRow(p->max_accesses_per_row),
84 frontendLatency(p->static_frontend_latency),
85 backendLatency(p->static_backend_latency),
86 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE),

--- 467 unchanged lines hidden (view full) ---

554 "tRCD %d ticks\n" \
555 "tCL %d ticks\n" \
556 "tRP %d ticks\n" \
557 "tBURST %d ticks\n" \
558 "tRFC %d ticks\n" \
559 "tREFI %d ticks\n" \
560 "tWTR %d ticks\n" \
561 "tRTW %d ticks\n" \
79 tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD),
80 tXAW(p->tXAW), activationLimit(p->activation_limit),
81 memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping),
82 pageMgmt(p->page_policy),
83 maxAccessesPerRow(p->max_accesses_per_row),
84 frontendLatency(p->static_frontend_latency),
85 backendLatency(p->static_backend_latency),
86 busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE),

--- 467 unchanged lines hidden (view full) ---

554 "tRCD %d ticks\n" \
555 "tCL %d ticks\n" \
556 "tRP %d ticks\n" \
557 "tBURST %d ticks\n" \
558 "tRFC %d ticks\n" \
559 "tREFI %d ticks\n" \
560 "tWTR %d ticks\n" \
561 "tRTW %d ticks\n" \
562 "tWR %d ticks\n" \
562 "tXAW (%d) %d ticks\n",
563 name(), tRCD, tCL, tRP, tBURST, tRFC, tREFI, tWTR,
563 "tXAW (%d) %d ticks\n",
564 name(), tRCD, tCL, tRP, tBURST, tRFC, tREFI, tWTR,
564 tRTW, activationLimit, tXAW);
565 tRTW, tWR, activationLimit, tXAW);
565}
566
567void
568DRAMCtrl::printQs() const {
569 DPRINTF(DRAM, "===READ QUEUE===\n\n");
570 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) {
571 DPRINTF(DRAM, "Read %lu\n", (*i)->addr);
572 }

--- 229 unchanged lines hidden (view full) ---

802 // *after* that does it take to finish the request, depending
803 // on bank status and page open policy. Note that this method
804 // considers only the time taken for the actual read or write
805 // to complete, NOT any additional time thereafter for tRAS or
806 // tRP.
807 Tick accLat = 0;
808 Tick bankLat = 0;
809 rowHitFlag = false;
566}
567
568void
569DRAMCtrl::printQs() const {
570 DPRINTF(DRAM, "===READ QUEUE===\n\n");
571 for (auto i = readQueue.begin() ; i != readQueue.end() ; ++i) {
572 DPRINTF(DRAM, "Read %lu\n", (*i)->addr);
573 }

--- 229 unchanged lines hidden (view full) ---

803 // *after* that does it take to finish the request, depending
804 // on bank status and page open policy. Note that this method
805 // considers only the time taken for the actual read or write
806 // to complete, NOT any additional time thereafter for tRAS or
807 // tRP.
808 Tick accLat = 0;
809 Tick bankLat = 0;
810 rowHitFlag = false;
810 Tick potentialActTick;
811
812 const Bank& bank = dram_pkt->bankRef;
813
814 if (bank.openRow == dram_pkt->row) {
815 // When we have a row-buffer hit,
816 // we don't care about tRAS having expired or not,
817 // but do care about bank being free for access
818 rowHitFlag = true;

--- 8 unchanged lines hidden (view full) ---

827 accLat += bank.freeAt - inTime;
828 bankLat += 0;
829 } else {
830 // CAS latency only
831 accLat += tCL;
832 bankLat += tCL;
833 }
834 } else {
811
812 const Bank& bank = dram_pkt->bankRef;
813
814 if (bank.openRow == dram_pkt->row) {
815 // When we have a row-buffer hit,
816 // we don't care about tRAS having expired or not,
817 // but do care about bank being free for access
818 rowHitFlag = true;

--- 8 unchanged lines hidden (view full) ---

827 accLat += bank.freeAt - inTime;
828 bankLat += 0;
829 } else {
830 // CAS latency only
831 accLat += tCL;
832 bankLat += tCL;
833 }
834 } else {
835 // Row-buffer miss, need to close existing row
836 // once tRAS has expired, then open the new one,
837 // then add cas latency.
838 Tick freeTime = std::max(bank.tRASDoneAt, bank.freeAt);
835 // Row-buffer miss, need to potentially close an existing row,
836 // then open the new one, then add CAS latency
837 Tick free_at = bank.freeAt;
838 Tick precharge_delay = 0;
839
839
840 if (freeTime > inTime)
841 accLat += freeTime - inTime;
840 // Check if we first need to precharge
841 if (bank.openRow != Bank::NO_ROW) {
842 free_at = std::max(bank.preAllowedAt, free_at);
843 precharge_delay = tRP;
844 }
842
845
843 // If the there is no open row, then there is no precharge
844 // delay, otherwise go with tRP
845 Tick precharge_delay = bank.openRow == Bank::NO_ROW ? 0 : tRP;
846 // If the earliest time to issue the command is in the future,
847 // add it to the access latency
848 if (free_at > inTime)
849 accLat += free_at - inTime;
846
850
847 //The bank is free, and you may be able to activate
848 potentialActTick = inTime + accLat + precharge_delay;
849 if (potentialActTick < bank.actAllowedAt)
850 accLat += bank.actAllowedAt - potentialActTick;
851 // We also need to account for the earliest activation time,
852 // and potentially add that as well to the access latency
853 Tick act_at = inTime + accLat + precharge_delay;
854 if (act_at < bank.actAllowedAt)
855 accLat += bank.actAllowedAt - act_at;
851
852 accLat += precharge_delay + tRCD + tCL;
853 bankLat += precharge_delay + tRCD + tCL;
854 }
855
856 DPRINTF(DRAM, "Returning < %lld, %lld > from estimateLatency()\n",
857 bankLat, accLat);
858
859 return make_pair(bankLat, accLat);
860}
861
862void
856
857 accLat += precharge_delay + tRCD + tCL;
858 bankLat += precharge_delay + tRCD + tCL;
859 }
860
861 DPRINTF(DRAM, "Returning < %lld, %lld > from estimateLatency()\n",
862 bankLat, accLat);
863
864 return make_pair(bankLat, accLat);
865}
866
867void
863DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
864 uint16_t row)
868DRAMCtrl::activateBank(Tick act_tick, uint8_t rank, uint8_t bank,
869 uint16_t row, Bank& bank_ref)
865{
866 assert(0 <= rank && rank < ranksPerChannel);
867 assert(actTicks[rank].size() == activationLimit);
868
869 DPRINTF(DRAM, "Activate at tick %d\n", act_tick);
870
871 // update the open row
870{
871 assert(0 <= rank && rank < ranksPerChannel);
872 assert(actTicks[rank].size() == activationLimit);
873
874 DPRINTF(DRAM, "Activate at tick %d\n", act_tick);
875
876 // update the open row
872 assert(banks[rank][bank].openRow == Bank::NO_ROW);
873 banks[rank][bank].openRow = row;
877 assert(bank_ref.openRow == Bank::NO_ROW);
878 bank_ref.openRow = row;
874
875 // start counting anew, this covers both the case when we
876 // auto-precharged, and when this access is forced to
877 // precharge
879
880 // start counting anew, this covers both the case when we
881 // auto-precharged, and when this access is forced to
882 // precharge
878 banks[rank][bank].bytesAccessed = 0;
879 banks[rank][bank].rowAccesses = 0;
883 bank_ref.bytesAccessed = 0;
884 bank_ref.rowAccesses = 0;
880
881 ++numBanksActive;
882 assert(numBanksActive <= banksPerRank * ranksPerChannel);
883
884 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n",
885 act_tick, numBanksActive);
886
887 // start by enforcing tRRD
888 for(int i = 0; i < banksPerRank; i++) {
885
886 ++numBanksActive;
887 assert(numBanksActive <= banksPerRank * ranksPerChannel);
888
889 DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n",
890 act_tick, numBanksActive);
891
892 // start by enforcing tRRD
893 for(int i = 0; i < banksPerRank; i++) {
889 // next activate must not happen before tRRD
890 banks[rank][i].actAllowedAt = act_tick + tRRD;
894 // next activate to any bank in this rank must not happen
895 // before tRRD
896 banks[rank][i].actAllowedAt = std::max(act_tick + tRRD,
897 banks[rank][i].actAllowedAt);
891 }
892
898 }
899
893 // tRC should be added to activation tick of the bank currently accessed,
894 // where tRC = tRAS + tRP, this is just for a check as actAllowedAt for same
895 // bank is already captured by bank.freeAt and bank.tRASDoneAt
896 banks[rank][bank].actAllowedAt = act_tick + tRAS + tRP;
897
898 // next, we deal with tXAW, if the activation limit is disabled
899 // then we are done
900 if (actTicks[rank].empty())
901 return;
902
903 // sanity check
904 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
900 // next, we deal with tXAW, if the activation limit is disabled
901 // then we are done
902 if (actTicks[rank].empty())
903 return;
904
905 // sanity check
906 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
905 // @todo For now, stick with a warning
906 warn("Got %d activates in window %d (%d - %d) which is smaller "
907 "than %d\n", activationLimit, act_tick - actTicks[rank].back(),
908 act_tick, actTicks[rank].back(), tXAW);
907 panic("Got %d activates in window %d (%llu - %llu) which is smaller "
908 "than %llu\n", activationLimit, act_tick - actTicks[rank].back(),
909 act_tick, actTicks[rank].back(), tXAW);
909 }
910
911 // shift the times used for the book keeping, the last element
912 // (highest index) is the oldest one and hence the lowest value
913 actTicks[rank].pop_back();
914
915 // record an new activation (in the future)
916 actTicks[rank].push_front(act_tick);
917
918 // cannot activate more than X times in time window tXAW, push the
919 // next one (the X + 1'st activate) to be tXAW away from the
920 // oldest in our window of X
921 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
922 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier "
910 }
911
912 // shift the times used for the book keeping, the last element
913 // (highest index) is the oldest one and hence the lowest value
914 actTicks[rank].pop_back();
915
916 // record an new activation (in the future)
917 actTicks[rank].push_front(act_tick);
918
919 // cannot activate more than X times in time window tXAW, push the
920 // next one (the X + 1'st activate) to be tXAW away from the
921 // oldest in our window of X
922 if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
923 DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier "
923 "than %d\n", activationLimit, actTicks[rank].back() + tXAW);
924 "than %llu\n", activationLimit, actTicks[rank].back() + tXAW);
924 for(int j = 0; j < banksPerRank; j++)
925 // next activate must not happen before end of window
925 for(int j = 0; j < banksPerRank; j++)
926 // next activate must not happen before end of window
926 banks[rank][j].actAllowedAt = actTicks[rank].back() + tXAW;
927 banks[rank][j].actAllowedAt =
928 std::max(actTicks[rank].back() + tXAW,
929 banks[rank][j].actAllowedAt);
927 }
928
929 // at the point when this activate takes place, make sure we
930 // transition to the active power state
931 if (!activateEvent.scheduled())
932 schedule(activateEvent, act_tick);
933 else if (activateEvent.when() > act_tick)
934 // move it sooner in time

--- 66 unchanged lines hidden (view full) ---

1001 Tick bankLat = lat.first;
1002 Tick accessLat = lat.second;
1003 Tick actTick;
1004
1005 // This request was woken up at this time based on a prior call
1006 // to estimateLatency(). However, between then and now, both the
1007 // accessLatency and/or busBusyUntil may have changed. We need
1008 // to correct for that.
930 }
931
932 // at the point when this activate takes place, make sure we
933 // transition to the active power state
934 if (!activateEvent.scheduled())
935 schedule(activateEvent, act_tick);
936 else if (activateEvent.when() > act_tick)
937 // move it sooner in time

--- 66 unchanged lines hidden (view full) ---

1004 Tick bankLat = lat.first;
1005 Tick accessLat = lat.second;
1006 Tick actTick;
1007
1008 // This request was woken up at this time based on a prior call
1009 // to estimateLatency(). However, between then and now, both the
1010 // accessLatency and/or busBusyUntil may have changed. We need
1011 // to correct for that.
1009
1010 Tick addDelay = (curTick() + accessLat < busBusyUntil) ?
1011 busBusyUntil - (curTick() + accessLat) : 0;
1012
1012 Tick addDelay = (curTick() + accessLat < busBusyUntil) ?
1013 busBusyUntil - (curTick() + accessLat) : 0;
1014
1015 // Update request parameters
1016 dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST;
1017
1018 DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \
1019 "readytime is %lld busbusyuntil is %lld. " \
1020 "Scheduling at readyTime\n", dram_pkt->addr,
1021 curTick(), accessLat, dram_pkt->readyTime, busBusyUntil);
1022
1023 // Make sure requests are not overlapping on the databus
1024 assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
1025
1013 Bank& bank = dram_pkt->bankRef;
1014
1015 // Update bank state
1016 if (rowHitFlag) {
1017 bank.freeAt = curTick() + addDelay + accessLat;
1018 } else {
1019 // If there is a page open, precharge it.
1020 if (bank.openRow != Bank::NO_ROW) {
1021 prechargeBank(bank, std::max(std::max(bank.freeAt,
1026 Bank& bank = dram_pkt->bankRef;
1027
1028 // Update bank state
1029 if (rowHitFlag) {
1030 bank.freeAt = curTick() + addDelay + accessLat;
1031 } else {
1032 // If there is a page open, precharge it.
1033 if (bank.openRow != Bank::NO_ROW) {
1034 prechargeBank(bank, std::max(std::max(bank.freeAt,
1022 bank.tRASDoneAt),
1035 bank.preAllowedAt),
1023 curTick()) + tRP);
1024 }
1025
1026 // Any precharge is already part of the latency
1027 // estimation, so update the bank free time
1028 bank.freeAt = curTick() + addDelay + accessLat;
1029
1030 // any waiting for banks account for in freeAt
1031 actTick = bank.freeAt - tCL - tRCD;
1032
1036 curTick()) + tRP);
1037 }
1038
1039 // Any precharge is already part of the latency
1040 // estimation, so update the bank free time
1041 bank.freeAt = curTick() + addDelay + accessLat;
1042
1043 // any waiting for banks account for in freeAt
1044 actTick = bank.freeAt - tCL - tRCD;
1045
1033 // If you activated a new row do to this access, the next access
1034 // will have to respect tRAS for this bank
1035 bank.tRASDoneAt = actTick + tRAS;
1046 // The next access has to respect tRAS for this bank
1047 bank.preAllowedAt = actTick + tRAS;
1036
1048
1037 recordActivate(actTick, dram_pkt->rank, dram_pkt->bank,
1038 dram_pkt->row);
1049 // Record the activation and deal with all the global timing
1050 // constraints caused be a new activation (tRRD and tXAW)
1051 activateBank(actTick, dram_pkt->rank, dram_pkt->bank,
1052 dram_pkt->row, bank);
1053
1039 }
1040
1054 }
1055
1056 // If this is a write, we also need to respect the write
1057 // recovery time before a precharge
1058 if (!dram_pkt->isRead) {
1059 bank.preAllowedAt = std::max(bank.preAllowedAt,
1060 dram_pkt->readyTime + tWR);
1061 }
1062
1063 // We also have to respect tRP, and any constraints on when we may
1064 // precharge the bank, in the case of reads this is really only
1065 // going to cause any change if we did not have a row hit and are
1066 // now forced to respect tRAS
1067 bank.actAllowedAt = std::max(bank.actAllowedAt,
1068 bank.preAllowedAt + tRP);
1069
1041 // increment the bytes accessed and the accesses per row
1042 bank.bytesAccessed += burstSize;
1043 ++bank.rowAccesses;
1044
1045 // if we reached the max, then issue with an auto-precharge
1046 bool auto_precharge = pageMgmt == Enums::close ||
1047 bank.rowAccesses == maxAccessesPerRow;
1048

--- 40 unchanged lines hidden (view full) ---

1089 // 2) close_adaptive policy and we have not got any more hits
1090 auto_precharge = !got_more_hits &&
1091 (got_bank_conflict || pageMgmt == Enums::close_adaptive);
1092 }
1093
1094 // if this access should use auto-precharge, then we are
1095 // closing the row
1096 if (auto_precharge) {
1070 // increment the bytes accessed and the accesses per row
1071 bank.bytesAccessed += burstSize;
1072 ++bank.rowAccesses;
1073
1074 // if we reached the max, then issue with an auto-precharge
1075 bool auto_precharge = pageMgmt == Enums::close ||
1076 bank.rowAccesses == maxAccessesPerRow;
1077

--- 40 unchanged lines hidden (view full) ---

1118 // 2) close_adaptive policy and we have not got any more hits
1119 auto_precharge = !got_more_hits &&
1120 (got_bank_conflict || pageMgmt == Enums::close_adaptive);
1121 }
1122
1123 // if this access should use auto-precharge, then we are
1124 // closing the row
1125 if (auto_precharge) {
1097 prechargeBank(bank, std::max(bank.freeAt, bank.tRASDoneAt) + tRP);
1126 prechargeBank(bank, std::max(bank.freeAt, bank.preAllowedAt) + tRP);
1098
1099 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId);
1100 }
1101
1102 DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt);
1103
1127
1128 DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId);
1129 }
1130
1131 DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt);
1132
1104 // Update request parameters
1105 dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST;
1106
1107 DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \
1108 "readytime is %lld busbusyuntil is %lld. " \
1109 "Scheduling at readyTime\n", dram_pkt->addr,
1110 curTick(), accessLat, dram_pkt->readyTime, busBusyUntil);
1111
1112 // Make sure requests are not overlapping on the databus
1113 assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
1114
1115 // Update bus state
1116 busBusyUntil = dram_pkt->readyTime;
1117
1118 DPRINTF(DRAM,"Access time is %lld\n",
1119 dram_pkt->readyTime - dram_pkt->entryTime);
1120
1121 // Update the minimum timing between the requests, this is a
1122 // conservative estimate of when we have to schedule the next

--- 284 unchanged lines hidden (view full) ---

1407 DPRINTF(DRAM, "Precharging all\n");
1408 for (int i = 0; i < ranksPerChannel; i++) {
1409 for (int j = 0; j < banksPerRank; j++) {
1410 if (banks[i][j].openRow != Bank::NO_ROW) {
1411 // respect both causality and any existing bank
1412 // constraints
1413 Tick free_at =
1414 std::max(std::max(banks[i][j].freeAt,
1133 // Update bus state
1134 busBusyUntil = dram_pkt->readyTime;
1135
1136 DPRINTF(DRAM,"Access time is %lld\n",
1137 dram_pkt->readyTime - dram_pkt->entryTime);
1138
1139 // Update the minimum timing between the requests, this is a
1140 // conservative estimate of when we have to schedule the next

--- 284 unchanged lines hidden (view full) ---

1425 DPRINTF(DRAM, "Precharging all\n");
1426 for (int i = 0; i < ranksPerChannel; i++) {
1427 for (int j = 0; j < banksPerRank; j++) {
1428 if (banks[i][j].openRow != Bank::NO_ROW) {
1429 // respect both causality and any existing bank
1430 // constraints
1431 Tick free_at =
1432 std::max(std::max(banks[i][j].freeAt,
1415 banks[i][j].tRASDoneAt),
1433 banks[i][j].preAllowedAt),
1416 curTick()) + tRP;
1417
1418 prechargeBank(banks[i][j], free_at);
1419 }
1420 }
1421 }
1422 } else {
1423 DPRINTF(DRAM, "All banks already precharged, starting refresh\n");

--- 496 unchanged lines hidden --- 		1434 curTick()) + tRP;
1435
1436 prechargeBank(banks[i][j], free_at);
1437 }
1438 }
1439 }
1440 } else {
1441 DPRINTF(DRAM, "All banks already precharged, starting refresh\n");

--- 496 unchanged lines hidden ---