2c2
< * Copyright (c) 2010-2017 ARM Limited
---
> * Copyright (c) 2010-2018 ARM Limited
87c87,88
< tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST),
---
> tCK(p->tCK), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST),
> tCCD_L_WR(p->tCCD_L_WR),
91c92,93
< activationLimit(p->activation_limit),
---
> activationLimit(p->activation_limit), rankToRankDly(tCS + tBURST),
> wrToRdDly(tCL + tBURST + p->tWTR), rdToWrDly(tRTW + tBURST),
97c99
< busBusyUntil(0), prevArrival(0),
---
> nextBurstAt(0), prevArrival(0),
166a169,174
> // tCCD_L_WR should be greater than minimal, back-to-back burst delay
> if (tCCD_L_WR <= tBURST) {
> fatal("tCCD_L_WR (%d) should be larger than tBURST (%d) when "
> "bank groups per rank (%d) is greater than 1\n",
> tCCD_L_WR, tBURST, bankGroupsPerRank);
> }
250c258
< busBusyUntil = curTick() + tRP + tRCD + tCL;
---
> nextBurstAt = curTick() + tRP + tRCD;
776c784,788
< uint64_t earliest_banks = 0;
---
> vector<uint32_t> earliest_banks(ranksPerChannel, 0);
>
> // Has minBankPrep been called to populate earliest_banks?
> bool filled_earliest_banks = false;
> // can the PRE/ACT sequence be done without impacting utlization?
797,798c809
< const Tick min_col_at = std::max(busBusyUntil - tCL + extra_col_delay,
< curTick());
---
> const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick());
802a814,815
> const Tick col_allowed_at = dram_pkt->isRead ? bank.rdAllowedAt :
> bank.wrAllowedAt;
812c825
< if (bank.colAllowedAt <= min_col_at) {
---
> if (col_allowed_at <= min_col_at) {
833c846
< if (earliest_banks == 0) {
---
> if (!filled_earliest_banks) {
835c848
< pair<uint64_t, bool> bankStatus =
---
> std::tie(earliest_banks, hidden_bank_prep) =
837,838c850
< earliest_banks = bankStatus.first;
< hidden_bank_prep = bankStatus.second;
---
> filled_earliest_banks = true;
844c856,857
< if (bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) {
---
> if (bits(earliest_banks[dram_pkt->rank],
> dram_pkt->bank, dram_pkt->bank)) {
940,941c953,955
< // Respect the row-to-column command delay
< bank_ref.colAllowedAt = std::max(act_tick + tRCD, bank_ref.colAllowedAt);
---
> // Respect the row-to-column command delay for both read and write cmds
> bank_ref.rdAllowedAt = std::max(act_tick + tRCD, bank_ref.rdAllowedAt);
> bank_ref.wrAllowedAt = std::max(act_tick + tRCD, bank_ref.wrAllowedAt);
1077,1079d1090
< // respect any constraints on the command (e.g. tRCD or tCCD)
< Tick cmd_at = std::max(bank.colAllowedAt, curTick());
<
1098,1100d1108
<
< // issue the command as early as possible
< cmd_at = bank.colAllowedAt;
1102a1111,1114
> // respect any constraints on the command (e.g. tRCD or tCCD)
> const Tick col_allowed_at = dram_pkt->isRead ?
> bank.rdAllowedAt : bank.wrAllowedAt;
>
1104,1105c1116,1117
< // the command
< cmd_at = std::max(cmd_at, busBusyUntil - tCL);
---
> // the command; need minimum of tBURST between commands
> Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()});
1110,1112d1121
< // only one burst can use the bus at any one point in time
< assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
<
1114,1115c1123,1125
< // bank (add a max with tCCD/tCCD_L here)
< Tick cmd_dly;
---
> // bank (add a max with tCCD/tCCD_L/tCCD_L_WR here)
> Tick dly_to_rd_cmd;
> Tick dly_to_wr_cmd;
1126,1127c1136,1142
< // Use tCCD_L in this case
< cmd_dly = tCCD_L;
---
> // tCCD_L is default requirement for same BG timing
> // tCCD_L_WR is required for write-to-write
> // Need to also take bus turnaround delays into account
> dly_to_rd_cmd = dram_pkt->isRead ?
> tCCD_L : std::max(tCCD_L, wrToRdDly);
> dly_to_wr_cmd = dram_pkt->isRead ?
> std::max(tCCD_L, rdToWrDly) : tCCD_L_WR;
1129,1133c1144,1147
< // use tBURST (equivalent to tCCD_S), the shorter
< // cas-to-cas delay value, when either:
< // 1) bank group architecture is not supportted
< // 2) bank is in a different bank group
< cmd_dly = tBURST;
---
> // tBURST is default requirement for diff BG timing
> // Need to also take bus turnaround delays into account
> dly_to_rd_cmd = dram_pkt->isRead ? tBURST : wrToRdDly;
> dly_to_wr_cmd = dram_pkt->isRead ? rdToWrDly : tBURST;
1136,1140c1150,1154
< // different rank is by default in a different bank group
< // use tBURST (equivalent to tCCD_S), which is the shorter
< // cas-to-cas delay in this case
< // Add tCS to account for rank-to-rank bus delay requirements
< cmd_dly = tBURST + tCS;
---
> // different rank is by default in a different bank group and
> // doesn't require longer tCCD or additional RTW, WTR delays
> // Need to account for rank-to-rank switching with tCS
> dly_to_wr_cmd = rankToRankDly;
> dly_to_rd_cmd = rankToRankDly;
1142,1143c1156,1159
< ranks[j]->banks[i].colAllowedAt = std::max(cmd_at + cmd_dly,
< ranks[j]->banks[i].colAllowedAt);
---
> ranks[j]->banks[i].rdAllowedAt = std::max(cmd_at + dly_to_rd_cmd,
> ranks[j]->banks[i].rdAllowedAt);
> ranks[j]->banks[i].wrAllowedAt = std::max(cmd_at + dly_to_wr_cmd,
> ranks[j]->banks[i].wrAllowedAt);
1218,1219c1234,1235
< // Update bus state
< busBusyUntil = dram_pkt->readyTime;
---
> // Update bus state to reflect when previous command was issued
> nextBurstAt = cmd_at + tBURST;
1221,1222c1237,1238
< DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n",
< dram_pkt->addr, dram_pkt->readyTime, busBusyUntil);
---
> DPRINTF(DRAM, "Access to %lld, ready at %lld next burst at %lld.\n",
> dram_pkt->addr, dram_pkt->readyTime, nextBurstAt);
1244c1260
< nextReqTime = busBusyUntil - (tRP + tRCD + tCL);
---
> nextReqTime = nextBurstAt - (tRP + tRCD);
1377,1378c1393
< found_read = chooseNext(readQueue,
< switched_cmd_type ? tCS : 0);
---
> found_read = chooseNext(readQueue, switched_cmd_type ? tCS : 0);
1391,1399d1405
< // here we get a bit creative and shift the bus busy time not
< // just the tWTR, but also a CAS latency to capture the fact
< // that we are allowed to prepare a new bank, but not issue a
< // read command until after tWTR, in essence we capture a
< // bubble on the data bus that is tWTR + tCL
< if (switched_cmd_type && dram_pkt->rank == activeRank) {
< busBusyUntil += tWTR + tCL;
< }
<
1459,1466d1464
< // add a bubble to the data bus, as defined by the
< // tRTW when access is to the same rank as previous burst
< // Different rank timing is handled with tCS, which is
< // applied to colAllowedAt
< if (switched_cmd_type && dram_pkt->rank == activeRank) {
< busBusyUntil += tRTW;
< }
<
1525c1523
< pair<uint64_t, bool>
---
> pair<vector<uint32_t>, bool>
1529d1526
< uint64_t bank_mask = 0;
1530a1528
> vector<uint32_t> bank_mask(ranksPerChannel, 0);
1570,1571c1568,1571
< Tick col_at = std::max(ranks[i]->banks[j].colAllowedAt,
< act_at + tRCD);
---
> const Tick col_allowed_at = (busState == READ) ?
> ranks[i]->banks[j].rdAllowedAt :
> ranks[i]->banks[j].wrAllowedAt;
> Tick col_at = std::max(col_allowed_at, act_at + tRCD);
1589c1589
< bank_mask = 0;
---
> std::fill(bank_mask.begin(), bank_mask.end(), 0);
1598c1598
< replaceBits(bank_mask, bank_id, bank_id, 1);
---
> replaceBits(bank_mask[i], j, j, 1);
2071c2071,2072
< b.colAllowedAt = std::max(wake_up_tick + exit_delay, b.colAllowedAt);
---
> b.wrAllowedAt = std::max(wake_up_tick + exit_delay, b.wrAllowedAt);
> b.rdAllowedAt = std::max(wake_up_tick + exit_delay, b.rdAllowedAt);
< * Copyright (c) 2010-2017 ARM Limited
---
> * Copyright (c) 2010-2018 ARM Limited
87c87,88
< tCK(p->tCK), tWTR(p->tWTR), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST),
---
> tCK(p->tCK), tRTW(p->tRTW), tCS(p->tCS), tBURST(p->tBURST),
> tCCD_L_WR(p->tCCD_L_WR),
91c92,93
< activationLimit(p->activation_limit),
---
> activationLimit(p->activation_limit), rankToRankDly(tCS + tBURST),
> wrToRdDly(tCL + tBURST + p->tWTR), rdToWrDly(tRTW + tBURST),
97c99
< busBusyUntil(0), prevArrival(0),
---
> nextBurstAt(0), prevArrival(0),
166a169,174
> // tCCD_L_WR should be greater than minimal, back-to-back burst delay
> if (tCCD_L_WR <= tBURST) {
> fatal("tCCD_L_WR (%d) should be larger than tBURST (%d) when "
> "bank groups per rank (%d) is greater than 1\n",
> tCCD_L_WR, tBURST, bankGroupsPerRank);
> }
250c258
< busBusyUntil = curTick() + tRP + tRCD + tCL;
---
> nextBurstAt = curTick() + tRP + tRCD;
776c784,788
< uint64_t earliest_banks = 0;
---
> vector<uint32_t> earliest_banks(ranksPerChannel, 0);
>
> // Has minBankPrep been called to populate earliest_banks?
> bool filled_earliest_banks = false;
> // can the PRE/ACT sequence be done without impacting utlization?
797,798c809
< const Tick min_col_at = std::max(busBusyUntil - tCL + extra_col_delay,
< curTick());
---
> const Tick min_col_at = std::max(nextBurstAt + extra_col_delay, curTick());
802a814,815
> const Tick col_allowed_at = dram_pkt->isRead ? bank.rdAllowedAt :
> bank.wrAllowedAt;
812c825
< if (bank.colAllowedAt <= min_col_at) {
---
> if (col_allowed_at <= min_col_at) {
833c846
< if (earliest_banks == 0) {
---
> if (!filled_earliest_banks) {
835c848
< pair<uint64_t, bool> bankStatus =
---
> std::tie(earliest_banks, hidden_bank_prep) =
837,838c850
< earliest_banks = bankStatus.first;
< hidden_bank_prep = bankStatus.second;
---
> filled_earliest_banks = true;
844c856,857
< if (bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) {
---
> if (bits(earliest_banks[dram_pkt->rank],
> dram_pkt->bank, dram_pkt->bank)) {
940,941c953,955
< // Respect the row-to-column command delay
< bank_ref.colAllowedAt = std::max(act_tick + tRCD, bank_ref.colAllowedAt);
---
> // Respect the row-to-column command delay for both read and write cmds
> bank_ref.rdAllowedAt = std::max(act_tick + tRCD, bank_ref.rdAllowedAt);
> bank_ref.wrAllowedAt = std::max(act_tick + tRCD, bank_ref.wrAllowedAt);
1077,1079d1090
< // respect any constraints on the command (e.g. tRCD or tCCD)
< Tick cmd_at = std::max(bank.colAllowedAt, curTick());
<
1098,1100d1108
<
< // issue the command as early as possible
< cmd_at = bank.colAllowedAt;
1102a1111,1114
> // respect any constraints on the command (e.g. tRCD or tCCD)
> const Tick col_allowed_at = dram_pkt->isRead ?
> bank.rdAllowedAt : bank.wrAllowedAt;
>
1104,1105c1116,1117
< // the command
< cmd_at = std::max(cmd_at, busBusyUntil - tCL);
---
> // the command; need minimum of tBURST between commands
> Tick cmd_at = std::max({col_allowed_at, nextBurstAt, curTick()});
1110,1112d1121
< // only one burst can use the bus at any one point in time
< assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
<
1114,1115c1123,1125
< // bank (add a max with tCCD/tCCD_L here)
< Tick cmd_dly;
---
> // bank (add a max with tCCD/tCCD_L/tCCD_L_WR here)
> Tick dly_to_rd_cmd;
> Tick dly_to_wr_cmd;
1126,1127c1136,1142
< // Use tCCD_L in this case
< cmd_dly = tCCD_L;
---
> // tCCD_L is default requirement for same BG timing
> // tCCD_L_WR is required for write-to-write
> // Need to also take bus turnaround delays into account
> dly_to_rd_cmd = dram_pkt->isRead ?
> tCCD_L : std::max(tCCD_L, wrToRdDly);
> dly_to_wr_cmd = dram_pkt->isRead ?
> std::max(tCCD_L, rdToWrDly) : tCCD_L_WR;
1129,1133c1144,1147
< // use tBURST (equivalent to tCCD_S), the shorter
< // cas-to-cas delay value, when either:
< // 1) bank group architecture is not supportted
< // 2) bank is in a different bank group
< cmd_dly = tBURST;
---
> // tBURST is default requirement for diff BG timing
> // Need to also take bus turnaround delays into account
> dly_to_rd_cmd = dram_pkt->isRead ? tBURST : wrToRdDly;
> dly_to_wr_cmd = dram_pkt->isRead ? rdToWrDly : tBURST;
1136,1140c1150,1154
< // different rank is by default in a different bank group
< // use tBURST (equivalent to tCCD_S), which is the shorter
< // cas-to-cas delay in this case
< // Add tCS to account for rank-to-rank bus delay requirements
< cmd_dly = tBURST + tCS;
---
> // different rank is by default in a different bank group and
> // doesn't require longer tCCD or additional RTW, WTR delays
> // Need to account for rank-to-rank switching with tCS
> dly_to_wr_cmd = rankToRankDly;
> dly_to_rd_cmd = rankToRankDly;
1142,1143c1156,1159
< ranks[j]->banks[i].colAllowedAt = std::max(cmd_at + cmd_dly,
< ranks[j]->banks[i].colAllowedAt);
---
> ranks[j]->banks[i].rdAllowedAt = std::max(cmd_at + dly_to_rd_cmd,
> ranks[j]->banks[i].rdAllowedAt);
> ranks[j]->banks[i].wrAllowedAt = std::max(cmd_at + dly_to_wr_cmd,
> ranks[j]->banks[i].wrAllowedAt);
1218,1219c1234,1235
< // Update bus state
< busBusyUntil = dram_pkt->readyTime;
---
> // Update bus state to reflect when previous command was issued
> nextBurstAt = cmd_at + tBURST;
1221,1222c1237,1238
< DPRINTF(DRAM, "Access to %lld, ready at %lld bus busy until %lld.\n",
< dram_pkt->addr, dram_pkt->readyTime, busBusyUntil);
---
> DPRINTF(DRAM, "Access to %lld, ready at %lld next burst at %lld.\n",
> dram_pkt->addr, dram_pkt->readyTime, nextBurstAt);
1244c1260
< nextReqTime = busBusyUntil - (tRP + tRCD + tCL);
---
> nextReqTime = nextBurstAt - (tRP + tRCD);
1377,1378c1393
< found_read = chooseNext(readQueue,
< switched_cmd_type ? tCS : 0);
---
> found_read = chooseNext(readQueue, switched_cmd_type ? tCS : 0);
1391,1399d1405
< // here we get a bit creative and shift the bus busy time not
< // just the tWTR, but also a CAS latency to capture the fact
< // that we are allowed to prepare a new bank, but not issue a
< // read command until after tWTR, in essence we capture a
< // bubble on the data bus that is tWTR + tCL
< if (switched_cmd_type && dram_pkt->rank == activeRank) {
< busBusyUntil += tWTR + tCL;
< }
<
1459,1466d1464
< // add a bubble to the data bus, as defined by the
< // tRTW when access is to the same rank as previous burst
< // Different rank timing is handled with tCS, which is
< // applied to colAllowedAt
< if (switched_cmd_type && dram_pkt->rank == activeRank) {
< busBusyUntil += tRTW;
< }
<
1525c1523
< pair<uint64_t, bool>
---
> pair<vector<uint32_t>, bool>
1529d1526
< uint64_t bank_mask = 0;
1530a1528
> vector<uint32_t> bank_mask(ranksPerChannel, 0);
1570,1571c1568,1571
< Tick col_at = std::max(ranks[i]->banks[j].colAllowedAt,
< act_at + tRCD);
---
> const Tick col_allowed_at = (busState == READ) ?
> ranks[i]->banks[j].rdAllowedAt :
> ranks[i]->banks[j].wrAllowedAt;
> Tick col_at = std::max(col_allowed_at, act_at + tRCD);
1589c1589
< bank_mask = 0;
---
> std::fill(bank_mask.begin(), bank_mask.end(), 0);
1598c1598
< replaceBits(bank_mask, bank_id, bank_id, 1);
---
> replaceBits(bank_mask[i], j, j, 1);
2071c2071,2072
< b.colAllowedAt = std::max(wake_up_tick + exit_delay, b.colAllowedAt);
---
> b.wrAllowedAt = std::max(wake_up_tick + exit_delay, b.wrAllowedAt);
> b.rdAllowedAt = std::max(wake_up_tick + exit_delay, b.rdAllowedAt);