dram_ctrl.cc (10889:c4c13fced000) dram_ctrl.cc (10890:bac38d2a4acb)
1/*
2 * Copyright (c) 2010-2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 697 unchanged lines hidden (view full) ---

706 // so if there is a read that was forced to wait, retry now
707 if (retryRdReq) {
708 retryRdReq = false;
709 port.sendRetryReq();
710 }
711}
712
713bool
1/*
2 * Copyright (c) 2010-2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 697 unchanged lines hidden (view full) ---

706 // so if there is a read that was forced to wait, retry now
707 if (retryRdReq) {
708 retryRdReq = false;
709 port.sendRetryReq();
710 }
711}
712
713bool
714DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, bool switched_cmd_type)
714DRAMCtrl::chooseNext(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
715{
716 // This method does the arbitration between requests. The chosen
717 // packet is simply moved to the head of the queue. The other
718 // methods know that this is the place to look. For example, with
719 // FCFS, this method does nothing
720 assert(!queue.empty());
721
722 // bool to indicate if a packet to an available rank is found

--- 17 unchanged lines hidden (view full) ---

740 if (ranks[dram_pkt->rank]->isAvailable()) {
741 queue.erase(i);
742 queue.push_front(dram_pkt);
743 found_packet = true;
744 break;
745 }
746 }
747 } else if (memSchedPolicy == Enums::frfcfs) {
715{
716 // This method does the arbitration between requests. The chosen
717 // packet is simply moved to the head of the queue. The other
718 // methods know that this is the place to look. For example, with
719 // FCFS, this method does nothing
720 assert(!queue.empty());
721
722 // bool to indicate if a packet to an available rank is found

--- 17 unchanged lines hidden (view full) ---

740 if (ranks[dram_pkt->rank]->isAvailable()) {
741 queue.erase(i);
742 queue.push_front(dram_pkt);
743 found_packet = true;
744 break;
745 }
746 }
747 } else if (memSchedPolicy == Enums::frfcfs) {
748 found_packet = reorderQueue(queue, switched_cmd_type);
748 found_packet = reorderQueue(queue, extra_col_delay);
749 } else
750 panic("No scheduling policy chosen\n");
751 return found_packet;
752}
753
754bool
749 } else
750 panic("No scheduling policy chosen\n");
751 return found_packet;
752}
753
754bool
755DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, bool switched_cmd_type)
755DRAMCtrl::reorderQueue(std::deque<DRAMPacket*>& queue, Tick extra_col_delay)
756{
756{
757 // Only determine this when needed
757 // Only determine this if needed
758 uint64_t earliest_banks = 0;
758 uint64_t earliest_banks = 0;
759 bool hidden_bank_prep = false;
759
760
760 // Search for row hits first, if no row hit is found then schedule the
761 // packet to one of the earliest banks available
762 bool found_packet = false;
761 // search for seamless row hits first, if no seamless row hit is
762 // found then determine if there are other packets that can be issued
763 // without incurring additional bus delay due to bank timing
764 // Will select closed rows first to enable more open row possibilies
765 // in future selections
766 bool found_hidden_bank = false;
767
768 // remember if we found a row hit, not seamless, but bank prepped
769 // and ready
770 bool found_prepped_pkt = false;
771
772 // if we have no row hit, prepped or not, and no seamless packet,
773 // just go for the earliest possible
763 bool found_earliest_pkt = false;
774 bool found_earliest_pkt = false;
764 bool found_prepped_diff_rank_pkt = false;
775
765 auto selected_pkt_it = queue.end();
766
776 auto selected_pkt_it = queue.end();
777
778 // time we need to issue a column command to be seamless
779 const Tick min_col_at = std::max(busBusyUntil - tCL + extra_col_delay,
780 curTick());
781
767 for (auto i = queue.begin(); i != queue.end() ; ++i) {
768 DRAMPacket* dram_pkt = *i;
769 const Bank& bank = dram_pkt->bankRef;
782 for (auto i = queue.begin(); i != queue.end() ; ++i) {
783 DRAMPacket* dram_pkt = *i;
784 const Bank& bank = dram_pkt->bankRef;
770 // check if rank is busy. If this is the case jump to the next packet
771 // Check if it is a row hit
785
786 // check if rank is available, if not, jump to the next packet
772 if (dram_pkt->rankRef.isAvailable()) {
787 if (dram_pkt->rankRef.isAvailable()) {
788 // check if it is a row hit
773 if (bank.openRow == dram_pkt->row) {
789 if (bank.openRow == dram_pkt->row) {
774 if (dram_pkt->rank == activeRank || switched_cmd_type) {
775 // FCFS within the hits, giving priority to commands
776 // that access the same rank as the previous burst
777 // to minimize bus turnaround delays
778 // Only give rank prioity when command type is
779 // not changing
780 DPRINTF(DRAM, "Row buffer hit\n");
790 // no additional rank-to-rank or same bank-group
791 // delays, or we switched read/write and might as well
792 // go for the row hit
793 if (bank.colAllowedAt <= min_col_at) {
794 // FCFS within the hits, giving priority to
795 // commands that can issue seamlessly, without
796 // additional delay, such as same rank accesses
797 // and/or different bank-group accesses
798 DPRINTF(DRAM, "Seamless row buffer hit\n");
781 selected_pkt_it = i;
799 selected_pkt_it = i;
800 // no need to look through the remaining queue entries
782 break;
801 break;
783 } else if (!found_prepped_diff_rank_pkt) {
784 // found row hit for command on different rank
785 // than prev burst
802 } else if (!found_hidden_bank && !found_prepped_pkt) {
803 // if we did not find a packet to a closed row that can
804 // issue the bank commands without incurring delay, and
805 // did not yet find a packet to a prepped row, remember
806 // the current one
786 selected_pkt_it = i;
807 selected_pkt_it = i;
787 found_prepped_diff_rank_pkt = true;
808 found_prepped_pkt = true;
809 DPRINTF(DRAM, "Prepped row buffer hit\n");
788 }
810 }
789 } else if (!found_earliest_pkt & !found_prepped_diff_rank_pkt) {
790 // packet going to a rank which is currently not waiting for a
791 // refresh, No row hit and
792 // haven't found an entry with a row hit to a new rank
793 if (earliest_banks == 0)
794 // Determine entries with earliest bank prep delay
795 // Function will give priority to commands that access the
796 // same rank as previous burst and can prep
797 // the bank seamlessly
798 earliest_banks = minBankPrep(queue, switched_cmd_type);
811 } else if (!found_earliest_pkt) {
812 // if we have not initialised the bank status, do it
813 // now, and only once per scheduling decisions
814 if (earliest_banks == 0) {
815 // determine entries with earliest bank delay
816 pair<uint64_t, bool> bankStatus =
817 minBankPrep(queue, min_col_at);
818 earliest_banks = bankStatus.first;
819 hidden_bank_prep = bankStatus.second;
820 }
799
821
800 // FCFS - Bank is first available bank
801 if (bits(earliest_banks, dram_pkt->bankId,
802 dram_pkt->bankId)) {
803 // Remember the packet to be scheduled to one of
804 // the earliest banks available, FCFS amongst the
805 // earliest banks
806 selected_pkt_it = i;
807 //if the packet found is going to a rank that is currently
808 //not busy then update the found_packet to true
822 // bank is amongst first available banks
823 // minBankPrep will give priority to packets that can
824 // issue seamlessly
825 if (bits(earliest_banks, dram_pkt->bankId, dram_pkt->bankId)) {
809 found_earliest_pkt = true;
826 found_earliest_pkt = true;
827 found_hidden_bank = hidden_bank_prep;
828
829 // give priority to packets that can issue
830 // bank commands 'behind the scenes'
831 // any additional delay if any will be due to
832 // col-to-col command requirements
833 if (hidden_bank_prep || !found_prepped_pkt)
834 selected_pkt_it = i;
810 }
811 }
812 }
813 }
814
815 if (selected_pkt_it != queue.end()) {
816 DRAMPacket* selected_pkt = *selected_pkt_it;
817 queue.erase(selected_pkt_it);
818 queue.push_front(selected_pkt);
835 }
836 }
837 }
838 }
839
840 if (selected_pkt_it != queue.end()) {
841 DRAMPacket* selected_pkt = *selected_pkt_it;
842 queue.erase(selected_pkt_it);
843 queue.push_front(selected_pkt);
819 found_packet = true;
844 return true;
820 }
845 }
821 return found_packet;
846
847 return false;
822}
823
824void
825DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency)
826{
827 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr());
828
829 bool needsResponse = pkt->needsResponse();

--- 456 unchanged lines hidden (view full) ---

1286 return;
1287 }
1288 } else {
1289 // bool to check if there is a read to a free rank
1290 bool found_read = false;
1291
1292 // Figure out which read request goes next, and move it to the
1293 // front of the read queue
848}
849
850void
851DRAMCtrl::accessAndRespond(PacketPtr pkt, Tick static_latency)
852{
853 DPRINTF(DRAM, "Responding to Address %lld.. ",pkt->getAddr());
854
855 bool needsResponse = pkt->needsResponse();

--- 456 unchanged lines hidden (view full) ---

1312 return;
1313 }
1314 } else {
1315 // bool to check if there is a read to a free rank
1316 bool found_read = false;
1317
1318 // Figure out which read request goes next, and move it to the
1319 // front of the read queue
1294 found_read = chooseNext(readQueue, switched_cmd_type);
1320 // If we are changing command type, incorporate the minimum
1321 // bus turnaround delay which will be tCS (different rank) case
1322 found_read = chooseNext(readQueue,
1323 switched_cmd_type ? tCS : 0);
1295
1296 // if no read to an available rank is found then return
1297 // at this point. There could be writes to the available ranks
1298 // which are above the required threshold. However, to
1299 // avoid adding more complexity to the code, return and wait
1300 // for a refresh event to kick things into action again.
1301 if (!found_read)
1302 return;

--- 42 unchanged lines hidden (view full) ---

1345 if (switch_to_writes) {
1346 // transition to writing
1347 busState = READ_TO_WRITE;
1348 }
1349 } else {
1350 // bool to check if write to free rank is found
1351 bool found_write = false;
1352
1324
1325 // if no read to an available rank is found then return
1326 // at this point. There could be writes to the available ranks
1327 // which are above the required threshold. However, to
1328 // avoid adding more complexity to the code, return and wait
1329 // for a refresh event to kick things into action again.
1330 if (!found_read)
1331 return;

--- 42 unchanged lines hidden (view full) ---

1374 if (switch_to_writes) {
1375 // transition to writing
1376 busState = READ_TO_WRITE;
1377 }
1378 } else {
1379 // bool to check if write to free rank is found
1380 bool found_write = false;
1381
1353 found_write = chooseNext(writeQueue, switched_cmd_type);
1382 // If we are changing command type, incorporate the minimum
1383 // bus turnaround delay
1384 found_write = chooseNext(writeQueue,
1385 switched_cmd_type ? std::min(tRTW, tCS) : 0);
1354
1355 // if no writes to an available rank are found then return.
1356 // There could be reads to the available ranks. However, to avoid
1357 // adding more complexity to the code, return at this point and wait
1358 // for a refresh event to kick things into action again.
1359 if (!found_write)
1360 return;
1361

--- 43 unchanged lines hidden (view full) ---

1405 // cause a nextReqEvent to be scheduled before we do so as part of
1406 // the next request processing
1407 if (retryWrReq && writeQueue.size() < writeBufferSize) {
1408 retryWrReq = false;
1409 port.sendRetryReq();
1410 }
1411}
1412
1386
1387 // if no writes to an available rank are found then return.
1388 // There could be reads to the available ranks. However, to avoid
1389 // adding more complexity to the code, return at this point and wait
1390 // for a refresh event to kick things into action again.
1391 if (!found_write)
1392 return;
1393

--- 43 unchanged lines hidden (view full) ---

1437 // cause a nextReqEvent to be scheduled before we do so as part of
1438 // the next request processing
1439 if (retryWrReq && writeQueue.size() < writeBufferSize) {
1440 retryWrReq = false;
1441 port.sendRetryReq();
1442 }
1443}
1444
1413uint64_t
1445pair<uint64_t, bool>
1414DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue,
1446DRAMCtrl::minBankPrep(const deque<DRAMPacket*>& queue,
1415 bool switched_cmd_type) const
1447 Tick min_col_at) const
1416{
1417 uint64_t bank_mask = 0;
1418 Tick min_act_at = MaxTick;
1419
1448{
1449 uint64_t bank_mask = 0;
1450 Tick min_act_at = MaxTick;
1451
1420 uint64_t bank_mask_same_rank = 0;
1421 Tick min_act_at_same_rank = MaxTick;
1452 // latest Tick for which ACT can occur without incurring additoinal
1453 // delay on the data bus
1454 const Tick hidden_act_max = std::max(min_col_at - tRCD, curTick());
1422
1455
1423 // Give precedence to commands that access same rank as previous command
1424 bool same_rank_match = false;
1456 // Flag condition when burst can issue back-to-back with previous burst
1457 bool found_seamless_bank = false;
1425
1458
1459 // Flag condition when bank can be opened without incurring additional
1460 // delay on the data bus
1461 bool hidden_bank_prep = false;
1462
1426 // determine if we have queued transactions targetting the
1427 // bank in question
1428 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false);
1429 for (const auto& p : queue) {
1430 if(p->rankRef.isAvailable())
1431 got_waiting[p->bankId] = true;
1432 }
1433
1463 // determine if we have queued transactions targetting the
1464 // bank in question
1465 vector<bool> got_waiting(ranksPerChannel * banksPerRank, false);
1466 for (const auto& p : queue) {
1467 if(p->rankRef.isAvailable())
1468 got_waiting[p->bankId] = true;
1469 }
1470
1471 // Find command with optimal bank timing
1472 // Will prioritize commands that can issue seamlessly.
1434 for (int i = 0; i < ranksPerChannel; i++) {
1435 for (int j = 0; j < banksPerRank; j++) {
1436 uint16_t bank_id = i * banksPerRank + j;
1437
1438 // if we have waiting requests for the bank, and it is
1439 // amongst the first available, update the mask
1440 if (got_waiting[bank_id]) {
1441 // make sure this rank is not currently refreshing.
1442 assert(ranks[i]->isAvailable());
1443 // simplistic approximation of when the bank can issue
1444 // an activate, ignoring any rank-to-rank switching
1445 // cost in this calculation
1446 Tick act_at = ranks[i]->banks[j].openRow == Bank::NO_ROW ?
1473 for (int i = 0; i < ranksPerChannel; i++) {
1474 for (int j = 0; j < banksPerRank; j++) {
1475 uint16_t bank_id = i * banksPerRank + j;
1476
1477 // if we have waiting requests for the bank, and it is
1478 // amongst the first available, update the mask
1479 if (got_waiting[bank_id]) {
1480 // make sure this rank is not currently refreshing.
1481 assert(ranks[i]->isAvailable());
1482 // simplistic approximation of when the bank can issue
1483 // an activate, ignoring any rank-to-rank switching
1484 // cost in this calculation
1485 Tick act_at = ranks[i]->banks[j].openRow == Bank::NO_ROW ?
1447 ranks[i]->banks[j].actAllowedAt :
1486 std::max(ranks[i]->banks[j].actAllowedAt, curTick()) :
1448 std::max(ranks[i]->banks[j].preAllowedAt, curTick()) + tRP;
1449
1487 std::max(ranks[i]->banks[j].preAllowedAt, curTick()) + tRP;
1488
1450 // prioritize commands that access the
1451 // same rank as previous burst
1452 // Calculate bank mask separately for the case and
1453 // evaluate after loop iterations complete
1454 if (i == activeRank && ranksPerChannel > 1) {
1455 if (act_at <= min_act_at_same_rank) {
1456 // reset same rank bank mask if new minimum is found
1457 // and previous minimum could not immediately send ACT
1458 if (act_at < min_act_at_same_rank &&
1459 min_act_at_same_rank > curTick())
1460 bank_mask_same_rank = 0;
1489 // When is the earliest the R/W burst can issue?
1490 Tick col_at = std::max(ranks[i]->banks[j].colAllowedAt,
1491 act_at + tRCD);
1461
1492
1462 // Set flag indicating that a same rank
1463 // opportunity was found
1464 same_rank_match = true;
1493 // bank can issue burst back-to-back (seamlessly) with
1494 // previous burst
1495 bool new_seamless_bank = col_at <= min_col_at;
1465
1496
1466 // set the bit corresponding to the available bank
1467 replaceBits(bank_mask_same_rank, bank_id, bank_id, 1);
1468 min_act_at_same_rank = act_at;
1497 // if we found a new seamless bank or we have no
1498 // seamless banks, and got a bank with an earlier
1499 // activate time, it should be added to the bit mask
1500 if (new_seamless_bank ||
1501 (!found_seamless_bank && act_at <= min_act_at)) {
1502 // if we did not have a seamless bank before, and
1503 // we do now, reset the bank mask, also reset it
1504 // if we have not yet found a seamless bank and
1505 // the activate time is smaller than what we have
1506 // seen so far
1507 if (!found_seamless_bank &&
1508 (new_seamless_bank || act_at < min_act_at)) {
1509 bank_mask = 0;
1469 }
1510 }
1470 } else {
1471 if (act_at <= min_act_at) {
1472 // reset bank mask if new minimum is found
1473 // and either previous minimum could not immediately send ACT
1474 if (act_at < min_act_at && min_act_at > curTick())
1475 bank_mask = 0;
1476 // set the bit corresponding to the available bank
1477 replaceBits(bank_mask, bank_id, bank_id, 1);
1478 min_act_at = act_at;
1479 }
1511
1512 found_seamless_bank |= new_seamless_bank;
1513
1514 // ACT can occur 'behind the scenes'
1515 hidden_bank_prep = act_at <= hidden_act_max;
1516
1517 // set the bit corresponding to the available bank
1518 replaceBits(bank_mask, bank_id, bank_id, 1);
1519 min_act_at = act_at;
1480 }
1481 }
1482 }
1483 }
1484
1520 }
1521 }
1522 }
1523 }
1524
1485 // Determine the earliest time when the next burst can issue based
1486 // on the current busBusyUntil delay.
1487 // Offset by tRCD to correlate with ACT timing variables
1488 Tick min_cmd_at = busBusyUntil - tCL - tRCD;
1489
1490 // if we have multiple ranks and all
1491 // waiting packets are accessing a rank which was previously active
1492 // then bank_mask_same_rank will be set to a value while bank_mask will
1493 // remain 0. In this case, the function should return the value of
1494 // bank_mask_same_rank.
1495 // else if waiting packets access a rank which was previously active and
1496 // other ranks, prioritize same rank accesses that can issue B2B
1497 // Only optimize for same ranks when the command type
1498 // does not change; do not want to unnecessarily incur tWTR
1499 //
1500 // Resulting FCFS prioritization Order is:
1501 // 1) Commands that access the same rank as previous burst
1502 // and can prep the bank seamlessly.
1503 // 2) Commands (any rank) with earliest bank prep
1504 if ((bank_mask == 0) || (!switched_cmd_type && same_rank_match &&
1505 min_act_at_same_rank <= min_cmd_at)) {
1506 bank_mask = bank_mask_same_rank;
1507 }
1508
1509 return bank_mask;
1525 return make_pair(bank_mask, hidden_bank_prep);
1510}
1511
1512DRAMCtrl::Rank::Rank(DRAMCtrl& _memory, const DRAMCtrlParams* _p)
1513 : EventManager(&_memory), memory(_memory),
1514 pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), pwrStateTick(0),
1515 refreshState(REF_IDLE), refreshDueAt(0),
1516 power(_p, false), numBanksActive(0),
1517 activateEvent(*this), prechargeEvent(*this),

--- 729 unchanged lines hidden --- 		1526}
1527
1528DRAMCtrl::Rank::Rank(DRAMCtrl& _memory, const DRAMCtrlParams* _p)
1529 : EventManager(&_memory), memory(_memory),
1530 pwrStateTrans(PWR_IDLE), pwrState(PWR_IDLE), pwrStateTick(0),
1531 refreshState(REF_IDLE), refreshDueAt(0),
1532 power(_p, false), numBanksActive(0),
1533 activateEvent(*this), prechargeEvent(*this),

--- 729 unchanged lines hidden ---