cache.cc (12715:0c8b4f376378) cache.cc (12719:68a20fbd07a6)
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 1308 unchanged lines hidden (view full) ---

1317
1318 // Reset the bus additional time as it is now accounted for
1319 pkt->headerDelay = pkt->payloadDelay = 0;
1320
1321 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1322}
1323
1324void
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 1308 unchanged lines hidden (view full) ---

1317
1318 // Reset the bus additional time as it is now accounted for
1319 pkt->headerDelay = pkt->payloadDelay = 0;
1320
1321 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1322}
1323
1324void
1325Cache::recvTimingResp(PacketPtr pkt)
1325Cache::serviceMSHRTargets(MSHR *mshr, const PacketPtr pkt, CacheBlk *blk,
1326 PacketList &writebacks)
1326{
1327{
1327 assert(pkt->isResponse());
1328
1329 // all header delay should be paid for by the crossbar, unless
1330 // this is a prefetch response from above
1331 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1332 "%s saw a non-zero packet delay\n", name());
1333
1334 bool is_error = pkt->isError();
1335
1336 if (is_error) {
1337 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1338 pkt->print());
1339 }
1340
1341 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1342 pkt->print());
1343
1344 // if this is a write, we should be looking at an uncacheable
1345 // write
1346 if (pkt->isWrite()) {
1347 assert(pkt->req->isUncacheable());
1348 handleUncacheableWriteResp(pkt);
1349 return;
1350 }
1351
1352 // we have dealt with any (uncacheable) writes above, from here on
1353 // we know we are dealing with an MSHR due to a miss or a prefetch
1354 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1355 assert(mshr);
1356
1357 if (mshr == noTargetMSHR) {
1358 // we always clear at least one target
1359 clearBlocked(Blocked_NoTargets);
1360 noTargetMSHR = nullptr;
1361 }
1362
1363 // Initial target is used just for stats
1364 MSHR::Target *initial_tgt = mshr->getTarget();
1328 MSHR::Target *initial_tgt = mshr->getTarget();
1365 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1366 Tick miss_latency = curTick() - initial_tgt->recvTime;
1329 // First offset for critical word first calculations
1330 const int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1367
1331
1368 if (pkt->req->isUncacheable()) {
1369 assert(pkt->req->masterId() < system->maxMasters());
1370 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1371 miss_latency;
1372 } else {
1373 assert(pkt->req->masterId() < system->maxMasters());
1374 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1375 miss_latency;
1376 }
1377
1378 bool wasFull = mshrQueue.isFull();
1379
1380 PacketList writebacks;
1381
1382 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1383
1332 const bool is_error = pkt->isError();
1384 bool is_fill = !mshr->isForward &&
1385 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1333 bool is_fill = !mshr->isForward &&
1334 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1386
1387 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1388 const bool valid_blk = blk && blk->isValid();
1389 // If the response indicates that there are no sharers and we
1390 // either had the block already or the response is filling we can
1391 // promote our copy to writable
1392 if (!pkt->hasSharers() &&
1393 (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1394 mshr->promoteWritable();
1395 }
1396
1397 if (is_fill && !is_error) {
1398 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1399 pkt->getAddr());
1400
1401 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1402 assert(blk != nullptr);
1403 }
1404
1405 // allow invalidation responses originating from write-line
1406 // requests to be discarded
1407 bool is_invalidate = pkt->isInvalidate();
1408
1335 // allow invalidation responses originating from write-line
1336 // requests to be discarded
1337 bool is_invalidate = pkt->isInvalidate();
1338
1409 // The block was marked as not readable while there was a pending
1410 // cache maintenance operation, restore its flag.
1411 if (pkt->isClean() && !is_invalidate && valid_blk) {
1412 blk->status |= BlkReadable;
1413 }
1414
1415 // First offset for critical word first calculations
1416 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1417
1418 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1419 for (auto &target: targets) {
1420 Packet *tgt_pkt = target.pkt;
1421 switch (target.source) {
1422 case MSHR::Target::FromCPU:
1423 Tick completion_time;
1424 // Here we charge on completion_time the delay of the xbar if the
1425 // packet comes from it, charged on headerDelay.

--- 19 unchanged lines hidden (view full) ---

1445 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1446 assert(!is_error);
1447 // we got the block in a writable state, so promote
1448 // any deferred targets if possible
1449 mshr->promoteWritable();
1450 // NB: we use the original packet here and not the response!
1451 blk = handleFill(tgt_pkt, blk, writebacks,
1452 targets.allocOnFill);
1339 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1340 for (auto &target: targets) {
1341 Packet *tgt_pkt = target.pkt;
1342 switch (target.source) {
1343 case MSHR::Target::FromCPU:
1344 Tick completion_time;
1345 // Here we charge on completion_time the delay of the xbar if the
1346 // packet comes from it, charged on headerDelay.

--- 19 unchanged lines hidden (view full) ---

1366 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1367 assert(!is_error);
1368 // we got the block in a writable state, so promote
1369 // any deferred targets if possible
1370 mshr->promoteWritable();
1371 // NB: we use the original packet here and not the response!
1372 blk = handleFill(tgt_pkt, blk, writebacks,
1373 targets.allocOnFill);
1453 assert(blk != nullptr);
1374 assert(blk);
1454
1455 // treat as a fill, and discard the invalidation
1456 // response
1457 is_fill = true;
1458 is_invalidate = false;
1459 }
1460
1461 if (is_fill) {

--- 112 unchanged lines hidden (view full) ---

1574 // should not invalidate the block, so check if the
1575 // invalidation should be discarded
1576 if (is_invalidate || mshr->hasPostInvalidate()) {
1577 invalidateBlock(blk);
1578 } else if (mshr->hasPostDowngrade()) {
1579 blk->status &= ~BlkWritable;
1580 }
1581 }
1375
1376 // treat as a fill, and discard the invalidation
1377 // response
1378 is_fill = true;
1379 is_invalidate = false;
1380 }
1381
1382 if (is_fill) {

--- 112 unchanged lines hidden (view full) ---

1495 // should not invalidate the block, so check if the
1496 // invalidation should be discarded
1497 if (is_invalidate || mshr->hasPostInvalidate()) {
1498 invalidateBlock(blk);
1499 } else if (mshr->hasPostDowngrade()) {
1500 blk->status &= ~BlkWritable;
1501 }
1502 }
1503}
1582
1504
1505void
1506Cache::recvTimingResp(PacketPtr pkt)
1507{
1508 assert(pkt->isResponse());
1509
1510 // all header delay should be paid for by the crossbar, unless
1511 // this is a prefetch response from above
1512 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1513 "%s saw a non-zero packet delay\n", name());
1514
1515 const bool is_error = pkt->isError();
1516
1517 if (is_error) {
1518 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1519 pkt->print());
1520 }
1521
1522 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1523 pkt->print());
1524
1525 // if this is a write, we should be looking at an uncacheable
1526 // write
1527 if (pkt->isWrite()) {
1528 assert(pkt->req->isUncacheable());
1529 handleUncacheableWriteResp(pkt);
1530 return;
1531 }
1532
1533 // we have dealt with any (uncacheable) writes above, from here on
1534 // we know we are dealing with an MSHR due to a miss or a prefetch
1535 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1536 assert(mshr);
1537
1538 if (mshr == noTargetMSHR) {
1539 // we always clear at least one target
1540 clearBlocked(Blocked_NoTargets);
1541 noTargetMSHR = nullptr;
1542 }
1543
1544 // Initial target is used just for stats
1545 MSHR::Target *initial_tgt = mshr->getTarget();
1546 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1547 Tick miss_latency = curTick() - initial_tgt->recvTime;
1548
1549 if (pkt->req->isUncacheable()) {
1550 assert(pkt->req->masterId() < system->maxMasters());
1551 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1552 miss_latency;
1553 } else {
1554 assert(pkt->req->masterId() < system->maxMasters());
1555 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1556 miss_latency;
1557 }
1558
1559 PacketList writebacks;
1560
1561 bool is_fill = !mshr->isForward &&
1562 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1563
1564 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1565
1566 if (is_fill && !is_error) {
1567 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1568 pkt->getAddr());
1569
1570 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1571 assert(blk != nullptr);
1572 }
1573
1574 if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) {
1575 // The block was marked not readable while there was a pending
1576 // cache maintenance operation, restore its flag.
1577 blk->status |= BlkReadable;
1578 }
1579
1580 if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) {
1581 // If at this point the referenced block is writable and the
1582 // response is not a cache invalidate, we promote targets that
1583 // were deferred as we couldn't guarrantee a writable copy
1584 mshr->promoteWritable();
1585 }
1586
1587 serviceMSHRTargets(mshr, pkt, blk, writebacks);
1588
1583 if (mshr->promoteDeferredTargets()) {
1584 // avoid later read getting stale data while write miss is
1585 // outstanding.. see comment in timingAccess()
1586 if (blk) {
1587 blk->status &= ~BlkReadable;
1588 }
1589 mshrQueue.markPending(mshr);
1590 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1591 } else {
1589 if (mshr->promoteDeferredTargets()) {
1590 // avoid later read getting stale data while write miss is
1591 // outstanding.. see comment in timingAccess()
1592 if (blk) {
1593 blk->status &= ~BlkReadable;
1594 }
1595 mshrQueue.markPending(mshr);
1596 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1597 } else {
1598 // while we deallocate an mshr from the queue we still have to
1599 // check the isFull condition before and after as we might
1600 // have been using the reserved entries already
1601 const bool was_full = mshrQueue.isFull();
1592 mshrQueue.deallocate(mshr);
1602 mshrQueue.deallocate(mshr);
1593 if (wasFull && !mshrQueue.isFull()) {
1603 if (was_full && !mshrQueue.isFull()) {
1594 clearBlocked(Blocked_NoMSHRs);
1595 }
1596
1597 // Request the bus for a prefetch if this deallocation freed enough
1598 // MSHRs for a prefetch to take place
1599 if (prefetcher && mshrQueue.canPrefetch()) {
1600 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1601 clockEdge());
1602 if (next_pf_time != MaxTick)
1603 schedMemSideSendEvent(next_pf_time);
1604 }
1605 }
1604 clearBlocked(Blocked_NoMSHRs);
1605 }
1606
1607 // Request the bus for a prefetch if this deallocation freed enough
1608 // MSHRs for a prefetch to take place
1609 if (prefetcher && mshrQueue.canPrefetch()) {
1610 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1611 clockEdge());
1612 if (next_pf_time != MaxTick)
1613 schedMemSideSendEvent(next_pf_time);
1614 }
1615 }
1606 // reset the xbar additional timinig as it is now accounted for
1607 pkt->headerDelay = pkt->payloadDelay = 0;
1608
1609 // if we used temp block, check to see if its valid and then clear it out
1610 if (blk == tempBlock && tempBlock->isValid()) {
1611 PacketPtr wb_pkt = tempBlock->isDirty() || writebackClean ?
1612 writebackBlk(blk) : cleanEvictBlk(blk);
1613 writebacks.push_back(wb_pkt);
1614 invalidateBlock(tempBlock);
1615 }
1616
1616
1617 // if we used temp block, check to see if its valid and then clear it out
1618 if (blk == tempBlock && tempBlock->isValid()) {
1619 PacketPtr wb_pkt = tempBlock->isDirty() || writebackClean ?
1620 writebackBlk(blk) : cleanEvictBlk(blk);
1621 writebacks.push_back(wb_pkt);
1622 invalidateBlock(tempBlock);
1623 }
1624
1625 const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1617 // copy writebacks to write buffer
1618 doWritebacks(writebacks, forward_time);
1619
1620 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1621 delete pkt;
1622}
1623
1624PacketPtr

--- 1235 unchanged lines hidden --- 		1626 // copy writebacks to write buffer
1627 doWritebacks(writebacks, forward_time);
1628
1629 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1630 delete pkt;
1631}
1632
1633PacketPtr

--- 1235 unchanged lines hidden ---