smmu_v3_caches.cc revision 14039:4991b2a345a1
1/* 2 * Copyright (c) 2014, 2018-2019 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Stan Czerniawski 38 * Damian Richardson 39 */ 40 41#include "dev/arm/smmu_v3_caches.hh" 42 43#include <numeric> 44 45#include "base/bitfield.hh" 46#include "base/intmath.hh" 47#include "base/logging.hh" 48#include "sim/stats.hh" 49 50 51// taken from hex expansion of pi 52#define SMMUTLB_SEED 0xEA752DFE 53#define ARMARCHTLB_SEED 0x8B021FA1 54#define IPACACHE_SEED 0xE5A0CC0F 55#define CONFIGCACHE_SEED 0xB56F74E8 56#define WALKCACHE_SEED 0x18ACF3D6 57 58/* 59 * BaseCache 60 * 61 * TODO: move more code into this base class to reduce duplication. 62 */ 63 64SMMUv3BaseCache::SMMUv3BaseCache(const std::string &policy_name, uint32_t seed) : 65 replacementPolicy(decodePolicyName(policy_name)), 66 nextToReplace(0), 67 random(seed), 68 useStamp(0) 69{} 70 71int 72SMMUv3BaseCache::decodePolicyName(const std::string &policy_name) 73{ 74 if (policy_name == "rr") { 75 return SMMU_CACHE_REPL_ROUND_ROBIN; 76 } else if (policy_name == "rand") { 77 return SMMU_CACHE_REPL_RANDOM; 78 } else if (policy_name == "lru") { 79 return SMMU_CACHE_REPL_LRU; 80 } else { 81 panic("Unknown cache replacement policy '%s'\n", policy_name); 82 } 83} 84 85void 86SMMUv3BaseCache::regStats(const std::string &name) 87{ 88 using namespace Stats; 89 90 91 averageLookups 92 .name(name + ".averageLookups") 93 .desc("Average number lookups per second") 94 .flags(pdf); 95 96 totalLookups 97 .name(name + ".totalLookups") 98 .desc("Total number of lookups") 99 .flags(pdf); 100 101 averageLookups = totalLookups / simSeconds; 102 103 104 averageMisses 105 .name(name + ".averageMisses") 106 .desc("Average number misses per second") 107 .flags(pdf); 108 109 totalMisses 110 .name(name + ".totalMisses") 111 .desc("Total number of misses") 112 .flags(pdf); 113 114 averageMisses = totalMisses / simSeconds; 115 116 117 averageUpdates 118 .name(name + ".averageUpdates") 119 .desc("Average number updates per second") 120 .flags(pdf); 121 122 totalUpdates 123 .name(name + ".totalUpdates") 124 .desc("Total number of updates") 125 .flags(pdf); 126 127 averageUpdates = totalUpdates / simSeconds; 128 129 130 averageHitRate 131 .name(name + ".averageHitRate") 132 .desc("Average hit rate") 133 .flags(pdf); 134 135 averageHitRate = (totalLookups - totalMisses) / totalLookups; 136 137 insertions 138 .name(name + ".insertions") 139 .desc("Number of insertions (not replacements)") 140 .flags(pdf); 141} 142 143 144 145/* 146 * SMMUTLB 147 */ 148 149SMMUTLB::SMMUTLB(unsigned numEntries, unsigned _associativity, 150 const std::string &policy) 151: 152 SMMUv3BaseCache(policy, SMMUTLB_SEED), 153 associativity(_associativity) 154{ 155 if (associativity == 0) 156 associativity = numEntries; // fully associative 157 158 if (numEntries == 0) 159 fatal("SMMUTLB must have at least one entry\n"); 160 161 if (associativity > numEntries) 162 fatal("SMMUTLB associativity cannot be higher than " 163 "its number of entries\n"); 164 165 unsigned num_sets = numEntries / associativity; 166 167 if (num_sets*associativity != numEntries) 168 fatal("Number of SMMUTLB entries must be divisible " 169 "by its associativity\n"); 170 171 Entry e; 172 e.valid = false; 173 174 Set set(associativity, e); 175 sets.resize(num_sets, set); 176} 177 178const SMMUTLB::Entry* 179SMMUTLB::lookup(uint32_t sid, uint32_t ssid, 180 Addr va, bool updStats) 181{ 182 const Entry *result = NULL; 183 184 Set &set = sets[pickSetIdx(va)]; 185 186 for (size_t i = 0; i < set.size(); i++) { 187 const Entry &e = set[i]; 188 189 if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && 190 e.sid==sid && e.ssid==ssid) 191 { 192 if (result != NULL) 193 panic("SMMUTLB: duplicate entry found!\n"); 194 195 result = &e; 196 break; 197 } 198 } 199 200 if (updStats) { 201 if (result) 202 result->lastUsed = useStamp++; 203 204 totalLookups++; 205 if (result == NULL) 206 totalMisses++; 207 } 208 209 return result; 210} 211 212const SMMUTLB::Entry* 213SMMUTLB::lookupAnyVA(uint32_t sid, uint32_t ssid, bool updStats) 214{ 215 const Entry *result = NULL; 216 217 for (size_t s = 0; s < sets.size(); s++) { 218 Set &set = sets[s]; 219 220 for (size_t i = 0; i < set.size(); i++) { 221 const Entry &e = set[i]; 222 223 if (e.valid && e.sid==sid && e.ssid==ssid) { 224 result = &e; 225 break; 226 } 227 } 228 } 229 230 if (updStats) { 231 totalLookups++; 232 if (result == NULL) 233 totalMisses++; 234 } 235 236 return result; 237} 238 239void 240SMMUTLB::store(const Entry &incoming, AllocPolicy alloc) 241{ 242 if (!incoming.valid) 243 panic("Tried to store an invalid entry\n"); 244 245 incoming.lastUsed = 0; 246 247 const Entry *existing = 248 lookup(incoming.sid, incoming.ssid, incoming.va, false); 249 250 if (existing) { 251 *const_cast<Entry *> (existing) = incoming; 252 } else { 253 Set &set = sets[pickSetIdx(incoming.va)]; 254 set[pickEntryIdxToReplace(set, alloc)] = incoming; 255 } 256 257 totalUpdates++; 258} 259 260void 261SMMUTLB::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) 262{ 263 Set &set = sets[pickSetIdx(va)]; 264 265 for (size_t i = 0; i < set.size(); i++) { 266 Entry &e = set[i]; 267 268 if ((e.va & e.vaMask) == (va & e.vaMask) && 269 e.asid==asid && e.vmid==vmid) 270 { 271 e.valid = false; 272 } 273 } 274} 275 276void 277SMMUTLB::invalidateVAA(Addr va, uint16_t vmid) 278{ 279 Set &set = sets[pickSetIdx(va)]; 280 281 for (size_t i = 0; i < set.size(); i++) { 282 Entry &e = set[i]; 283 284 if ((e.va & e.vaMask) == (va & e.vaMask) && e.vmid==vmid) 285 e.valid = false; 286 } 287} 288 289void 290SMMUTLB::invalidateASID(uint16_t asid, uint16_t vmid) 291{ 292 for (size_t s = 0; s < sets.size(); s++) { 293 Set &set = sets[s]; 294 295 for (size_t i = 0; i < set.size(); i++) { 296 Entry &e = set[i]; 297 298 if (e.asid==asid && e.vmid==vmid) 299 e.valid = false; 300 } 301 } 302} 303 304void 305SMMUTLB::invalidateVMID(uint16_t vmid) 306{ 307 for (size_t s = 0; s < sets.size(); s++) { 308 Set &set = sets[s]; 309 310 for (size_t i = 0; i < set.size(); i++) { 311 Entry &e = set[i]; 312 313 if (e.vmid == vmid) 314 e.valid = false; 315 } 316 } 317} 318 319void 320SMMUTLB::invalidateAll() 321{ 322 for (size_t s = 0; s < sets.size(); s++) { 323 Set &set = sets[s]; 324 325 for (size_t i = 0; i < set.size(); i++) 326 set[i].valid = false; 327 } 328} 329 330size_t 331SMMUTLB::pickSetIdx(Addr va) const 332{ 333 return (va >> 12) % sets.size(); 334} 335 336size_t 337SMMUTLB::pickEntryIdxToReplace(const Set &set, AllocPolicy alloc) 338{ 339 if (alloc == ALLOC_LAST_WAY) 340 return associativity - 1; 341 342 uint32_t lru_tick = UINT32_MAX; 343 size_t lru_idx = 0; 344 size_t max_idx = 345 alloc==ALLOC_ANY_BUT_LAST_WAY ? 346 set.size()-1 : set.size(); 347 348 for (size_t i = 0; i < max_idx; i++) { 349 if (!set[i].valid) { 350 insertions++; 351 return i; 352 } 353 354 if (set[i].lastUsed < lru_tick) { 355 lru_idx = i; 356 lru_tick = set[i].lastUsed; 357 } 358 } 359 360 switch (replacementPolicy) { 361 case SMMU_CACHE_REPL_ROUND_ROBIN: 362 switch (alloc) { 363 case ALLOC_ANY_WAY: 364 return nextToReplace = ((nextToReplace+1) % associativity); 365 case ALLOC_ANY_BUT_LAST_WAY: 366 return nextToReplace = ((nextToReplace+1) % (associativity-1)); 367 default: 368 panic("Unknown allocation mode %d\n", alloc); 369 } 370 371 case SMMU_CACHE_REPL_RANDOM: 372 switch (alloc) { 373 case ALLOC_ANY_WAY: 374 return random.random<size_t>(0, associativity-1); 375 case ALLOC_ANY_BUT_LAST_WAY: 376 return random.random<size_t>(0, associativity-2); 377 default: 378 panic("Unknown allocation mode %d\n", alloc); 379 } 380 381 case SMMU_CACHE_REPL_LRU: 382 return lru_idx; 383 384 default: 385 panic("Unknown replacement policy %d\n", replacementPolicy); 386 } 387} 388 389 390 391/* 392 * ARMArchTLB 393 */ 394 395ARMArchTLB::ARMArchTLB(unsigned numEntries, unsigned _associativity, 396 const std::string &policy) 397: 398 SMMUv3BaseCache(policy, ARMARCHTLB_SEED), 399 associativity(_associativity) 400{ 401 if (associativity == 0) 402 associativity = numEntries; // fully associative 403 404 if (numEntries == 0) 405 fatal("ARMArchTLB must have at least one entry\n"); 406 407 if (associativity > numEntries) 408 fatal("ARMArchTLB associativity cannot be higher than " 409 "its number of entries\n"); 410 411 unsigned num_sets = numEntries / associativity; 412 413 if (num_sets*associativity != numEntries) 414 fatal("Number of ARMArchTLB entries must be divisible " 415 "by its associativity\n"); 416 417 Entry e; 418 e.valid = false; 419 420 Set set(associativity, e); 421 sets.resize(num_sets, set); 422} 423 424const ARMArchTLB::Entry * 425ARMArchTLB::lookup(Addr va, uint16_t asid, uint16_t vmid, bool updStats) 426{ 427 const Entry *result = NULL; 428 429 Set &set = sets[pickSetIdx(va, asid, vmid)]; 430 431 for (size_t i = 0; i < set.size(); i++) { 432 const Entry &e = set[i]; 433 434 if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && 435 e.asid==asid && e.vmid==vmid) 436 { 437 if (result != NULL) 438 panic("ARMArchTLB: duplicate entry found!\n"); 439 440 result = &e; 441 break; 442 } 443 } 444 445 if (updStats) { 446 if (result) 447 result->lastUsed = useStamp++; 448 449 totalLookups++; 450 if (result == NULL) 451 totalMisses++; 452 } 453 454 return result; 455} 456 457void 458ARMArchTLB::store(const Entry &incoming) 459{ 460 if (!incoming.valid) 461 panic("Tried to store an invalid entry\n"); 462 463 incoming.lastUsed = 0; 464 465 const Entry *existing = 466 lookup(incoming.va, incoming.asid, incoming.vmid, false); 467 468 if (existing) { 469 *const_cast<Entry *> (existing) = incoming; 470 } else { 471 Set &set = sets[pickSetIdx(incoming.va, incoming.asid, incoming.vmid)]; 472 set[pickEntryIdxToReplace(set)] = incoming; 473 } 474 475 totalUpdates++; 476} 477 478void 479ARMArchTLB::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) 480{ 481 Set &set = sets[pickSetIdx(va, asid, vmid)]; 482 483 for (size_t i = 0; i < set.size(); i++) { 484 Entry &e = set[i]; 485 486 if ((e.va & e.vaMask) == (va & e.vaMask) && 487 e.asid==asid && e.vmid==vmid) 488 { 489 e.valid = false; 490 } 491 } 492} 493 494void 495ARMArchTLB::invalidateVAA(Addr va, uint16_t vmid) 496{ 497 for (size_t s = 0; s < sets.size(); s++) { 498 Set &set = sets[s]; 499 500 for (size_t i = 0; i < set.size(); i++) { 501 Entry &e = set[i]; 502 503 if ((e.va & e.vaMask) == (va & e.vaMask) && e.vmid==vmid) 504 e.valid = false; 505 } 506 } 507} 508 509void 510ARMArchTLB::invalidateASID(uint16_t asid, uint16_t vmid) 511{ 512 for (size_t s = 0; s < sets.size(); s++) { 513 Set &set = sets[s]; 514 515 for (size_t i = 0; i < set.size(); i++) { 516 Entry &e = set[i]; 517 518 if (e.asid==asid && e.vmid==vmid) 519 e.valid = false; 520 } 521 } 522} 523 524void 525ARMArchTLB::invalidateVMID(uint16_t vmid) 526{ 527 for (size_t s = 0; s < sets.size(); s++) { 528 Set &set = sets[s]; 529 530 for (size_t i = 0; i < set.size(); i++) { 531 Entry &e = set[i]; 532 533 if (e.vmid == vmid) 534 e.valid = false; 535 } 536 } 537} 538 539void 540ARMArchTLB::invalidateAll() 541{ 542 for (size_t s = 0; s < sets.size(); s++) { 543 Set &set = sets[s]; 544 545 for (size_t i = 0; i < set.size(); i++) 546 set[i].valid = false; 547 } 548} 549 550size_t 551ARMArchTLB::pickSetIdx(Addr va, uint16_t asid, uint16_t vmid) const 552{ 553 return ((va >> 12) ^ asid ^ vmid) % sets.size(); 554} 555 556size_t 557ARMArchTLB::pickEntryIdxToReplace(const Set &set) 558{ 559 size_t lru_idx = 0; 560 uint32_t lru_tick = UINT32_MAX; 561 562 for (size_t i = 0; i < set.size(); i++) { 563 if (!set[i].valid) { 564 insertions++; 565 return i; 566 } 567 568 if (set[i].lastUsed < lru_tick) { 569 lru_idx = i; 570 lru_tick = set[i].lastUsed; 571 } 572 } 573 574 switch (replacementPolicy) { 575 case SMMU_CACHE_REPL_ROUND_ROBIN: 576 return nextToReplace = ((nextToReplace+1) % associativity); 577 578 case SMMU_CACHE_REPL_RANDOM: 579 return random.random<size_t>(0, associativity-1); 580 581 case SMMU_CACHE_REPL_LRU: 582 return lru_idx; 583 584 default: 585 panic("Unknown replacement policy %d\n", replacementPolicy); 586 } 587 588} 589 590/* 591 * IPACache 592 */ 593 594IPACache::IPACache(unsigned numEntries, unsigned _associativity, 595 const std::string &policy) 596: 597 SMMUv3BaseCache(policy, IPACACHE_SEED), 598 associativity(_associativity) 599{ 600 if (associativity == 0) 601 associativity = numEntries; // fully associative 602 603 if (numEntries == 0) 604 fatal("IPACache must have at least one entry\n"); 605 606 if (associativity > numEntries) 607 fatal("IPACache associativity cannot be higher than " 608 "its number of entries\n"); 609 610 unsigned num_sets = numEntries / associativity; 611 612 if (num_sets*associativity != numEntries) 613 fatal("Number of IPACache entries must be divisible " 614 "by its associativity\n"); 615 616 Entry e; 617 e.valid = false; 618 619 Set set(associativity, e); 620 sets.resize(num_sets, set); 621} 622 623const IPACache::Entry* 624IPACache::lookup(Addr ipa, uint16_t vmid, bool updStats) 625{ 626 const Entry *result = NULL; 627 628 Set &set = sets[pickSetIdx(ipa, vmid)]; 629 630 for (size_t i = 0; i < set.size(); i++) { 631 const Entry &e = set[i]; 632 633 if (e.valid && (e.ipa & e.ipaMask) == (ipa & e.ipaMask) && 634 e.vmid==vmid) 635 { 636 if (result != NULL) 637 panic("IPACache: duplicate entry found!\n"); 638 639 result = &e; 640 break; 641 } 642 } 643 644 if (updStats) { 645 if (result) 646 result->lastUsed = useStamp++; 647 648 totalLookups++; 649 if (result == NULL) 650 totalMisses++; 651 } 652 653 return result; 654} 655 656void 657IPACache::store(const Entry &incoming) 658{ 659 if (!incoming.valid) 660 panic("Tried to store an invalid entry\n"); 661 662 incoming.lastUsed = 0; 663 664 const Entry *existing = lookup(incoming.ipa, incoming.vmid, false); 665 666 if (existing) { 667 *const_cast<Entry *> (existing) = incoming; 668 } else { 669 Set &set = sets[pickSetIdx(incoming.ipa, incoming.vmid)]; 670 set[pickEntryIdxToReplace(set)] = incoming; 671 } 672 673 totalUpdates++; 674} 675 676void 677IPACache::invalidateIPA(Addr ipa, uint16_t vmid) 678{ 679 Set &set = sets[pickSetIdx(ipa, vmid)]; 680 681 for (size_t i = 0; i < set.size(); i++) { 682 Entry &e = set[i]; 683 684 if ((e.ipa & e.ipaMask) == (ipa & e.ipaMask) && e.vmid==vmid) 685 e.valid = false; 686 } 687} 688 689void 690IPACache::invalidateIPAA(Addr ipa) 691{ 692 for (size_t s = 0; s < sets.size(); s++) { 693 Set &set = sets[s]; 694 695 for (size_t i = 0; i < set.size(); i++) { 696 Entry &e = set[i]; 697 698 if ((e.ipa & e.ipaMask) == (ipa & e.ipaMask)) 699 e.valid = false; 700 } 701 } 702} 703 704void 705IPACache::invalidateVMID(uint16_t vmid) 706{ 707 for (size_t s = 0; s < sets.size(); s++) { 708 Set &set = sets[s]; 709 710 for (size_t i = 0; i < set.size(); i++) { 711 Entry &e = set[i]; 712 713 if (e.vmid == vmid) 714 e.valid = false; 715 } 716 } 717} 718 719void 720IPACache::invalidateAll() 721{ 722 for (size_t s = 0; s < sets.size(); s++) { 723 Set &set = sets[s]; 724 725 for (size_t i = 0; i < set.size(); i++) 726 set[i].valid = false; 727 } 728} 729 730size_t 731IPACache::pickSetIdx(Addr va, uint16_t vmid) const 732{ 733 return ((va >> 12) ^ vmid) % sets.size(); 734} 735 736size_t 737IPACache::pickEntryIdxToReplace(const Set &set) 738{ 739 size_t lru_idx = 0; 740 uint32_t lru_tick = UINT32_MAX; 741 742 for (size_t i = 0; i < set.size(); i++) { 743 if (!set[i].valid) { 744 insertions++; 745 return i; 746 } 747 748 if (set[i].lastUsed < lru_tick) { 749 lru_idx = i; 750 lru_tick = set[i].lastUsed; 751 } 752 } 753 754 switch (replacementPolicy) { 755 case SMMU_CACHE_REPL_ROUND_ROBIN: 756 return nextToReplace = ((nextToReplace+1) % associativity); 757 758 case SMMU_CACHE_REPL_RANDOM: 759 return random.random<size_t>(0, associativity-1); 760 761 case SMMU_CACHE_REPL_LRU: 762 return lru_idx; 763 764 default: 765 panic("Unknown replacement policy %d\n", replacementPolicy); 766 } 767 768} 769 770/* 771 * ConfigCache 772 */ 773 774ConfigCache::ConfigCache(unsigned numEntries, unsigned _associativity, 775 const std::string &policy) 776: 777 SMMUv3BaseCache(policy, CONFIGCACHE_SEED), 778 associativity(_associativity) 779{ 780 if (associativity == 0) 781 associativity = numEntries; // fully associative 782 783 if (numEntries == 0) 784 fatal("ConfigCache must have at least one entry\n"); 785 786 if (associativity > numEntries) 787 fatal("ConfigCache associativity cannot be higher than " 788 "its number of entries\n"); 789 790 unsigned num_sets = numEntries / associativity; 791 792 if (num_sets*associativity != numEntries) 793 fatal("Number of ConfigCache entries must be divisible " 794 "by its associativity\n"); 795 796 Entry e; 797 e.valid = false; 798 799 Set set(associativity, e); 800 sets.resize(num_sets, set); 801} 802 803const ConfigCache::Entry * 804ConfigCache::lookup(uint32_t sid, uint32_t ssid, bool updStats) 805{ 806 const Entry *result = NULL; 807 808 Set &set = sets[pickSetIdx(sid, ssid)]; 809 810 for (size_t i = 0; i < set.size(); i++) { 811 const Entry &e = set[i]; 812 813 if (e.valid && e.sid==sid && e.ssid==ssid) 814 { 815 if (result != NULL) 816 panic("ConfigCache: duplicate entry found!\n"); 817 818 result = &e; 819 break; 820 } 821 } 822 823 if (updStats) { 824 if (result) 825 result->lastUsed = useStamp++; 826 827 totalLookups++; 828 if (result == NULL) 829 totalMisses++; 830 } 831 832 return result; 833} 834 835void 836ConfigCache::store(const Entry &incoming) 837{ 838 if (!incoming.valid) 839 panic("Tried to store an invalid entry\n"); 840 841 incoming.lastUsed = 0; 842 843 const Entry *existing = lookup(incoming.sid, incoming.ssid, false); 844 845 if (existing) { 846 *const_cast<Entry *> (existing) = incoming; 847 } else { 848 Set &set = sets[pickSetIdx(incoming.sid, incoming.ssid)]; 849 set[pickEntryIdxToReplace(set)] = incoming; 850 } 851 852 totalUpdates++; 853} 854 855void 856ConfigCache::invalidateSSID(uint32_t sid, uint32_t ssid) 857{ 858 Set &set = sets[pickSetIdx(sid, ssid)]; 859 860 for (size_t i = 0; i < set.size(); i++) { 861 Entry &e = set[i]; 862 863 if (e.sid==sid && e.ssid==ssid) 864 e.valid = false; 865 } 866} 867 868void 869ConfigCache::invalidateSID(uint32_t sid) 870{ 871 for (size_t s = 0; s < sets.size(); s++) { 872 Set &set = sets[s]; 873 874 for (size_t i = 0; i < set.size(); i++) { 875 Entry &e = set[i]; 876 877 if (e.sid == sid) 878 e.valid = false; 879 } 880 } 881} 882 883void 884ConfigCache::invalidateAll() 885{ 886 for (size_t s = 0; s < sets.size(); s++) { 887 Set &set = sets[s]; 888 889 for (size_t i = 0; i < set.size(); i++) 890 set[i].valid = false; 891 } 892} 893 894size_t 895ConfigCache::pickSetIdx(uint32_t sid, uint32_t ssid) const 896{ 897 return (sid^ssid) % sets.size(); 898} 899 900size_t 901ConfigCache::pickEntryIdxToReplace(const Set &set) 902{ 903 size_t lru_idx = 0; 904 uint32_t lru_tick = UINT32_MAX; 905 906 for (size_t i = 0; i < set.size(); i++) { 907 if (!set[i].valid) { 908 insertions++; 909 return i; 910 } 911 912 if (set[i].lastUsed < lru_tick) { 913 lru_idx = i; 914 lru_tick = set[i].lastUsed; 915 } 916 } 917 918 switch (replacementPolicy) { 919 case SMMU_CACHE_REPL_ROUND_ROBIN: 920 return nextToReplace = ((nextToReplace+1) % associativity); 921 922 case SMMU_CACHE_REPL_RANDOM: 923 return random.random<size_t>(0, associativity-1); 924 925 case SMMU_CACHE_REPL_LRU: 926 return lru_idx; 927 928 default: 929 panic("Unknown replacement policy %d\n", replacementPolicy); 930 } 931 932} 933 934/* 935 * WalkCache 936 */ 937 938WalkCache::WalkCache(const std::array<unsigned, 2*WALK_CACHE_LEVELS> &_sizes, 939 unsigned _associativity, const std::string &policy) : 940 SMMUv3BaseCache(policy, WALKCACHE_SEED), 941 associativity(_associativity), 942 sizes() 943{ 944 unsigned numEntries = std::accumulate(&_sizes[0], 945 &_sizes[2*WALK_CACHE_LEVELS], 0); 946 947 if (associativity == 0) 948 associativity = numEntries; // fully associative 949 950 if (numEntries == 0) 951 fatal("WalkCache must have at least one entry\n"); 952 953 for (size_t i = 0; i < 2*WALK_CACHE_LEVELS; i++){ 954 if (_sizes[i] % associativity != 0) 955 fatal("Number of WalkCache entries at each level must be " 956 "divisible by WalkCache associativity\n"); 957 958 sizes[i] = _sizes[i] / associativity; 959 offsets[i] = i==0 ? 0 : offsets[i-1] + sizes[i-1]; 960 } 961 962 if (associativity > numEntries) 963 fatal("WalkCache associativity cannot be higher than " 964 "its number of entries\n"); 965 966 unsigned num_sets = numEntries / associativity; 967 968 if (num_sets*associativity != numEntries) 969 fatal("Number of WalkCache entries must be divisible " 970 "by its associativity\n"); 971 972 Entry e; 973 e.valid = false; 974 975 Set set(associativity, e); 976 sets.resize(num_sets, set); 977} 978 979const WalkCache::Entry* 980WalkCache::lookup(Addr va, Addr vaMask, 981 uint16_t asid, uint16_t vmid, 982 unsigned stage, unsigned level, 983 bool updStats) 984{ 985 const Entry *result = NULL; 986 987 Set &set = sets[pickSetIdx(va, vaMask, stage, level)]; 988 989 for (size_t i = 0; i < set.size(); i++) { 990 const Entry &e = set[i]; 991 992 if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && 993 e.asid==asid && e.vmid==vmid && e.stage==stage && e.level==level) 994 { 995 if (result != NULL) 996 panic("WalkCache: duplicate entry found!\n"); 997 998 result = &e; 999 break; 1000 } 1001 } 1002 1003 if (updStats) { 1004 if (result) 1005 result->lastUsed = useStamp++; 1006 1007 totalLookups++; 1008 if (result == NULL) 1009 totalMisses++; 1010 1011 lookupsByStageLevel[stage-1][level]++; 1012 totalLookupsByStageLevel[stage-1][level]++; 1013 if (result == NULL) { 1014 missesByStageLevel[stage-1][level]++; 1015 totalMissesByStageLevel[stage-1][level]++; 1016 } 1017 } 1018 1019 return result; 1020} 1021 1022void 1023WalkCache::store(const Entry &incoming) 1024{ 1025 if (!incoming.valid) 1026 panic("Tried to store an invalid entry\n"); 1027 1028 assert(incoming.stage==1 || incoming.stage==2); 1029 assert(incoming.level<=WALK_CACHE_LEVELS); 1030 1031 incoming.lastUsed = 0; 1032 1033 const Entry *existing = lookup(incoming.va, incoming.vaMask, 1034 incoming.asid, incoming.vmid, 1035 incoming.stage, incoming.level, false); 1036 1037 if (existing) { 1038 *const_cast<Entry *> (existing) = incoming; 1039 } else { 1040 Set &set = sets[pickSetIdx(incoming.va, incoming.vaMask, 1041 incoming.stage, incoming.level)]; 1042 set[pickEntryIdxToReplace(set, incoming.stage, incoming.level)] = 1043 incoming; 1044 } 1045 1046 totalUpdates++; 1047 updatesByStageLevel[incoming.stage-1][incoming.level]++; 1048 totalUpdatesByStageLevel[incoming.stage-1][incoming.level]++; 1049} 1050 1051void 1052WalkCache::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) 1053{ 1054 panic("%s unimplemented\n", __func__); 1055} 1056 1057void 1058WalkCache::invalidateVAA(Addr va, uint16_t vmid) 1059{ 1060 panic("%s unimplemented\n", __func__); 1061} 1062 1063void 1064WalkCache::invalidateASID(uint16_t asid, uint16_t vmid) 1065{ 1066 panic("%s unimplemented\n", __func__); 1067} 1068 1069void 1070WalkCache::invalidateVMID(uint16_t vmid) 1071{ 1072 for (size_t s = 0; s < sets.size(); s++) { 1073 Set &set = sets[s]; 1074 1075 for (size_t i = 0; i < set.size(); i++) { 1076 Entry &e = set[i]; 1077 1078 if (e.vmid == vmid) 1079 e.valid = false; 1080 } 1081 } 1082} 1083 1084void 1085WalkCache::invalidateAll() 1086{ 1087 for (size_t s = 0; s < sets.size(); s++) { 1088 Set &set = sets[s]; 1089 1090 for (size_t i = 0; i < set.size(); i++) 1091 set[i].valid = false; 1092 } 1093} 1094 1095size_t 1096WalkCache::pickSetIdx(Addr va, Addr vaMask, 1097 unsigned stage, unsigned level) const 1098{ 1099 (void) stage; 1100 1101 int size, offset; 1102 1103 switch (stage) { 1104 case 1: 1105 assert (level<=3); 1106 size = sizes[0*WALK_CACHE_LEVELS + level]; 1107 offset = offsets[0*WALK_CACHE_LEVELS + level]; 1108 break; 1109 1110 case 2: 1111 assert (level<=3); 1112 size = sizes[1*WALK_CACHE_LEVELS + level]; 1113 offset = offsets[1*WALK_CACHE_LEVELS + level]; 1114 break; 1115 1116 default: 1117 panic("bad stage"); 1118 } 1119 1120 return ((va >> findLsbSet(vaMask)) % size) + offset; 1121} 1122 1123size_t 1124WalkCache::pickEntryIdxToReplace(const Set &set, 1125 unsigned stage, unsigned level) 1126{ 1127 size_t lru_idx = 0; 1128 uint32_t lru_tick = UINT32_MAX; 1129 1130 for (size_t i = 0; i < set.size(); i++) { 1131 if (!set[i].valid) { 1132 insertions++; 1133 insertionsByStageLevel[stage-1][level]++; 1134 return i; 1135 } 1136 1137 if (set[i].lastUsed < lru_tick) { 1138 lru_idx = i; 1139 lru_tick = set[i].lastUsed; 1140 } 1141 } 1142 1143 switch (replacementPolicy) { 1144 case SMMU_CACHE_REPL_ROUND_ROBIN: 1145 return nextToReplace = ((nextToReplace+1) % associativity); 1146 1147 case SMMU_CACHE_REPL_RANDOM: 1148 return random.random<size_t>(0, associativity-1); 1149 1150 case SMMU_CACHE_REPL_LRU: 1151 return lru_idx; 1152 1153 default: 1154 panic("Unknown replacement policy %d\n", replacementPolicy); 1155 } 1156 1157} 1158 1159void 1160WalkCache::regStats(const std::string &name) 1161{ 1162 using namespace Stats; 1163 1164 SMMUv3BaseCache::regStats(name); 1165 1166 for (int s = 0; s < 2; s++) { 1167 for (int l = 0; l < WALK_CACHE_LEVELS; l++) { 1168 averageLookupsByStageLevel[s][l] 1169 .name(csprintf("%s.averageLookupsS%dL%d", name, s+1, l)) 1170 .desc("Average number lookups per second") 1171 .flags(pdf); 1172 1173 totalLookupsByStageLevel[s][l] 1174 .name(csprintf("%s.totalLookupsS%dL%d", name, s+1, l)) 1175 .desc("Total number of lookups") 1176 .flags(pdf); 1177 1178 averageLookupsByStageLevel[s][l] = 1179 totalLookupsByStageLevel[s][l] / simSeconds; 1180 1181 1182 averageMissesByStageLevel[s][l] 1183 .name(csprintf("%s.averageMissesS%dL%d", name, s+1, l)) 1184 .desc("Average number misses per second") 1185 .flags(pdf); 1186 1187 totalMissesByStageLevel[s][l] 1188 .name(csprintf("%s.totalMissesS%dL%d", name, s+1, l)) 1189 .desc("Total number of misses") 1190 .flags(pdf); 1191 1192 averageMissesByStageLevel[s][l] = 1193 totalMissesByStageLevel[s][l] / simSeconds; 1194 1195 1196 averageUpdatesByStageLevel[s][l] 1197 .name(csprintf("%s.averageUpdatesS%dL%d", name, s+1, l)) 1198 .desc("Average number updates per second") 1199 .flags(pdf); 1200 1201 totalUpdatesByStageLevel[s][l] 1202 .name(csprintf("%s.totalUpdatesS%dL%d", name, s+1, l)) 1203 .desc("Total number of updates") 1204 .flags(pdf); 1205 1206 averageUpdatesByStageLevel[s][l] = 1207 totalUpdatesByStageLevel[s][l] / simSeconds; 1208 1209 1210 averageHitRateByStageLevel[s][l] 1211 .name(csprintf("%s.averageHitRateS%dL%d", name, s+1, l)) 1212 .desc("Average hit rate") 1213 .flags(pdf); 1214 1215 averageHitRateByStageLevel[s][l] = 1216 (totalLookupsByStageLevel[s][l] - 1217 totalMissesByStageLevel[s][l]) 1218 / totalLookupsByStageLevel[s][l]; 1219 1220 insertionsByStageLevel[s][l] 1221 .name(csprintf("%s.insertionsS%dL%d", name, s+1, l)) 1222 .desc("Number of insertions (not replacements)") 1223 .flags(pdf); 1224 } 1225 } 1226} 1227