vm.c revision 12771
14120Sgblack@eecs.umich.edu// See LICENSE for license details. 24120Sgblack@eecs.umich.edu 34120Sgblack@eecs.umich.edu#include <stdint.h> 44120Sgblack@eecs.umich.edu#include <string.h> 54120Sgblack@eecs.umich.edu#include <stdio.h> 64120Sgblack@eecs.umich.edu 74120Sgblack@eecs.umich.edu#include "riscv_test.h" 84120Sgblack@eecs.umich.edu 94120Sgblack@eecs.umich.eduvoid trap_entry(); 104120Sgblack@eecs.umich.eduvoid pop_tf(trapframe_t*); 114120Sgblack@eecs.umich.edu 124120Sgblack@eecs.umich.eduvolatile uint64_t tohost; 134120Sgblack@eecs.umich.eduvolatile uint64_t fromhost; 144120Sgblack@eecs.umich.edu 154120Sgblack@eecs.umich.edustatic void do_tohost(uint64_t tohost_value) 164120Sgblack@eecs.umich.edu{ 174120Sgblack@eecs.umich.edu while (tohost) 184120Sgblack@eecs.umich.edu fromhost = 0; 194120Sgblack@eecs.umich.edu tohost = tohost_value; 204120Sgblack@eecs.umich.edu} 214120Sgblack@eecs.umich.edu 224120Sgblack@eecs.umich.edu#define pa2kva(pa) ((void*)(pa) - DRAM_BASE - MEGAPAGE_SIZE) 234120Sgblack@eecs.umich.edu#define uva2kva(pa) ((void*)(pa) - MEGAPAGE_SIZE) 244120Sgblack@eecs.umich.edu 254120Sgblack@eecs.umich.edu#define flush_page(addr) asm volatile ("sfence.vma %0" : : "r" (addr) : "memory") 264120Sgblack@eecs.umich.edu 274120Sgblack@eecs.umich.edustatic uint64_t lfsr63(uint64_t x) 284120Sgblack@eecs.umich.edu{ 294120Sgblack@eecs.umich.edu uint64_t bit = (x ^ (x >> 1)) & 1; 304120Sgblack@eecs.umich.edu return (x >> 1) | (bit << 62); 314120Sgblack@eecs.umich.edu} 324120Sgblack@eecs.umich.edu 334120Sgblack@eecs.umich.edustatic void cputchar(int x) 344120Sgblack@eecs.umich.edu{ 354120Sgblack@eecs.umich.edu do_tohost(0x0101000000000000 | (unsigned char)x); 364120Sgblack@eecs.umich.edu} 374120Sgblack@eecs.umich.edu 384120Sgblack@eecs.umich.edustatic void cputstring(const char* s) 394120Sgblack@eecs.umich.edu{ 404120Sgblack@eecs.umich.edu while (*s) 414120Sgblack@eecs.umich.edu cputchar(*s++); 424120Sgblack@eecs.umich.edu} 434120Sgblack@eecs.umich.edu 444120Sgblack@eecs.umich.edustatic void terminate(int code) 454120Sgblack@eecs.umich.edu{ 464120Sgblack@eecs.umich.edu do_tohost(code); 474120Sgblack@eecs.umich.edu while (1); 484120Sgblack@eecs.umich.edu} 494120Sgblack@eecs.umich.edu 504120Sgblack@eecs.umich.eduvoid wtf() 514120Sgblack@eecs.umich.edu{ 524120Sgblack@eecs.umich.edu terminate(841); 534120Sgblack@eecs.umich.edu} 544120Sgblack@eecs.umich.edu 554120Sgblack@eecs.umich.edu#define stringify1(x) #x 564120Sgblack@eecs.umich.edu#define stringify(x) stringify1(x) 574120Sgblack@eecs.umich.edu#define assert(x) do { \ 584120Sgblack@eecs.umich.edu if (x) break; \ 594120Sgblack@eecs.umich.edu cputstring("Assertion failed: " stringify(x) "\n"); \ 604120Sgblack@eecs.umich.edu terminate(3); \ 614148Sgblack@eecs.umich.edu} while(0) 624148Sgblack@eecs.umich.edu 634154Sgblack@eecs.umich.edu#define l1pt pt[0] 644148Sgblack@eecs.umich.edu#define user_l2pt pt[1] 654148Sgblack@eecs.umich.edu#if __riscv_xlen == 64 664120Sgblack@eecs.umich.edu# define NPT 4 674120Sgblack@eecs.umich.edu#define kernel_l2pt pt[2] 684120Sgblack@eecs.umich.edu# define user_l3pt pt[3] 694148Sgblack@eecs.umich.edu#else 704148Sgblack@eecs.umich.edu# define NPT 2 714148Sgblack@eecs.umich.edu# define user_l3pt user_l2pt 724148Sgblack@eecs.umich.edu#endif 734148Sgblack@eecs.umich.edupte_t pt[NPT][PTES_PER_PT] __attribute__((aligned(PGSIZE))); 744148Sgblack@eecs.umich.edu 754148Sgblack@eecs.umich.edutypedef struct { pte_t addr; void* next; } freelist_t; 764148Sgblack@eecs.umich.edu 774148Sgblack@eecs.umich.edufreelist_t user_mapping[MAX_TEST_PAGES]; 784148Sgblack@eecs.umich.edufreelist_t freelist_nodes[MAX_TEST_PAGES]; 794148Sgblack@eecs.umich.edufreelist_t *freelist_head, *freelist_tail; 804148Sgblack@eecs.umich.edu 814148Sgblack@eecs.umich.eduvoid printhex(uint64_t x) 824148Sgblack@eecs.umich.edu{ 834148Sgblack@eecs.umich.edu char str[17]; 844148Sgblack@eecs.umich.edu for (int i = 0; i < 16; i++) 854148Sgblack@eecs.umich.edu { 864148Sgblack@eecs.umich.edu str[15-i] = (x & 0xF) + ((x & 0xF) < 10 ? '0' : 'a'-10); 874148Sgblack@eecs.umich.edu x >>= 4; 884148Sgblack@eecs.umich.edu } 894148Sgblack@eecs.umich.edu str[16] = 0; 904148Sgblack@eecs.umich.edu 914148Sgblack@eecs.umich.edu cputstring(str); 924148Sgblack@eecs.umich.edu} 934148Sgblack@eecs.umich.edu 944148Sgblack@eecs.umich.edustatic void evict(unsigned long addr) 954148Sgblack@eecs.umich.edu{ 964148Sgblack@eecs.umich.edu assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 974148Sgblack@eecs.umich.edu addr = addr/PGSIZE*PGSIZE; 984148Sgblack@eecs.umich.edu 994148Sgblack@eecs.umich.edu freelist_t* node = &user_mapping[addr/PGSIZE]; 1004148Sgblack@eecs.umich.edu if (node->addr) 1014148Sgblack@eecs.umich.edu { 1024148Sgblack@eecs.umich.edu // check accessed and dirty bits 1034148Sgblack@eecs.umich.edu assert(user_l3pt[addr/PGSIZE] & PTE_A); 1044148Sgblack@eecs.umich.edu uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 1054148Sgblack@eecs.umich.edu if (memcmp((void*)addr, uva2kva(addr), PGSIZE)) { 1064148Sgblack@eecs.umich.edu assert(user_l3pt[addr/PGSIZE] & PTE_D); 1074148Sgblack@eecs.umich.edu memcpy((void*)addr, uva2kva(addr), PGSIZE); 1084148Sgblack@eecs.umich.edu } 1094148Sgblack@eecs.umich.edu write_csr(sstatus, sstatus); 1104148Sgblack@eecs.umich.edu 1114148Sgblack@eecs.umich.edu user_mapping[addr/PGSIZE].addr = 0; 1124148Sgblack@eecs.umich.edu 1134148Sgblack@eecs.umich.edu if (freelist_tail == 0) 1144148Sgblack@eecs.umich.edu freelist_head = freelist_tail = node; 1154148Sgblack@eecs.umich.edu else 1164148Sgblack@eecs.umich.edu { 1174148Sgblack@eecs.umich.edu freelist_tail->next = node; 1184148Sgblack@eecs.umich.edu freelist_tail = node; 1194148Sgblack@eecs.umich.edu } 1204148Sgblack@eecs.umich.edu } 1214148Sgblack@eecs.umich.edu} 1224148Sgblack@eecs.umich.edu 1234148Sgblack@eecs.umich.eduvoid handle_fault(uintptr_t addr, uintptr_t cause) 1244148Sgblack@eecs.umich.edu{ 1254148Sgblack@eecs.umich.edu assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 1264148Sgblack@eecs.umich.edu addr = addr/PGSIZE*PGSIZE; 1274148Sgblack@eecs.umich.edu 1284148Sgblack@eecs.umich.edu if (user_l3pt[addr/PGSIZE]) { 1294194Ssaidi@eecs.umich.edu if (!(user_l3pt[addr/PGSIZE] & PTE_A)) { 1304194Ssaidi@eecs.umich.edu user_l3pt[addr/PGSIZE] |= PTE_A; 1314194Ssaidi@eecs.umich.edu } else { 1324194Ssaidi@eecs.umich.edu assert(!(user_l3pt[addr/PGSIZE] & PTE_D) && cause == CAUSE_STORE_PAGE_FAULT); 1334194Ssaidi@eecs.umich.edu user_l3pt[addr/PGSIZE] |= PTE_D; 1344120Sgblack@eecs.umich.edu } 1354120Sgblack@eecs.umich.edu flush_page(addr); 1364120Sgblack@eecs.umich.edu return; 137 } 138 139 freelist_t* node = freelist_head; 140 assert(node); 141 freelist_head = node->next; 142 if (freelist_head == freelist_tail) 143 freelist_tail = 0; 144 145 uintptr_t new_pte = (node->addr >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V | PTE_U | PTE_R | PTE_W | PTE_X; 146 user_l3pt[addr/PGSIZE] = new_pte | PTE_A | PTE_D; 147 flush_page(addr); 148 149 assert(user_mapping[addr/PGSIZE].addr == 0); 150 user_mapping[addr/PGSIZE] = *node; 151 152 uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 153 memcpy((void*)addr, uva2kva(addr), PGSIZE); 154 write_csr(sstatus, sstatus); 155 156 user_l3pt[addr/PGSIZE] = new_pte; 157 flush_page(addr); 158 159 __builtin___clear_cache(0,0); 160} 161 162void handle_trap(trapframe_t* tf) 163{ 164 if (tf->cause == CAUSE_USER_ECALL) 165 { 166 int n = tf->gpr[10]; 167 168 for (long i = 1; i < MAX_TEST_PAGES; i++) 169 evict(i*PGSIZE); 170 171 terminate(n); 172 } 173 else if (tf->cause == CAUSE_ILLEGAL_INSTRUCTION) 174 { 175 assert(tf->epc % 4 == 0); 176 177 int* fssr; 178 asm ("jal %0, 1f; fssr x0; 1:" : "=r"(fssr)); 179 180 if (*(int*)tf->epc == *fssr) 181 terminate(1); // FP test on non-FP hardware. "succeed." 182 else 183 assert(!"illegal instruction"); 184 tf->epc += 4; 185 } 186 else if (tf->cause == CAUSE_FETCH_PAGE_FAULT || tf->cause == CAUSE_LOAD_PAGE_FAULT || tf->cause == CAUSE_STORE_PAGE_FAULT) 187 handle_fault(tf->badvaddr, tf->cause); 188 else 189 assert(!"unexpected exception"); 190 191 pop_tf(tf); 192} 193 194static void coherence_torture() 195{ 196 // cause coherence misses without affecting program semantics 197 unsigned int random = ENTROPY; 198 while (1) { 199 uintptr_t paddr = DRAM_BASE + ((random % (2 * (MAX_TEST_PAGES + 1) * PGSIZE)) & -4); 200#ifdef __riscv_atomic 201 if (random & 1) // perform a no-op write 202 asm volatile ("amoadd.w zero, zero, (%0)" :: "r"(paddr)); 203 else // perform a read 204#endif 205 asm volatile ("lw zero, (%0)" :: "r"(paddr)); 206 random = lfsr63(random); 207 } 208} 209 210void vm_boot(uintptr_t test_addr) 211{ 212 unsigned int random = ENTROPY; 213 if (read_csr(mhartid) > 0) 214 coherence_torture(); 215 216 _Static_assert(SIZEOF_TRAPFRAME_T == sizeof(trapframe_t), "???"); 217 218#if (MAX_TEST_PAGES > PTES_PER_PT) || (DRAM_BASE % MEGAPAGE_SIZE) != 0 219# error 220#endif 221 // map user to lowermost megapage 222 l1pt[0] = ((pte_t)user_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 223 // map kernel to uppermost megapage 224#if __riscv_xlen == 64 225 l1pt[PTES_PER_PT-1] = ((pte_t)kernel_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 226 kernel_l2pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 227 user_l2pt[0] = ((pte_t)user_l3pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 228 uintptr_t vm_choice = SATP_MODE_SV39; 229#else 230 l1pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 231 uintptr_t vm_choice = SATP_MODE_SV32; 232#endif 233 write_csr(sptbr, ((uintptr_t)l1pt >> PGSHIFT) | 234 (vm_choice * (SATP_MODE & ~(SATP_MODE<<1)))); 235 236 // Set up PMPs if present, ignoring illegal instruction trap if not. 237 uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X; 238 asm volatile ("la t0, 1f\n\t" 239 "csrrw t0, mtvec, t0\n\t" 240 "csrw pmpaddr0, %1\n\t" 241 "csrw pmpcfg0, %0\n\t" 242 ".align 2\n\t" 243 "1:" 244 : : "r" (pmpc), "r" (-1UL) : "t0"); 245 246 // set up supervisor trap handling 247 write_csr(stvec, pa2kva(trap_entry)); 248 write_csr(sscratch, pa2kva(read_csr(mscratch))); 249 write_csr(medeleg, 250 (1 << CAUSE_USER_ECALL) | 251 (1 << CAUSE_FETCH_PAGE_FAULT) | 252 (1 << CAUSE_LOAD_PAGE_FAULT) | 253 (1 << CAUSE_STORE_PAGE_FAULT)); 254 // FPU on; accelerator on; allow supervisor access to user memory access 255 write_csr(mstatus, MSTATUS_FS | MSTATUS_XS); 256 write_csr(mie, 0); 257 258 random = 1 + (random % MAX_TEST_PAGES); 259 freelist_head = pa2kva((void*)&freelist_nodes[0]); 260 freelist_tail = pa2kva(&freelist_nodes[MAX_TEST_PAGES-1]); 261 for (long i = 0; i < MAX_TEST_PAGES; i++) 262 { 263 freelist_nodes[i].addr = DRAM_BASE + (MAX_TEST_PAGES + random)*PGSIZE; 264 freelist_nodes[i].next = pa2kva(&freelist_nodes[i+1]); 265 random = LFSR_NEXT(random); 266 } 267 freelist_nodes[MAX_TEST_PAGES-1].next = 0; 268 269 trapframe_t tf; 270 memset(&tf, 0, sizeof(tf)); 271 tf.epc = test_addr - DRAM_BASE; 272 pop_tf(&tf); 273} 274