vm.c revision 12771:75508af5d8dc
1// See LICENSE for license details. 2 3#include <stdint.h> 4#include <string.h> 5#include <stdio.h> 6 7#include "riscv_test.h" 8 9void trap_entry(); 10void pop_tf(trapframe_t*); 11 12volatile uint64_t tohost; 13volatile uint64_t fromhost; 14 15static void do_tohost(uint64_t tohost_value) 16{ 17 while (tohost) 18 fromhost = 0; 19 tohost = tohost_value; 20} 21 22#define pa2kva(pa) ((void*)(pa) - DRAM_BASE - MEGAPAGE_SIZE) 23#define uva2kva(pa) ((void*)(pa) - MEGAPAGE_SIZE) 24 25#define flush_page(addr) asm volatile ("sfence.vma %0" : : "r" (addr) : "memory") 26 27static uint64_t lfsr63(uint64_t x) 28{ 29 uint64_t bit = (x ^ (x >> 1)) & 1; 30 return (x >> 1) | (bit << 62); 31} 32 33static void cputchar(int x) 34{ 35 do_tohost(0x0101000000000000 | (unsigned char)x); 36} 37 38static void cputstring(const char* s) 39{ 40 while (*s) 41 cputchar(*s++); 42} 43 44static void terminate(int code) 45{ 46 do_tohost(code); 47 while (1); 48} 49 50void wtf() 51{ 52 terminate(841); 53} 54 55#define stringify1(x) #x 56#define stringify(x) stringify1(x) 57#define assert(x) do { \ 58 if (x) break; \ 59 cputstring("Assertion failed: " stringify(x) "\n"); \ 60 terminate(3); \ 61} while(0) 62 63#define l1pt pt[0] 64#define user_l2pt pt[1] 65#if __riscv_xlen == 64 66# define NPT 4 67#define kernel_l2pt pt[2] 68# define user_l3pt pt[3] 69#else 70# define NPT 2 71# define user_l3pt user_l2pt 72#endif 73pte_t pt[NPT][PTES_PER_PT] __attribute__((aligned(PGSIZE))); 74 75typedef struct { pte_t addr; void* next; } freelist_t; 76 77freelist_t user_mapping[MAX_TEST_PAGES]; 78freelist_t freelist_nodes[MAX_TEST_PAGES]; 79freelist_t *freelist_head, *freelist_tail; 80 81void printhex(uint64_t x) 82{ 83 char str[17]; 84 for (int i = 0; i < 16; i++) 85 { 86 str[15-i] = (x & 0xF) + ((x & 0xF) < 10 ? '0' : 'a'-10); 87 x >>= 4; 88 } 89 str[16] = 0; 90 91 cputstring(str); 92} 93 94static void evict(unsigned long addr) 95{ 96 assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 97 addr = addr/PGSIZE*PGSIZE; 98 99 freelist_t* node = &user_mapping[addr/PGSIZE]; 100 if (node->addr) 101 { 102 // check accessed and dirty bits 103 assert(user_l3pt[addr/PGSIZE] & PTE_A); 104 uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 105 if (memcmp((void*)addr, uva2kva(addr), PGSIZE)) { 106 assert(user_l3pt[addr/PGSIZE] & PTE_D); 107 memcpy((void*)addr, uva2kva(addr), PGSIZE); 108 } 109 write_csr(sstatus, sstatus); 110 111 user_mapping[addr/PGSIZE].addr = 0; 112 113 if (freelist_tail == 0) 114 freelist_head = freelist_tail = node; 115 else 116 { 117 freelist_tail->next = node; 118 freelist_tail = node; 119 } 120 } 121} 122 123void handle_fault(uintptr_t addr, uintptr_t cause) 124{ 125 assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 126 addr = addr/PGSIZE*PGSIZE; 127 128 if (user_l3pt[addr/PGSIZE]) { 129 if (!(user_l3pt[addr/PGSIZE] & PTE_A)) { 130 user_l3pt[addr/PGSIZE] |= PTE_A; 131 } else { 132 assert(!(user_l3pt[addr/PGSIZE] & PTE_D) && cause == CAUSE_STORE_PAGE_FAULT); 133 user_l3pt[addr/PGSIZE] |= PTE_D; 134 } 135 flush_page(addr); 136 return; 137 } 138 139 freelist_t* node = freelist_head; 140 assert(node); 141 freelist_head = node->next; 142 if (freelist_head == freelist_tail) 143 freelist_tail = 0; 144 145 uintptr_t new_pte = (node->addr >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V | PTE_U | PTE_R | PTE_W | PTE_X; 146 user_l3pt[addr/PGSIZE] = new_pte | PTE_A | PTE_D; 147 flush_page(addr); 148 149 assert(user_mapping[addr/PGSIZE].addr == 0); 150 user_mapping[addr/PGSIZE] = *node; 151 152 uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 153 memcpy((void*)addr, uva2kva(addr), PGSIZE); 154 write_csr(sstatus, sstatus); 155 156 user_l3pt[addr/PGSIZE] = new_pte; 157 flush_page(addr); 158 159 __builtin___clear_cache(0,0); 160} 161 162void handle_trap(trapframe_t* tf) 163{ 164 if (tf->cause == CAUSE_USER_ECALL) 165 { 166 int n = tf->gpr[10]; 167 168 for (long i = 1; i < MAX_TEST_PAGES; i++) 169 evict(i*PGSIZE); 170 171 terminate(n); 172 } 173 else if (tf->cause == CAUSE_ILLEGAL_INSTRUCTION) 174 { 175 assert(tf->epc % 4 == 0); 176 177 int* fssr; 178 asm ("jal %0, 1f; fssr x0; 1:" : "=r"(fssr)); 179 180 if (*(int*)tf->epc == *fssr) 181 terminate(1); // FP test on non-FP hardware. "succeed." 182 else 183 assert(!"illegal instruction"); 184 tf->epc += 4; 185 } 186 else if (tf->cause == CAUSE_FETCH_PAGE_FAULT || tf->cause == CAUSE_LOAD_PAGE_FAULT || tf->cause == CAUSE_STORE_PAGE_FAULT) 187 handle_fault(tf->badvaddr, tf->cause); 188 else 189 assert(!"unexpected exception"); 190 191 pop_tf(tf); 192} 193 194static void coherence_torture() 195{ 196 // cause coherence misses without affecting program semantics 197 unsigned int random = ENTROPY; 198 while (1) { 199 uintptr_t paddr = DRAM_BASE + ((random % (2 * (MAX_TEST_PAGES + 1) * PGSIZE)) & -4); 200#ifdef __riscv_atomic 201 if (random & 1) // perform a no-op write 202 asm volatile ("amoadd.w zero, zero, (%0)" :: "r"(paddr)); 203 else // perform a read 204#endif 205 asm volatile ("lw zero, (%0)" :: "r"(paddr)); 206 random = lfsr63(random); 207 } 208} 209 210void vm_boot(uintptr_t test_addr) 211{ 212 unsigned int random = ENTROPY; 213 if (read_csr(mhartid) > 0) 214 coherence_torture(); 215 216 _Static_assert(SIZEOF_TRAPFRAME_T == sizeof(trapframe_t), "???"); 217 218#if (MAX_TEST_PAGES > PTES_PER_PT) || (DRAM_BASE % MEGAPAGE_SIZE) != 0 219# error 220#endif 221 // map user to lowermost megapage 222 l1pt[0] = ((pte_t)user_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 223 // map kernel to uppermost megapage 224#if __riscv_xlen == 64 225 l1pt[PTES_PER_PT-1] = ((pte_t)kernel_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 226 kernel_l2pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 227 user_l2pt[0] = ((pte_t)user_l3pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 228 uintptr_t vm_choice = SATP_MODE_SV39; 229#else 230 l1pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 231 uintptr_t vm_choice = SATP_MODE_SV32; 232#endif 233 write_csr(sptbr, ((uintptr_t)l1pt >> PGSHIFT) | 234 (vm_choice * (SATP_MODE & ~(SATP_MODE<<1)))); 235 236 // Set up PMPs if present, ignoring illegal instruction trap if not. 237 uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X; 238 asm volatile ("la t0, 1f\n\t" 239 "csrrw t0, mtvec, t0\n\t" 240 "csrw pmpaddr0, %1\n\t" 241 "csrw pmpcfg0, %0\n\t" 242 ".align 2\n\t" 243 "1:" 244 : : "r" (pmpc), "r" (-1UL) : "t0"); 245 246 // set up supervisor trap handling 247 write_csr(stvec, pa2kva(trap_entry)); 248 write_csr(sscratch, pa2kva(read_csr(mscratch))); 249 write_csr(medeleg, 250 (1 << CAUSE_USER_ECALL) | 251 (1 << CAUSE_FETCH_PAGE_FAULT) | 252 (1 << CAUSE_LOAD_PAGE_FAULT) | 253 (1 << CAUSE_STORE_PAGE_FAULT)); 254 // FPU on; accelerator on; allow supervisor access to user memory access 255 write_csr(mstatus, MSTATUS_FS | MSTATUS_XS); 256 write_csr(mie, 0); 257 258 random = 1 + (random % MAX_TEST_PAGES); 259 freelist_head = pa2kva((void*)&freelist_nodes[0]); 260 freelist_tail = pa2kva(&freelist_nodes[MAX_TEST_PAGES-1]); 261 for (long i = 0; i < MAX_TEST_PAGES; i++) 262 { 263 freelist_nodes[i].addr = DRAM_BASE + (MAX_TEST_PAGES + random)*PGSIZE; 264 freelist_nodes[i].next = pa2kva(&freelist_nodes[i+1]); 265 random = LFSR_NEXT(random); 266 } 267 freelist_nodes[MAX_TEST_PAGES-1].next = 0; 268 269 trapframe_t tf; 270 memset(&tf, 0, sizeof(tf)); 271 tf.epc = test_addr - DRAM_BASE; 272 pop_tf(&tf); 273} 274