vm.c revision 12771
112771Sqtt2@cornell.edu// See LICENSE for license details. 212771Sqtt2@cornell.edu 312771Sqtt2@cornell.edu#include <stdint.h> 412771Sqtt2@cornell.edu#include <string.h> 512771Sqtt2@cornell.edu#include <stdio.h> 612771Sqtt2@cornell.edu 712771Sqtt2@cornell.edu#include "riscv_test.h" 812771Sqtt2@cornell.edu 912771Sqtt2@cornell.eduvoid trap_entry(); 1012771Sqtt2@cornell.eduvoid pop_tf(trapframe_t*); 1112771Sqtt2@cornell.edu 1212771Sqtt2@cornell.eduvolatile uint64_t tohost; 1312771Sqtt2@cornell.eduvolatile uint64_t fromhost; 1412771Sqtt2@cornell.edu 1512771Sqtt2@cornell.edustatic void do_tohost(uint64_t tohost_value) 1612771Sqtt2@cornell.edu{ 1712771Sqtt2@cornell.edu while (tohost) 1812771Sqtt2@cornell.edu fromhost = 0; 1912771Sqtt2@cornell.edu tohost = tohost_value; 2012771Sqtt2@cornell.edu} 2112771Sqtt2@cornell.edu 2212771Sqtt2@cornell.edu#define pa2kva(pa) ((void*)(pa) - DRAM_BASE - MEGAPAGE_SIZE) 2312771Sqtt2@cornell.edu#define uva2kva(pa) ((void*)(pa) - MEGAPAGE_SIZE) 2412771Sqtt2@cornell.edu 2512771Sqtt2@cornell.edu#define flush_page(addr) asm volatile ("sfence.vma %0" : : "r" (addr) : "memory") 2612771Sqtt2@cornell.edu 2712771Sqtt2@cornell.edustatic uint64_t lfsr63(uint64_t x) 2812771Sqtt2@cornell.edu{ 2912771Sqtt2@cornell.edu uint64_t bit = (x ^ (x >> 1)) & 1; 3012771Sqtt2@cornell.edu return (x >> 1) | (bit << 62); 3112771Sqtt2@cornell.edu} 3212771Sqtt2@cornell.edu 3312771Sqtt2@cornell.edustatic void cputchar(int x) 3412771Sqtt2@cornell.edu{ 3512771Sqtt2@cornell.edu do_tohost(0x0101000000000000 | (unsigned char)x); 3612771Sqtt2@cornell.edu} 3712771Sqtt2@cornell.edu 3812771Sqtt2@cornell.edustatic void cputstring(const char* s) 3912771Sqtt2@cornell.edu{ 4012771Sqtt2@cornell.edu while (*s) 4112771Sqtt2@cornell.edu cputchar(*s++); 4212771Sqtt2@cornell.edu} 4312771Sqtt2@cornell.edu 4412771Sqtt2@cornell.edustatic void terminate(int code) 4512771Sqtt2@cornell.edu{ 4612771Sqtt2@cornell.edu do_tohost(code); 4712771Sqtt2@cornell.edu while (1); 4812771Sqtt2@cornell.edu} 4912771Sqtt2@cornell.edu 5012771Sqtt2@cornell.eduvoid wtf() 5112771Sqtt2@cornell.edu{ 5212771Sqtt2@cornell.edu terminate(841); 5312771Sqtt2@cornell.edu} 5412771Sqtt2@cornell.edu 5512771Sqtt2@cornell.edu#define stringify1(x) #x 5612771Sqtt2@cornell.edu#define stringify(x) stringify1(x) 5712771Sqtt2@cornell.edu#define assert(x) do { \ 5812771Sqtt2@cornell.edu if (x) break; \ 5912771Sqtt2@cornell.edu cputstring("Assertion failed: " stringify(x) "\n"); \ 6012771Sqtt2@cornell.edu terminate(3); \ 6112771Sqtt2@cornell.edu} while(0) 6212771Sqtt2@cornell.edu 6312771Sqtt2@cornell.edu#define l1pt pt[0] 6412771Sqtt2@cornell.edu#define user_l2pt pt[1] 6512771Sqtt2@cornell.edu#if __riscv_xlen == 64 6612771Sqtt2@cornell.edu# define NPT 4 6712771Sqtt2@cornell.edu#define kernel_l2pt pt[2] 6812771Sqtt2@cornell.edu# define user_l3pt pt[3] 6912771Sqtt2@cornell.edu#else 7012771Sqtt2@cornell.edu# define NPT 2 7112771Sqtt2@cornell.edu# define user_l3pt user_l2pt 7212771Sqtt2@cornell.edu#endif 7312771Sqtt2@cornell.edupte_t pt[NPT][PTES_PER_PT] __attribute__((aligned(PGSIZE))); 7412771Sqtt2@cornell.edu 7512771Sqtt2@cornell.edutypedef struct { pte_t addr; void* next; } freelist_t; 7612771Sqtt2@cornell.edu 7712771Sqtt2@cornell.edufreelist_t user_mapping[MAX_TEST_PAGES]; 7812771Sqtt2@cornell.edufreelist_t freelist_nodes[MAX_TEST_PAGES]; 7912771Sqtt2@cornell.edufreelist_t *freelist_head, *freelist_tail; 8012771Sqtt2@cornell.edu 8112771Sqtt2@cornell.eduvoid printhex(uint64_t x) 8212771Sqtt2@cornell.edu{ 8312771Sqtt2@cornell.edu char str[17]; 8412771Sqtt2@cornell.edu for (int i = 0; i < 16; i++) 8512771Sqtt2@cornell.edu { 8612771Sqtt2@cornell.edu str[15-i] = (x & 0xF) + ((x & 0xF) < 10 ? '0' : 'a'-10); 8712771Sqtt2@cornell.edu x >>= 4; 8812771Sqtt2@cornell.edu } 8912771Sqtt2@cornell.edu str[16] = 0; 9012771Sqtt2@cornell.edu 9112771Sqtt2@cornell.edu cputstring(str); 9212771Sqtt2@cornell.edu} 9312771Sqtt2@cornell.edu 9412771Sqtt2@cornell.edustatic void evict(unsigned long addr) 9512771Sqtt2@cornell.edu{ 9612771Sqtt2@cornell.edu assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 9712771Sqtt2@cornell.edu addr = addr/PGSIZE*PGSIZE; 9812771Sqtt2@cornell.edu 9912771Sqtt2@cornell.edu freelist_t* node = &user_mapping[addr/PGSIZE]; 10012771Sqtt2@cornell.edu if (node->addr) 10112771Sqtt2@cornell.edu { 10212771Sqtt2@cornell.edu // check accessed and dirty bits 10312771Sqtt2@cornell.edu assert(user_l3pt[addr/PGSIZE] & PTE_A); 10412771Sqtt2@cornell.edu uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 10512771Sqtt2@cornell.edu if (memcmp((void*)addr, uva2kva(addr), PGSIZE)) { 10612771Sqtt2@cornell.edu assert(user_l3pt[addr/PGSIZE] & PTE_D); 10712771Sqtt2@cornell.edu memcpy((void*)addr, uva2kva(addr), PGSIZE); 10812771Sqtt2@cornell.edu } 10912771Sqtt2@cornell.edu write_csr(sstatus, sstatus); 11012771Sqtt2@cornell.edu 11112771Sqtt2@cornell.edu user_mapping[addr/PGSIZE].addr = 0; 11212771Sqtt2@cornell.edu 11312771Sqtt2@cornell.edu if (freelist_tail == 0) 11412771Sqtt2@cornell.edu freelist_head = freelist_tail = node; 11512771Sqtt2@cornell.edu else 11612771Sqtt2@cornell.edu { 11712771Sqtt2@cornell.edu freelist_tail->next = node; 11812771Sqtt2@cornell.edu freelist_tail = node; 11912771Sqtt2@cornell.edu } 12012771Sqtt2@cornell.edu } 12112771Sqtt2@cornell.edu} 12212771Sqtt2@cornell.edu 12312771Sqtt2@cornell.eduvoid handle_fault(uintptr_t addr, uintptr_t cause) 12412771Sqtt2@cornell.edu{ 12512771Sqtt2@cornell.edu assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE); 12612771Sqtt2@cornell.edu addr = addr/PGSIZE*PGSIZE; 12712771Sqtt2@cornell.edu 12812771Sqtt2@cornell.edu if (user_l3pt[addr/PGSIZE]) { 12912771Sqtt2@cornell.edu if (!(user_l3pt[addr/PGSIZE] & PTE_A)) { 13012771Sqtt2@cornell.edu user_l3pt[addr/PGSIZE] |= PTE_A; 13112771Sqtt2@cornell.edu } else { 13212771Sqtt2@cornell.edu assert(!(user_l3pt[addr/PGSIZE] & PTE_D) && cause == CAUSE_STORE_PAGE_FAULT); 13312771Sqtt2@cornell.edu user_l3pt[addr/PGSIZE] |= PTE_D; 13412771Sqtt2@cornell.edu } 13512771Sqtt2@cornell.edu flush_page(addr); 13612771Sqtt2@cornell.edu return; 13712771Sqtt2@cornell.edu } 13812771Sqtt2@cornell.edu 13912771Sqtt2@cornell.edu freelist_t* node = freelist_head; 14012771Sqtt2@cornell.edu assert(node); 14112771Sqtt2@cornell.edu freelist_head = node->next; 14212771Sqtt2@cornell.edu if (freelist_head == freelist_tail) 14312771Sqtt2@cornell.edu freelist_tail = 0; 14412771Sqtt2@cornell.edu 14512771Sqtt2@cornell.edu uintptr_t new_pte = (node->addr >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V | PTE_U | PTE_R | PTE_W | PTE_X; 14612771Sqtt2@cornell.edu user_l3pt[addr/PGSIZE] = new_pte | PTE_A | PTE_D; 14712771Sqtt2@cornell.edu flush_page(addr); 14812771Sqtt2@cornell.edu 14912771Sqtt2@cornell.edu assert(user_mapping[addr/PGSIZE].addr == 0); 15012771Sqtt2@cornell.edu user_mapping[addr/PGSIZE] = *node; 15112771Sqtt2@cornell.edu 15212771Sqtt2@cornell.edu uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM); 15312771Sqtt2@cornell.edu memcpy((void*)addr, uva2kva(addr), PGSIZE); 15412771Sqtt2@cornell.edu write_csr(sstatus, sstatus); 15512771Sqtt2@cornell.edu 15612771Sqtt2@cornell.edu user_l3pt[addr/PGSIZE] = new_pte; 15712771Sqtt2@cornell.edu flush_page(addr); 15812771Sqtt2@cornell.edu 15912771Sqtt2@cornell.edu __builtin___clear_cache(0,0); 16012771Sqtt2@cornell.edu} 16112771Sqtt2@cornell.edu 16212771Sqtt2@cornell.eduvoid handle_trap(trapframe_t* tf) 16312771Sqtt2@cornell.edu{ 16412771Sqtt2@cornell.edu if (tf->cause == CAUSE_USER_ECALL) 16512771Sqtt2@cornell.edu { 16612771Sqtt2@cornell.edu int n = tf->gpr[10]; 16712771Sqtt2@cornell.edu 16812771Sqtt2@cornell.edu for (long i = 1; i < MAX_TEST_PAGES; i++) 16912771Sqtt2@cornell.edu evict(i*PGSIZE); 17012771Sqtt2@cornell.edu 17112771Sqtt2@cornell.edu terminate(n); 17212771Sqtt2@cornell.edu } 17312771Sqtt2@cornell.edu else if (tf->cause == CAUSE_ILLEGAL_INSTRUCTION) 17412771Sqtt2@cornell.edu { 17512771Sqtt2@cornell.edu assert(tf->epc % 4 == 0); 17612771Sqtt2@cornell.edu 17712771Sqtt2@cornell.edu int* fssr; 17812771Sqtt2@cornell.edu asm ("jal %0, 1f; fssr x0; 1:" : "=r"(fssr)); 17912771Sqtt2@cornell.edu 18012771Sqtt2@cornell.edu if (*(int*)tf->epc == *fssr) 18112771Sqtt2@cornell.edu terminate(1); // FP test on non-FP hardware. "succeed." 18212771Sqtt2@cornell.edu else 18312771Sqtt2@cornell.edu assert(!"illegal instruction"); 18412771Sqtt2@cornell.edu tf->epc += 4; 18512771Sqtt2@cornell.edu } 18612771Sqtt2@cornell.edu else if (tf->cause == CAUSE_FETCH_PAGE_FAULT || tf->cause == CAUSE_LOAD_PAGE_FAULT || tf->cause == CAUSE_STORE_PAGE_FAULT) 18712771Sqtt2@cornell.edu handle_fault(tf->badvaddr, tf->cause); 18812771Sqtt2@cornell.edu else 18912771Sqtt2@cornell.edu assert(!"unexpected exception"); 19012771Sqtt2@cornell.edu 19112771Sqtt2@cornell.edu pop_tf(tf); 19212771Sqtt2@cornell.edu} 19312771Sqtt2@cornell.edu 19412771Sqtt2@cornell.edustatic void coherence_torture() 19512771Sqtt2@cornell.edu{ 19612771Sqtt2@cornell.edu // cause coherence misses without affecting program semantics 19712771Sqtt2@cornell.edu unsigned int random = ENTROPY; 19812771Sqtt2@cornell.edu while (1) { 19912771Sqtt2@cornell.edu uintptr_t paddr = DRAM_BASE + ((random % (2 * (MAX_TEST_PAGES + 1) * PGSIZE)) & -4); 20012771Sqtt2@cornell.edu#ifdef __riscv_atomic 20112771Sqtt2@cornell.edu if (random & 1) // perform a no-op write 20212771Sqtt2@cornell.edu asm volatile ("amoadd.w zero, zero, (%0)" :: "r"(paddr)); 20312771Sqtt2@cornell.edu else // perform a read 20412771Sqtt2@cornell.edu#endif 20512771Sqtt2@cornell.edu asm volatile ("lw zero, (%0)" :: "r"(paddr)); 20612771Sqtt2@cornell.edu random = lfsr63(random); 20712771Sqtt2@cornell.edu } 20812771Sqtt2@cornell.edu} 20912771Sqtt2@cornell.edu 21012771Sqtt2@cornell.eduvoid vm_boot(uintptr_t test_addr) 21112771Sqtt2@cornell.edu{ 21212771Sqtt2@cornell.edu unsigned int random = ENTROPY; 21312771Sqtt2@cornell.edu if (read_csr(mhartid) > 0) 21412771Sqtt2@cornell.edu coherence_torture(); 21512771Sqtt2@cornell.edu 21612771Sqtt2@cornell.edu _Static_assert(SIZEOF_TRAPFRAME_T == sizeof(trapframe_t), "???"); 21712771Sqtt2@cornell.edu 21812771Sqtt2@cornell.edu#if (MAX_TEST_PAGES > PTES_PER_PT) || (DRAM_BASE % MEGAPAGE_SIZE) != 0 21912771Sqtt2@cornell.edu# error 22012771Sqtt2@cornell.edu#endif 22112771Sqtt2@cornell.edu // map user to lowermost megapage 22212771Sqtt2@cornell.edu l1pt[0] = ((pte_t)user_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 22312771Sqtt2@cornell.edu // map kernel to uppermost megapage 22412771Sqtt2@cornell.edu#if __riscv_xlen == 64 22512771Sqtt2@cornell.edu l1pt[PTES_PER_PT-1] = ((pte_t)kernel_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 22612771Sqtt2@cornell.edu kernel_l2pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 22712771Sqtt2@cornell.edu user_l2pt[0] = ((pte_t)user_l3pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V; 22812771Sqtt2@cornell.edu uintptr_t vm_choice = SATP_MODE_SV39; 22912771Sqtt2@cornell.edu#else 23012771Sqtt2@cornell.edu l1pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; 23112771Sqtt2@cornell.edu uintptr_t vm_choice = SATP_MODE_SV32; 23212771Sqtt2@cornell.edu#endif 23312771Sqtt2@cornell.edu write_csr(sptbr, ((uintptr_t)l1pt >> PGSHIFT) | 23412771Sqtt2@cornell.edu (vm_choice * (SATP_MODE & ~(SATP_MODE<<1)))); 23512771Sqtt2@cornell.edu 23612771Sqtt2@cornell.edu // Set up PMPs if present, ignoring illegal instruction trap if not. 23712771Sqtt2@cornell.edu uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X; 23812771Sqtt2@cornell.edu asm volatile ("la t0, 1f\n\t" 23912771Sqtt2@cornell.edu "csrrw t0, mtvec, t0\n\t" 24012771Sqtt2@cornell.edu "csrw pmpaddr0, %1\n\t" 24112771Sqtt2@cornell.edu "csrw pmpcfg0, %0\n\t" 24212771Sqtt2@cornell.edu ".align 2\n\t" 24312771Sqtt2@cornell.edu "1:" 24412771Sqtt2@cornell.edu : : "r" (pmpc), "r" (-1UL) : "t0"); 24512771Sqtt2@cornell.edu 24612771Sqtt2@cornell.edu // set up supervisor trap handling 24712771Sqtt2@cornell.edu write_csr(stvec, pa2kva(trap_entry)); 24812771Sqtt2@cornell.edu write_csr(sscratch, pa2kva(read_csr(mscratch))); 24912771Sqtt2@cornell.edu write_csr(medeleg, 25012771Sqtt2@cornell.edu (1 << CAUSE_USER_ECALL) | 25112771Sqtt2@cornell.edu (1 << CAUSE_FETCH_PAGE_FAULT) | 25212771Sqtt2@cornell.edu (1 << CAUSE_LOAD_PAGE_FAULT) | 25312771Sqtt2@cornell.edu (1 << CAUSE_STORE_PAGE_FAULT)); 25412771Sqtt2@cornell.edu // FPU on; accelerator on; allow supervisor access to user memory access 25512771Sqtt2@cornell.edu write_csr(mstatus, MSTATUS_FS | MSTATUS_XS); 25612771Sqtt2@cornell.edu write_csr(mie, 0); 25712771Sqtt2@cornell.edu 25812771Sqtt2@cornell.edu random = 1 + (random % MAX_TEST_PAGES); 25912771Sqtt2@cornell.edu freelist_head = pa2kva((void*)&freelist_nodes[0]); 26012771Sqtt2@cornell.edu freelist_tail = pa2kva(&freelist_nodes[MAX_TEST_PAGES-1]); 26112771Sqtt2@cornell.edu for (long i = 0; i < MAX_TEST_PAGES; i++) 26212771Sqtt2@cornell.edu { 26312771Sqtt2@cornell.edu freelist_nodes[i].addr = DRAM_BASE + (MAX_TEST_PAGES + random)*PGSIZE; 26412771Sqtt2@cornell.edu freelist_nodes[i].next = pa2kva(&freelist_nodes[i+1]); 26512771Sqtt2@cornell.edu random = LFSR_NEXT(random); 26612771Sqtt2@cornell.edu } 26712771Sqtt2@cornell.edu freelist_nodes[MAX_TEST_PAGES-1].next = 0; 26812771Sqtt2@cornell.edu 26912771Sqtt2@cornell.edu trapframe_t tf; 27012771Sqtt2@cornell.edu memset(&tf, 0, sizeof(tf)); 27112771Sqtt2@cornell.edu tf.epc = test_addr - DRAM_BASE; 27212771Sqtt2@cornell.edu pop_tf(&tf); 27312771Sqtt2@cornell.edu} 274