1// See LICENSE for license details.
2
3#include <stdint.h>
4#include <string.h>
5#include <stdio.h>
6
7#include "riscv_test.h"
8
9void trap_entry();
10void pop_tf(trapframe_t*);
11
12volatile uint64_t tohost;
13volatile uint64_t fromhost;
14
15static void do_tohost(uint64_t tohost_value)
16{
17  while (tohost)
18    fromhost = 0;
19  tohost = tohost_value;
20}
21
22#define pa2kva(pa) ((void*)(pa) - DRAM_BASE - MEGAPAGE_SIZE)
23#define uva2kva(pa) ((void*)(pa) - MEGAPAGE_SIZE)
24
25#define flush_page(addr) asm volatile ("sfence.vma %0" : : "r" (addr) : "memory")
26
27static uint64_t lfsr63(uint64_t x)
28{
29  uint64_t bit = (x ^ (x >> 1)) & 1;
30  return (x >> 1) | (bit << 62);
31}
32
33static void cputchar(int x)
34{
35  do_tohost(0x0101000000000000 | (unsigned char)x);
36}
37
38static void cputstring(const char* s)
39{
40  while (*s)
41    cputchar(*s++);
42}
43
44static void terminate(int code)
45{
46  do_tohost(code);
47  while (1);
48}
49
50void wtf()
51{
52  terminate(841);
53}
54
55#define stringify1(x) #x
56#define stringify(x) stringify1(x)
57#define assert(x) do { \
58  if (x) break; \
59  cputstring("Assertion failed: " stringify(x) "\n"); \
60  terminate(3); \
61} while(0)
62
63#define l1pt pt[0]
64#define user_l2pt pt[1]
65#if __riscv_xlen == 64
66# define NPT 4
67#define kernel_l2pt pt[2]
68# define user_l3pt pt[3]
69#else
70# define NPT 2
71# define user_l3pt user_l2pt
72#endif
73pte_t pt[NPT][PTES_PER_PT] __attribute__((aligned(PGSIZE)));
74
75typedef struct { pte_t addr; void* next; } freelist_t;
76
77freelist_t user_mapping[MAX_TEST_PAGES];
78freelist_t freelist_nodes[MAX_TEST_PAGES];
79freelist_t *freelist_head, *freelist_tail;
80
81void printhex(uint64_t x)
82{
83  char str[17];
84  for (int i = 0; i < 16; i++)
85  {
86    str[15-i] = (x & 0xF) + ((x & 0xF) < 10 ? '0' : 'a'-10);
87    x >>= 4;
88  }
89  str[16] = 0;
90
91  cputstring(str);
92}
93
94static void evict(unsigned long addr)
95{
96  assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE);
97  addr = addr/PGSIZE*PGSIZE;
98
99  freelist_t* node = &user_mapping[addr/PGSIZE];
100  if (node->addr)
101  {
102    // check accessed and dirty bits
103    assert(user_l3pt[addr/PGSIZE] & PTE_A);
104    uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM);
105    if (memcmp((void*)addr, uva2kva(addr), PGSIZE)) {
106      assert(user_l3pt[addr/PGSIZE] & PTE_D);
107      memcpy((void*)addr, uva2kva(addr), PGSIZE);
108    }
109    write_csr(sstatus, sstatus);
110
111    user_mapping[addr/PGSIZE].addr = 0;
112
113    if (freelist_tail == 0)
114      freelist_head = freelist_tail = node;
115    else
116    {
117      freelist_tail->next = node;
118      freelist_tail = node;
119    }
120  }
121}
122
123void handle_fault(uintptr_t addr, uintptr_t cause)
124{
125  assert(addr >= PGSIZE && addr < MAX_TEST_PAGES * PGSIZE);
126  addr = addr/PGSIZE*PGSIZE;
127
128  if (user_l3pt[addr/PGSIZE]) {
129    if (!(user_l3pt[addr/PGSIZE] & PTE_A)) {
130      user_l3pt[addr/PGSIZE] |= PTE_A;
131    } else {
132      assert(!(user_l3pt[addr/PGSIZE] & PTE_D) && cause == CAUSE_STORE_PAGE_FAULT);
133      user_l3pt[addr/PGSIZE] |= PTE_D;
134    }
135    flush_page(addr);
136    return;
137  }
138
139  freelist_t* node = freelist_head;
140  assert(node);
141  freelist_head = node->next;
142  if (freelist_head == freelist_tail)
143    freelist_tail = 0;
144
145  uintptr_t new_pte = (node->addr >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V | PTE_U | PTE_R | PTE_W | PTE_X;
146  user_l3pt[addr/PGSIZE] = new_pte | PTE_A | PTE_D;
147  flush_page(addr);
148
149  assert(user_mapping[addr/PGSIZE].addr == 0);
150  user_mapping[addr/PGSIZE] = *node;
151
152  uintptr_t sstatus = set_csr(sstatus, SSTATUS_SUM);
153  memcpy((void*)addr, uva2kva(addr), PGSIZE);
154  write_csr(sstatus, sstatus);
155
156  user_l3pt[addr/PGSIZE] = new_pte;
157  flush_page(addr);
158
159  __builtin___clear_cache(0,0);
160}
161
162void handle_trap(trapframe_t* tf)
163{
164  if (tf->cause == CAUSE_USER_ECALL)
165  {
166    int n = tf->gpr[10];
167
168    for (long i = 1; i < MAX_TEST_PAGES; i++)
169      evict(i*PGSIZE);
170
171    terminate(n);
172  }
173  else if (tf->cause == CAUSE_ILLEGAL_INSTRUCTION)
174  {
175    assert(tf->epc % 4 == 0);
176
177    int* fssr;
178    asm ("jal %0, 1f; fssr x0; 1:" : "=r"(fssr));
179
180    if (*(int*)tf->epc == *fssr)
181      terminate(1); // FP test on non-FP hardware.  "succeed."
182    else
183      assert(!"illegal instruction");
184    tf->epc += 4;
185  }
186  else if (tf->cause == CAUSE_FETCH_PAGE_FAULT || tf->cause == CAUSE_LOAD_PAGE_FAULT || tf->cause == CAUSE_STORE_PAGE_FAULT)
187    handle_fault(tf->badvaddr, tf->cause);
188  else
189    assert(!"unexpected exception");
190
191  pop_tf(tf);
192}
193
194static void coherence_torture()
195{
196  // cause coherence misses without affecting program semantics
197  unsigned int random = ENTROPY;
198  while (1) {
199    uintptr_t paddr = DRAM_BASE + ((random % (2 * (MAX_TEST_PAGES + 1) * PGSIZE)) & -4);
200#ifdef __riscv_atomic
201    if (random & 1) // perform a no-op write
202      asm volatile ("amoadd.w zero, zero, (%0)" :: "r"(paddr));
203    else // perform a read
204#endif
205      asm volatile ("lw zero, (%0)" :: "r"(paddr));
206    random = lfsr63(random);
207  }
208}
209
210void vm_boot(uintptr_t test_addr)
211{
212  unsigned int random = ENTROPY;
213  if (read_csr(mhartid) > 0)
214    coherence_torture();
215
216  _Static_assert(SIZEOF_TRAPFRAME_T == sizeof(trapframe_t), "???");
217
218#if (MAX_TEST_PAGES > PTES_PER_PT) || (DRAM_BASE % MEGAPAGE_SIZE) != 0
219# error
220#endif
221  // map user to lowermost megapage
222  l1pt[0] = ((pte_t)user_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V;
223  // map kernel to uppermost megapage
224#if __riscv_xlen == 64
225  l1pt[PTES_PER_PT-1] = ((pte_t)kernel_l2pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V;
226  kernel_l2pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D;
227  user_l2pt[0] = ((pte_t)user_l3pt >> PGSHIFT << PTE_PPN_SHIFT) | PTE_V;
228  uintptr_t vm_choice = SATP_MODE_SV39;
229#else
230  l1pt[PTES_PER_PT-1] = (DRAM_BASE/RISCV_PGSIZE << PTE_PPN_SHIFT) | PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D;
231  uintptr_t vm_choice = SATP_MODE_SV32;
232#endif
233  write_csr(sptbr, ((uintptr_t)l1pt >> PGSHIFT) |
234                   (vm_choice * (SATP_MODE & ~(SATP_MODE<<1))));
235
236  // Set up PMPs if present, ignoring illegal instruction trap if not.
237  uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X;
238  asm volatile ("la t0, 1f\n\t"
239                "csrrw t0, mtvec, t0\n\t"
240                "csrw pmpaddr0, %1\n\t"
241                "csrw pmpcfg0, %0\n\t"
242                ".align 2\n\t"
243                "1:"
244                : : "r" (pmpc), "r" (-1UL) : "t0");
245
246  // set up supervisor trap handling
247  write_csr(stvec, pa2kva(trap_entry));
248  write_csr(sscratch, pa2kva(read_csr(mscratch)));
249  write_csr(medeleg,
250    (1 << CAUSE_USER_ECALL) |
251    (1 << CAUSE_FETCH_PAGE_FAULT) |
252    (1 << CAUSE_LOAD_PAGE_FAULT) |
253    (1 << CAUSE_STORE_PAGE_FAULT));
254  // FPU on; accelerator on; allow supervisor access to user memory access
255  write_csr(mstatus, MSTATUS_FS | MSTATUS_XS);
256  write_csr(mie, 0);
257
258  random = 1 + (random % MAX_TEST_PAGES);
259  freelist_head = pa2kva((void*)&freelist_nodes[0]);
260  freelist_tail = pa2kva(&freelist_nodes[MAX_TEST_PAGES-1]);
261  for (long i = 0; i < MAX_TEST_PAGES; i++)
262  {
263    freelist_nodes[i].addr = DRAM_BASE + (MAX_TEST_PAGES + random)*PGSIZE;
264    freelist_nodes[i].next = pa2kva(&freelist_nodes[i+1]);
265    random = LFSR_NEXT(random);
266  }
267  freelist_nodes[MAX_TEST_PAGES-1].next = 0;
268
269  trapframe_t tf;
270  memset(&tf, 0, sizeof(tf));
271  tf.epc = test_addr - DRAM_BASE;
272  pop_tf(&tf);
273}
274