==> src/interp.c <==1/* Compiling, instantiating, and the run loop. */
2#include "wasm/interp.h"
3
4#include "wasm/decode.h"
5#include "wasm/fuse.h"
6#include "wasm/instr.h"
7#include "wasm/lower.h"
8#include "wasm/reader.h"
9#include "wasm/validate.h"
10
11#include <stdarg.h>
12#include <stdio.h>
13#include <stdlib.h>
14#include <string.h>
15
16_Static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "linear memory is accessed in place: a little-endian host");
17
18struct wasm_compiled {
19 uint8_t *bytes;
20 size_t len;
21 wasm_module m;
22 wasm_fn_code *fns; /* defined functions */
23 uint32_t *canon; /* canonical type ids */
24};
25
26typedef struct {
27 uint64_t *elems; /* function index + 1, or 0 for null; externrefs as given */
28 uint32_t size;
29 uint64_t max;
30} table;
31
32typedef struct {
33 const wasm_fn_code *fn;
34 const wasm_ins *pc; /* where to continue in the caller */
35 uint64_t *fp;
36} frame;
37
38struct wasm_instance {
39 const wasm_compiled *c;
40 const wasm_host_func **hosts; /* per imported function */
41 uint8_t *mem;
42 uint64_t mem_size;
43 uint64_t mem_max; /* bytes: the module's maximum and the store's limit */
44 bool has_mem;
45 uint64_t *globals;
46 table *tables;
47 uint32_t ntables;
48 uint64_t table_limit;
49 uint64_t **elem_values; /* each element segment's references; NULL once dropped */
50 uint32_t *elem_len;
51 bool *data_dropped;
52 uint64_t *stack;
53 uint32_t stack_slots;
54 frame *frames;
55 uint32_t max_frames;
56 const _Atomic uint64_t *epoch;
57 uint64_t deadline;
58 void *user;
59};
60
61static void say(char *err, size_t errlen, const char *fmt, ...) {
62 if (!err || !errlen) return;
63 va_list ap;
64 va_start(ap, fmt);
65 vsnprintf(err, errlen, fmt, ap);
66 va_end(ap);
67}
68
69/* ---- compiling ---------------------------------------------------------------------------- */
70
71void wasm_compiled_free(wasm_compiled *c) {
72 if (!c) return;
73 if (c->fns)
74 for (uint32_t i = 0; i < c->m.ncodes; i++) wasm_fn_code_free(&c->fns[i]);
75 free(c->fns);
76 free(c->canon);
77 wasm_module_free(&c->m);
78 free(c->bytes);
79 free(c);
80}
81
82const wasm_module *wasm_compiled_module(const wasm_compiled *c) { return &c->m; }
83
84wasm_compiled *wasm_compile(const uint8_t *bytes, size_t len, wasm_features features, char *err, size_t errlen) {
85 wasm_compiled *c = calloc(1, sizeof *c);
86 if (!c || !(c->bytes = malloc(len ? len : 1))) {
87 free(c);
88 say(err, errlen, "out of memory");
89 return NULL;
90 }
91 if (len) memcpy(c->bytes, bytes, len);
92 c->len = len;
93 if (!wasm_decode(c->bytes, len, &c->m, err, errlen) || !wasm_validate(&c->m, features, NULL, err, errlen)) goto fail;
94 const wasm_module *m = &c->m;
95 if (m->nmems > 1) {
96 say(err, errlen, "multi-memory is not supported by this executor");
97 goto fail;
98 }
99 if (m->nmems && (m->mems[0].flags & (WASM_LIMITS_64 | WASM_LIMITS_SHARED))) {
100 say(err, errlen, "%s memories are not supported by this executor",
101 (m->mems[0].flags & WASM_LIMITS_64) ? "64-bit" : "shared");
102 goto fail;
103 }
104 if (!(c->canon = wasm_canonical_types(m)) || !(c->fns = calloc(m->ncodes ? m->ncodes : 1, sizeof *c->fns))) {
105 say(err, errlen, "out of memory");
106 goto fail;
107 }
108 for (uint32_t i = 0; i < m->ncodes; i++) {
109 if (!wasm_lower(m, m->nfunc_imports + i, &c->fns[i], err, errlen)) goto fail;
110 if (!wasm_fuse(&c->fns[i])) {
111 say(err, errlen, "out of memory");
112 goto fail;
113 }
114 }
115 return c;
116fail:
117 wasm_compiled_free(c);
118 return NULL;
119}
120
121/* ---- instantiating ------------------------------------------------------------------------ */
122
123static bool sig_matches(const wasm_functype *t, const char *sig) {
124 const char *gt = strchr(sig, '>');
125 if (!gt || (size_t)(gt - sig) != t->nparams || strlen(gt + 1) != t->nresults) return false;
126 for (uint32_t i = 0; i < t->nparams; i++)
127 if (wasm_sig_type(sig[i]) != t->params[i]) return false;
128 for (uint32_t i = 0; i < t->nresults; i++)
129 if (wasm_sig_type(gt[1 + i]) != t->results[i]) return false;
130 return true;
131}
132
133static bool name_is(const uint8_t *s, uint32_t n, const char *z) { return strlen(z) == n && memcmp(s, z, n) == 0; }
134
135/* A constant expression's value (validated: the ops are constant, the types right). */
136static uint64_t eval_const(const wasm_instance *I, const wasm_expr *e) {
137 wasm_reader r = wasm_reader_at(e->start, e->start, e->start + e->len, NULL, 0);
138 uint64_t stack[64];
139 uint32_t sp = 0;
140 wasm_instr in;
141 while (wasm_rd_instr(&r, &in) && in.op != WASM_OP_END && sp < 64) {
142 switch (in.op) {
143 case WASM_OP_I32_CONST: stack[sp++] = (uint32_t)in.i; break;
144 case WASM_OP_I64_CONST: stack[sp++] = (uint64_t)in.i; break;
145 case WASM_OP_F32_CONST: stack[sp++] = in.f32; break;
146 case WASM_OP_F64_CONST: stack[sp++] = in.f64; break;
147 case WASM_OP_REF_NULL: stack[sp++] = 0; break;
148 case WASM_OP_REF_FUNC: stack[sp++] = (uint64_t)in.a + 1; break;
149 case WASM_OP_GLOBAL_GET: stack[sp++] = I->globals[in.a]; break;
150 case 0x6a: sp--; stack[sp - 1] = (uint32_t)(stack[sp - 1] + stack[sp]); break;
151 case 0x6b: sp--; stack[sp - 1] = (uint32_t)(stack[sp - 1] - stack[sp]); break;
152 case 0x6c: sp--; stack[sp - 1] = (uint32_t)((uint32_t)stack[sp - 1] * (uint32_t)stack[sp]); break;
153 case 0x7c: sp--; stack[sp - 1] = stack[sp - 1] + stack[sp]; break;
154 case 0x7d: sp--; stack[sp - 1] = stack[sp - 1] - stack[sp]; break;
155 case 0x7e: sp--; stack[sp - 1] = stack[sp - 1] * stack[sp]; break;
156 default: break;
157 }
158 }
159 return sp ? stack[sp - 1] : 0;
160}
161
162void wasm_instance_free(wasm_instance *I) {
163 if (!I) return;
164 free(I->hosts);
165 free(I->mem);
166 free(I->globals);
167 if (I->tables)
168 for (uint32_t i = 0; i < I->ntables; i++) free(I->tables[i].elems);
169 free(I->tables);
170 if (I->elem_values)
171 for (uint32_t i = 0; i < I->c->m.nelems; i++) free(I->elem_values[i]);
172 free(I->elem_values);
173 free(I->elem_len);
174 free(I->data_dropped);
175 free(I->stack);
176 free(I->frames);
177 free(I);
178}
179
180wasm_instance *wasm_instantiate(const wasm_compiled *c, const wasm_host_func *imports, size_t nimports,
181 const wasm_store_limits *lim, void *user, char *err, size_t errlen) {
182 const wasm_module *m = &c->m;
183 wasm_store_limits L = lim ? *lim : (wasm_store_limits){0};
184 wasm_instance *I = calloc(1, sizeof *I);
185 if (!I) goto oom;
186 I->c = c;
187 I->user = user;
188 I->stack_slots = L.stack_slots ? L.stack_slots : 128u << 10;
189 I->max_frames = L.call_depth ? L.call_depth : 10000;
190 I->table_limit = WASM_MAX_TABLE;
191 if (!(I->hosts = calloc(m->nfunc_imports ? m->nfunc_imports : 1, sizeof *I->hosts))) goto oom;
192 uint32_t f = 0;
193 for (uint32_t i = 0; i < m->nimports; i++) {
194 const wasm_import *im = &m->imports[i];
195 if (im->kind != WASM_EXTERN_FUNC) {
196 say(err, errlen, "unknown import: `%.*s::%.*s` has not been defined", (int)im->module_len, im->module,
197 (int)im->name_len, im->name);
198 goto fail;
199 }
200 const wasm_host_func *h = NULL;
201 for (size_t k = 0; k < nimports && !h; k++)
202 if (name_is(im->module, im->module_len, imports[k].module) && name_is(im->name, im->name_len, imports[k].name))
203 h = &imports[k];
204 if (!h) {
205 say(err, errlen, "unknown import: `%.*s::%.*s` has not been defined", (int)im->module_len, im->module,
206 (int)im->name_len, im->name);
207 goto fail;
208 }
209 if (!sig_matches(&m->types[im->func_type], h->sig)) {
210 say(err, errlen, "incompatible import type for `%.*s::%.*s`", (int)im->module_len, im->module,
211 (int)im->name_len, im->name);
212 goto fail;
213 }
214 I->hosts[f++] = h;
215 }
216 if (L.memories && m->nmems > L.memories) {
217 say(err, errlen, "resource limit exceeded: memory count too high at %u", m->nmems);
218 goto fail;
219 }
220 if (L.tables && m->ntables > L.tables) {
221 say(err, errlen, "resource limit exceeded: table count too high at %u", m->ntables);
222 goto fail;
223 }
224 if (m->nmems) {
225 const wasm_limits *ml = &m->mems[0];
226 I->has_mem = true;
227 uint64_t max_pages = (ml->flags & WASM_LIMITS_HAS_MAX) ? ml->max : WASM_MAX_PAGES;
228 I->mem_max = max_pages * WASM_PAGE;
229 if (L.memory_bytes && L.memory_bytes < I->mem_max) I->mem_max = L.memory_bytes;
230 I->mem_size = ml->min * WASM_PAGE;
231 if (L.memory_bytes && I->mem_size > L.memory_bytes) {
232 say(err, errlen, "memory minimum size of %llu pages exceeds memory limits", (unsigned long long)ml->min);
233 goto fail;
234 }
235 if (!(I->mem = calloc(I->mem_size ? I->mem_size : 1, 1))) goto oom;
236 }
237 if (!(I->globals = calloc(m->nglobals ? m->nglobals : 1, sizeof *I->globals))) goto oom;
238 for (uint32_t g = m->nglobal_imports; g < m->nglobals; g++)
239 I->globals[g] = eval_const(I, &m->global_inits[g - m->nglobal_imports]);
240 I->ntables = m->ntables;
241 if (!(I->tables = calloc(m->ntables ? m->ntables : 1, sizeof *I->tables))) goto oom;
242 for (uint32_t t = 0; t < m->ntables; t++) {
243 I->tables[t].size = (uint32_t)m->tables[t].limits.min;
244 I->tables[t].max = (m->tables[t].limits.flags & WASM_LIMITS_HAS_MAX) ? m->tables[t].limits.max : WASM_MAX_TABLE;
245 if (!(I->tables[t].elems = calloc(I->tables[t].size ? I->tables[t].size : 1, sizeof(uint64_t)))) goto oom;
246 }
247 if (!(I->elem_values = calloc(m->nelems ? m->nelems : 1, sizeof *I->elem_values)) ||
248 !(I->elem_len = calloc(m->nelems ? m->nelems : 1, sizeof *I->elem_len)) ||
249 !(I->data_dropped = calloc(m->ndatas ? m->ndatas : 1, sizeof *I->data_dropped)))
250 goto oom;
251 for (uint32_t i = 0; i < m->nelems; i++) {
252 const wasm_elem *e = &m->elems[i];
253 if (!(I->elem_values[i] = calloc(e->count ? e->count : 1, sizeof(uint64_t)))) goto oom;
254 I->elem_len[i] = e->count;
255 for (uint32_t k = 0; k < e->count; k++)
256 I->elem_values[i][k] = e->funcs ? (uint64_t)e->funcs[k] + 1 : eval_const(I, &e->exprs[k]);
257 }
258 /* Active segments, in order; then drop them (and the declarative ones). */
259 for (uint32_t i = 0; i < m->nelems; i++) {
260 const wasm_elem *e = &m->elems[i];
261 if (e->mode == WASM_SEG_ACTIVE) {
262 uint64_t at = (uint32_t)eval_const(I, &e->offset);
263 table *t = &I->tables[e->table];
264 if (at + e->count > t->size) {
265 say(err, errlen, "%s", wasm_trap_message(WASM_TRAP_TABLE_OUT_OF_BOUNDS));
266 goto fail;
267 }
268 if (e->count) memcpy(t->elems + at, I->elem_values[i], e->count * sizeof(uint64_t));
269 }
270 if (e->mode != WASM_SEG_PASSIVE) I->elem_len[i] = 0;
271 }
272 for (uint32_t i = 0; i < m->ndatas; i++) {
273 const wasm_data *d = &m->datas[i];
274 if (d->mode != WASM_SEG_ACTIVE) continue;
275 uint64_t at = (uint32_t)eval_const(I, &d->offset);
276 if (at + d->len > I->mem_size) {
277 say(err, errlen, "%s", wasm_trap_message(WASM_TRAP_MEMORY_OUT_OF_BOUNDS));
278 goto fail;
279 }
280 if (d->len) memcpy(I->mem + at, d->bytes, d->len);
281 I->data_dropped[i] = true;
282 }
283 if (!(I->stack = calloc(I->stack_slots, sizeof *I->stack)) || !(I->frames = calloc(I->max_frames, sizeof *I->frames)))
284 goto oom;
285 if (m->has_start) {
286 uint64_t io[1];
287 int t = wasm_call(I, m->start, io);
288 if (t != WASM_NUM_OK) {
289 say(err, errlen, "wasm trap: %s (in the start function)", wasm_trap_message(t));
290 goto fail;
291 }
292 }
293 return I;
294oom:
295 say(err, errlen, "out of memory");
296fail:
297 wasm_instance_free(I);
298 return NULL;
299}
300
301void *wasm_instance_user(const wasm_instance *I) { return I->user; }
302const wasm_compiled *wasm_instance_compiled(const wasm_instance *I) { return I->c; }
303
304int64_t wasm_instance_func(const wasm_instance *I, const char *name, const char *sig, char *err, size_t errlen) {
305 const wasm_module *m = &I->c->m;
306 const wasm_export *e = wasm_module_export(m, name, WASM_EXTERN_FUNC);
307 if (!e) {
308 say(err, errlen, "failed to find function export `%s`", name);
309 return -1;
310 }
311 if (!sig_matches(wasm_func_type(m, e->index), sig)) {
312 say(err, errlen, "type mismatch with parameters or results of function export `%s`", name);
313 return -1;
314 }
315 return e->index;
316}
317
318uint8_t *wasm_memory(const wasm_instance *I, uint64_t *size) {
319 if (size) *size = I->has_mem ? I->mem_size : 0;
320 return I->has_mem ? I->mem : NULL;
321}
322
323void wasm_set_epoch(wasm_instance *I, const _Atomic uint64_t *epoch) { I->epoch = epoch; }
324void wasm_set_deadline(wasm_instance *I, uint64_t deadline) { I->deadline = deadline; }
325uint64_t wasm_deadline(const wasm_instance *I) { return I->deadline; }
326
327/* ---- running ------------------------------------------------------------------------------ */
328
329static int grow_memory(wasm_instance *I, uint32_t pages, uint32_t *old) {
330 uint64_t before = I->mem_size / WASM_PAGE;
331 *old = (uint32_t)before;
332 uint64_t want = (before + pages) * WASM_PAGE;
333 if (want > I->mem_max) return 0;
334 if (pages == 0) return 1;
335 uint8_t *p = realloc(I->mem, want);
336 if (!p) return 0;
337 memset(p + I->mem_size, 0, want - I->mem_size);
338 I->mem = p;
339 I->mem_size = want;
340 return 1;
341}
342
343static int grow_table(wasm_instance *I, table *t, uint32_t n, uint64_t init, uint32_t *old) {
344 *old = t->size;
345 uint64_t want = (uint64_t)t->size + n;
346 if (want > t->max || want > I->table_limit) return 0;
347 if (n == 0) return 1;
348 uint64_t *p = realloc(t->elems, want * sizeof *p);
349 if (!p) return 0;
350 for (uint64_t k = t->size; k < want; k++) p[k] = init;
351 t->elems = p;
352 t->size = (uint32_t)want;
353 return 1;
354}
355
356static inline float F32(uint64_t s) { return wasm_f32_from((uint32_t)s); }
357static inline double F64(uint64_t s) { return wasm_f64_from(s); }
358static inline uint64_t SF32(float f) { return wasm_f32_bits(f); }
359static inline uint64_t SF64(double d) { return wasm_f64_bits(d); }
360
361#define TRAP(code) \
362 do { \
363 trap = (code); \
364 goto trapped; \
365 } while (0)
366#define CHECK_TRAP(expr) \
367 do { \
368 int t_ = (expr); \
369 if (t_ != WASM_NUM_OK) TRAP(t_); \
370 } while (0)
371#define INTERRUPTED() (I->epoch && atomic_load_explicit(I->epoch, memory_order_relaxed) >= I->deadline)
372
373/* Memory access: the effective address of an access of n bytes at the popped i32 + the offset. */
374#define EA(n) \
375 uint64_t ea = (uint64_t)(uint32_t)sp[-1] + pc->b; \
376 if (ea + (n) > memsize) TRAP(WASM_TRAP_MEMORY_OUT_OF_BOUNDS)
377#define LOAD(T, n, conv) \
378 { \
379 EA(n); \
380 T v_; \
381 memcpy(&v_, mem + ea, n); \
382 sp[-1] = conv; \
383 break; \
384 }
385#define STORE(T, n, val) \
386 { \
387 uint64_t x_ = sp[-1]; \
388 sp--; \
389 EA(n); \
390 T v_ = (T)(val); \
391 memcpy(mem + ea, &v_, n); \
392 sp--; \
393 break; \
394 }
395
396#define BIN32(expr) \
397 { \
398 uint32_t b = (uint32_t)sp[-1], a = (uint32_t)sp[-2]; \
399 sp--; \
400 sp[-1] = (uint32_t)(expr); \
401 break; \
402 }
403#define BIN64(expr) \
404 { \
405 uint64_t b = sp[-1], a = sp[-2]; \
406 sp--; \
407 sp[-1] = (uint64_t)(expr); \
408 break; \
409 }
410#define CMP32(expr) BIN32((expr) ? 1u : 0u)
411#define CMP64(expr) \
412 { \
413 uint64_t b = sp[-1], a = sp[-2]; \
414 sp--; \
415 sp[-1] = (expr) ? 1u : 0u; \
416 break; \
417 }
418#define FBIN32(expr) \
419 { \
420 float b = F32(sp[-1]), a = F32(sp[-2]); \
421 sp--; \
422 sp[-1] = SF32(expr); \
423 break; \
424 }
425#define FBIN64(expr) \
426 { \
427 double b = F64(sp[-1]), a = F64(sp[-2]); \
428 sp--; \
429 sp[-1] = SF64(expr); \
430 break; \
431 }
432#define FCMP32(expr) \
433 { \
434 float b = F32(sp[-1]), a = F32(sp[-2]); \
435 sp--; \
436 sp[-1] = (expr) ? 1u : 0u; \
437 break; \
438 }
439#define FCMP64(expr) \
440 { \
441 double b = F64(sp[-1]), a = F64(sp[-2]); \
442 sp--; \
443 sp[-1] = (expr) ? 1u : 0u; \
444 break; \
445 }
446#define UN(expr) \
447 { \
448 uint64_t a = sp[-1]; \
449 sp[-1] = (expr); \
450 break; \
451 }
452#define TRUNC(fn, conv, T) \
453 { \
454 T r_ = 0; \
455 CHECK_TRAP(fn(conv, &r_)); \
456 sp[-1] = r_; \
457 break; \
458 }
459
460/* Moves `values` slots from the top of the stack to fp[height..], and the stack's top after them. */
461#define BRANCH(height, values) \
462 do { \
463 uint32_t n_ = (values); \
464 uint64_t *dst_ = fp + (height); \
465 if (n_ && dst_ != sp - n_) memmove(dst_, sp - n_, n_ * 8); \
466 sp = dst_ + n_; \
467 } while (0)
468
469
470/* A plain instruction's case, with a label fused instructions jump to (wasm/fuse.h). */
471#define OP(n) case 0x##n: op_##n
472/* Every plain instruction fusion may give an operand to. */
473#define FUSIBLE(X) X(28) X(29) X(2a) X(2b) X(2c) X(2d) X(2e) X(2f) X(30) X(31) X(32) X(33) X(34) X(35) X(36) X(37) X(38) X(39) X(3a) X(3b) X(3c) X(3d) X(3e) X(45) X(46) X(47) X(48) X(49) X(4a) X(4b) X(4c) X(4d) X(4e) X(4f) X(50) X(51) X(52) X(53) X(54) X(55) X(56) X(57) X(58) X(59) X(5a) X(5b) X(5c) X(5d) X(5e) X(5f) X(60) X(61) X(62) X(63) X(64) X(65) X(66) X(67) X(68) X(69) X(6a) X(6b) X(6c) X(6d) X(6e) X(6f) X(70) X(71) X(72) X(73) X(74) X(75) X(76) X(77) X(78) X(79) X(7a) X(7b) X(7c) X(7d) X(7e) X(7f) X(80) X(81) X(82) X(83) X(84) X(85) X(86) X(87) X(88) X(89) X(8a) X(8b) X(8c) X(8d) X(8e) X(8f) X(90) X(91) X(92) X(93) X(94) X(95) X(96) X(97) X(98) X(99) X(9a) X(9b) X(9c) X(9d) X(9e) X(9f) X(a0) X(a1) X(a2) X(a3) X(a4) X(a5) X(a6) X(a7) X(a8) X(a9) X(aa) X(ab) X(ac) X(ad) X(ae) X(af) X(b0) X(b1) X(b2) X(b3) X(b4) X(b5) X(b6) X(b7) X(b8) X(b9) X(ba) X(bb) X(bc) X(bd) X(be) X(bf) X(c0) X(c1) X(c2) X(c3) X(c4) X(100) X(101) X(102) X(103) X(104) X(105) X(106) X(107)
474/* The constant of an I-fused instruction: in a, for a load or store (their b is the offset). */
475#define IMMV(n) ((0x##n >= 0x28 && 0x##n <= 0x3e) ? (uint64_t)pc->a : pc->b)
476#define FUSED(n) \
477 case WASM_INS_L + 0x##n: \
478 *sp++ = fp[pc->a]; \
479 goto op_##n; \
480 case WASM_INS_I + 0x##n: \
481 *sp++ = IMMV(n); \
482 goto op_##n;
483
484int wasm_call(wasm_instance *I, uint32_t func, uint64_t *io) {
485 const wasm_compiled *C = I->c;
486 const wasm_module *M = &C->m;
487 uint32_t nimp = M->nfunc_imports;
488 if (func < nimp) {
489 const wasm_host_func *h = I->hosts[func];
490 return h->fn(I, h->ctx, io);
491 }
492 uint64_t *stack_end = I->stack + I->stack_slots;
493 const wasm_fn_code *fn = &C->fns[func - nimp];
494 uint64_t *fp = I->stack;
495 if (fp + fn->frame_slots > stack_end) return WASM_TRAP_STACK_OVERFLOW;
496 if (fn->nparams) memcpy(fp, io, fn->nparams * 8);
497 memset(fp + fn->nparams, 0, (fn->nlocals - fn->nparams) * 8);
498 if (INTERRUPTED()) return WASM_TRAP_INTERRUPT;
499 uint64_t *sp = fp + fn->nlocals;
500 const wasm_ins *pc = fn->code;
501 uint32_t depth = 0;
502 uint8_t *mem = I->mem;
503 uint64_t memsize = I->mem_size;
504 int trap = WASM_NUM_OK;
505 uint32_t target;
506
507 for (;;) {
508 switch (pc->op) {
509 case 0x00: TRAP(WASM_TRAP_UNREACHABLE);
510 case WASM_INS_INTERRUPT:
511 if (INTERRUPTED()) TRAP(WASM_TRAP_INTERRUPT);
512 break;
513 case WASM_INS_JUMP: pc = fn->code + pc->a; continue;
514 case WASM_INS_BR:
515 BRANCH(pc->b >> 32, (uint32_t)pc->b);
516 pc = fn->code + pc->a;
517 continue;
518 case WASM_INS_BR_IF:
519 if ((uint32_t)*--sp) {
520 BRANCH(pc->b >> 32, (uint32_t)pc->b);
521 pc = fn->code + pc->a;
522 continue;
523 }
524 break;
525 case WASM_INS_BR_UNLESS:
526 if (!(uint32_t)*--sp) {
527 pc = fn->code + pc->a;
528 continue;
529 }
530 break;
531 case WASM_INS_BR_TABLE: {
532 uint32_t i = (uint32_t)*--sp;
533 if (i >= pc->b - 1) i = (uint32_t)pc->b - 1;
534 const wasm_br_entry *e = &fn->brs[pc->a + i];
535 BRANCH(e->height, e->values);
536 pc = fn->code + e->target;
537 continue;
538 }
539 case WASM_INS_RETURN: {
540 uint32_t n = pc->a;
541 if (n && fp != sp - n) memmove(fp, sp - n, n * 8);
542 if (depth == 0) {
543 if (n) memcpy(io, fp, n * 8);
544 return WASM_NUM_OK;
545 }
546 sp = fp + n;
547 frame *f = &I->frames[--depth];
548 fn = f->fn;
549 pc = f->pc;
550 fp = f->fp;
551 continue;
552 }
553 case WASM_INS_CALL_INDIRECT: {
554 table *t = &I->tables[pc->b];
555 uint32_t i = (uint32_t)*--sp;
556 if (i >= t->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
557 uint64_t ref = t->elems[i];
558 if (!ref) TRAP(WASM_TRAP_INDIRECT_CALL_TO_NULL);
559 target = (uint32_t)(ref - 1);
560 if (C->canon[M->funcs[target]] != C->canon[pc->a]) TRAP(WASM_TRAP_BAD_SIGNATURE);
561 goto call;
562 }
563 case WASM_INS_CALL:
564 target = pc->a;
565 call:
566 if (target < nimp) {
567 const wasm_host_func *h = I->hosts[target];
568 const wasm_functype *ft = &M->types[M->funcs[target]];
569 uint64_t *args = sp - ft->nparams;
570 /* Room for the results where the arguments were. */
571 if (args + ft->nresults > stack_end) TRAP(WASM_TRAP_STACK_OVERFLOW);
572 CHECK_TRAP(h->fn(I, h->ctx, args));
573 sp = args + ft->nresults;
574 mem = I->mem;
575 memsize = I->mem_size;
576 break;
577 } else {
578 const wasm_fn_code *callee = &C->fns[target - nimp];
579 uint64_t *nfp = sp - callee->nparams;
580 if (depth == I->max_frames || nfp + callee->frame_slots > stack_end) TRAP(WASM_TRAP_STACK_OVERFLOW);
581 I->frames[depth++] = (frame){fn, pc + 1, fp};
582 fn = callee;
583 fp = nfp;
584 memset(fp + fn->nparams, 0, (fn->nlocals - fn->nparams) * 8);
585 sp = fp + fn->nlocals;
586 pc = fn->code;
587 if (INTERRUPTED()) TRAP(WASM_TRAP_INTERRUPT);
588 continue;
589 }
590 case WASM_OP_DROP: sp--; break;
591 case WASM_OP_SELECT: {
592 uint32_t c = (uint32_t)sp[-1];
593 sp -= 2;
594 if (!c) sp[-1] = sp[0];
595 break;
596 }
597 case WASM_OP_LOCAL_GET: *sp++ = fp[pc->a]; break;
598 case WASM_INS_GET2:
599 sp[0] = fp[pc->a];
600 sp[1] = fp[pc->b];
601 sp += 2;
602 break;
603 FUSIBLE(FUSED)
604 case WASM_OP_LOCAL_SET: fp[pc->a] = *--sp; break;
605 case WASM_OP_LOCAL_TEE: fp[pc->a] = sp[-1]; break;
606 case WASM_OP_GLOBAL_GET: *sp++ = I->globals[pc->a]; break;
607 case WASM_OP_GLOBAL_SET: I->globals[pc->a] = *--sp; break;
608 case WASM_OP_TABLE_GET: {
609 table *t = &I->tables[pc->a];
610 uint32_t i = (uint32_t)sp[-1];
611 if (i >= t->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
612 sp[-1] = t->elems[i];
613 break;
614 }
615 case WASM_OP_TABLE_SET: {
616 table *t = &I->tables[pc->a];
617 uint32_t i = (uint32_t)sp[-2];
618 if (i >= t->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
619 t->elems[i] = sp[-1];
620 sp -= 2;
621 break;
622 }
623 OP(28): LOAD(uint32_t, 4, v_)
624 OP(29): LOAD(uint64_t, 8, v_)
625 OP(2a): LOAD(uint32_t, 4, v_)
626 OP(2b): LOAD(uint64_t, 8, v_)
627 OP(2c): LOAD(int8_t, 1, (uint32_t)(int32_t)v_)
628 OP(2d): LOAD(uint8_t, 1, (uint32_t)v_)
629 OP(2e): LOAD(int16_t, 2, (uint32_t)(int32_t)v_)
630 OP(2f): LOAD(uint16_t, 2, (uint32_t)v_)
631 OP(30): LOAD(int8_t, 1, (uint64_t)(int64_t)v_)
632 OP(31): LOAD(uint8_t, 1, (uint64_t)v_)
633 OP(32): LOAD(int16_t, 2, (uint64_t)(int64_t)v_)
634 OP(33): LOAD(uint16_t, 2, (uint64_t)v_)
635 OP(34): LOAD(int32_t, 4, (uint64_t)(int64_t)v_)
636 OP(35): LOAD(uint32_t, 4, (uint64_t)v_)
637 OP(36): STORE(uint32_t, 4, x_)
638 OP(37): STORE(uint64_t, 8, x_)
639 OP(38): STORE(uint32_t, 4, x_)
640 OP(39): STORE(uint64_t, 8, x_)
641 OP(3a): STORE(uint8_t, 1, x_)
642 OP(3b): STORE(uint16_t, 2, x_)
643 OP(3c): STORE(uint8_t, 1, x_)
644 OP(3d): STORE(uint16_t, 2, x_)
645 OP(3e): STORE(uint32_t, 4, x_)
646 case WASM_OP_MEMORY_SIZE: *sp++ = (uint32_t)(memsize / WASM_PAGE); break;
647 case WASM_OP_MEMORY_GROW: {
648 uint32_t old;
649 sp[-1] = grow_memory(I, (uint32_t)sp[-1], &old) ? old : 0xffffffffu;
650 mem = I->mem;
651 memsize = I->mem_size;
652 break;
653 }
654 case WASM_OP_I32_CONST:
655 case WASM_OP_I64_CONST:
656 case WASM_OP_F32_CONST:
657 case WASM_OP_F64_CONST: *sp++ = pc->b; break;
658 OP(45): UN((uint32_t)a == 0)
659 OP(46): CMP32(a == b)
660 OP(47): CMP32(a != b)
661 OP(48): CMP32((int32_t)a < (int32_t)b)
662 OP(49): CMP32(a < b)
663 OP(4a): CMP32((int32_t)a > (int32_t)b)
664 OP(4b): CMP32(a > b)
665 OP(4c): CMP32((int32_t)a <= (int32_t)b)
666 OP(4d): CMP32(a <= b)
667 OP(4e): CMP32((int32_t)a >= (int32_t)b)
668 OP(4f): CMP32(a >= b)
669 OP(50): UN(a == 0)
670 OP(51): CMP64(a == b)
671 OP(52): CMP64(a != b)
672 OP(53): CMP64((int64_t)a < (int64_t)b)
673 OP(54): CMP64(a < b)
674 OP(55): CMP64((int64_t)a > (int64_t)b)
675 OP(56): CMP64(a > b)
676 OP(57): CMP64((int64_t)a <= (int64_t)b)
677 OP(58): CMP64(a <= b)
678 OP(59): CMP64((int64_t)a >= (int64_t)b)
679 OP(5a): CMP64(a >= b)
680 OP(5b): FCMP32(a == b)
681 OP(5c): FCMP32(a != b)
682 OP(5d): FCMP32(a < b)
683 OP(5e): FCMP32(a > b)
684 OP(5f): FCMP32(a <= b)
685 OP(60): FCMP32(a >= b)
686 OP(61): FCMP64(a == b)
687 OP(62): FCMP64(a != b)
688 OP(63): FCMP64(a < b)
689 OP(64): FCMP64(a > b)
690 OP(65): FCMP64(a <= b)
691 OP(66): FCMP64(a >= b)
692 OP(67): UN(wasm_i32_clz((uint32_t)a))
693 OP(68): UN(wasm_i32_ctz((uint32_t)a))
694 OP(69): UN(wasm_i32_popcnt((uint32_t)a))
695 OP(6a): BIN32(a + b)
696 OP(6b): BIN32(a - b)
697 OP(6c): BIN32(a * b)
698 OP(6d): {
699 uint32_t r;
700 CHECK_TRAP(wasm_i32_div_s((uint32_t)sp[-2], (uint32_t)sp[-1], &r));
701 sp--;
702 sp[-1] = r;
703 break;
704 }
705 OP(6e): {
706 uint32_t r;
707 CHECK_TRAP(wasm_i32_div_u((uint32_t)sp[-2], (uint32_t)sp[-1], &r));
708 sp--;
709 sp[-1] = r;
710 break;
711 }
712 OP(6f): {
713 uint32_t r;
714 CHECK_TRAP(wasm_i32_rem_s((uint32_t)sp[-2], (uint32_t)sp[-1], &r));
715 sp--;
716 sp[-1] = r;
717 break;
718 }
719 OP(70): {
720 uint32_t r;
721 CHECK_TRAP(wasm_i32_rem_u((uint32_t)sp[-2], (uint32_t)sp[-1], &r));
722 sp--;
723 sp[-1] = r;
724 break;
725 }
726 OP(71): BIN32(a & b)
727 OP(72): BIN32(a | b)
728 OP(73): BIN32(a ^ b)
729 OP(74): BIN32(a << (b & 31))
730 OP(75): BIN32(wasm_i32_shr_s(a, b))
731 OP(76): BIN32(a >> (b & 31))
732 OP(77): BIN32(wasm_i32_rotl(a, b))
733 OP(78): BIN32(wasm_i32_rotr(a, b))
734 OP(79): UN(wasm_i64_clz(a))
735 OP(7a): UN(wasm_i64_ctz(a))
736 OP(7b): UN(wasm_i64_popcnt(a))
737 OP(7c): BIN64(a + b)
738 OP(7d): BIN64(a - b)
739 OP(7e): BIN64(a * b)
740 OP(7f): {
741 uint64_t r;
742 CHECK_TRAP(wasm_i64_div_s(sp[-2], sp[-1], &r));
743 sp--;
744 sp[-1] = r;
745 break;
746 }
747 OP(80): {
748 uint64_t r;
749 CHECK_TRAP(wasm_i64_div_u(sp[-2], sp[-1], &r));
750 sp--;
751 sp[-1] = r;
752 break;
753 }
754 OP(81): {
755 uint64_t r;
756 CHECK_TRAP(wasm_i64_rem_s(sp[-2], sp[-1], &r));
757 sp--;
758 sp[-1] = r;
759 break;
760 }
761 OP(82): {
762 uint64_t r;
763 CHECK_TRAP(wasm_i64_rem_u(sp[-2], sp[-1], &r));
764 sp--;
765 sp[-1] = r;
766 break;
767 }
768 OP(83): BIN64(a & b)
769 OP(84): BIN64(a | b)
770 OP(85): BIN64(a ^ b)
771 OP(86): BIN64(a << (b & 63))
772 OP(87): BIN64(wasm_i64_shr_s(a, b))
773 OP(88): BIN64(a >> (b & 63))
774 OP(89): BIN64(wasm_i64_rotl(a, b))
775 OP(8a): BIN64(wasm_i64_rotr(a, b))
776 OP(8b): UN(wasm_f32_abs((uint32_t)a))
777 OP(8c): UN(wasm_f32_neg((uint32_t)a))
778 OP(8d): UN(SF32(wasm_f32_ceil(F32(a))))
779 OP(8e): UN(SF32(wasm_f32_floor(F32(a))))
780 OP(8f): UN(SF32(wasm_f32_trunc(F32(a))))
781 OP(90): UN(SF32(wasm_f32_nearest(F32(a))))
782 OP(91): UN(SF32(wasm_f32_sqrt(F32(a))))
783 OP(92): FBIN32(wasm_f32_add(a, b))
784 OP(93): FBIN32(wasm_f32_sub(a, b))
785 OP(94): FBIN32(wasm_f32_mul(a, b))
786 OP(95): FBIN32(wasm_f32_div(a, b))
787 OP(96): FBIN32(wasm_f32_min(a, b))
788 OP(97): FBIN32(wasm_f32_max(a, b))
789 OP(98): BIN32(wasm_f32_copysign(a, b))
790 OP(99): UN(wasm_f64_abs(a))
791 OP(9a): UN(wasm_f64_neg(a))
792 OP(9b): UN(SF64(wasm_f64_ceil(F64(a))))
793 OP(9c): UN(SF64(wasm_f64_floor(F64(a))))
794 OP(9d): UN(SF64(wasm_f64_trunc(F64(a))))
795 OP(9e): UN(SF64(wasm_f64_nearest(F64(a))))
796 OP(9f): UN(SF64(wasm_f64_sqrt(F64(a))))
797 OP(a0): FBIN64(wasm_f64_add(a, b))
798 OP(a1): FBIN64(wasm_f64_sub(a, b))
799 OP(a2): FBIN64(wasm_f64_mul(a, b))
800 OP(a3): FBIN64(wasm_f64_div(a, b))
801 OP(a4): FBIN64(wasm_f64_min(a, b))
802 OP(a5): FBIN64(wasm_f64_max(a, b))
803 OP(a6): BIN64(wasm_f64_copysign(a, b))
804 OP(a7): UN((uint32_t)a)
805 OP(a8): TRUNC(wasm_i32_trunc_f32_s, F32(sp[-1]), uint32_t)
806 OP(a9): TRUNC(wasm_i32_trunc_f32_u, F32(sp[-1]), uint32_t)
807 OP(aa): TRUNC(wasm_i32_trunc_f64_s, F64(sp[-1]), uint32_t)
808 OP(ab): TRUNC(wasm_i32_trunc_f64_u, F64(sp[-1]), uint32_t)
809 OP(ac): UN((uint64_t)(int64_t)(int32_t)(uint32_t)a)
810 OP(ad): UN((uint32_t)a)
811 OP(ae): TRUNC(wasm_i64_trunc_f32_s, F32(sp[-1]), uint64_t)
812 OP(af): TRUNC(wasm_i64_trunc_f32_u, F32(sp[-1]), uint64_t)
813 OP(b0): TRUNC(wasm_i64_trunc_f64_s, F64(sp[-1]), uint64_t)
814 OP(b1): TRUNC(wasm_i64_trunc_f64_u, F64(sp[-1]), uint64_t)
815 OP(b2): UN(SF32((float)(int32_t)(uint32_t)a))
816 OP(b3): UN(SF32((float)(uint32_t)a))
817 OP(b4): UN(SF32((float)(int64_t)a))
818 OP(b5): UN(SF32((float)a))
819 OP(b6): UN(SF32(wasm_f32_demote(F64(a))))
820 OP(b7): UN(SF64((double)(int32_t)(uint32_t)a))
821 OP(b8): UN(SF64((double)(uint32_t)a))
822 OP(b9): UN(SF64((double)(int64_t)a))
823 OP(ba): UN(SF64((double)a))
824 OP(bb): UN(SF64(wasm_f64_promote(F32(a))))
825 OP(bc): /* reinterprets: the slot holds the bits already */
826 OP(bd):
827 OP(be):
828 OP(bf): break;
829 OP(c0): UN((uint32_t)(int32_t)(int8_t)(uint8_t)a)
830 OP(c1): UN((uint32_t)(int32_t)(int16_t)(uint16_t)a)
831 OP(c2): UN((uint64_t)(int64_t)(int8_t)(uint8_t)a)
832 OP(c3): UN((uint64_t)(int64_t)(int16_t)(uint16_t)a)
833 OP(c4): UN((uint64_t)(int64_t)(int32_t)(uint32_t)a)
834 case WASM_OP_REF_NULL: *sp++ = 0; break;
835 case WASM_OP_REF_IS_NULL: sp[-1] = sp[-1] == 0; break;
836 case WASM_OP_REF_FUNC: *sp++ = (uint64_t)pc->a + 1; break;
837 OP(100): UN(wasm_i32_trunc_sat_f32_s(F32(a)))
838 OP(101): UN(wasm_i32_trunc_sat_f32_u(F32(a)))
839 OP(102): UN(wasm_i32_trunc_sat_f64_s(F64(a)))
840 OP(103): UN(wasm_i32_trunc_sat_f64_u(F64(a)))
841 OP(104): UN(wasm_i64_trunc_sat_f32_s(F32(a)))
842 OP(105): UN(wasm_i64_trunc_sat_f32_u(F32(a)))
843 OP(106): UN(wasm_i64_trunc_sat_f64_s(F64(a)))
844 OP(107): UN(wasm_i64_trunc_sat_f64_u(F64(a)))
845 case WASM_INS_FC + 8: { /* memory.init */
846 uint64_t n = (uint32_t)sp[-1], s = (uint32_t)sp[-2], d = (uint32_t)sp[-3];
847 sp -= 3;
848 const wasm_data *seg = &M->datas[pc->a];
849 uint64_t len = I->data_dropped[pc->a] ? 0 : seg->len;
850 if (s + n > len || d + n > memsize) TRAP(WASM_TRAP_MEMORY_OUT_OF_BOUNDS);
851 if (n) memcpy(mem + d, seg->bytes + s, n);
852 break;
853 }
854 case WASM_INS_FC + 9: I->data_dropped[pc->a] = true; break;
855 case WASM_INS_FC + 10: { /* memory.copy */
856 uint64_t n = (uint32_t)sp[-1], s = (uint32_t)sp[-2], d = (uint32_t)sp[-3];
857 sp -= 3;
858 if (s + n > memsize || d + n > memsize) TRAP(WASM_TRAP_MEMORY_OUT_OF_BOUNDS);
859 if (n) memmove(mem + d, mem + s, n);
860 break;
861 }
862 case WASM_INS_FC + 11: { /* memory.fill */
863 uint64_t n = (uint32_t)sp[-1], d = (uint32_t)sp[-3];
864 uint8_t v = (uint8_t)sp[-2];
865 sp -= 3;
866 if (d + n > memsize) TRAP(WASM_TRAP_MEMORY_OUT_OF_BOUNDS);
867 if (n) memset(mem + d, v, n);
868 break;
869 }
870 case WASM_INS_FC + 12: { /* table.init elem a, table b */
871 uint64_t n = (uint32_t)sp[-1], s = (uint32_t)sp[-2], d = (uint32_t)sp[-3];
872 sp -= 3;
873 table *t = &I->tables[pc->b];
874 if (s + n > I->elem_len[pc->a] || d + n > t->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
875 if (n) memcpy(t->elems + d, I->elem_values[pc->a] + s, n * sizeof(uint64_t));
876 break;
877 }
878 case WASM_INS_FC + 13: I->elem_len[pc->a] = 0; break;
879 case WASM_INS_FC + 14: { /* table.copy dst a, src b */
880 uint64_t n = (uint32_t)sp[-1], s = (uint32_t)sp[-2], d = (uint32_t)sp[-3];
881 sp -= 3;
882 table *td = &I->tables[pc->a], *ts = &I->tables[pc->b];
883 if (s + n > ts->size || d + n > td->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
884 if (n) memmove(td->elems + d, ts->elems + s, n * sizeof(uint64_t));
885 break;
886 }
887 case WASM_INS_FC + 15: { /* table.grow */
888 uint32_t old;
889 uint32_t n = (uint32_t)sp[-1];
890 uint64_t init = sp[-2];
891 sp--;
892 sp[-1] = grow_table(I, &I->tables[pc->a], n, init, &old) ? old : 0xffffffffu;
893 break;
894 }
895 case WASM_INS_FC + 16: *sp++ = I->tables[pc->a].size; break;
896 case WASM_INS_FC + 17: { /* table.fill */
897 uint64_t n = (uint32_t)sp[-1], d = (uint32_t)sp[-3];
898 uint64_t v = sp[-2];
899 sp -= 3;
900 table *t = &I->tables[pc->a];
901 if (d + n > t->size) TRAP(WASM_TRAP_TABLE_OUT_OF_BOUNDS);
902 for (uint64_t k = 0; k < n; k++) t->elems[d + k] = v;
903 break;
904 }
905 default: TRAP(WASM_TRAP_UNREACHABLE); /* lowering emits nothing else */
906 }
907 pc++;
908 }
909trapped:
910 return trap;
911}