1/* @ajax/mesh-rt-sim: mesh-rt's device primitives on the CPU, and a launcher that runs a
2 * kernel's nodes as threads, so the same C program can be tested without a GPU.
3 *
4 * Each node is `warps` x 32 POSIX threads with a shared-memory buffer; the
5 * block barrier is a barrier over the node's threads, and every warp
6 * collective (shuffle, ballot, warp barrier) is an exchange through a
7 * per-warp array between two barriers over its 32 lanes, so programs that
8 * follow mesh-rt's collective rules behave as on the GPU. Memory ordering is
9 * C11 atomics with the same orders; the global timer is CLOCK_MONOTONIC.
10 * Every simulated thread runs with the f32 flush-to-zero mode of the cluster
11 * float mode (x86 MXCSR FTZ|DAZ, AArch64 FPCR.FZ), as the PTX's .ftz ops do.
12 *
13 * Empty on nvptx64 (@ajax/mesh-rt-ptx has the real ones there). */
14#ifndef MESH_RT_SIM_H
15#define MESH_RT_SIM_H
16
17#if !defined(__NVPTX__)
18
19#include <stdbool.h>
20#include <stdint.h>
21
22#ifndef MESH_RT_INLINE
23#define MESH_RT_INLINE static inline __attribute__((always_inline))
24#endif
25
26/* The primitives: the same names and meaning as @ajax/mesh-rt-ptx. */
27uint32_t mesh_rt_smid(void);
28uint32_t mesh_rt_tid(void);
29uint32_t mesh_rt_lane(void);
30uint32_t mesh_rt_dyn_smem(void);
31uint32_t mesh_rt_ntid(void);
32uint32_t mesh_rt_ctaid(void);
33uint64_t mesh_rt_gtimer(void);
34uint32_t mesh_rt_ld_acquire(const uint8_t *p);
35uint64_t mesh_rt_ld_acquire64(const uint8_t *p);
36uint32_t mesh_rt_ld_relaxed(const uint8_t *p);
37uint64_t mesh_rt_ld_relaxed64(const uint8_t *p);
38void mesh_rt_st_release(uint8_t *p, uint32_t v);
39void mesh_rt_st_release64(uint8_t *p, uint64_t v);
40void mesh_rt_st_relaxed(uint8_t *p, uint32_t v);
41uint32_t mesh_rt_atom_add(uint8_t *p, uint32_t v);
42uint32_t mesh_rt_atom_or(uint8_t *p, uint32_t v);
43uint32_t mesh_rt_atom_and(uint8_t *p, uint32_t v);
44uint32_t mesh_rt_atom_cas(uint8_t *p, uint32_t cmp, uint32_t desired);
45void mesh_rt_red_add(uint8_t *p, uint32_t v);
46void mesh_rt_fence_sc(void);
47void mesh_rt_fence_acq_rel(void);
48void mesh_rt_nanosleep(uint32_t ns);
49void mesh_rt_warp_sync(void);
50void mesh_rt_bar_sync(void);
51uint8_t *mesh_rt_smem_base(void);
52uint32_t mesh_rt_shfl_idx(uint32_t v, uint32_t src);
53uint32_t mesh_rt_shfl_bfly(uint32_t v, uint32_t m);
54uint32_t mesh_rt_shfl_up(uint32_t v, uint32_t d);
55uint32_t mesh_rt_ballot(bool p);
56_Noreturn void mesh_rt_trap(void);
57
58MESH_RT_INLINE uint32_t mesh_rt_rd(const uint8_t *p) { return *(const volatile uint32_t *)(const volatile void *)p; }
59MESH_RT_INLINE uint64_t mesh_rt_rd64(const uint8_t *p) { return *(const volatile uint64_t *)(const volatile void *)p; }
60MESH_RT_INLINE void mesh_rt_wr(uint8_t *p, uint32_t v) { *(volatile uint32_t *)(volatile void *)p = v; }
61MESH_RT_INLINE void mesh_rt_wrb(uint8_t *p, uint8_t v) { *(volatile uint8_t *)p = v; }
62MESH_RT_INLINE void mesh_rt_wr64(uint8_t *p, uint64_t v) { *(volatile uint64_t *)(volatile void *)p = v; }
63
64/* ---- the launcher */
65
66/* A kernel: `mesh_main` of a program compiled for the host (MESH_PROGRAM). */
67typedef void (*mesh_sim_kernel)(uint8_t *mb, uint32_t slot);
68typedef struct mesh_sim_launch mesh_sim_launch;
69
70/* Starts `nodes` blocks of 32 x `warps` threads running kernel(mb, slot), each
71 * node with RT_SHARED_BYTES + shared_bytes of zeroed shared memory (as meshd's
72 * cuLaunchKernel). Returns at once; NULL if threads could not be created. */
73mesh_sim_launch *mesh_sim_start(mesh_sim_kernel kernel, uint8_t *mb, uint32_t slot, uint32_t nodes, uint32_t warps,
74                                uint32_t shared_bytes);
75/* Waits until every thread of the launch has returned, then frees it. */
76void mesh_sim_join(mesh_sim_launch *l);
77
78#endif /* !__NVPTX__ */
79#endif