==> include/mesh/rt/loop.h <==1/* @ajax/mesh-rt-loop: the node loop, the body of every program's kernel (`mesh_main`).
2 *
3 * At block scope: thread 0 of the node takes a ticket from the slot's ring and
4 * waits for its publication (acquire), then writes the descriptor (ticket, op,
5 * arg, completion slot, or "exit") to the runtime's shared area; a block
6 * barrier hands it to every thread (thread 0's sys-scope acquire, then the
7 * barrier, puts every thread's reads after the host's payload). The whole
8 * block runs the op; a barrier (every thread's outputs before it), then thread
9 * 0 completes it: result, release of the mark, P_DONE, the node record and
10 * the trace. The node exits at a task boundary once P_SHUTDOWN is set (the
11 * decision is thread 0's, broadcast like a ticket, so every thread leaves the
12 * loop at the same barrier), and takes no new ticket while H_BLINK is set.
13 *
14 * While idle, thread 0 also measures how much of the GPU the context gets
15 * (the node load records): every poll turn reads the global timer, and a turn
16 * that took over GAP_NS was time the context spent off the GPU.
17 * Ported from alpha-rust-mesh crates/mesh-rt/src/ring.rs. */
18#ifndef MESH_RT_LOOP_H
19#define MESH_RT_LOOP_H
20
21#include <stdint.h>
22
23#include "mesh/abi/control_block.h"
24#include "mesh/abi/header.h"
25#include "mesh/abi/node.h"
26#include "mesh/abi/node_load.h"
27#include "mesh/abi/ops.h"
28#include "mesh/abi/ring.h"
29#include "mesh/abi/trace.h"
30#include "mesh/rt/block.h"
31#include "mesh/rt/ctx.h"
32#include "mesh/rt/prim.h"
33
34/* Idle poll period (publish->start latency is what the host sees); heartbeat
35 * at most every HEARTBEAT_NS while idle. */
36#define MESH_RT_POLL_NS 1000u
37#define MESH_RT_HEARTBEAT_NS UINT64_C(2000000)
38#define MESH_RT_SHUTDOWN_CHECK_NS UINT64_C(20000)
39
40typedef void (*mesh_init_fn)(const mesh_ctx *ctx);
41typedef uint32_t (*mesh_task_fn)(const mesh_ctx *ctx, uint32_t op, uint32_t arg);
42
43MESH_RT_INLINE uint8_t *mesh_rt_entry(uint8_t *ctl, uint32_t t) { return ctl + MESH_P_RING + MESH_ENTRY_BYTES * (t % MESH_PROG_RING); }
44
45/* The body of mesh_main: mb is the mailbox's device address and slot a slot meshd set up. */
46MESH_RT_INLINE void mesh_rt_run(uint8_t *mb, uint32_t slot, mesh_init_fn init, mesh_task_fn task) {
47 mesh_ctx ctx = mesh_ctx_new(mb, slot);
48 /* Thread 0 of the node does everything the host sees (l == 0 below). */
49 uint32_t l = ctx.thread;
50 uint32_t node = ctx.node;
51 uint8_t *desc = mesh_rt_smem_base() + MESH_RT_DESC;
52 uint8_t *ctl = mb + mesh_prog_ctl(slot);
53 /* node_rec(slot, node) as one 32-bit index times NODE_STRIDE: ptxas 13.0
54 * (sm_121) miscompiles mb + (slot << 10) + node * 32 on its uniform
55 * datapath, dropping the high word of mb; a single wide multiply-add is
56 * compiled correctly. */
57 uint8_t *rec = mb + MESH_PROG_NODES + (size_t)MESH_NODE_STRIDE * (slot * (uint32_t)MESH_PROG_MAX_NODES + node);
58 /* The node load record, indexed the same way (one wide multiply-add). */
59 uint8_t *lrec = mb + MESH_NODE_LOAD + (size_t)MESH_NL_STRIDE * (slot * (uint32_t)MESH_PROG_MAX_NODES + node);
60 /* Thread 0's running sums, continued across relaunches like the heartbeat. */
61 uint64_t obs = 0, lost = 0;
62 uint32_t gaps = 0, max_gap = 0, win = 0;
63 if (l == 0) {
64 obs = mesh_rt_rd64(lrec + MESH_NL_OBS_NS);
65 lost = mesh_rt_rd64(lrec + MESH_NL_LOST_NS);
66 gaps = mesh_rt_rd(lrec + MESH_NL_GAPS);
67 max_gap = mesh_rt_rd(lrec + MESH_NL_MAX_GAP_NS);
68 win = mesh_rt_rd(lrec + MESH_NL_WIN);
69 mesh_rt_wr(rec + MESH_N_SM, mesh_rt_smid());
70 mesh_rt_fence_acq_rel();
71 mesh_rt_atom_add(ctl + MESH_P_ONLINE, 1);
72 }
73 /* The heartbeat continues across relaunches (viewers diff it). */
74 uint32_t beat = l == 0 ? mesh_rt_rd(rec + MESH_N_HEARTBEAT) : 0;
75 for (;;) {
76 uint32_t t = 0, go = 1;
77 if (l == 0) {
78 bool have = false;
79 uint64_t beat_at = mesh_rt_gtimer(), check_at = beat_at, prev = beat_at;
80 /* Poll relaxed (an acquire per poll costs fabric bandwidth idle nodes
81 * share); shutdown and heartbeat go by the clock. */
82 for (;;) {
83 if (!have && mesh_rt_ld_relaxed(mb + MESH_H_BLINK) == 0) {
84 t = mesh_rt_atom_add(ctl + MESH_P_RING_HEAD, 1);
85 have = true;
86 }
87 uint8_t *seq = mesh_rt_entry(ctl, t) + MESH_E_SEQ;
88 if (have && mesh_rt_ld_relaxed(seq) == t + 1) break;
89 uint64_t now = mesh_rt_gtimer();
90 /* A turn is a few us; one over GAP_NS is a time slice the context did not get. */
91 uint64_t d = now - prev;
92 prev = now;
93 obs += d;
94 if (d > (uint64_t)MESH_GAP_NS) {
95 lost += d;
96 gaps += 1;
97 uint32_t w = mesh_rt_ld_relaxed(mb + MESH_H_LOAD_WIN);
98 if (w != win) { win = w; max_gap = 0; }
99 uint32_t d32 = d > UINT32_MAX ? UINT32_MAX : (uint32_t)d;
100 if (d32 > max_gap) max_gap = d32;
101 mesh_rt_wr(lrec + MESH_NL_MAX_GAP_NS, max_gap);
102 mesh_rt_wr(lrec + MESH_NL_WIN, win);
103 mesh_rt_wr(lrec + MESH_NL_GAPS, gaps);
104 mesh_rt_wr64(lrec + MESH_NL_LOST_NS, lost);
105 mesh_rt_wr64(lrec + MESH_NL_OBS_NS, obs);
106 }
107 if (now - check_at >= MESH_RT_SHUTDOWN_CHECK_NS) {
108 check_at = now;
109 /* A ticket published before the shutdown still runs. */
110 if (mesh_rt_ld_relaxed(ctl + MESH_P_SHUTDOWN) != 0 && !(have && mesh_rt_ld_relaxed(seq) == t + 1)) {
111 go = 0;
112 break;
113 }
114 }
115 if (now - beat_at >= MESH_RT_HEARTBEAT_NS) {
116 beat_at = now;
117 beat += 1;
118 mesh_rt_st_relaxed(rec + MESH_N_HEARTBEAT, beat);
119 mesh_rt_wr64(lrec + MESH_NL_OBS_NS, obs);
120 }
121 mesh_rt_nanosleep(MESH_RT_POLL_NS);
122 }
123 mesh_rt_wr64(lrec + MESH_NL_OBS_NS, obs);
124 /* The descriptor: the published entry, read after the acquire. */
125 mesh_rt_wr(desc + MESH_RT_D_GO, go);
126 if (go != 0) {
127 uint8_t *e = mesh_rt_entry(ctl, t);
128 (void)mesh_rt_ld_acquire(e + MESH_E_SEQ);
129 mesh_rt_wr(desc + MESH_RT_D_T, t);
130 mesh_rt_wr(desc + MESH_RT_D_OP, mesh_rt_rd(e + MESH_E_OP));
131 mesh_rt_wr(desc + MESH_RT_D_ARG, mesh_rt_rd(e + MESH_E_ARG));
132 mesh_rt_wr(desc + MESH_RT_D_CSLOT, mesh_rt_rd(e + MESH_E_SLOT));
133 }
134 }
135 /* Thread 0's acquire, then the block barrier, puts every thread's reads after the host's payload. */
136 mesh_rt_bar_sync();
137 if (mesh_rt_rd(desc + MESH_RT_D_GO) == 0) break;
138 uint32_t tk = mesh_rt_rd(desc + MESH_RT_D_T), op = mesh_rt_rd(desc + MESH_RT_D_OP), arg = mesh_rt_rd(desc + MESH_RT_D_ARG);
139 uint32_t cslot = mesh_rt_rd(desc + MESH_RT_D_CSLOT) % MESH_PROG_RESULTS;
140
141 ctx.ticket = tk;
142 uint64_t start = mesh_rt_gtimer();
143 uint32_t result;
144 if (op == MESH_OP_INIT) { init(&ctx); result = MESH_R_NONE; }
145 else result = task(&ctx, op, arg);
146
147 /* The barrier orders every thread's outputs before thread 0's release of
148 * the mark (and lets thread 0 rewrite the descriptor: every thread has read it). */
149 mesh_rt_bar_sync();
150 if (l == 0) {
151 uint64_t end = mesh_rt_gtimer();
152 mesh_rt_wr(ctl + MESH_P_RESULTS + 4 * cslot, result);
153 mesh_rt_st_release(ctl + MESH_P_MARKS + 4 * cslot, tk + 1);
154 mesh_rt_red_add(ctl + MESH_P_DONE, 1);
155 mesh_rt_wr(rec + MESH_N_TASKS, mesh_rt_rd(rec + MESH_N_TASKS) + 1);
156 mesh_rt_wr(rec + MESH_N_LAST_OP, op);
157 mesh_rt_wr64(rec + MESH_N_BUSY_NS, mesh_rt_rd64(rec + MESH_N_BUSY_NS) + (end - start));
158 beat += 1;
159 mesh_rt_st_relaxed(rec + MESH_N_HEARTBEAT, beat);
160 uint32_t n = mesh_rt_atom_add(ctl + MESH_P_TRACE_N, 1);
161 uint8_t *tr = mb + mesh_trace_rec(slot, n);
162 mesh_rt_wr(tr, slot | node << 8 | op << 16);
163 mesh_rt_wr(tr + 4, tk);
164 mesh_rt_wr(tr + 8, (uint32_t)start);
165 mesh_rt_wr(tr + 12, (uint32_t)end);
166 }
167 }
168 /* Every thread left at the same barrier (go == 0 for all). */
169 if (l == 0) {
170 mesh_rt_fence_acq_rel();
171 mesh_rt_atom_add(ctl + MESH_P_HALTED, 1);
172 }
173}
174
175#endif