1/* The driver's state (driver.rs Driver, programs.rs Image/Slot/Phase), shared by its source files. */
2#ifndef MESH_DRIVER_INT_H
3#define MESH_DRIVER_INT_H
4
5#include "mesh/driver.h"
6
7#include "mesh/abi.h"
8#include "mesh/buf.h"
9#include "mesh/jobs/frame.h"
10#include "mesh/jobs/ranges.h"
11#include "mesh/log.h"
12#include "mesh/manifest_encode.h"
13#include "mesh/placer.h"
14#include "mesh/prog_table.h"
15#include "mesh/publish.h"
16#include "mesh/view_alloc.h"
17
18#include <stdbool.h>
19#include <stddef.h>
20#include <stdint.h>
21
22/* Ticks and bounds, ns. */
23#define TICK_NS 500000ll
24#define FAST_TICK_NS 50000ll
25#define DRAIN_NS 3000000000ll
26#define HALT_NS 3000000000ll
27#define ONLINE_NS 2000000000ll
28#define MAX_QUEUED 256
29#define PIPE_SAMPLE_NS 2000000ll
30#define PIPE_DEADLINE_NS 500000000ll
31
32/* A program image loaded into the context (programs.rs Image). */
33typedef struct {
34    mesh_manifest manifest;
35    char *dir;
36    mesh_kernel *kernel;
37    uint64_t hash;
38    uint32_t code_bytes, regs, per_sm, reg_cap, local, rt_abi, exec;
39} dimage;
40
41typedef enum { PH_STOPPED, PH_STARTING, PH_RUNNING, PH_DRAINING, PH_HALTING } dphase;
42
43typedef struct {
44    uint32_t op, arg;
45    mesh_reply *reply;
46} dtask;
47
48typedef struct {
49    uint8_t *bytes;
50    size_t len;
51    mesh_reply *reply;
52} dreq;
53
54typedef struct {
55    dimage image;
56    bool has_prev;
57    dimage prev;
58    mesh_stream *stream;
59    uint32_t view_off, loads, reads_mask, feeds_mask, fed_mask;
60    dphase phase;
61    int64_t phase_t0;
62    bool want_run;
63    uint32_t run_seq;
64    bool needs_init;
65    char *failed; /* NULL: not failed */
66    bool unloading;
67    mesh_pring ring;
68    int64_t last_auto;
69    uint32_t last_done;
70    float rate;
71    dtask *tasks;
72    size_t ntasks, ctasks;
73    dreq *reqs;
74    size_t nreqs, creqs;
75} dslot;
76
77/* Lifecycle work that needs the JIT worker first. */
78typedef struct {
79    bool upgrade;
80    mesh_src src;
81    int run;        /* load: -1 none, 0, 1 */
82    uint32_t slot;  /* upgrade */
83    mesh_reply *reply;
84} djob;
85
86typedef enum { OP_LOAD, OP_UPGRADE, OP_ROLLBACK, OP_RESET, OP_FREE, OP_REBIND } dop_kind;
87
88typedef struct {
89    dop_kind k;
90    mesh_prepared *p; /* LOAD, UPGRADE */
91    int run;          /* LOAD */
92    uint32_t slot;    /* UPGRADE, ROLLBACK, RESET */
93    mesh_reply *reply;
94    bool has_scope;   /* false: the whole GPU context */
95    uint32_t scope;
96    uint32_t exec;
97    int64_t since;
98} dblink;
99
100typedef struct {
101    uint32_t slot;
102    bool on, unload, now;
103    mesh_reply *reply;
104    int64_t t0;
105} dwaiter;
106
107typedef struct {
108    mesh_kernel *kernel; /* NULL: only a view */
109    uint32_t view_off, view_bytes;
110    uint32_t exec;
111    bool has_inbox; /* the view held an inbox */
112    uint32_t fed;
113    int64_t since;
114    bool warned;
115} dzombie;
116
117typedef struct {
118    uint32_t consumer;
119    mesh_pipe_inbox inbox;
120    uint32_t side, producer, pos;
121} ddead;
122
123typedef struct {
124    uint64_t lease, job;
125    uint32_t op;
126    mj_setup *setup;
127    size_t nsetup;
128    mj_ranges items;
129} dlease;
130
131typedef struct {
132    dlease *q;
133    size_t n, cap;
134    bool has_ctx;
135    uint64_t ctx_job, ctx_window;
136    mesh_setup_state setup;
137    uint32_t inflight;
138} djobslot;
139
140/* A placeable program's placement (place.rs), peered only (driver.rs PSlot). */
141typedef struct {
142    mpl_placer *p;
143    char *name;
144    bool has_follow; /* the follow batch to publish (split, a follower): its frame */
145    uint32_t follow;
146    bool stopped;    /* the placement stopped it here (pin elsewhere): it runs again when that ends */
147    uint32_t part_ns;
148} dplace;
149
150/* A `place` request waiting for its change (by handle). */
151typedef struct {
152    uint64_t h;
153    mesh_reply *reply;
154} dplace_reply;
155
156struct mesh_driver {
157    mesh_shared *sh;
158    mesh_backend *gpu;
159    mesh_backend **cpus;
160    size_t ncpus;
161    mesh_driver_config cfg;
162    int64_t last_stats, last_log;
163    char *fault; /* a kernel faulted: the CUDA context is lost */
164    dslot *slots[MESH_PROG_SLOTS];
165    mesh_view_heap heap;
166    dzombie *zombies;
167    size_t nzombies, czombies;
168    djob *jobs; /* FIFO */
169    size_t njobs, cjobs;
170    bool working;
171    mesh_reply *work_rx;
172    djob work_job;
173    dblink *blinks; /* FIFO */
174    size_t nblinks, cblinks;
175    dwaiter *waiters;
176    size_t nwaiters, cwaiters;
177    uint32_t epoch;
178    uint32_t lock_hi, lock_lo;
179    bool state_dirty, restoring;
180    /* A GET /load/<name> from the http program: its slot, HV_LOAD_SEQ, the load's answer. */
181    bool gpu_load;
182    uint32_t gpu_load_slot, gpu_load_seq;
183    mesh_reply *gpu_load_rx;
184    bool halting;
185    int64_t halting_t;
186    char *startup_error;
187    struct { uintptr_t key; uint64_t cpu; } native_cpu[MESH_PROG_SLOTS];
188    uint32_t me, n;
189    uint32_t fleet_up;
190    bool has_fleet_leader;
191    uint32_t fleet_leader;
192    djobslot jobw[MESH_PROG_SLOTS];
193    /* Per slot: its placement (programs whose manifest declares `place`, when peered). */
194    dplace *place[MESH_PROG_SLOTS];
195    dplace_reply *place_replies;
196    size_t nplace_replies, cplace_replies;
197    uint64_t place_handle;
198    /* The fleet's placement requests (FleetView). */
199    mesh_place_req *fleet_places;
200    size_t nfleet_places;
201    ddead *dead;
202    size_t ndead, cdead;
203};
204
205/* driver.c */
206uint32_t drv_gpu_kind(const mesh_driver *d);
207mesh_backend *drv_be(const mesh_driver *d, uint32_t exec);
208uint32_t drv_exec_of(const mesh_driver *d, uint32_t s);
209bool drv_in_scope(const mesh_driver *d, bool has_scope, uint32_t scope, uint32_t s);
210int drv_find(const mesh_driver *d, const char *name);
211const char *drv_name(const dslot *x);
212bool drv_live(const dslot *x);
213uint32_t drv_state(const dslot *x);
214void drv_entry(const dslot *x, uint8_t out[MESH_PT_BYTES]);
215void drv_pipe(const dslot *x, uint8_t out[MESH_PP_BYTES]);
216void drv_ext(const dslot *x, uint8_t out[MESH_PX_BYTES]);
217bool drv_view_compatible(const mesh_manifest *a, const mesh_manifest *b);
218void drv_answer(mesh_reply *reply, bool ok, const char *text);
219void drv_startup_failed(mesh_driver *d, const mesh_reply *reply, const char *e);
220void drv_image_free(dimage *im);
221void drv_push_blink(mesh_driver *d, dblink b);
222void drv_push_job(mesh_driver *d, djob j);
223void drv_program_cmd(mesh_driver *d, mesh_cmd *c);
224
225/* lifecycle.c */
226void drv_programs_tick(mesh_driver *d, int64_t now);
227void drv_phase_step(mesh_driver *d, uint32_t s, int64_t now, bool in_blink);
228bool drv_halted(mesh_driver *d, uint32_t s, bool *halted, char **err);
229bool drv_launch_slot(mesh_driver *d, uint32_t s, char **err);
230void drv_running(mesh_driver *d, uint32_t s, int64_t t0);
231void drv_stopped(mesh_driver *d, uint32_t s);
232void drv_fail(mesh_driver *d, uint32_t s, char *why);
233void drv_resolve(mesh_driver *d, uint32_t s, bool on, bool ok, const char *text);
234void drv_remove(mesh_driver *d, uint32_t s);
235void drv_collect(mesh_driver *d, uint32_t s);
236void drv_context_fault(mesh_driver *d, const char *e);
237void drv_not_run(mesh_driver *d, uint32_t s, mesh_out *outs, size_t n);
238int drv_input(mesh_driver *d, uint32_t s, const uint8_t *bytes, size_t n, char **text);
239void drv_producer_stopped(mesh_driver *d, uint32_t s);
240uint32_t drv_pipe_quiet(mesh_driver *d, uint32_t fed, int64_t limit_ns);
241void drv_zero_exchange(mesh_driver *d);
242void drv_connect_pipes(mesh_driver *d);
243bool drv_pipe_check(mesh_driver *d, const mesh_manifest *m, bool has_except, uint32_t except, char **err);
244
245/* blink.c */
246void drv_jobs_tick(mesh_driver *d);
247void drv_blink_tick(mesh_driver *d);
248bool drv_op_reset(mesh_driver *d, uint32_t s, char **text);
249bool drv_precheck(mesh_driver *d, const mesh_manifest *m, char **err);
250
251/* place.c */
252const char *drv_role_word(mpm_role r);
253char *drv_placed_here(const mesh_driver *d, uint32_t s);
254void drv_place_words(mesh_driver *d, uint32_t s);
255void drv_place_acts(mesh_driver *d, bool has_s, uint32_t s, mpl_acts acts);
256void drv_sync_place(mesh_driver *d, uint32_t s);
257void drv_place_drop(mesh_driver *d, uint32_t s, const char *name);
258void drv_place_free_all(mesh_driver *d);
259bool drv_place_busy(const mesh_driver *d);
260void drv_place_part_ns(mesh_driver *d);
261void drv_place_log(mesh_driver *d);
262void drv_place_tick(mesh_driver *d);
263void drv_place_frame(mesh_driver *d, uint32_t s, bool own, bool follow);
264void drv_place_follow_lost(mesh_driver *d, uint32_t s);
265bool drv_place_stopped(const mesh_driver *d, const char *name);
266void drv_place_publish(mesh_driver *d, uint32_t s, bool *paused, mpm_role *role, uint32_t *parts);
267void drv_cmd_place(mesh_driver *d, mesh_cmd *c, mesh_reply *reply);
268void drv_cmd_peer_place(mesh_driver *d, mesh_cmd *c);
269void drv_cmd_peer_frame(mesh_driver *d, mesh_cmd *c);
270void drv_cmd_peer_input(mesh_driver *d, mesh_cmd *c);
271void drv_cmd_fleet_view(mesh_driver *d, mesh_cmd *c);
272uint32_t drv_input_to(const mesh_driver *d, uint32_t s);
273
274/* jobs.c */
275void drv_publish_jobs(mesh_driver *d, uint32_t s);
276void drv_job_done(mesh_driver *d, uint64_t lease, uint64_t job, uint64_t item, uint32_t status, uint32_t result, const char *out);
277void drv_jobs_returned(mesh_driver *d, const dlease *l, uint32_t status, const char *why);
278void drv_lease_free(dlease *l);
279uint32_t drv_job_cap(const mesh_manifest *m, uint32_t exec);
280
281#endif