1/* @ajax/mesh-abi-peer-frames: Peer link frame types (F_*), the link version and port, and liveness timing.
2 *
3 * Part of alpha-c-mesh, a C port of alpha-rust-mesh (README.md says from where). */
4#ifndef MESH_ABI_PEER_FRAMES_H
5#define MESH_ABI_PEER_FRAMES_H
67#include <stdbool.h>
8#include <stddef.h>
9#include <stdint.h>
1011#define MESH_PEER_PORT 7701
12/* ---------------------------------------------------------------- peer frames v6 */
13/* The peer link's version, in F_JOIN and F_HELLO. 3 added F_PLACE, F_FRAME,
14 * F_INPUT (placement); 4 is the N-machine cluster (F_JOIN, the cluster size in
15 * F_HELLO, F_PLACE's assignment); 5 added F_LOADREP; 6 is the fleet (plan F3):
16 * partial membership, a leader, the desired state (F_BEAT, F_VOTE_REQ,
17 * F_VOTE, F_STATE, F_STATE_ACK, F_COMMIT, F_CTL, F_CTL_REPLY, F_IMAGE_REQ,
18 * F_IMAGE) and lockstep groups (F_HELLO, F_ROUND per group); F_RUN, F_RESET,
19 * F_TASK, F_LOAD, F_LOADED and F_PING are gone; 7 was the scheduler (plan F4:
20 * F_PERF, F_VIEW_REQ, F_VIEW, 18..20) or the jobs (plan F5: F_JOB_AVAIL ..
21 * F_RESULT, 21..28) alone; 8 is both; 9 carries a move's outcome in F_BEAT;
22 * 10 adds payload results (F_BLOB_PUT .. F_BLOB, 29..32; a result's blob
23 * reference in F_RESULT and the job log); 11 names the job in F_BLOB_ACK (blobs are
24 * referenced per job); 12 adds a node's limits to F_JOIN (max_threads, max_shared:
25 * multi-warp nodes). A peer of another version is refused at JOIN. */
26#define MESH_PEER_VERSION 12u
27/* The fleet: N systems (2..=MAX_SYSTEMS), a full mesh of TCP links. Every
28 * system k > 0 listens on its peer address; system k dials every system j > k.
29 * Each link starts with F_JOIN both ways (ids, version, size, capabilities)
30 * and lives on its own: a member is *up* while its link is (every member
31 * beats every other at PEER_PING_MS; a link silent for PEER_DEAD_MS closes).
32 * Members elect a leader with a majority of N (mesh-host fleet.rs; N = 2: system
33 * 0 always), the leader owns the desired state (the document) and commits
34 * every change on a majority, and each member reconciles its programs to the
35 * committed document. Lockstep groups exchange point to point among their
36 * members (F_HELLO, F_ROUND); placement frames go point to point too.
37 * u8 type then the body:
38 * F_HELLO (1): u32 system, u32 PEER_VERSION, [u8; 16] group, u32 epoch, u32 cycle,
39 * u32 member mask, [u8; 8] rank order (system ids, 0xff past the end),
40 * u64 image hash -- a group member, ready for (epoch, cycle) of its group,
41 * to every other member; the members run once every one's HELLO matches
42 * F_ROUND (2): u32 epoch, u32 cycle, u64 counter, EX_REGION_BYTES -- the sender's
43 * ex_counter(rank) and ex_slots(rank); the receiver drops a round whose
44 * (epoch, cycle) is not its own, and writes it at the sender's rank
45 * F_BEAT (3): u64 term, u32 flags (BEAT_LEADER), u32 leader (PEER_NONE), u64 commit_term,
46 * u64 commit_version, u64 accepted_term, u64 accepted_version, u64 applied
47 * version, u32 RS_* reconcile status, u32 up mask (whom it hears), u32 group
48 * epoch, u32 GL_* group state, u32 reason_len (<= BEAT_REASON_MAX), reason,
49 * u32 moves (<= BEAT_MOVES_MAX), then per program a move placed here (a
50 * migration or a restart): [u8; 16] name, u64 version, u32 ok, u32 gap_us
51 * (BEAT_MOVE_BYTES each; the leader's verdict on that move) --
52 * every member to every member each PEER_PING_MS (liveness, the leader's
53 * authority, catch-up, and the status every `fleet` shows)
54 * F_VOTE_REQ (4): u32 pre (1: a pre-vote), u64 term, u64 accepted_term, u64 accepted_version
55 * F_VOTE (5): u32 pre, u64 term, u32 granted
56 * F_STATE (6): u64 term, u32 len (<= F_STATE_MAX), the document (text) -- the leader's
57 * F_STATE_ACK (7): u64 term, u64 doc_term, u64 doc_version -- persisted
58 * F_COMMIT (8): u64 term, u64 doc_term, u64 doc_version -- committed on a majority
59 * F_PLACE (9): u32 PL_*, u32 id, [u8; 16] name, u32 PLACE_*, u32 sys, [u8; PLACE_PARTS_MAX]
60 * assignment (split: part b's system; else PART_NONE), u32 aux (PL_PAUSED: the
61 * sender's mean ns per part, 0 unknown; PL_SET: the member mask of the
62 * change in bits 0..8 and the system whose state continues in bits 8..16), u32 len, then len bytes (<= F_PLACE_MAX): the state regions
63 * (PL_PAUSED, PL_SET) or a reason (PL_FAIL). A placement change, coordinated
64 * by the leader among the up members
65 * F_FRAME (10): [u8; 16] name, u32 frame, u32 parts (bit b: part b is in data), u32 sys
66 * (the sender), u32 state_len, u32 data_len, then the state regions and,
67 * per part in ascending order, that part of each place_out region
68 * (<= F_PLACE_MAX in all)
69 * F_INPUT (11): [u8; 16] name, u32 len (<= F_INPUT_MAX), bytes -- input forwarded to the
70 * machine that applies it (pin k: k; split: the owner)
71 * F_JOIN (12): u32 system, u32 PEER_VERSION, u32 systems, u32 cc (major * 10 + minor),
72 * u32 sms, u32 sms_total, u32 blocks_per_sm, u32 backend (H_BACKEND; 0 CUDA), [u8; 32] gpu name,
73 * [u8; 16] host arch, [u8; 32] hostname, u32 max_threads, u32 max_shared (a node's
74 * limits: H_MAX_TG_THREADS, H_MAX_SHARED; 0 without a GPU) -- the first frame each way on a
75 * link; a version or size mismatch, or an unexpected system, closes it
76 * F_LOADREP (13): LOADREP_BYTES: the sender's load summary (its member load record
77 * from ML_SHARE on), after each load sample (~1 s), to every linked member;
78 * the receiver stores it at mload(sender) with its own receive time
79 * F_CTL (14): u32 id, u32 len (<= F_CTL_MAX), a control request line -- a change asked
80 * for on a member that does not lead, forwarded to the leader
81 * F_CTL_REPLY (15): u32 id, u32 ok, u32 len (<= F_CTL_REPLY_MAX), the reply text
82 * F_IMAGE_REQ (16): u32 id, [u8; 16] name, u64 hash, u32 dir_len (<= 64), dir -- a member
83 * reconciling asks for an image its catalog does not have at that hash
84 * F_IMAGE (17): u32 id, u64 hash, u32 ok, u32 dir_len, u32 manifest_len, u32 code_len,
85 * then the dir, the manifest and the PTX (ok), or the reason in the manifest
86 * field (not ok); <= F_LOAD_MAX in all
87 * F_PERF (18): u32 len (<= F_PERF_MAX), text -- the sender's scheduler report after each
88 * load sample (~1 s), to every linked member: `caps ...` (CPUs, RAM), `prog
89 * <name> hash .. exec .. task_us ..` per loaded program, `rtt <k> <us>` per
90 * member it measured, `echo <k> <t_us> <held_us>` per member it heard (link
91 * RTT: the receiver of an echo of its own send time t computes now - t - held)
92 * F_VIEW_REQ (19): u32 id, [u8; 16] name, u64 hash -- a migration's target asks the
93 * source (the desired state names both) for the program's view; the source
94 * stops it and answers with F_VIEW
95 * F_VIEW (20): u32 id, u32 ok, u32 view_abi, u32 total, u32 off, u32 len, then len bytes
96 * (<= F_VIEW_CHUNK): the view in chunks (ok), or the reason (not ok)
97 * Jobs (plan F5; mesh-host jobs.rs has the bodies): u8 type, u32 len (<= F_JOB_MAX),
98 * then the body, which starts with the u64 term of the leader it is from or for:
99 * F_JOB_AVAIL (21): member -> leader: the job programs running here (name, executor,
100 * tickets in flight) and the leases it holds (for a new leader to adopt)
101 * F_JOB_STATE (22): leader -> member: the replicated job table: a delta (ops) of seq n,
102 * a snapshot or a term's first entry in numbered parts (the end says how
103 * many: a part lost is never taken for a whole table), or the commit point
104 * F_JOB_ACK (23): member -> leader: entries up to n persisted (or: a gap, send a snapshot)
105 * F_JOB_PULL (24): a new leader -> member: send your whole table
106 * F_JOB_HAVE (25): member -> the new leader: its committed table (with its commit point)
107 * and its entries not committed yet (each with its leadership and number),
108 * in numbered parts, then how many and its leases
109 * F_LEASE (26): leader -> member: run these items (job, attempt, program, op, setup, ranges)
110 * F_LEASE_CUT (27): leader -> member: drop the lease's items from `cut` on (0: all)
111 * F_RESULT (28): member -> leader: item results (ok, failed with a reason, returned unrun);
112 * a payload item's result names its blob (sha256, length, holders)
113 * Blobs (an item's payload, off the job log; mesh-host blobd.rs), u8 type, u32 len
114 * (<= F_BLOB_MAX), then the body (a 32-byte sha256 first). PUT and BLOB carry bytes
115 * and go through the link's bulk queue, behind its control frames (Shared::send_bulk):
116 * F_BLOB_PUT (29): hash, u64 job, u32 len, the bytes -- keep this blob (durably) for that job
117 * F_BLOB_ACK (30): hash, u32 ok, u64 job -- that job's copy stored and synced (ok), or refused (full)
118 * F_BLOB_GET (31): hash, u64 job -- send me this blob
119 * F_BLOB (32): hash, u32 found, the bytes (found) -- the answer */
120#define MESH_F_HELLO 1
121#define MESH_F_ROUND 2
122#define MESH_F_BEAT 3
123#define MESH_F_VOTE_REQ 4
124#define MESH_F_VOTE 5
125#define MESH_F_STATE 6
126#define MESH_F_STATE_ACK 7
127#define MESH_F_COMMIT 8
128#define MESH_F_PLACE 9
129#define MESH_F_FRAME 10
130#define MESH_F_INPUT 11
131#define MESH_F_JOIN 12
132#define MESH_F_LOADREP 13
133#define MESH_F_CTL 14
134#define MESH_F_CTL_REPLY 15
135#define MESH_F_IMAGE_REQ 16
136#define MESH_F_IMAGE 17
137#define MESH_F_PERF 18
138#define MESH_F_VIEW_REQ 19
139#define MESH_F_VIEW 20
140#define MESH_F_JOB_AVAIL 21
141#define MESH_F_JOB_STATE 22
142#define MESH_F_JOB_ACK 23
143#define MESH_F_JOB_PULL 24
144#define MESH_F_JOB_HAVE 25
145#define MESH_F_LEASE 26
146#define MESH_F_LEASE_CUT 27
147#define MESH_F_RESULT 28
148#define MESH_F_BLOB_PUT 29
149#define MESH_F_BLOB_ACK 30
150#define MESH_F_BLOB_GET 31
151#define MESH_F_BLOB 32
152/* Every peer frame type, in id order: the guard below holds them unique and
153 * dense (1..=32), so a branch that adds a frame cannot reuse another's id. */
154#define MESH_FRAMES_LEN 32
155extern const uint8_t MESH_FRAMES[32];
156/* Liveness (defaults; a cluster config's `ping_ms` / `dead_ms` override them):
157 * each member beats every other every PEER_PING_MS; a link that received
158 * nothing for PEER_DEAD_MS closes (a peer that vanished without a FIN: power
159 * loss, a SIGSTOP, a pulled cable), and a member silent for a third of it is
160 * *suspect* (shown, still up). */
161#define MESH_PEER_PING_MS UINT64_C(250)
162#define MESH_PEER_DEAD_MS UINT64_C(3000)
163164/* The Rust const asserts that only involve this package. */
165_Static_assert(MESH_F_VIEW < MESH_F_JOB_AVAIL, "F_VIEW < F_JOB_AVAIL");
166_Static_assert(MESH_F_RESULT == MESH_F_JOB_AVAIL + 7, "F_RESULT == F_JOB_AVAIL + 7");
167_Static_assert(MESH_F_BLOB_PUT == MESH_F_RESULT + 1, "F_BLOB_PUT == F_RESULT + 1");
168_Static_assert(MESH_F_BLOB == MESH_F_BLOB_PUT + 3, "F_BLOB == F_BLOB_PUT + 3");
169170#endif