==> src/fleet.c <==1/* ui_fleet.rs and ui.rs's draw_fleet_snap, ported. */
2#include "mesh/top_ui_fleet.h"
3
4#include <math.h>
5#include <stdlib.h>
6#include <string.h>
7
8const char *mt_bar(mt_frame *f, double frac, size_t w) {
9 double fr = isfinite(frac) ? (frac < 0.0 ? 0.0 : frac > 1.0 ? 1.0 : frac) : 0.0;
10 size_t eighths = (size_t)mt_sat_u64(round(fr * (double)(w * 8)));
11 static const char *const part[8] = {" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"};
12 char *o = tui_arena_alloc(f->a, 3 * (w + eighths / 8 + 2) + 1);
13 size_t chars = 0;
14 for (size_t i = 0; i < eighths / 8; i++, chars++) strcat(o, "█");
15 if (eighths % 8 != 0 && chars < w) {
16 strcat(o, part[eighths % 8]);
17 chars++;
18 }
19 while (chars < w) {
20 strcat(o, " ");
21 chars++;
22 }
23 return o;
24}
25
26const char *mt_cut(mt_frame *f, const char *s, size_t w) {
27 if (rsfmt_chars(s) <= w) return tui_arena_strdup(f->a, s);
28 if (w == 0) return "";
29 return MT_FMT(f, "%s…", mt_take(f, s, w - 1));
30}
31
32/* Rows of area, top to bottom: each a height (cut to what is left), -1 the rest (at most one). */
33static void split_v(tui_rect area, const int *hs, size_t n, tui_rect *out) {
34 uint32_t fixed = 0;
35 for (size_t i = 0; i < n; i++)
36 if (hs[i] >= 0) fixed += (uint32_t)hs[i];
37 uint16_t rest = area.height > fixed ? (uint16_t)(area.height - fixed) : 0;
38 uint16_t y = area.y, end = (uint16_t)(area.y + area.height);
39 for (size_t i = 0; i < n; i++) {
40 uint16_t h = hs[i] >= 0 ? (uint16_t)hs[i] : rest;
41 if (h > end - y) h = (uint16_t)(end - y);
42 out[i] = mt_rect(area.x, y, area.width, h);
43 y = (uint16_t)(y + h);
44 }
45}
46
47static const char *secs_text(mt_frame *f, double x) {
48 char a[64], b[64];
49 if (x >= 3600.0) {
50 rsfmt_fixed(a, sizeof a, floor(x / 3600.0), 0, 0);
51 rsfmt_fixed_w(b, sizeof b, fmod(x, 3600.0) / 60.0, 0, 2, '>', RSFMT_ZERO);
52 return MT_FMT(f, "%sh%sm", a, b);
53 }
54 if (x >= 60.0) {
55 rsfmt_fixed(a, sizeof a, floor(x / 60.0), 0, 0);
56 rsfmt_fixed_w(b, sizeof b, fmod(x, 60.0), 0, 2, '>', RSFMT_ZERO);
57 return MT_FMT(f, "%sm%ss", a, b);
58 }
59 return MT_FMT(f, "%ss", mt_ff(f, x, 1));
60}
61
62static const char *pct3(mt_frame *f, double v) {
63 char b[64];
64 rsfmt_fixed_w(b, sizeof b, v, 0, 3, '>', 0);
65 return MT_FMT(f, " %s%%", b);
66}
67
68/* ---- view records */
69
70static const mt_vrec EMPTY_REC = {"", NULL, 0, ""};
71
72static const char *rget(const mt_vrec *r, const char *k) { return mt_vrec_get(r, k); }
73static const char *rs(const mt_vrec *r, const char *k) { return mt_vrec_s(r, k); }
74static bool ru(const mt_vrec *r, const char *k, uint64_t *v) { return mt_vrec_u(r, k, v); }
75static bool rf(const mt_vrec *r, const char *k, double *v) { return mt_vrec_f(r, k, v); }
76static bool streq_opt(const char *a, const char *b) { return a == b || (a && b && strcmp(a, b) == 0); }
77
78/* The records of one kind. */
79static size_t all(mt_frame *f, const mt_view *v, const char *kind, const mt_vrec ***out) {
80 const mt_vrec **o = tui_arena_alloc(f->a, (v->n ? v->n : 1) * sizeof *o);
81 size_t n = 0;
82 for (size_t i = 0; i < v->n; i++)
83 if (strcmp(v->recs[i].kind, kind) == 0) o[n++] = &v->recs[i];
84 *out = o;
85 return n;
86}
87
88static tui_span source(mt_frame *f, const mt_app *app, const mt_view *v, mt_instant now) {
89 const char *m = v->machine < app->nmachines ? mt_name(f, &app->machines[v->machine]) : "?";
90 if (v->err) return mt_fgs(MT_FMT(f, " %s: %s ", m, v->err), MT_BAD);
91 mt_duration d = now > v->at ? now - v->at : 0;
92 return mt_label(MT_FMT(f, " view of %s, %s s ago ", m, mt_ff(f, mt_secs_f64(d), 1)));
93}
94
95static void par_wrap(mt_frame *f, tui_text t, const tui_block *b, tui_rect area) {
96 tui_paragraph p = tui_paragraph_new(t);
97 p.wrap = TUI_WRAP_TRIM;
98 p.block = b;
99 tui_paragraph_render(f->a, &p, area, f->buf);
100}
101
102static void par_block(mt_frame *f, tui_text t, const tui_block *b, tui_rect area) {
103 tui_paragraph p = tui_paragraph_new(t);
104 p.block = b;
105 tui_paragraph_render(f->a, &p, area, f->buf);
106}
107
108/* Waiting for a view: which machine is asked. */
109static void waiting(mt_frame *f, tui_rect area, const mt_app *app, size_t view) {
110 size_t mi = mt_app_view_machine(app, view);
111 const char *m = mi < app->nmachines ? mt_name(f, &app->machines[mi]) : "?";
112 const char *vn = MT_VIEWS[view];
113 tui_line t = MT_LINE(f, mt_label("asking "), mt_val(m),
114 mt_label(MT_FMT(f, " for `view %s` (meshd's %s view; http /%s serves the same)", vn, vn, vn)));
115 tui_block b = mt_panel(f, tui_line_span(f->a, mt_label(MT_FMT(f, " %s ", vn))), MT_FAINT);
116 par_wrap(f, tui_text_line(f->a, t), &b, area);
117}
118
119static tui_style vstate_style(const char *s) {
120 static const char *const good[] = {"up", "running", "in-sync", "chosen", "eligible", "done"};
121 static const char *const warn[] = {"suspect", "pending", "linked", "hello", "feasible", "cancelling", "leader-merging"};
122 static const char *const bad[] = {"down", "failed", "paused", "rejected", "deadline", "done-with-failures", "cancelled"};
123 for (size_t i = 0; i < sizeof good / sizeof *good; i++)
124 if (strcmp(s, good[i]) == 0) return tui_fg(MT_GOOD);
125 for (size_t i = 0; i < sizeof warn / sizeof *warn; i++)
126 if (strcmp(s, warn[i]) == 0) return tui_fg(MT_WARN);
127 for (size_t i = 0; i < sizeof bad / sizeof *bad; i++)
128 if (strcmp(s, bad[i]) == 0) return tui_fg(MT_BAD);
129 return tui_fg(MT_DIM);
130}
131
132static const char *mb_gb(mt_frame *f, const mt_vrec *r, const char *k) {
133 uint64_t m;
134 return ru(r, k, &m) ? mt_ff(f, (double)m / 1024.0, 0) : "-";
135}
136
137static void draw_table(mt_frame *f, const mt_rows *rows, const char *const *header, size_t nh, const tui_constraint *w, size_t nw,
138 const tui_block *b, tui_rect area, tui_table_state *st) {
139 tui_table_row hdr = mt_header(f, header, nh);
140 tui_table t = tui_table_new(rows->rows, rows->n, w, nw);
141 t.header = &hdr;
142 t.column_spacing = 1;
143 t.block = b;
144 tui_table_state none = {0};
145 tui_table_render(f->a, &t, area, f->buf, st ? st : &none);
146}
147
148/* ---------------------------------------------------------------- 9 fleet */
149
150/* The placement matrix's cell: a glyph (running ●, placed but not running ○, running but not
151 * placed ◌, nothing ·), the executor's letter, then flags. */
152static const char *cell_text(mt_frame *f, const mt_vrec *c, tui_style *st) {
153 bool placed = mt_vrec_flag(c, "placed");
154 const char *state = rs(c, "state");
155 bool running = strcmp(state, "running") == 0;
156 const char *glyph = placed && running ? "●" : placed ? "○" : (running || strcmp(state, "stopped") == 0) ? "◌" : "·";
157 const char *e = rget(c, "exec");
158 char exec = !e ? ' ' : !strcmp(e, "cuda") ? 'C' : !strcmp(e, "metal") ? 'M' : !strcmp(e, "wasm") ? 'W' : !strcmp(e, "native") ? 'N' : ' ';
159 char flags[8];
160 size_t n = 0;
161 if (mt_vrec_flag(c, "pinned")) flags[n++] = 'p';
162 const char *mv = rget(c, "moving");
163 if (mv && !strcmp(mv, "to")) flags[n++] = '>';
164 else if (mv && !strcmp(mv, "from")) flags[n++] = '<';
165 if (rget(c, "reserved")) flags[n++] = 'r';
166 flags[n] = 0;
167 if (mv) *st = tui_fg_bold(MT_ACCENT);
168 else if (placed && running) *st = tui_fg(MT_GOOD);
169 else if (placed) *st = tui_fg(MT_WARN);
170 else if (running) *st = tui_fg(MT_BAD);
171 else *st = tui_fg(MT_FAINT);
172 return MT_FMT(f, "%s%c%s", glyph, exec, mt_pad(f, mt_cut(f, flags, 2), 2, '<'));
173}
174
175void mt_draw_fleet(mt_frame *f, tui_rect area, const mt_app *app, mt_instant now) {
176 const mt_view *v = mt_app_view(app, 0);
177 if (!v) {
178 if (area.height >= 12) {
179 tui_rect r[2];
180 tui_vsplit(area, 2, (tui_constraint[]){TUI_LENGTH(3), TUI_FILL(1)}, r);
181 waiting(f, r[0], app, 0);
182 mt_draw_fleet_snap(f, r[1], app);
183 } else {
184 waiting(f, area, app, 0);
185 }
186 return;
187 }
188 const mt_vrec *head = mt_view_first(v, "fleet");
189 if (!head) head = &EMPTY_REC;
190 bool single = streq_opt(rget(head, "role"), "single");
191 const mt_vrec **members, **progs;
192 size_t nmem = all(f, v, "member", &members), nprog = all(f, v, "program", &progs);
193 char *sel_p;
194 bool has_sel_m;
195 uint32_t sel_m = 0;
196 mt_app_fleet_sel(app, &sel_p, &has_sel_m, &sel_m);
197 const char *sel_pa = sel_p ? tui_arena_strdup(f->a, sel_p) : NULL;
198 free(sel_p);
199 /* The headline: the leader, the desired state, the scheduler's switches. */
200 uint64_t leader;
201 bool has_leader = ru(head, "leader", &leader);
202 tui_line l1 = MT_LINE(f, mt_label("leader "),
203 has_leader ? mt_fgb(MT_FMT(f, "%u", (uint32_t)leader), mt_sys_color((uint32_t)leader)) : mt_fgb("none", MT_BAD),
204 mt_label(" term "), mt_val(rs(head, "term")), mt_label(" version "), mt_val(rs(head, "version")), mt_label(" applied "),
205 mt_fgs(rs(head, "applied"), streq_opt(rget(head, "applied"), rget(head, "version")) ? MT_GOOD : MT_WARN),
206 mt_label(" quorum "), mt_val(rs(head, "quorum")), mt_label(" up "), mt_val(rs(head, "up")), mt_label(" suspect "),
207 mt_fgs(rs(head, "suspect"), rget(head, "suspect") ? MT_WARN : MT_DIM), mt_label(" majority "),
208 mt_styled(rs(head, "majority"), streq_opt(rget(head, "majority"), "no") ? tui_fg_bold(MT_BAD) : tui_fg(MT_GOOD)));
209 if (single || area.height < 6 || area.width < 40) {
210 /* Tiny (or a single meshd): the headline only. */
211 mt_lines lines = {0};
212 mt_lines_push(f, &lines, l1);
213 if (single) {
214 mt_lines_push(f, &lines, tui_line_span(f->a, mt_label(head->text)));
215 } else {
216 size_t up = 0;
217 for (size_t i = 0; i < nmem; i++) up += streq_opt(rget(members[i], "state"), "up");
218 mt_lines_push(f, &lines, MT_LINE(f, mt_label("members up "), mt_val(MT_FMT(f, "%zu/%zu", up, nmem)), mt_label(" programs "),
219 mt_val(MT_FMT(f, "%zu", nprog))));
220 }
221 par_wrap(f, mt_text(f, &lines), NULL, area);
222 return;
223 }
224 bool paused = mt_vrec_flag(head, "paused");
225 const char *gap = rget(head, "last_gap_ms");
226 tui_line l2 = MT_LINE(f, mt_label("scheduler "), mt_styled(paused ? "PAUSED" : "running", paused ? tui_fg_bold(MT_BAD) : tui_fg(MT_GOOD)),
227 mt_label(" autoreform "), mt_val(mt_vrec_flag(head, "autoreform") ? "on" : "off"), mt_label(" cordon "),
228 mt_styled(rs(head, "cordon"), rget(head, "cordon") ? tui_fg_bold(MT_WARN) : tui_fg(MT_DIM)), mt_label(" busy "),
229 mt_styled(rs(head, "busy_members"), rget(head, "busy_members") ? tui_fg_bold(MT_WARN) : tui_fg(MT_DIM)),
230 mt_label(" migrations "), mt_val(rs(head, "migrations")), mt_label(" last gap "),
231 mt_val(gap ? MT_FMT(f, "%s ms", gap) : "-"), source(f, app, v, now));
232 const mt_vrec **groups;
233 size_t ngroups = all(f, v, "group", &groups);
234 int want_groups = ngroups == 0 ? 0 : (int)(ngroups * 2 + 2 < 6 ? ngroups * 2 + 2 : 6);
235 tui_rect sp[4];
236 split_v(area, (int[]){2, (int)nmem + 3, -1, area.height >= 24 ? want_groups : 0}, 4, sp);
237 tui_rect hd = sp[0], mem = sp[1], mid = sp[2], grp = sp[3];
238 mt_par(f, tui_text_of(f->a, 2, (tui_line[]){l1, l2}), hd);
239 /* The members: link state, flags, executor and caps, the smoothed load, reconcile. */
240 bool wide = area.width >= 96;
241 const char *header[12];
242 size_t nh = 0;
243 const char *h0[5] = {"sys", "host", "state", "flags", "exec"};
244 for (size_t i = 0; i < 5; i++) header[nh++] = h0[i];
245 if (wide) {
246 header[nh++] = "cpus";
247 header[nh++] = "mem GB";
248 }
249 header[nh++] = "gpu share";
250 header[nh++] = "cpu";
251 header[nh++] = "psi";
252 header[nh++] = "reconcile (applied) / why";
253 mt_rows rows = {0};
254 for (size_t i = 0; i < nmem; i++) {
255 const mt_vrec *m = members[i];
256 uint64_t id = 0;
257 ru(m, "id", &id);
258 uint32_t k = (uint32_t)id;
259 char flags[4];
260 size_t fl = 0;
261 if (mt_vrec_flag(m, "leader")) flags[fl++] = 'L';
262 if (mt_vrec_flag(m, "cordoned")) flags[fl++] = 'C';
263 if (mt_vrec_flag(m, "busy")) flags[fl++] = 'B';
264 flags[fl] = 0;
265 mt_cells cells = {0};
266 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgb(MT_FMT(f, "%u", k), mt_sys_color(k))));
267 mt_cells_push(f, &cells, mt_cell_str(f, mt_cut(f, rs(m, "host"), 14)));
268 mt_cells_push(f, &cells, mt_cell_span(f, mt_styled(rs(m, "state"), vstate_style(rs(m, "state")))));
269 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgb(tui_arena_strdup(f->a, flags),
270 mt_vrec_flag(m, "cordoned") || mt_vrec_flag(m, "busy") ? MT_WARN : MT_ACCENT)));
271 mt_cells_push(f, &cells, mt_cell_str(f, rs(m, "gpu")));
272 if (wide) {
273 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%s/%s", rs(m, "big"), rs(m, "little"))));
274 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%s/%s", mb_gb(f, m, "avail_mb"), mb_gb(f, m, "ram_mb"))));
275 }
276 double x;
277 if (rf(m, "share", &x))
278 mt_cells_push(f, &cells, mt_cell_line(f, MT_LINE(f, mt_styled(mt_bar(f, x, 5), mt_share_style((float)x)),
279 mt_styled(MT_FMT(f, " %s", mt_pct(f, (float)x)), mt_share_style((float)x)))));
280 else
281 mt_cells_push(f, &cells, mt_cell_line(f, tui_line_span(f->a, mt_fgs("-", MT_FAINT))));
282 mt_cells_push(f, &cells, mt_cell_span(f, rf(m, "cpu", &x) ? mt_styled(mt_pct(f, (float)x), mt_busy_style((float)x)) : mt_fgs("-", MT_FAINT)));
283 mt_cells_push(f, &cells, mt_cell_span(f, rf(m, "psi_cpu", &x) ? mt_styled(mt_ff(f, x, 2), mt_busy_style((float)(x * 4.0))) : mt_fgs("-", MT_FAINT)));
284 const char *rec = rs(m, "reconcile");
285 tui_line rc = MT_LINE(f, mt_styled(rec, vstate_style(rec)), mt_label(MT_FMT(f, " (%s)", rs(m, "applied"))));
286 if (*m->text) tui_line_push(f->a, &rc, mt_fgs(MT_FMT(f, " %s", m->text), mt_vrec_flag(m, "busy") ? MT_WARN : MT_DIM));
287 mt_cells_push(f, &cells, mt_cell_line(f, rc));
288 tui_table_row row = mt_row(f, cells.c, cells.n);
289 if (has_sel_m && sel_m == k) row.style = tui_style_bg(tui_style_new(), MT_FAINT);
290 mt_rows_push(f, &rows, row);
291 }
292 tui_constraint widths[12];
293 size_t nw = 0;
294 const uint16_t w0[5] = {3, 14, 7, 5, 6};
295 for (size_t i = 0; i < 5; i++) widths[nw++] = TUI_LENGTH(w0[i]);
296 if (wide) {
297 widths[nw++] = TUI_LENGTH(5);
298 widths[nw++] = TUI_LENGTH(7);
299 }
300 widths[nw++] = TUI_LENGTH(10);
301 widths[nw++] = TUI_LENGTH(4);
302 widths[nw++] = TUI_LENGTH(5);
303 widths[nw++] = TUI_FILL(1);
304 tui_block mb = mt_panel(f, MT_LINE(f, mt_fgb(" members ", MT_FG), mt_label("L leader C cordoned B busy (owner policy) ")), MT_FAINT);
305 draw_table(f, &rows, header, nh, widths, nw, &mb, mem, NULL);
306 if (mid.height < 3) return;
307 /* The placement matrix (programs x members) and, beside it, the selected program's explain
308 * and the decisions. */
309 uint32_t *ids = tui_arena_alloc(f->a, (nmem ? nmem : 1) * sizeof *ids);
310 size_t nids = 0;
311 for (size_t i = 0; i < nmem; i++) {
312 uint64_t id;
313 if (ru(members[i], "id", &id)) ids[nids++] = (uint32_t)id;
314 }
315 size_t name_w = 7;
316 if (nprog) {
317 name_w = 0;
318 for (size_t i = 0; i < nprog; i++) {
319 size_t c = rsfmt_chars(rs(progs[i], "name"));
320 if (c > name_w) name_w = c;
321 }
322 }
323 if (name_w < 7) name_w = 7;
324 if (name_w > 14) name_w = 14;
325 uint16_t mat_w = tui_min16((uint16_t)(name_w + 2 + 5 * nids + 6), mid.width);
326 bool side = mid.width >= mat_w + 30;
327 tui_rect hs[2];
328 tui_hsplit(mid, 2, (tui_constraint[]){TUI_LENGTH(side ? mat_w : mid.width), TUI_FILL(1)}, hs);
329 tui_rect mat = hs[0], why = hs[1];
330 mt_lines lines = {0};
331 tui_line hl = tui_line_span(f->a, mt_fgs(mt_pad(f, "program", (int)name_w + 1, '<'), MT_DIM));
332 for (size_t i = 0; i < nids; i++) tui_line_push(f->a, &hl, mt_fgb(mt_pad(f, MT_FMT(f, "%u", ids[i]), 5, '<'), mt_sys_color(ids[i])));
333 mt_lines_push(f, &lines, hl);
334 size_t rows_fit = tui_sat_sub(mat.height, 3);
335 size_t a = rows_fit > 0 ? rows_fit - 1 : 0;
336 size_t first = app->fsel[0] > a ? app->fsel[0] - a : 0;
337 size_t cap = nprog > (rows_fit > 1 ? rows_fit : 1) ? nprog - (rows_fit > 1 ? rows_fit : 1) : 0;
338 if (first > cap) first = cap;
339 const mt_vrec **cells_r;
340 size_t ncells = all(f, v, "cell", &cells_r);
341 for (size_t pi = first; pi < nprog && pi < first + rows_fit; pi++) {
342 const char *name = rs(progs[pi], "name");
343 bool selp = sel_pa && strcmp(sel_pa, name) == 0;
344 tui_line l = tui_line_span(f->a, mt_styled(MT_FMT(f, "%s ", mt_pad(f, mt_cut(f, name, name_w), (int)name_w, '<')),
345 selp ? mt_fg_bg_bold(MT_BG, MT_ACCENT) : tui_fg(MT_FG)));
346 for (size_t ki = 0; ki < nids; ki++) {
347 const mt_vrec *c = NULL;
348 for (size_t q = 0; q < ncells && !c; q++) {
349 uint64_t mm;
350 if (streq_opt(rget(cells_r[q], "program"), name) && ru(cells_r[q], "member", &mm) && mm == ids[ki]) c = cells_r[q];
351 }
352 tui_style st = tui_fg(MT_FAINT);
353 const char *t = c ? cell_text(f, c, &st) : "· ";
354 if (selp && has_sel_m && sel_m == ids[ki]) st = tui_style_add(st, TUI_REVERSED);
355 tui_line_push(f->a, &l, mt_styled(MT_FMT(f, "%s ", t), st));
356 }
357 mt_lines_push(f, &lines, l);
358 }
359 if (nprog == 0) mt_lines_push(f, &lines, tui_line_span(f->a, mt_label("no programs in the desired state")));
360 size_t shown = nprog < rows_fit ? nprog : rows_fit;
361 const char *more = shown < nprog ? MT_FMT(f, "%zu-%zu of %zu (↑↓) ", first + 1, first + shown, nprog) : "";
362 tui_block pb = mt_panel(f, MT_LINE(f, mt_fgb(" placement ", MT_FG), mt_fgs(more, MT_WARN),
363 mt_label("● run ○ placed ◌ leaving; C/M/W/N exec; p pin >< moving r reserved ")),
364 MT_FAINT);
365 par_block(f, mt_text(f, &lines), &pb, mat);
366 if (side) {
367 mt_lines l = {0};
368 const mt_vrec *p = NULL;
369 for (size_t i = 0; i < nprog && sel_pa && !p; i++)
370 if (streq_opt(rget(progs[i], "name"), sel_pa)) p = progs[i];
371 if (p) {
372 mt_lines_push(f, &l, MT_LINE(f, mt_fgb(MT_FMT(f, "%s ", rs(p, "name")), MT_FG), mt_label("at "), mt_val(rs(p, "at")), mt_label(" pin "),
373 mt_val(rs(p, "pin")), mt_label(" replicas "), mt_val(rs(p, "replicas")), mt_label(" migrate "),
374 mt_val(rs(p, "migrate"))));
375 if (*p->text) mt_lines_push(f, &l, tui_line_span(f->a, mt_fgs(p->text, MT_DIM)));
376 const mt_vrec **ex;
377 size_t nex = all(f, v, "explain", &ex);
378 for (size_t i = 0; i < nex; i++) {
379 if (!streq_opt(rget(ex[i], "program"), rget(p, "name"))) continue;
380 uint64_t km = 0;
381 ru(ex[i], "member", &km);
382 uint32_t k = (uint32_t)km;
383 const char *verdict = rs(ex[i], "verdict");
384 mt_lines_push(f, &l, MT_LINE(f, mt_fgb(MT_FMT(f, "%u ", k), mt_sys_color(k)), mt_styled(mt_pad(f, verdict, 9, '<'), vstate_style(verdict)),
385 mt_fgs(ex[i]->text, strcmp(verdict, "rejected") == 0 ? MT_DIM : MT_FG)));
386 }
387 }
388 mt_lines notes = {0};
389 const mt_vrec **rr;
390 size_t nr = all(f, v, "proposal", &rr);
391 for (size_t i = 0; i < nr; i++) mt_lines_push(f, ¬es, MT_LINE(f, mt_fgb("proposal ", MT_WARN), tui_span_raw(rr[i]->text)));
392 nr = all(f, v, "migration", &rr);
393 for (size_t i = 0; i < nr; i++)
394 mt_lines_push(f, ¬es, MT_LINE(f, mt_fgb("migrating ", MT_ACCENT),
395 mt_val(MT_FMT(f, "%s %s→%s ", rs(rr[i], "programs"), rs(rr[i], "from"), rs(rr[i], "to"))),
396 mt_label(MT_FMT(f, "%s s", rs(rr[i], "secs")))));
397 nr = all(f, v, "note", &rr);
398 for (size_t i = 0; i < nr; i++) mt_lines_push(f, ¬es, MT_LINE(f, mt_label("note "), tui_span_raw(rr[i]->text)));
399 if (notes.n) {
400 mt_lines_push(f, &l, tui_line_from_str(f->a, ""));
401 for (size_t i = 0; i < notes.n; i++) mt_lines_push(f, &l, notes.l[i]);
402 }
403 nr = all(f, v, "decision", &rr);
404 if (nr) {
405 mt_lines_push(f, &l, tui_line_span(f->a, mt_fgb("decisions (newest first)", MT_FG)));
406 for (size_t i = 0; i < nr; i++)
407 mt_lines_push(f, &l, MT_LINE(f, mt_label(MT_FMT(f, "%ss ", mt_pad(f, rs(rr[i], "at_s"), 7, '>'))), tui_span_raw(rr[i]->text)));
408 }
409 tui_block wb = mt_panel(f, MT_LINE(f, mt_fgb(" why ", MT_FG), mt_label("the selected program's explain, the scheduler's decisions ")), MT_FAINT);
410 par_wrap(f, mt_text(f, &l), &wb, why);
411 }
412 if (grp.height > 0) {
413 mt_lines l = {0};
414 const mt_vrec **gms;
415 size_t ngm = all(f, v, "gmember", &gms);
416 for (size_t gi = 0; gi < ngroups; gi++) {
417 const mt_vrec *g = groups[gi];
418 const char *name = rs(g, "name");
419 mt_lines_push(f, &l, MT_LINE(f, mt_fgb(MT_FMT(f, "%s ", name), MT_FG), mt_label("epoch "), mt_val(rs(g, "epoch")), mt_label(" cycle "),
420 mt_val(rs(g, "cycle")), mt_label(" members "), mt_val(rs(g, "members")), mt_label(" state "),
421 mt_styled(rs(g, "state"), vstate_style(rs(g, "state"))), mt_label(" on loss "),
422 mt_val(streq_opt(rget(g, "on_loss"), "reform") ? MT_FMT(f, "reform after %s ms", rs(g, "auto_ms")) : "pause")));
423 tui_line per = tui_line_span(f->a, mt_label("ranks "));
424 for (size_t i = 0; i < ngm; i++) {
425 if (!streq_opt(rget(gms[i], "group"), name)) continue;
426 uint64_t km = 0;
427 ru(gms[i], "member", &km);
428 uint32_t k = (uint32_t)km;
429 tui_line_push(f->a, &per, mt_fgb(MT_FMT(f, "%u", k), mt_sys_color(k)));
430 tui_line_push(f->a, &per, mt_label(MT_FMT(f, "#%s ", rs(gms[i], "rank"))));
431 tui_line_push(f->a, &per, mt_styled(MT_FMT(f, "%s@%s ", rs(gms[i], "state"), rs(gms[i], "epoch")), vstate_style(rs(gms[i], "state"))));
432 }
433 mt_lines_push(f, &l, per);
434 }
435 tui_block gb = mt_panel(f, tui_line_span(f->a, mt_fgb(" lockstep groups ", MT_FG)), MT_FAINT);
436 par_block(f, mt_text(f, &l), &gb, grp);
437 }
438}
439
440/* ---------------------------------------------------------------- 0 jobs */
441
442static const char *rate_or_dash(mt_frame *f, const mt_vrec *r, const char *k) {
443 double x;
444 return rf(r, k, &x) ? mt_fmt_rate(f, (float)x) : "-";
445}
446
447void mt_draw_jobs(mt_frame *f, tui_rect area, const mt_app *app, mt_instant now) {
448 const mt_view *v = mt_app_view(app, 1);
449 if (!v) {
450 waiting(f, area, app, 1);
451 return;
452 }
453 const mt_vrec *head = mt_view_first(v, "jobs");
454 if (!head) head = &EMPTY_REC;
455 const mt_vrec **jobs;
456 size_t nj = all(f, v, "job", &jobs);
457 size_t active = 0;
458 for (size_t i = 0; i < nj; i++) active += streq_opt(rget(jobs[i], "state"), "running") || streq_opt(rget(jobs[i], "state"), "cancelling");
459 tui_line l1 = MT_LINE(f, mt_label("jobs "), mt_val(MT_FMT(f, "%zu", nj)), mt_label(" active "), mt_val(MT_FMT(f, "%zu", active)), mt_label(" leader "),
460 mt_val(rs(head, "leader")), mt_label(" ("), mt_fgs(rs(head, "role"), MT_ACCENT), mt_label(") commit "),
461 mt_val(rs(head, "commit")), mt_label(" items/s "), mt_val(rate_or_dash(f, head, "rate")), mt_label(" leases held "),
462 mt_val(rs(head, "held")), source(f, app, v, now));
463 if (area.height < 6 || area.width < 40) {
464 mt_lines lines = {0};
465 mt_lines_push(f, &lines, l1);
466 if (nj) {
467 const mt_vrec *j = jobs[0];
468 double p;
469 const char *prog = rf(j, "progress", &p) ? MT_FMT(f, "%s%%", mt_ff(f, p * 100.0, 0)) : "endless";
470 mt_lines_push(f, &lines, MT_LINE(f, mt_val(rs(j, "job")), mt_label(" "), mt_val(rs(j, "program")), mt_label(" "),
471 mt_styled(rs(j, "state"), vstate_style(rs(j, "state"))), mt_label(MT_FMT(f, " %s", prog))));
472 }
473 par_wrap(f, mt_text(f, &lines), NULL, area);
474 return;
475 }
476 char *sel_m = mt_app_job_sel(app);
477 const char *sel = sel_m ? tui_arena_strdup(f->a, sel_m) : NULL;
478 free(sel_m);
479 int tblh = (int)(nj ? nj : 1) + 3;
480 tui_rect sp[3];
481 split_v(area, (int[]){1, tblh < 12 ? tblh : 12, -1}, 3, sp);
482 tui_rect hd = sp[0], tbl = sp[1], detail = sp[2];
483 mt_par_line(f, l1, hd);
484 bool wide = area.width >= 110, narrow = area.width < 96;
485 const char *header[11];
486 size_t nh = 0;
487 const char *h0[6] = {"job", "program", "state", "progress", "done / items", "items/s"};
488 for (size_t i = 0; i < 6; i++) header[nh++] = h0[i];
489 if (!narrow) {
490 header[nh++] = "ETA";
491 header[nh++] = "failed";
492 header[nh++] = "retried";
493 header[nh++] = "dupes";
494 }
495 if (wide) header[nh++] = "prio";
496 mt_rows rows = {0};
497 for (size_t i = 0; i < nj; i++) {
498 const mt_vrec *j = jobs[i];
499 const char *st = rs(j, "state");
500 double p;
501 mt_cells cells = {0};
502 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgb(rs(j, "job"), MT_FG)));
503 mt_cells_push(f, &cells, mt_cell_str(f, mt_cut(f, rs(j, "program"), 10)));
504 mt_cells_push(f, &cells, mt_cell_span(f, mt_styled(mt_cut(f, st, 9), vstate_style(st))));
505 if (rf(j, "progress", &p))
506 mt_cells_push(f, &cells, mt_cell_line(f, MT_LINE(f, mt_fgs(mt_bar(f, p, 8), strcmp(st, "running") == 0 ? MT_ACCENT : MT_GOOD),
507 mt_label(pct3(f, p * 100.0)))));
508 else
509 mt_cells_push(f, &cells, mt_cell_line(f, tui_line_span(f->a, mt_fgs("endless", MT_DIM))));
510 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%s / %s", rs(j, "done"), rs(j, "items"))));
511 mt_cells_push(f, &cells, mt_cell_str(f, rate_or_dash(f, j, "rate")));
512 if (!narrow) {
513 double eta;
514 uint64_t u;
515 mt_cells_push(f, &cells, mt_cell_str(f, rf(j, "eta_s", &eta) ? secs_text(f, eta) : "-"));
516 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(rs(j, "failed"), ru(j, "failed", &u) && u > 0 ? MT_BAD : MT_DIM)));
517 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(rs(j, "retried"), ru(j, "retried", &u) && u > 0 ? MT_WARN : MT_DIM)));
518 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(rs(j, "dupes"), ru(j, "dupes", &u) && u > 0 ? MT_WARN : MT_DIM)));
519 }
520 if (wide) mt_cells_push(f, &cells, mt_cell_str(f, rs(j, "priority")));
521 tui_table_row row = mt_row(f, cells.c, cells.n);
522 if (streq_opt(sel, rget(j, "job"))) row.style = tui_style_bg(tui_style_new(), MT_FAINT);
523 mt_rows_push(f, &rows, row);
524 }
525 tui_constraint widths[11];
526 size_t nw = 0;
527 const uint16_t w0[6] = {5, 10, 9, 13, 17, 7};
528 for (size_t i = 0; i < 6; i++) widths[nw++] = TUI_LENGTH(w0[i]);
529 if (!narrow) {
530 widths[nw++] = TUI_LENGTH(6);
531 widths[nw++] = TUI_LENGTH(6);
532 widths[nw++] = TUI_LENGTH(7);
533 widths[nw++] = TUI_LENGTH(5);
534 }
535 if (wide) widths[nw++] = TUI_LENGTH(4);
536 tui_block jb = mt_panel(f, MT_LINE(f, mt_fgb(" jobs ", MT_FG), mt_label("active and finished, as this member's committed table has them ")), MT_FAINT);
537 tui_table_state ts = {0};
538 for (size_t i = 0; i < nj && sel; i++)
539 if (streq_opt(rget(jobs[i], "job"), sel)) {
540 ts.has_selected = true;
541 ts.selected = i;
542 break;
543 }
544 draw_table(f, &rows, header, nh, widths, nw, &jb, tbl, &ts);
545 if (detail.height < 3) return;
546 if (!sel) {
547 tui_block b = mt_panel(f, tui_line_from_str(f->a, " job "), MT_FAINT);
548 par_block(f, tui_text_line(f->a, tui_line_span(f->a, mt_label("no jobs: `job submit <program> items <n>` on any member"))), &b, detail);
549 return;
550 }
551 const mt_vrec **shares, **picks, **rr;
552 size_t ns = 0, np = 0;
553 {
554 size_t n = all(f, v, "share", &rr);
555 shares = tui_arena_alloc(f->a, (n ? n : 1) * sizeof *shares);
556 for (size_t i = 0; i < n; i++)
557 if (streq_opt(rget(rr[i], "job"), sel)) shares[ns++] = rr[i];
558 n = all(f, v, "pick", &rr);
559 picks = tui_arena_alloc(f->a, (n ? n : 1) * sizeof *picks);
560 for (size_t i = 0; i < n; i++)
561 if (streq_opt(rget(rr[i], "job"), sel)) picks[np++] = rr[i];
562 }
563 bool side = detail.width >= 90;
564 tui_rect hs[2];
565 tui_hsplit(detail, 2, (tui_constraint[]){TUI_PERCENTAGE(side ? 55 : 100), TUI_FILL(1)}, hs);
566 /* Per member: done (its share of the job), what it holds now, its measured rate. */
567 uint64_t total = 0;
568 for (size_t i = 0; i < ns; i++) {
569 uint64_t d;
570 if (ru(shares[i], "done", &d)) total += d;
571 }
572 mt_rows srows = {0};
573 for (size_t i = 0; i < ns; i++) {
574 const mt_vrec *s = shares[i];
575 uint64_t km = 0, d = 0;
576 ru(s, "member", &km);
577 ru(s, "done", &d);
578 uint32_t k = (uint32_t)km;
579 double frac = (double)d / (double)(total > 1 ? total : 1);
580 const char *host = "";
581 for (size_t q = 0; q < app->nmachines; q++) {
582 uint32_t sk;
583 if (mt_machine_system(&app->machines[q], &sk) && sk == k) {
584 host = mt_name(f, &app->machines[q]);
585 break;
586 }
587 }
588 tui_table_cell c[6] = {
589 mt_cell_span(f, mt_fgb(MT_FMT(f, "%u", k), mt_sys_color(k))),
590 mt_cell_str(f, mt_cut(f, host, 10)),
591 mt_cell_line(f, MT_LINE(f, mt_fgs(mt_bar(f, frac, 6), mt_sys_color(k)), mt_val(pct3(f, frac * 100.0)))),
592 mt_cell_str(f, MT_FMT(f, "%llu", (unsigned long long)d)),
593 mt_cell_str(f, rs(s, "leased")),
594 mt_cell_str(f, rate_or_dash(f, s, "rate")),
595 };
596 mt_rows_push(f, &srows, mt_row(f, c, 6));
597 }
598 if (srows.n == 0) {
599 tui_table_cell c = mt_cell_span(f, mt_label("no member has run an item yet"));
600 mt_rows_push(f, &srows, mt_row(f, &c, 1));
601 }
602 uint16_t nrows = (uint16_t)srows.n;
603 static const char *const sh[6] = {"sys", "host", "share", "done", "leased", "items/s"};
604 static const tui_constraint sw[6] = {{TUI_C_LENGTH, 3, 0}, {TUI_C_LENGTH, 10, 0}, {TUI_C_LENGTH, 11, 0},
605 {TUI_C_LENGTH, 8, 0}, {TUI_C_LENGTH, 7, 0}, {TUI_C_FILL, 1, 0}};
606 mt_lines l = {0};
607 const mt_vrec **results, **failed;
608 size_t nres = 0, nfail = 0;
609 {
610 size_t n = all(f, v, "result", &rr);
611 results = tui_arena_alloc(f->a, (n ? n : 1) * sizeof *results);
612 for (size_t i = 0; i < n; i++)
613 if (streq_opt(rget(rr[i], "job"), sel)) results[nres++] = rr[i];
614 n = all(f, v, "failed", &rr);
615 failed = tui_arena_alloc(f->a, (n ? n : 1) * sizeof *failed);
616 for (size_t i = 0; i < n; i++)
617 if (streq_opt(rget(rr[i], "job"), sel)) failed[nfail++] = rr[i];
618 }
619 if (nres || nfail) {
620 mt_lines_push(f, &l, tui_line_span(f->a, mt_fgb("results (last) / failed (first)", MT_FG)));
621 for (size_t i = 0; i < nres; i++)
622 mt_lines_push(f, &l, MT_LINE(f, mt_label("item "), mt_val(rs(results[i], "item")), mt_label(" = "), mt_val(rs(results[i], "result")),
623 mt_label(MT_FMT(f, " by %s ", rs(results[i], "member"))), mt_fgs(rs(results[i], "out"), MT_DIM)));
624 for (size_t i = 0; i < nfail; i++)
625 mt_lines_push(f, &l, MT_LINE(f, mt_fgs("failed ", MT_BAD), mt_val(rs(failed[i], "item")),
626 mt_label(MT_FMT(f, " after %s tries: ", rs(failed[i], "tries"))), tui_span_raw(failed[i]->text)));
627 }
628 const mt_vrec *j = NULL;
629 for (size_t i = 0; i < nj && !j; i++)
630 if (streq_opt(rget(jobs[i], "job"), sel)) j = jobs[i];
631 tui_block block = mt_panel(f, MT_LINE(f, mt_fgb(MT_FMT(f, " job %s ", sel), MT_FG), mt_label(MT_FMT(f, "%s members ", j ? rs(j, "program") : "-"))),
632 MT_FAINT);
633 tui_rect inner = tui_block_inner(&block, hs[0]);
634 mt_block(f, &block, hs[0]);
635 tui_rect s2[2];
636 split_v(inner, (int[]){nrows + 1, -1}, 2, s2);
637 draw_table(f, &srows, sh, 6, sw, 6, NULL, s2[0], NULL);
638 par_wrap(f, mt_text(f, &l), NULL, s2[1]);
639 if (side) {
640 mt_lines pl = {0};
641 for (size_t i = 0; i < np; i++) {
642 uint64_t km = 0;
643 ru(picks[i], "member", &km);
644 uint32_t k = (uint32_t)km;
645 const char *verdict = rs(picks[i], "verdict");
646 mt_lines_push(f, &pl, MT_LINE(f, mt_fgb(MT_FMT(f, "%u ", k), mt_sys_color(k)),
647 mt_styled(mt_pad(f, mt_vrec_flag(picks[i], "busy") ? "busy" : verdict, 9, '<'), vstate_style(verdict)),
648 mt_fgs(picks[i]->text, strcmp(verdict, "rejected") == 0 ? MT_DIM : MT_FG)));
649 }
650 if (np == 0) {
651 bool lead = streq_opt(rget(head, "role"), "leader") || streq_opt(rget(head, "role"), "single");
652 mt_lines_push(f, &pl, tui_line_span(f->a, mt_label(lead ? "no lease to choose (nothing queued)" : "the leader's view has the lease choice")));
653 }
654 tui_block rb = mt_panel(f, MT_LINE(f, mt_fgb(" lease choice ", MT_FG), mt_label("F4's filter and score ")), MT_FAINT);
655 par_wrap(f, mt_text(f, &pl), &rb, hs[1]);
656 }
657}
658
659/* ---------------------------------------------------------------- - pipes */
660
661void mt_draw_pipes(mt_frame *f, tui_rect area, const mt_app *app, mt_instant now) {
662 const mt_view *v = mt_app_view(app, 2);
663 if (!v) {
664 waiting(f, area, app, 2);
665 return;
666 }
667 const mt_vrec **inboxes, **feeds;
668 size_t ni = all(f, v, "inbox", &inboxes), nf = all(f, v, "feed", &feeds);
669 size_t refusing = 0;
670 for (size_t i = 0; i < ni; i++) {
671 double x;
672 refusing += rf(inboxes[i], "refused_s", &x) && x > 0.0;
673 }
674 tui_line l1 = MT_LINE(f, mt_label("inboxes "), mt_val(MT_FMT(f, "%zu", ni)), mt_label(" producers "), mt_val(MT_FMT(f, "%zu", nf)),
675 mt_label(" refusing now (backpressure) "),
676 mt_styled(MT_FMT(f, "%zu", refusing), refusing > 0 ? tui_fg_bold(MT_WARN) : tui_fg(MT_DIM)), source(f, app, v, now));
677 if (area.height < 6 || area.width < 40) {
678 par_wrap(f, tui_text_line(f->a, l1), NULL, area);
679 return;
680 }
681 int fdh = (int)nf + 2;
682 tui_rect sp[3];
683 split_v(area, (int[]){1, -1, fdh < 6 ? fdh : 6}, 3, sp);
684 mt_par_line(f, l1, sp[0]);
685 bool wide = area.width >= 96;
686 const char *header[10];
687 size_t nh = 0;
688 const char *h0[7] = {"consumer", "type", "depth", "in/s", "out/s", "refused/s", "refused"};
689 for (size_t i = 0; i < 7; i++) header[nh++] = h0[i];
690 if (wide) {
691 header[nh++] = "empty";
692 header[nh++] = "rec";
693 }
694 header[nh++] = "fed by";
695 mt_rows rows = {0};
696 for (size_t i = 0; i < ni; i++) {
697 const mt_vrec *r = inboxes[i];
698 double fill = 0.0, rsv = 0.0;
699 rf(r, "fill", &fill);
700 rf(r, "refused_s", &rsv);
701 mt_cells cells = {0};
702 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgb(mt_cut(f, rs(r, "consumer"), 12), MT_FG)));
703 mt_cells_push(f, &cells, mt_cell_str(f, mt_cut(f, rs(r, "type"), 10)));
704 mt_cells_push(f, &cells, mt_cell_line(f, MT_LINE(f, mt_fgs(mt_bar(f, fill, 8), fill > 0.9 ? MT_BAD : fill > 0.5 ? MT_WARN : MT_GOOD),
705 mt_label(MT_FMT(f, " %s/%s", rs(r, "depth"), rs(r, "cap"))))));
706 mt_cells_push(f, &cells, mt_cell_str(f, rate_or_dash(f, r, "in_s")));
707 mt_cells_push(f, &cells, mt_cell_str(f, rate_or_dash(f, r, "out_s")));
708 mt_cells_push(f, &cells, mt_cell_span(f, mt_styled(mt_fmt_rate(f, (float)rsv), rsv > 0.0 ? tui_fg_bold(MT_WARN) : tui_fg(MT_DIM))));
709 mt_cells_push(f, &cells, mt_cell_str(f, rs(r, "refused")));
710 if (wide) {
711 mt_cells_push(f, &cells, mt_cell_str(f, rs(r, "empty")));
712 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%s B", rs(r, "rec"))));
713 }
714 mt_cells_push(f, &cells, mt_cell_str(f, rs(r, "fed_by")));
715 mt_rows_push(f, &rows, mt_row(f, cells.c, cells.n));
716 }
717 tui_constraint widths[10];
718 size_t nw = 0;
719 const uint16_t w0[7] = {12, 10, 20, 8, 8, 9, 9};
720 for (size_t i = 0; i < 7; i++) widths[nw++] = TUI_LENGTH(w0[i]);
721 if (wide) {
722 widths[nw++] = TUI_LENGTH(9);
723 widths[nw++] = TUI_LENGTH(6);
724 }
725 widths[nw++] = TUI_FILL(1);
726 if (ni == 0) {
727 tui_block b = mt_panel(f, tui_line_from_str(f->a, " inboxes "), MT_FAINT);
728 par_block(f, tui_text_line(f->a, tui_line_span(f->a, mt_label("no program with an inbox is loaded here (pipes are per machine: [ ] picks another)"))),
729 &b, sp[1]);
730 } else {
731 tui_block b = mt_panel(f, MT_LINE(f, mt_fgb(" inboxes ", MT_FG), mt_label("depth, rates, pushes refused while full (backpressure) ")), MT_FAINT);
732 draw_table(f, &rows, header, nh, widths, nw, &b, sp[1], NULL);
733 }
734 if (sp[2].height > 0) {
735 mt_lines l = {0};
736 for (size_t i = 0; i < nf; i++)
737 mt_lines_push(f, &l, MT_LINE(f, mt_val(rs(feeds[i], "producer")), mt_label(" → "), tui_span_raw(rs(feeds[i], "consumers"))));
738 tui_block b = mt_panel(f, tui_line_span(f->a, mt_fgb(" feeds ", MT_FG)), MT_FAINT);
739 par_block(f, mt_text(f, &l), &b, sp[2]);
740 }
741}
742
743/* ---------------------------------------------------------------- fleet from the snapshots */
744
745static tui_style rs_style(uint32_t st) {
746 switch (st) {
747 case MESH_RS_SYNC: return tui_fg(MT_GOOD);
748 case MESH_RS_PENDING: return tui_fg(MT_WARN);
749 case MESH_RS_FAILED: return tui_fg_bold(MT_BAD);
750 default: return tui_fg(MT_DIM);
751 }
752}
753
754static tui_style gl_style(uint32_t st) {
755 switch (st) {
756 case MESH_GL_RUNNING: return tui_fg(MT_GOOD);
757 case MESH_GL_HELLO: return tui_fg(MT_WARN);
758 case MESH_GL_PAUSED: return tui_fg(MT_BAD);
759 default: return tui_fg(MT_DIM);
760 }
761}
762
763static const char *gl_name(uint32_t g) { return g < MESH_GL_NAMES_LEN ? MESH_GL_NAMES[g] : "?"; }
764
765void mt_draw_fleet_snap(mt_frame *f, tui_rect area, const mt_app *app) {
766 uint16_t n = (uint16_t)app->nmachines;
767 /* The member table and group come from the focused machine, or the leader's machine when the
768 * focused one has no fleet. */
769 bool has_pick = false;
770 size_t src = 0;
771 mt_fleet fl;
772 const mt_machine *fm0 = mt_app_focused(app);
773 if (fm0->has_snap && mt_snapshot_fleet(&fm0->snap, &fl)) {
774 has_pick = true;
775 src = app->focus;
776 } else {
777 for (size_t i = 0; i < app->nmachines && !has_pick; i++) {
778 const mt_machine *m = &app->machines[i];
779 if (m->has_snap && mt_snapshot_fleet(&m->snap, &fl) && fl.has_leader && fl.role == MESH_ROLE_LEADER) {
780 has_pick = true;
781 src = i;
782 }
783 }
784 }
785 uint16_t nm = has_pick ? (uint16_t)fl.nmembers : 0;
786 tui_rect r[3];
787 tui_vsplit(area, 3, (tui_constraint[]){TUI_LENGTH(tui_min16((uint16_t)(n + 3), area.height)), TUI_LENGTH(nm + 3), TUI_FILL(1)}, r);
788 tui_rect views = r[0], members = r[1], group = r[2];
789 /* 1. Every machine's view: role, leader, term, version committed / applied, up, majority. */
790 static const char *const header[10] = {"machine", "system", "role", "leader", "term", "version", "applied", "up", "suspect", "majority"};
791 mt_rows rows = {0};
792 for (size_t i = 0; i < app->nmachines; i++) {
793 const mt_machine *m = &app->machines[i];
794 tui_table_cell name = mt_cell_span(f, mt_fgb(mt_name(f, m), MT_MACHINE[i % 8]));
795 mt_fleet x;
796 if (!m->has_snap) {
797 tui_table_cell c[2] = {name, mt_cell_span(f, mt_fgs(m->status, MT_DIM))};
798 mt_rows_push(f, &rows, mt_row(f, c, 2));
799 continue;
800 }
801 if (!mt_snapshot_fleet(&m->snap, &x)) {
802 tui_table_cell c[2] = {name, mt_cell_span(f, mt_label("single (not in a fleet)"))};
803 mt_rows_push(f, &rows, mt_row(f, c, 2));
804 continue;
805 }
806 tui_span maj = x.flags & MESH_FF_FIXED ? mt_fgs("fixed pair", MT_ACCENT) : x.flags & MESH_FF_QUORUM ? mt_fgs("yes", MT_GOOD) : mt_fgb("NO", MT_BAD);
807 uint32_t k;
808 bool has_k = mt_snapshot_system(&m->snap, &k);
809 tui_table_cell c[10] = {
810 name,
811 mt_cell_str(f, has_k ? MT_FMT(f, "%u", k) : "-"),
812 mt_cell_span(f, mt_styled(x.role < MESH_ROLE_NAMES_LEN ? MESH_ROLE_NAMES[x.role] : "?",
813 x.role == MESH_ROLE_LEADER ? tui_fg_bold(MT_ACCENT) : tui_fg(MT_FG))),
814 mt_cell_span(f, x.has_leader ? mt_fgb(MT_FMT(f, "%u", x.leader), mt_sys_color(x.leader)) : mt_fgb("none", MT_BAD)),
815 mt_cell_str(f, MT_FMT(f, "%llu", (unsigned long long)x.term)),
816 mt_cell_str(f, MT_FMT(f, "%llu", (unsigned long long)x.version)),
817 mt_cell_span(f, mt_fgs(MT_FMT(f, "%llu", (unsigned long long)x.applied), x.applied == x.version ? MT_GOOD : MT_WARN)),
818 mt_cell_str(f, mt_mask_list(f, x.up)),
819 mt_cell_span(f, mt_fgs(mt_mask_list(f, x.suspect), x.suspect != 0 ? MT_WARN : MT_DIM)),
820 mt_cell_span(f, maj),
821 };
822 mt_rows_push(f, &rows, mt_row(f, c, 10));
823 }
824 static const tui_constraint w[10] = {{TUI_C_LENGTH, 14, 0}, {TUI_C_LENGTH, 6, 0}, {TUI_C_LENGTH, 13, 0}, {TUI_C_LENGTH, 6, 0},
825 {TUI_C_LENGTH, 5, 0}, {TUI_C_LENGTH, 7, 0}, {TUI_C_LENGTH, 7, 0}, {TUI_C_LENGTH, 15, 0},
826 {TUI_C_LENGTH, 9, 0}, {TUI_C_FILL, 1, 0}};
827 tui_block vb = mt_panel(f, MT_LINE(f, mt_fgb(" fleet ", MT_FG), mt_label("each member's view: the leader, its term, the desired state committed and applied ")),
828 MT_FAINT);
829 draw_table(f, &rows, header, 10, w, 10, &vb, views, NULL);
830 if (!has_pick) {
831 tui_block b = mt_panel(f, tui_line_from_str(f->a, " members "), MT_FAINT);
832 par_block(f, tui_text_line(f->a, tui_line_span(f->a, mt_label("no machine is in a fleet"))), &b, members);
833 return;
834 }
835 const mt_snapshot *s = &app->machines[src].snap;
836 uint64_t now = mt_snapshot_uptime_ms(s);
837 /* 2. The members as machine src sees them. */
838 if (members.height > 0) {
839 static const char *const mh[9] = {"system", "host", "link", "", "reconcile", "applied", "heard", "group", "reason / last"};
840 mt_rows mrows = {0};
841 uint32_t me;
842 bool has_me = mt_snapshot_system(s, &me);
843 for (size_t ki = 0; ki < fl.nmembers; ki++) {
844 uint32_t k = (uint32_t)ki;
845 const mt_fleet_member *fm = &fl.members[ki];
846 mt_member mr = mt_snapshot_member(s, k);
847 const char *link = mr.state < MESH_MEMBER_STATES_LEN ? MESH_MEMBER_STATES[mr.state] : "?";
848 tui_style link_style = mr.state == MESH_MS_UP ? tui_fg(MT_GOOD)
849 : mr.state == MESH_MS_SUSPECT ? tui_fg_bold(MT_WARN)
850 : mr.state == MESH_MS_LINKED ? tui_fg(MT_WARN)
851 : tui_fg_bold(MT_BAD);
852 bool pending = fm->applied < fl.version && fm->status != MESH_RS_FAILED;
853 const char *rsw = pending ? "pending" : fm->status < MESH_RS_NAMES_LEN ? MESH_RS_NAMES[fm->status] : "?";
854 tui_style rss = strcmp(rsw, "pending") == 0 ? tui_fg(MT_WARN) : rs_style(fm->status);
855 const char *heard = has_me && me == k ? "-"
856 : fm->heard_ms == 0 ? "never"
857 : MT_FMT(f, "%llu ms", (unsigned long long)(now > fm->heard_ms ? now - fm->heard_ms : 0));
858 tui_table_cell c[9] = {
859 mt_cell_span(f, mt_fgb(MT_FMT(f, "%u", k), mt_sys_color(k))),
860 mt_cell_str(f, tui_arena_strdup(f->a, mr.host)),
861 mt_cell_span(f, mt_styled(link, link_style)),
862 mt_cell_span(f, mt_fgb(fl.has_leader && fl.leader == k ? "leader" : "", MT_ACCENT)),
863 mt_cell_span(f, mt_styled(rsw, rss)),
864 mt_cell_str(f, MT_FMT(f, "%llu", (unsigned long long)fm->applied)),
865 mt_cell_str(f, heard),
866 mt_cell_span(f, mt_styled(gl_name(fm->group), gl_style(fm->group))),
867 mt_cell_span(f, mt_fgs(tui_arena_strdup(f->a, fm->reason), fm->status == MESH_RS_FAILED ? MT_BAD : MT_DIM)),
868 };
869 mt_rows_push(f, &mrows, mt_row(f, c, 9));
870 }
871 static const tui_constraint mw[9] = {{TUI_C_LENGTH, 6, 0}, {TUI_C_LENGTH, 16, 0}, {TUI_C_LENGTH, 8, 0}, {TUI_C_LENGTH, 6, 0}, {TUI_C_LENGTH, 9, 0},
872 {TUI_C_LENGTH, 7, 0}, {TUI_C_LENGTH, 8, 0}, {TUI_C_LENGTH, 8, 0}, {TUI_C_FILL, 1, 0}};
873 tui_block mb = mt_panel(f, MT_LINE(f, mt_fgb(MT_FMT(f, " members as %s sees them ", mt_name(f, &app->machines[src])), MT_FG),
874 mt_label(MT_FMT(f, "dead after %u ms; majority %u ", fl.dead_ms, fl.quorum))),
875 MT_FAINT);
876 draw_table(f, &mrows, mh, 9, mw, 9, &mb, members, NULL);
877 }
878 /* 3. The lockstep group and the jobs. */
879 if (group.height > 0) {
880 mt_lines lines = {0};
881 if (fl.has_group) {
882 const mt_fleet_group *g = &fl.group;
883 const char *order = "";
884 for (size_t i = 0; i < MESH_MAX_SYSTEMS && g->order[i] != 0xff; i++) order = MT_FMT(f, "%s%s%u", order, i ? "," : "", g->order[i]);
885 const char *loss = g->reform ? (g->auto_ms > 0 ? MT_FMT(f, "reform after %u ms", g->auto_ms) : "reform (manual)") : "pause";
886 mt_lines_push(f, &lines, MT_LINE(f, mt_fgb(MT_FMT(f, "%s ", g->name), MT_FG), mt_label("epoch "), mt_val(MT_FMT(f, "%u", g->epoch)),
887 mt_label(" cycle "), mt_val(MT_FMT(f, "%u", g->cycle)), mt_label(" members (rank order) "), mt_val(order),
888 mt_label(" state "), mt_styled(g->running ? "running" : "paused", g->running ? tui_fg(MT_GOOD) : tui_fg_bold(MT_BAD)),
889 mt_label(" on loss "), mt_val(loss)));
890 tui_line per = tui_line_span(f->a, mt_label("data plane: "));
891 const char *out = "";
892 for (size_t k = 0; k < fl.nmembers; k++) {
893 if (!(g->mask >> k & 1)) {
894 out = MT_FMT(f, "%s%s%zu", out, *out ? "," : "", k);
895 continue;
896 }
897 tui_line_push(f->a, &per, mt_fgb(MT_FMT(f, "%zu ", k), mt_sys_color((uint32_t)k)));
898 tui_line_push(f->a, &per, mt_styled(MT_FMT(f, "%s ", gl_name(fl.members[k].group)), gl_style(fl.members[k].group)));
899 }
900 mt_lines_push(f, &lines, per);
901 if (*out) mt_lines_push(f, &lines, MT_LINE(f, mt_label("not in this epoch (unfed until the next reform): "), mt_val(out)));
902 } else {
903 mt_lines_push(f, &lines, tui_line_span(f->a, mt_label("no lockstep group in the desired state")));
904 }
905 /* The jobs as this machine's replica has them. */
906 mt_jobs j = mt_snapshot_jobs(s);
907 bool lead = j.flags & MESH_JF_LEADER;
908 mt_lines_push(f, &lines,
909 MT_LINE(f, mt_label("jobs: "), mt_val(MT_FMT(f, "%u active of %u", j.active, j.total)), mt_label(" items done "),
910 mt_val(MT_FMT(f, "%llu", (unsigned long long)j.done)), mt_label(" failed "),
911 mt_fgs(MT_FMT(f, "%llu", (unsigned long long)j.failed), j.failed > 0 ? MT_BAD : MT_FG), mt_label(" dupes dropped "),
912 mt_val(MT_FMT(f, "%llu", (unsigned long long)j.dupes)), mt_label(lead ? " leased " : " held here "),
913 mt_val(MT_FMT(f, "%u", j.leased)), mt_label(" items/s "), mt_val(mt_ff(f, j.rate, 0)),
914 mt_label(lead ? (j.flags & MESH_JF_ACTIVE ? " (leads the job table)" : " (leader: merging the tables)") : "")));
915 tui_block gb = mt_panel(f, tui_line_span(f->a, mt_fgb(" lockstep group and jobs ", MT_FG)), MT_FAINT);
916 par_block(f, mt_text(f, &lines), &gb, group);
917 }
918}