==> src/mesh.c <==1/* ui.rs's "1 mesh" section, ported. */
2#include "mesh/top_ui_mesh.h"
3
4#include "mesh/top_ui_load.h"
5#include "mesh/top_ui_net.h"
6#include "tui/sparkline.h"
7
8#include <stdlib.h>
9#include <string.h>
10
11static unsigned popcount(uint32_t x) {
12 unsigned n = 0;
13 for (; x; x &= x - 1) n++;
14 return n;
15}
16
17size_t mt_cluster_rows(const mt_app *app, mt_cluster_row **out) {
18 size_t cap = app->nmachines + MESH_MAX_SYSTEMS + 1, n = 0;
19 mt_cluster_row *r = calloc(cap, sizeof *r);
20 if (!r) abort();
21 bool clustered = false;
22 for (size_t i = 0; i < app->nmachines; i++) {
23 uint32_t k;
24 clustered |= mt_machine_system(&app->machines[i], &k);
25 }
26 if (clustered) {
27 uint32_t size = mt_app_cluster_size(app);
28 for (uint32_t k = 0; k < size; k++) {
29 if (n == cap) break;
30 bool found = false;
31 for (size_t i = 0; i < app->nmachines && !found; i++) {
32 uint32_t sk;
33 if (mt_machine_system(&app->machines[i], &sk) && sk == k) {
34 r[n++] = (mt_cluster_row){.has_system = true, .system = k, .has_machine = true, .machine = i};
35 found = true;
36 }
37 }
38 if (found) continue;
39 mt_cluster_row row = {.has_system = true, .system = k};
40 for (size_t i = 0; i < app->nmachines; i++) {
41 const mt_machine *m = &app->machines[i];
42 uint32_t sk;
43 if (!m->has_snap || !mt_snapshot_system(&m->snap, &sk) || !(k < mt_snapshot_systems(&m->snap))) continue;
44 mt_member mb = mt_snapshot_member(&m->snap, k);
45 /* max_by_key (state, gpu known): the last of equal maxima wins */
46 bool better = !row.has_member || mb.state > row.member.state ||
47 (mb.state == row.member.state && (*mb.gpu != 0) >= (*row.member.gpu != 0));
48 if (better) {
49 row.has_member = true;
50 row.member = mb;
51 }
52 }
53 r[n++] = row;
54 }
55 }
56 for (size_t i = 0; i < app->nmachines && n < cap; i++) {
57 uint32_t sk;
58 if (!mt_machine_system(&app->machines[i], &sk) || !clustered)
59 r[n++] = (mt_cluster_row){.has_machine = true, .machine = i};
60 }
61 *out = r;
62 return n;
63}
64
65static tui_table_cell dash(mt_frame *f) { return mt_cell_span(f, mt_fgs("-", MT_FAINT)); }
66
67static void draw_overview(mt_frame *f, tui_rect area, const mt_app *app, const mt_cluster_row *rows, size_t nrows) {
68 uint32_t n = mt_app_cluster_size(app);
69 tui_line title = MT_LINE(f, mt_fgb(" cluster ", MT_FG), mt_label(MT_FMT(f, "%u system%s ", n, n == 1 ? "" : "s")));
70 tui_block block = mt_panel(f, title, MT_FAINT);
71 bool wide = area.width >= 140;
72 /* The load columns: none below 100 columns, `GPU CPU` (and meshd when wide), all four from 230. */
73 mt_load_cols lt = area.width >= 230 ? MT_LOAD_COLS_FULL : wide ? MT_LOAD_COLS_PAIR : area.width >= 100 ? MT_LOAD_COLS_ONE : MT_LOAD_COLS_NONE;
74 const char *header[32];
75 size_t nh = 0;
76 const char *h0[5] = {"sys", "host", "GPU", "cc", "SMs"};
77 for (size_t i = 0; i < 5; i++) header[nh++] = h0[i];
78 if (wide) {
79 header[nh++] = "arch";
80 header[nh++] = "blk/SM";
81 }
82 /* Narrow: the linked count joins the peer cell, the step column goes. */
83 if (wide) {
84 header[nh++] = "peer";
85 header[nh++] = "linked";
86 header[nh++] = "step";
87 header[nh++] = "steps/s";
88 } else {
89 header[nh++] = "peer";
90 header[nh++] = "steps/s";
91 }
92 if (wide) header[nh++] = "RTT";
93 header[nh++] = "tasks/s";
94 nh += mt_load_cols_header(lt, header + nh);
95 if (wide) {
96 header[nh++] = "net rx";
97 header[nh++] = "net tx";
98 header[nh++] = "mesh";
99 header[nh++] = "net, 1 s/col";
100 } else {
101 header[nh++] = "net ↓ ↑";
102 header[nh++] = "mesh";
103 }
104 header[nh++] = "wait per round, by member";
105 tui_constraint widths[32];
106 size_t nw = 0;
107 widths[nw++] = TUI_LENGTH(4);
108 widths[nw++] = TUI_LENGTH(wide ? 14 : 10);
109 widths[nw++] = TUI_LENGTH(wide ? 16 : 10);
110 widths[nw++] = TUI_LENGTH(4);
111 widths[nw++] = TUI_LENGTH(6);
112 if (wide) {
113 widths[nw++] = TUI_LENGTH(8);
114 widths[nw++] = TUI_LENGTH(6);
115 widths[nw++] = TUI_LENGTH(8);
116 widths[nw++] = TUI_LENGTH(6);
117 widths[nw++] = TUI_LENGTH(9);
118 widths[nw++] = TUI_LENGTH(7);
119 widths[nw++] = TUI_LENGTH(8);
120 } else {
121 widths[nw++] = TUI_LENGTH(10);
122 widths[nw++] = TUI_LENGTH(7);
123 }
124 widths[nw++] = TUI_LENGTH(8);
125 nw += mt_load_cols_widths(lt, widths + nw);
126 if (wide) {
127 widths[nw++] = TUI_LENGTH(10);
128 widths[nw++] = TUI_LENGTH(10);
129 widths[nw++] = TUI_LENGTH(4);
130 widths[nw++] = TUI_LENGTH(MT_OV_SPARK);
131 } else {
132 widths[nw++] = TUI_LENGTH(12);
133 widths[nw++] = TUI_LENGTH(4);
134 }
135 widths[nw++] = TUI_FILL(1);
136 mt_rows trows = {0};
137 for (size_t ri = 0; ri < nrows; ri++) {
138 const mt_cluster_row *r = &rows[ri];
139 bool focused = r->has_machine && r->machine == app->focus;
140 mt_cells cells = {0};
141 mt_cells_push(f, &cells,
142 mt_cell_line(f, MT_LINE(f, mt_fgs(focused ? "◆" : " ", MT_ACCENT),
143 r->has_system ? mt_fgb(MT_FMT(f, "%u", r->system), mt_sys_color(r->system)) : mt_fgs("-", MT_DIM))));
144 if (r->has_machine) {
145 const mt_machine *m = &app->machines[r->machine];
146 if (!m->has_snap) {
147 mt_cells_push(f, &cells, mt_cell_str(f, m->endpoint));
148 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(m->status, MT_BAD)));
149 mt_rows_push(f, &trows, mt_row(f, cells.c, cells.n));
150 continue;
151 }
152 const mt_snapshot *s = &m->snap;
153 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgb(mt_name(f, m), m->online ? MT_FG : MT_BAD)));
154 char gpu[97], lab[32];
155 mt_snapshot_gpu(s, gpu);
156 mt_cells_push(f, &cells, mt_cell_str(f, mt_short_gpu(f, gpu)));
157 mt_gpu_label(mt_snapshot_head(s, MESH_H_CC), mt_snapshot_head(s, MESH_H_BACKEND), lab);
158 mt_cells_push(f, &cells, mt_cell_str(f, tui_arena_strdup(f->a, lab)));
159 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%u/%u", mt_snapshot_host_u32(s, MESH_S_SMS), mt_snapshot_host_u32(s, MESH_S_SMS_TOTAL))));
160 if (wide) {
161 char arch[49];
162 mt_snapshot_host_str(s, MESH_S_ARCH, 16, arch);
163 mt_cells_push(f, &cells, mt_cell_str(f, *arch ? tui_arena_strdup(f->a, arch) : "-"));
164 mt_blocks_label(mt_snapshot_head(s, MESH_H_BLOCKS_PER_SM), mt_snapshot_head(s, MESH_H_BACKEND), lab);
165 mt_cells_push(f, &cells, mt_cell_str(f, tui_arena_strdup(f->a, lab)));
166 }
167 uint32_t ps = mt_snapshot_host_u32(s, MESH_S_PEER_STATE);
168 tui_span peer = ps == 0 ? mt_fgs("single", MT_DIM)
169 : ps == 1 ? mt_fgb("waiting", MT_WARN)
170 : ps == 2 ? mt_fgb("up", MT_GOOD)
171 : mt_fgs(MT_FMT(f, "?%u", ps), MT_BAD);
172 unsigned linked = popcount(mt_snapshot_host_u32(s, MESH_S_LINKED));
173 uint32_t nn = mt_snapshot_systems(s);
174 if (nn < 1) nn = 1;
175 uint32_t me;
176 bool has_me = mt_snapshot_system(s, &me);
177 tui_color lc = !has_me ? MT_DIM : linked >= nn ? MT_GOOD : MT_WARN;
178 tui_span link = mt_fgs(has_me ? MT_FMT(f, "%u/%u", linked, nn) : "-", lc);
179 if (wide) {
180 mt_cells_push(f, &cells, mt_cell_span(f, peer));
181 mt_cells_push(f, &cells, mt_cell_span(f, link));
182 } else if (has_me) {
183 mt_cells_push(f, &cells, mt_cell_line(f, MT_LINE(f, peer, tui_span_raw(" "), link)));
184 } else {
185 mt_cells_push(f, &cells, mt_cell_span(f, peer));
186 }
187 const mt_section *t = mt_machine_program(m, "train");
188 if (t) {
189 if (wide) mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%llu", (unsigned long long)mt_train_next_step(t))));
190 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(mt_ff(f, mt_rate_per_s(&m->train_steps), 0), MT_WARN)));
191 } else if (wide) {
192 mt_cells_push(f, &cells, dash(f));
193 mt_cells_push(f, &cells, dash(f));
194 } else {
195 mt_cells_push(f, &cells, dash(f));
196 }
197 if (wide) {
198 float rtt = mt_snapshot_host_f32(s, MESH_S_PEER_RTT_US);
199 mt_cells_push(f, &cells, mt_cell_str(f, rtt > 0.0f ? MT_FMT(f, "%sus", mt_ff(f, rtt, 0)) : "-"));
200 }
201 mt_cells_push(f, &cells, mt_cell_str(f, mt_fmt_rate(f, mt_snapshot_host_f32(s, MESH_S_TASKS_PER_S))));
202 tui_table_cell lc4[4];
203 mt_load l;
204 mt_load_row lr;
205 bool has_l = mt_snapshot_load(s, &l);
206 if (has_l) lr = mt_load_row_from_load(&l);
207 size_t nl = mt_load_cells(f, has_l ? &lr : NULL, lt, lc4);
208 for (size_t i = 0; i < nl; i++) mt_cells_push(f, &cells, lc4[i]);
209 tui_table_cell nc[4];
210 size_t nnc = mt_net_cells(f, m, wide, nc);
211 for (size_t i = 0; i < nnc; i++) mt_cells_push(f, &cells, nc[i]);
212 /* The wait it sees for each other member; the longest (the one it waits for) highlighted. */
213 uint32_t slow_k;
214 float slow_us;
215 bool has_slow = mt_slowest_wait(s, &slow_k, &slow_us) && slow_us > 0.0f;
216 mt_member ms[MESH_MAX_SYSTEMS];
217 size_t nm = mt_snapshot_members(s, ms);
218 tui_line spans = tui_line_empty();
219 for (size_t i = 0; i < nm; i++) {
220 const mt_member *mb = &ms[i];
221 if (has_me && mb->system == me) continue;
222 tui_style st = mb->state != MESH_MS_UP ? tui_fg(MT_BAD)
223 : has_slow && mb->system == slow_k ? tui_fg_bold(MT_WARN)
224 : tui_fg(MT_FG);
225 tui_line_push(f->a, &spans, mt_fgs(MT_FMT(f, "%u:", mb->system), mt_sys_color(mb->system)));
226 const char *v = mb->state == MESH_MS_ABSENT ? "absent" : mb->state != MESH_MS_UP ? "linked" : mt_fmt_us(f, mb->wait_us);
227 tui_line_push(f->a, &spans, mt_styled(MT_FMT(f, "%s ", v), st));
228 }
229 mt_cells_push(f, &cells, spans.n ? mt_cell_line(f, spans) : dash(f));
230 } else if (r->has_member) {
231 const mt_member *mb = &r->member;
232 /* Known only through another member's record: what it told at JOIN. */
233 char lab[32];
234 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(*mb->host ? tui_arena_strdup(f->a, mb->host) : "?", MT_DIM)));
235 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(mt_short_gpu(f, mb->gpu), MT_DIM)));
236 mt_gpu_label(mb->cc, mb->backend, lab);
237 mt_cells_push(f, &cells, mt_cell_str(f, tui_arena_strdup(f->a, lab)));
238 mt_cells_push(f, &cells, mt_cell_str(f, MT_FMT(f, "%u/%u", mb->sms, mb->sms_total)));
239 if (wide) {
240 mt_cells_push(f, &cells, mt_cell_str(f, *mb->arch ? tui_arena_strdup(f->a, mb->arch) : "-"));
241 mt_blocks_label(mb->blocks_per_sm, mb->backend, lab);
242 mt_cells_push(f, &cells, mt_cell_str(f, tui_arena_strdup(f->a, lab)));
243 }
244 const char *st = mb->state < MESH_MEMBER_STATES_LEN ? MESH_MEMBER_STATES[mb->state] : "?";
245 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs(st, mb->state == MESH_MS_UP ? MT_GOOD : MT_BAD)));
246 if (wide) mt_cells_push(f, &cells, dash(f));
247 /* Steps/s, RTT, tasks/s: not reported by peers; its load is (F_LOADREP). */
248 if (wide) {
249 mt_cells_push(f, &cells, dash(f));
250 mt_cells_push(f, &cells, dash(f));
251 }
252 mt_cells_push(f, &cells, dash(f));
253 mt_cells_push(f, &cells, dash(f));
254 bool has_ml = false;
255 mt_member_load ml = {0};
256 for (size_t i = 0; i < app->nmachines; i++) {
257 const mt_machine *m = &app->machines[i];
258 uint32_t sk;
259 mt_member_load x;
260 if (!m->has_snap || !mt_snapshot_system(&m->snap, &sk) || !mt_snapshot_member_load(&m->snap, mb->system, &x)) continue;
261 if (!has_ml || x.age_ms < ml.age_ms) { /* min_by_key: the first of equal minima */
262 ml = x;
263 has_ml = true;
264 }
265 }
266 mt_load_row lr;
267 if (has_ml) lr = mt_load_row_from_member(&ml);
268 tui_table_cell lc4[4];
269 size_t nl = mt_load_cells(f, has_ml ? &lr : NULL, lt, lc4);
270 for (size_t i = 0; i < nl; i++) mt_cells_push(f, &cells, lc4[i]);
271 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs("via peers", MT_DIM)));
272 } else {
273 mt_cells_push(f, &cells, mt_cell_span(f, mt_fgs("unknown", MT_DIM)));
274 }
275 mt_rows_push(f, &trows, mt_row(f, cells.c, cells.n));
276 }
277 tui_table_row hdr = mt_header(f, header, nh);
278 tui_table t = tui_table_new(trows.rows, trows.n, widths, nw);
279 t.header = &hdr;
280 t.column_spacing = 1;
281 t.block = █
282 mt_table(f, &t, area);
283}
284
285bool mt_infer_spans(mt_frame *f, const mt_snapshot *s, tui_line *out) {
286 static const char *const MODES[4] = {"fast", "fast16", "exact", "exact-cpu"};
287 tui_line o = tui_line_empty();
288 for (size_t i = 0; i < s->nsections; i++) {
289 const mt_section *p = &s->sections[i];
290 const char *name = mt_sec_name(f, p);
291 if (strncmp(name, "gpt2", 4) != 0 || strcmp(name, "gpt2x") == 0) continue;
292 if (mt_section_view_len(p) < 0x44) continue;
293 uint32_t role = mt_section_view_u32(p, 0x04), req = mt_section_view_u32(p, 0x08), mi = mt_section_view_u32(p, 0x34);
294 const char *mode = mi < 4 ? MODES[mi] : "?";
295 float ttft = mt_section_view_f32(p, 0x10), pre = mt_section_view_f32(p, 0x14), tps = mt_section_view_f32(p, 0x18);
296 float kv_ms = mt_section_view_f32(p, 0x1c), kv_mb = mt_section_view_f32(p, 0x24), gbps = mt_section_view_f32(p, 0x40);
297 uint32_t peer = mt_section_view_u32(p, 0x30);
298 tui_line_push(f->a, &o, mt_fgs(MT_FMT(f, "%s ", name), MT_ACCENT));
299 tui_line_push(f->a, &o, mt_label(MT_FMT(f, "%u req ", req)));
300 if (req == 0) continue;
301 tui_line_push(f->a, &o, mt_label(MT_FMT(f, "%s ", mode)));
302 if (role == 1) {
303 tui_line_push(f->a, &o, mt_label("prefill "));
304 tui_line_push(f->a, &o, mt_val(MT_FMT(f, "%sms", mt_ff(f, pre, 1))));
305 tui_line_push(f->a, &o, mt_label(" KV out "));
306 tui_line_push(f->a, &o, mt_val(MT_FMT(f, "%sMB ", mt_ff(f, kv_mb, 1))));
307 } else if (role == 2) {
308 tui_line_push(f->a, &o, mt_label("ttft "));
309 tui_line_push(f->a, &o, mt_val(MT_FMT(f, "%sms ", mt_ff(f, ttft, 1))));
310 tui_line_push(f->a, &o, mt_val(mt_ff(f, tps, 0)));
311 tui_line_push(f->a, &o, mt_label(" tok/s KV "));
312 tui_line_push(f->a, &o, mt_val(MT_FMT(f, "%sMB/%sms %sGb/s", mt_ff(f, kv_mb, 1), mt_ff(f, kv_ms, 1), mt_ff(f, gbps, 1))));
313 if (peer != UINT32_MAX) {
314 tui_line_push(f->a, &o, mt_label(" from "));
315 tui_line_push(f->a, &o, mt_fgs(MT_FMT(f, "system %u ", peer), mt_sys_color(peer)));
316 }
317 } else {
318 tui_line_push(f->a, &o, mt_label("ttft "));
319 tui_line_push(f->a, &o, mt_val(MT_FMT(f, "%sms ", mt_ff(f, ttft, 1))));
320 tui_line_push(f->a, &o, mt_val(mt_ff(f, tps, 0)));
321 tui_line_push(f->a, &o, mt_label(" tok/s "));
322 }
323 }
324 if (!o.n) return false;
325 *out = o;
326 return true;
327}
328
329static uint16_t div_ceil16(uint16_t a, uint16_t b) { return (uint16_t)(a / b + (a % b != 0)); }
330
331/* One cell per node of every program, labelled with its %smid: hue = the program, brightness =
332 * utilisation, red = a running program's heartbeat stalled. */
333static void draw_sm_grid(mt_frame *f, tui_rect area, const mt_machine *m, const mt_snapshot *s, mt_instant now) {
334 if (area.height < 2 || area.width == 0) return;
335 tui_line head = tui_line_span(f->a, mt_label("nodes by program (%smid)"));
336 tui_buffer_set_line(f->buf, area.x, area.y, &head, area.width);
337 tui_rect cells = {area.x, (uint16_t)(area.y + 1), area.width, (uint16_t)(area.height - 1)};
338 uint16_t legend_y = (uint16_t)(cells.y + cells.height - 1);
339 size_t total = 0;
340 for (size_t i = 0; i < s->nsections; i++) total += mt_section_nodes(&s->sections[i]);
341 uint16_t n = (uint16_t)total;
342 uint16_t grid_h = tui_sat_sub(cells.height, 1);
343 uint16_t per_row = div_ceil16(n, tui_max16(grid_h, 1));
344 per_row = tui_max16(per_row, 8);
345 per_row = tui_min16(per_row, (uint16_t)(cells.width / 4));
346 per_row = tui_max16(per_row, 1);
347 uint16_t w = (uint16_t)(cells.width / per_row);
348 uint16_t h = (uint16_t)(grid_h / tui_max16(div_ceil16(n, per_row), 1));
349 if (h < 1) h = 1;
350 if (h > 4) h = 4;
351 uint16_t text_h = h >= 3 ? (uint16_t)(h - 1) : h;
352 uint16_t i = 0;
353 for (size_t pi = 0; pi < s->nsections; pi++) {
354 const mt_section *p = &s->sections[pi];
355 const char *name = mt_sec_name(f, p);
356 size_t nodes = mt_section_nodes(p);
357 for (uint32_t node = 0; node < nodes; node++, i++) {
358 uint16_t cx = (uint16_t)(cells.x + (i % per_row) * w), cy = (uint16_t)(cells.y + (i / per_row) * h);
359 if (cy + text_h > cells.y + grid_h) goto legend;
360 mt_node rec = mt_section_node(p, node);
361 const mt_node_track *t = mt_machine_node(m, mt_section_slot(p), node);
362 float util = t ? t->util : 0.0f;
363 bool stalled = mt_section_running(p) && mt_machine_stalled(m, mt_section_slot(p), node, now);
364 tui_style style;
365 if (stalled) {
366 style = mt_fg_bg_bold(TUI_WHITE, MT_BAD);
367 } else if (!mt_section_running(p) || rec.tasks == 0) {
368 style = tui_fg_bg(MT_DIM, MT_FAINT);
369 } else {
370 float b = 0.22f + 0.78f * util;
371 style = tui_fg_bg(b > 0.55f ? TUI_BLACK : TUI_WHITE, mt_prog_color(name, mt_section_slot(p), b));
372 }
373 size_t cw = tui_sat_sub(w, 1);
374 if (cw < 1) cw = 1;
375 const char *lines[3] = {
376 cw >= 6 ? MT_FMT(f, "sm %u", rec.sm) : MT_FMT(f, "%u", rec.sm),
377 MT_FMT(f, "%s%%", mt_ff(f, util * 100.0f, 0)),
378 stalled ? "STALL" : name,
379 };
380 for (uint16_t dy = 0; dy < text_h && dy < 3; dy++) {
381 const char *txt = mt_pad(f, lines[dy], (int)cw, '^');
382 tui_buffer_set_stringn(f->buf, cx, (uint16_t)(cy + dy), txt, strlen(txt), cw, style);
383 }
384 }
385 }
386legend:;
387 tui_line legend = tui_line_empty();
388 for (size_t pi = 0; pi < s->nsections; pi++) {
389 const mt_section *p = &s->sections[pi];
390 const char *name = mt_sec_name(f, p);
391 tui_line_push(f->a, &legend, mt_fgs("■", mt_prog_color(name, mt_section_slot(p), 1.0f)));
392 tui_line_push(f->a, &legend, mt_label(MT_FMT(f, "%s ", name)));
393 }
394 tui_line_push(f->a, &legend, mt_fgs("■", MT_BAD));
395 tui_line_push(f->a, &legend, mt_label("stalled bright=busy"));
396 tui_buffer_set_line(f->buf, cells.x, legend_y, &legend, cells.width);
397}
398
399/* `metal apple9` for the GPU line. */
400static const char *mesh_host_label(mt_frame *f, uint32_t word) {
401 uint32_t k = mesh_backend_kind(word);
402 const char *name = k < 4 ? MESH_BACKENDS[k] : "?";
403 return mesh_backend_family(word) > 0 ? MT_FMT(f, "%s apple%u", name, mesh_backend_family(word)) : name;
404}
405
406static void draw_machine(mt_frame *f, tui_rect area, const mt_machine *m, bool focused, mt_instant now) {
407 tui_color color;
408 tui_line title = mt_machine_title(f, m, focused, m->endpoint, &color);
409 tui_block block = mt_panel(f, title, color);
410 tui_rect inner = tui_block_inner(&block, area);
411 mt_block(f, &block, area);
412 if (!m->has_snap) {
413 mt_no_data(f, inner, m);
414 return;
415 }
416 const mt_snapshot *s = &m->snap;
417 /* Below the info lines: the Network box (full, else compact), the tasks/s sparkline (4 rows,
418 * else 2) and the node grid (at least 3 rows when it can). */
419 mt_lines net_full = {0}, net_compact = {0};
420 mt_net_lines(f, m, s, tui_sat_sub(inner.width, 2), false, &net_full);
421 mt_net_lines(f, m, s, tui_sat_sub(inner.width, 2), true, &net_compact);
422 uint16_t full_h = (uint16_t)(net_full.n + 2), compact_h = (uint16_t)(net_compact.n + 2);
423 /* The Load box: 3 lines, else 1 (compact), else none. */
424 mt_lines load_full = {0}, load_compact = {0};
425 mt_load l;
426 if (mt_snapshot_load(s, &l)) {
427 mt_load_lines(f, &l, false, &load_full);
428 mt_load_lines(f, &l, true, &load_compact);
429 } else {
430 mt_lines_push(f, &load_full, tui_line_span(f->a, mt_label("no load sample yet")));
431 mt_lines_push(f, &load_compact, tui_line_span(f->a, mt_label("no load sample yet")));
432 }
433 uint16_t lf_h = (uint16_t)(load_full.n + 2), lc_h = (uint16_t)(load_compact.n + 2);
434 uint16_t avail = tui_sat_sub(inner.height, 7);
435 /* In order of preference: (net full?, spark rows, load full?/compact/none); the grid keeps 3 rows (4 for the roomiest). */
436 static const struct {
437 bool nf;
438 uint16_t sp;
439 uint8_t ld;
440 uint16_t g;
441 } picks[5] = {{true, 4, 2, 4}, {true, 2, 2, 3}, {true, 2, 1, 3}, {false, 2, 1, 3}, {false, 2, 0, 3}};
442 uint16_t net_h, spark_h, load_h;
443 const mt_lines *net_rows, *load_rows;
444 mt_lines none = {0};
445 size_t pk = 5;
446 for (size_t i = 0; i < 5; i++) {
447 uint16_t lh[3] = {0, lc_h, lf_h};
448 if ((uint32_t)(picks[i].nf ? full_h : compact_h) + picks[i].sp + lh[picks[i].ld] + picks[i].g <= avail) {
449 pk = i;
450 break;
451 }
452 }
453 if (pk < 5) {
454 net_h = picks[pk].nf ? full_h : compact_h;
455 net_rows = picks[pk].nf ? &net_full : &net_compact;
456 spark_h = picks[pk].sp;
457 load_h = picks[pk].ld == 2 ? lf_h : picks[pk].ld == 1 ? lc_h : 0;
458 load_rows = picks[pk].ld == 2 ? &load_full : picks[pk].ld == 1 ? &load_compact : &none;
459 } else {
460 uint16_t h = tui_min16(compact_h, tui_sat_sub(avail, 2));
461 net_h = h;
462 spark_h = tui_min16(tui_sat_sub(avail, h), 2);
463 net_rows = &net_compact;
464 load_h = 0;
465 load_rows = &none;
466 }
467 tui_rect r[6];
468 tui_constraint cons[6] = {TUI_LENGTH(6), TUI_LENGTH(1), TUI_LENGTH(spark_h), TUI_LENGTH(net_h), TUI_LENGTH(load_h), TUI_FILL(1)};
469 tui_vsplit(inner, 6, cons, r);
470 tui_rect info = r[0], chips = r[1], spark = r[2], net = r[3], load = r[4], grid = r[5];
471
472 uint32_t sysk = mt_snapshot_host_u32(s, MESH_S_SYSTEM);
473 const char *sys = sysk == MESH_PEER_NONE ? "single" : MT_FMT(f, "system %u", sysk);
474 bool green = mt_snapshot_host_u32(s, MESH_S_GREEN) == 1;
475 size_t launched = 0;
476 uint32_t halted = 0, failed = 0;
477 for (size_t i = 0; i < s->nsections; i++) {
478 const mt_section *p = &s->sections[i];
479 if (mt_section_running(p)) launched += mt_section_nodes(p);
480 halted += mt_section_p(p, MESH_P_HALTED);
481 if (mt_section_state(p) == MESH_ST_FAILED) failed++;
482 }
483 bool cuda = mesh_backend_kind(mt_snapshot_head(s, MESH_H_BACKEND)) == MESH_BACKEND_CUDA;
484 uint32_t iso = mt_snapshot_host_u32(s, MESH_S_ISOLATION);
485 tui_line nodes_line = MT_LINE(f, mt_label("nodes "), mt_val(MT_FMT(f, "%u/%zu", mt_snapshot_nodes_online(s), launched)),
486 mt_label(cuda ? " online SMs " : " online cores "),
487 mt_val(MT_FMT(f, "%u/%u", mt_snapshot_host_u32(s, MESH_S_SMS), mt_snapshot_host_u32(s, MESH_S_SMS_TOTAL))),
488 mt_label(" green ctx "), green ? mt_fgb("yes", MT_GOOD) : mt_fgs("no", MT_DIM), mt_label(" isolation "),
489 mt_val(iso == MESH_ISOLATION_MPS ? "mps" : iso == MESH_ISOLATION_LEASE ? "lease" : "blink"));
490 if (failed > 0)
491 tui_line_push(f->a, &nodes_line, mt_fgb(MT_FMT(f, " %u failed", failed), MT_BAD));
492 else if (halted > 0 && launched == 0)
493 tui_line_push(f->a, &nodes_line, mt_fgs(" idle", MT_DIM));
494 float rtt = mt_snapshot_host_f32(s, MESH_S_PEER_RTT_US);
495 char pdir[193];
496 mt_snapshot_host_str(s, MESH_S_PROGRAMS_DIR, 64, pdir);
497 tui_line status = MT_LINE(f, mt_label("programs dir "), mt_fgs(tui_arena_strdup(f->a, pdir), MT_DIM));
498 tui_line inf;
499 if (mt_infer_spans(f, s, &inf)) status = inf;
500 if (!m->online || strcmp(m->status, "online") != 0) status = MT_LINE(f, mt_label("status "), mt_fgs(m->status, MT_BAD));
501 char gpu[97], cc[32];
502 mt_snapshot_gpu(s, gpu);
503 mt_cc_label(mt_snapshot_head(s, MESH_H_CC), cc);
504 const char *gl = cuda ? MT_FMT(f, " cc %s ", cc)
505 : MT_FMT(f, " %s, %u threads/threadgroup, leases ", mesh_host_label(f, mt_snapshot_head(s, MESH_H_BACKEND)),
506 mt_snapshot_head(s, MESH_H_MAX_TG_THREADS));
507 mt_jobs j = mt_snapshot_jobs(s);
508 uint32_t me;
509 bool has_me = mt_snapshot_system(s, &me);
510 tui_line lines[6] = {
511 MT_LINE(f, mt_label("GPU "), mt_val(tui_arena_strdup(f->a, gpu)), mt_label(gl), mt_fgs(sys, MT_ACCENT)),
512 nodes_line,
513 MT_LINE(f, mt_label("blinks "), mt_val(MT_FMT(f, "%u", mt_snapshot_host_u32(s, MESH_S_EPOCH))), mt_label(" last "),
514 mt_val(MT_FMT(f, "%uus", mt_snapshot_host_u32(s, MESH_S_BLINK_US))), mt_label(" up "),
515 mt_val(mt_fmt_uptime(f, mt_snapshot_host_u64(s, MESH_S_UPTIME_MS)))),
516 MT_LINE(f, mt_label("tasks/s "), mt_fgb(mt_fmt_rate(f, mt_snapshot_host_f32(s, MESH_S_TASKS_PER_S)), MT_WARN), mt_label(" programs "),
517 mt_val(MT_FMT(f, "%u", mt_snapshot_host_u32(s, MESH_S_PROGRAMS))), mt_label(" heap free "),
518 mt_val(MT_FMT(f, "%zu KiB", (size_t)mt_snapshot_host_u32(s, MESH_S_HEAP_FREE) * MESH_VIEW_PAGE / 1024)), mt_label(" jobs "),
519 mt_val(j.total == 0 ? "-" : MT_FMT(f, "%u active of %u", j.active, j.total))),
520 MT_LINE(f, mt_label("peer "), mt_peer_span(f, s), mt_label(has_me ? " linked " : ""),
521 mt_val(has_me ? MT_FMT(f, "%u/%u", popcount(mt_snapshot_host_u32(s, MESH_S_LINKED)), mt_snapshot_systems(s)) : ""),
522 mt_label(" RTT "), mt_val(rtt > 0.0f ? MT_FMT(f, "%sus", mt_ff(f, rtt, 1)) : "-")),
523 status,
524 };
525 mt_par(f, tui_text_of(f->a, 6, lines), info);
526
527 tui_line chip = tui_line_span(f->a, mt_label("programs "));
528 for (size_t i = 0; i < s->nsections; i++) {
529 const mt_section *p = &s->sections[i];
530 const char *name = mt_sec_name(f, p);
531 uint8_t c[3];
532 mt_prog_rgb(name, mt_section_slot(p), c);
533 tui_style st;
534 switch (mt_section_state(p)) {
535 case MESH_ST_RUNNING: st = mt_fg_bg_bold(MT_BG, mt_rgb(c, 1.0f)); break;
536 case MESH_ST_FAILED: st = mt_fg_bg_bold(TUI_WHITE, MT_BAD); break;
537 default: st = tui_fg_bg(mt_rgb(c, 0.8f), MT_FAINT); break;
538 }
539 tui_line_push(f->a, &chip, mt_styled(MT_FMT(f, "[%s]", name), st));
540 tui_line_push(f->a, &chip, tui_span_raw(" "));
541 }
542 mt_par_line(f, chip, chips);
543
544 size_t from = m->ntps > spark.width ? m->ntps - spark.width : 0;
545 uint64_t peak = 0;
546 for (size_t i = from; i < m->ntps; i++)
547 if (m->tps[i] > peak) peak = m->tps[i];
548 tui_block sb = tui_block_new();
549 sb.borders = TUI_BORDER_TOP;
550 sb.border_style = tui_fg(MT_FAINT);
551 tui_block_add_title(f->a, &sb, MT_LINE(f, mt_label(" tasks/s "), mt_fgs(MT_FMT(f, "peak %llu ", (unsigned long long)peak), MT_DIM)),
552 TUI_TITLE_DEFAULT);
553 tui_sparkline sp = {0};
554 sp.block = &sb;
555 sp.data = m->tps + from;
556 sp.n = m->ntps - from;
557 sp.style = tui_fg(MT_WARN);
558 tui_sparkline_render(f->a, &sp, spark, f->buf);
559 mt_draw_net_box(f, net, m, net_rows);
560 mt_draw_load_box(f, load, s, load_rows);
561 draw_sm_grid(f, grid, m, s, now);
562}
563
564void mt_draw_mesh(mt_frame *f, tui_rect area, const mt_app *app, mt_instant now) {
565 mt_cluster_row *rows;
566 size_t nrows = mt_cluster_rows(app, &rows);
567 /* The overview: one row per system (and per single machine), above the panels. */
568 uint16_t want = (uint16_t)(nrows + 3);
569 uint16_t ov_h = nrows >= 2 ? tui_max16(tui_min16(want, (uint16_t)(area.height / 2)), tui_min16(4, area.height)) : 0;
570 tui_rect r[2];
571 tui_constraint cons[2] = {TUI_LENGTH(ov_h), TUI_FILL(1)};
572 tui_vsplit(area, 2, cons, r);
573 if (ov_h > 0) draw_overview(f, r[0], app, rows, nrows);
574 free(rows);
575 tui_rect panels = r[1];
576 /* Every machine's panel side by side while each gets 40 columns; else the focused one. */
577 size_t n = app->nmachines;
578 if (n * 40 <= panels.width || n == 1) {
579 tui_constraint *c = tui_arena_alloc(f->a, n * sizeof *c);
580 tui_rect *cols = tui_arena_alloc(f->a, n * sizeof *cols);
581 for (size_t i = 0; i < n; i++) c[i] = TUI_FILL(1);
582 tui_hsplit(panels, n, c, cols);
583 for (size_t i = 0; i < n; i++) draw_machine(f, cols[i], &app->machines[i], i == app->focus, now);
584 } else {
585 draw_machine(f, panels, mt_app_focused(app), true, now);
586 }
587}