==> tests/tracker.c <==1/* sched_tests.rs: colocation_and_pins, hysteresis_and_budget, owner_policy_has_hysteresis,
2 * groups_are_proposed_not_moved, a_new_leader_takes_over_a_migration,
3 * the_moves_verdict_is_its_programs_not_the_whole_pass, a_pin_never_splits_a_colocated_pair (the
4 * scheduler's half), and the 10-minute simulated fleet. */
5#include "fixture.h"
6#include "mesh/check.h"
7#include "mesh/sched_tracker.h"
8
9static msp_input input(const mfd_doc *d, const msm_member *ms, const msm_perfs *perf, const msw_weights *w) { return (msp_input){d, ms, 4, perf, w}; }
10
11static mfd_doc pipe_doc(bool other) {
12 mfd_doc d = doc_new();
13 mesh_needs n = needs("cuda", "metal", NULL);
14 n.replicas = (mesh_replicas){false, 1};
15 msn_terms_push(&n.prefer, &n.nprefer, term(MESH_TERM_SYSTEM, NULL, 2));
16 msn_strs_push(&n.feeds, &n.nfeeds, "pipesink-w");
17 mfd_prog_put(&d, "pipehash", prog_of(n));
18 n = needs("wasm", NULL, NULL);
19 n.colocate_with = mt_strdup("pipehash");
20 msn_strs_push(&n.feeds, &n.nfeeds, "pipehash");
21 if (!other) msn_strs_push(&n.feeds, &n.nfeeds, "pipesink-w");
22 mfd_prog_put(&d, "pipesrc", prog_of(n));
23 n = needs("wasm", NULL, NULL);
24 n.colocate_with = mt_strdup("pipehash");
25 mfd_prog_put(&d, "pipesink-w", prog_of(n));
26 if (other) {
27 n = needs("wasm", NULL, NULL);
28 n.replicas = (mesh_replicas){false, 1};
29 mfd_prog_put(&d, "other", prog_of(n));
30 }
31 return d;
32}
33
34static void colocation_and_pins(void) {
35 msm_member ms[4];
36 fleet4(ms);
37 mfd_doc d = pipe_doc(false);
38 mesh_needs n = needs("wasm", NULL, NULL);
39 n.colocate_with = mt_strdup("absent");
40 mfd_prog_put(&d, "lone", prog_of(n));
41 mfd_doc p = placed(ms, 4, &d);
42 CHECK(at_is(&p, "pipehash", (uint32_t[]){2}, 1) && at_is(&p, "pipesrc", (uint32_t[]){2}, 1) && at_is(&p, "pipesink-w", (uint32_t[]){2}, 1));
43 CHECK(!mfd_prog_get(&p, "lone")->at.some);
44 mt_strs u = mst_unit(&p, "pipehash", 2);
45 CHECK(u.n == 3 && !strcmp(u.v[0], "pipesrc") && !strcmp(u.v[1], "pipehash") && !strcmp(u.v[2], "pipesink-w"));
46 mt_strs_free(&u);
47 /* A pin of a running stateful program is a migration (the leader's); restart programs move at once. */
48 mfd_doc dp = mfd_doc_clone(&p);
49 mfd_prog_get(&dp, "pipehash")->pin = ml_some((uint32_t[]){0}, 1);
50 mfd_doc q = placed(ms, 4, &dp);
51 CHECK(at_is(&q, "pipehash", (uint32_t[]){2}, 1));
52 mst_tracker *t = mst_new();
53 msm_perfs perf = {0};
54 msw_weights w = msw_default();
55 msp_input in = input(&q, ms, &perf, &w);
56 mst_move mv;
57 CHECK(mst_step(t, &in, 0, false, &mv));
58 CHECK(mv.forced && mv.from == 2 && mv.to == 0 && mv.programs.n == 3);
59 mfd_doc s = mst_start_move(&q, &mv, q.version + 1);
60 size_t from2 = 0;
61 for (size_t i = 0; i < s.nprogs; i++) from2 += s.prog[i].has_from && s.prog[i].from == 2;
62 CHECK(from2 == 3 && at_is(&s, "pipesrc", (uint32_t[]){0}, 1));
63 mfd_doc e = mst_end_move(&s, &mv, true, "done");
64 bool none_from = true;
65 for (size_t i = 0; i < e.nprogs; i++) none_from &= !e.prog[i].has_from;
66 CHECK(none_from && at_is(&e, "pipehash", (uint32_t[]){0}, 1));
67 mfd_doc back = mst_end_move(&s, &mv, false, "reverted");
68 CHECK(at_is(&back, "pipehash", (uint32_t[]){2}, 1));
69 mfd_prog_get(&dp, "pipehash")->needs.migrate = MESH_MIGRATE_RESTART;
70 mfd_doc r = placed(ms, 4, &dp);
71 CHECK(at_is(&r, "pipehash", (uint32_t[]){0}, 1));
72 mst_move_free(&mv);
73 mst_free(t);
74 mfd_doc_free(&d), mfd_doc_free(&p), mfd_doc_free(&dp), mfd_doc_free(&q), mfd_doc_free(&s), mfd_doc_free(&e), mfd_doc_free(&back), mfd_doc_free(&r);
75 free_members(ms, 4);
76}
77
78static void hysteresis_and_budget(void) {
79 msm_member ms[4];
80 fleet4(ms);
81 msw_weights w = msw_default();
82 w.hold_ms = 3000, w.dwell_ms = 10000, w.per_min = 1;
83 mfd_doc d0 = doc_new();
84 for (int i = 0; i < 2; i++) {
85 mesh_needs n = needs("wasm", NULL, NULL);
86 n.replicas = (mesh_replicas){false, 1};
87 msn_terms_push(&n.prefer, &n.nprefer, term(MESH_TERM_SYSTEM, NULL, 0));
88 mfd_prog_put(&d0, i ? "b" : "a", prog_of(n));
89 }
90 mfd_doc d = placed(ms, 4, &d0);
91 CHECK(at_is(&d, "a", (uint32_t[]){0}, 1));
92 msm_perfs perf = {0};
93 mst_tracker *t = mst_new();
94 mst_move m;
95 for (uint64_t now = 0; now < 10000; now += 1000) {
96 msp_input in = input(&d, ms, &perf, &w);
97 CHECK(!mst_step(t, &in, now, false, &m));
98 }
99 ms[0].cpu_util = 1.0f;
100 ms[0].psi_cpu = mof32_some(0.5f);
101 uint64_t at = 0;
102 bool got = false;
103 for (uint64_t now = 10000; now < 20000 && !got; now += 500) {
104 msp_input in = input(&d, ms, &perf, &w);
105 if (mst_step(t, &in, now, false, &m)) got = true, at = now;
106 }
107 CHECK(got && at >= 13000 && at <= 14000);
108 CHECK_U(m.from, 0);
109 mst_moved(t, &m, at);
110 mfd_doc s = mst_start_move(&d, &m, d.version + 1);
111 mfd_doc_free(&d);
112 d = mst_end_move(&s, &m, true, "done");
113 mfd_doc_free(&s);
114 mst_move_free(&m);
115 bool second = false;
116 for (uint64_t now = at; now < at + 30000 && !second; now += 500) {
117 msp_input in = input(&d, ms, &perf, &w);
118 if (mst_step(t, &in, now, false, &m)) second = true, mst_move_free(&m);
119 }
120 CHECK(!second);
121 bool budget = false;
122 for (size_t i = 0; i < mst_notes(t)->n; i++) budget |= mt_contains(mst_notes(t)->v[i], "budget");
123 CHECK(budget);
124 msp_input in = input(&d, ms, &perf, &w);
125 CHECK(!mst_step(t, &in, at + 70000, true, &m));
126 CHECK(mst_step(t, &in, at + 70000, false, &m));
127 mst_move_free(&m);
128 mst_free(t);
129 mfd_doc_free(&d), mfd_doc_free(&d0);
130 free_members(ms, 4);
131}
132
133static void owner_policy(void) {
134 msm_member ms[4];
135 fleet4(ms);
136 msw_weights w = msw_default();
137 w.owner_ms = 3000;
138 mst_tracker *t = mst_new();
139 ms[0].gpu_share = mof32_some(0.2f);
140 mst_owner(t, ms, 4, 0, &w);
141 CHECK(!ms[0].busy);
142 mst_owner(t, ms, 4, 2000, &w);
143 ms[0].gpu_share = mof32_some(0.5f);
144 mst_owner(t, ms, 4, 2500, &w);
145 ms[0].gpu_share = mof32_some(0.2f);
146 mst_owner(t, ms, 4, 3000, &w);
147 CHECK(!ms[0].busy);
148 mst_owner(t, ms, 4, 6100, &w);
149 CHECK(ms[0].busy != NULL);
150 CHECK_S(ms[0].busy, "other tenants load it (share 0.20, PSI cpu 0.010)");
151 ms[0].gpu_share = mof32_some(0.35f);
152 mst_owner(t, ms, 4, 7000, &w);
153 mst_owner(t, ms, 4, 20000, &w);
154 CHECK(ms[0].busy != NULL);
155 ms[0].gpu_share = mof32_some(0.6f);
156 mst_owner(t, ms, 4, 21000, &w);
157 mst_owner(t, ms, 4, 24100, &w);
158 CHECK(!ms[0].busy);
159 MSM_SET(ms[3].gpu, "wasm");
160 ms[3].gpu_share = mof32_some(0.1f);
161 mst_owner(t, ms, 4, 0, &w);
162 mst_owner(t, ms, 4, 5000, &w);
163 mst_owner(t, ms, 4, 10000, &w);
164 CHECK(!ms[3].busy);
165 mst_free(t);
166 free_members(ms, 4);
167}
168
169static void groups_proposed(void) {
170 msm_member ms[4];
171 fleet4(ms);
172 mfd_doc d = doc_new();
173 mesh_needs n = needs("cuda", "metal", "wasm");
174 n.lockstep = true;
175 n.group_size = 3;
176 msn_terms_push(&n.group_members, &n.ngroup_members, term(MESH_TERM_BACKEND, "cuda", 0));
177 mfd_prog_put(&d, "train", prog_of(n));
178 mfd_group_push(&d, (mfd_group){.name = mt_strdup("train"), .program = mt_strdup("train"), .epoch = 1, .members = ml_of((uint32_t[]){0, 1, 2}, 3),
179 .state = MFD_RUNNING, .on_loss = MFD_LOSS_PAUSE});
180 msm_perfs perf = {0};
181 msw_weights w = msw_default();
182 w.hold_ms = 2000;
183 msp_input in = input(&d, ms, &perf, &w);
184 ml_list ideal;
185 char *why;
186 CHECK(msp_group_choice(&in, &d.groups[0], &ideal, &why));
187 CHECK(ideal.n == 3 && ideal.v[0] == 0 && ideal.v[2] == 2);
188 ml_free(&ideal);
189 free(why);
190 mst_tracker *t = mst_new();
191 char *g;
192 ml_list m;
193 CHECK(!mst_group_step(t, &in, 0, &g, &m, &why));
194 ms[1].up = false;
195 CHECK(!mst_group_step(t, &in, 1000, &g, &m, &why));
196 CHECK(mst_group_step(t, &in, 3500, &g, &m, &why));
197 CHECK(m.n == 2 && m.v[0] == 0 && m.v[1] == 2);
198 free(g), free(why), ml_free(&m);
199 mt_strs ch;
200 msp_explains ex;
201 mfd_doc q = msp_place(&in, &ch, &ex);
202 CHECK(ch.n == 0 && !mfd_prog_get(&q, "train")->at.some);
203 mt_strs_free(&ch), msp_explains_free(&ex), mfd_doc_free(&q);
204 mst_move mv;
205 CHECK(!mst_step(t, &in, 10000, false, &mv));
206 ms[1].up = true;
207 mesh_needs *tn = &mfd_prog_get(&d, "train")->needs;
208 for (size_t i = 0; i < tn->ngroup_members; i++) mesh_term_free(&tn->group_members[i]);
209 free(tn->group_members), tn->group_members = NULL, tn->ngroup_members = 0;
210 for (int i = 0; i < 4; i++) {
211 for (int j = 0; j < 4; j++) ms[i].rtt_us[j] = mof32_some(250.0f);
212 ms[i].rtt_us[3] = mof32_some(1900.0f);
213 }
214 for (int j = 0; j < 4; j++) ms[3].rtt_us[j] = mof32_some(1900.0f);
215 CHECK(msp_group_choice(&in, &d.groups[0], &ideal, &why));
216 CHECK(ideal.n == 3 && ideal.v[0] == 0 && ideal.v[1] == 1 && ideal.v[2] == 2);
217 ml_free(&ideal), free(why);
218 mst_free(t);
219 mfd_doc_free(&d);
220 free_members(ms, 4);
221}
222
223/* The colocated unit pipesrc + pipehash + pipesink-w on system 2, moving to 0 (state), as version 8. */
224static mfd_doc moving(mst_move *mv) {
225 msm_member ms[4];
226 fleet4(ms);
227 mfd_doc d = pipe_doc(true);
228 mfd_doc p = placed(ms, 4, &d);
229 *mv = (mst_move){mst_unit(&p, "pipehash", 2), 2, 0, true, mt_strdup("test"), false};
230 mfd_doc s = mst_start_move(&p, mv, 8);
231 s.version = 8;
232 mfd_doc_free(&d), mfd_doc_free(&p);
233 free_members(ms, 4);
234 return s;
235}
236
237static mst_outcome out_of(const char *name, uint64_t v, bool ok) { return (mst_outcome){(char *)name, v, ok, ok ? 2500 : 0}; }
238
239static void new_leader_takes_over(void) {
240 mst_move mv;
241 mfd_doc s0 = moving(&mv);
242 size_t c = 0;
243 for (size_t i = 0; i < s0.nprogs; i++) c += s0.prog[i].has_from && s0.prog[i].from == 2 && s0.prog[i].has_to && s0.prog[i].to == 0 && s0.prog[i].since == 8;
244 CHECK_U(c, 3);
245 /* The document round-trips the move. */
246 char *text = mfd_encode(&s0), *err = NULL;
247 mfd_doc s;
248 CHECK(mfd_decode(text, &s, &err));
249 free(text);
250 mst_resumed r = mst_resume_moves(&s);
251 CHECK_U(r.n, 1);
252 const mst_move *m = &r.moves[0];
253 CHECK(m->from == 2 && m->to == 0 && r.since[0] == 8 && m->state && m->programs.n == 3);
254 for (size_t i = 0; i < mv.programs.n; i++) CHECK(mt_strs_has(&m->programs, mv.programs.v[i]));
255 mst_outcome ok[] = {out_of("pipesrc", 9, true), out_of("pipehash", 9, true), out_of("pipesink-w", 9, true)};
256 bool good;
257 char *why;
258 double gap;
259 CHECK(mst_mig_verdict(m, 8, true, ok, 3, "", 100, 90000, &good, &why, &gap) && good && gap == 2.5);
260 free(why);
261 mfd_doc e = mst_end_move(&s, m, true, "done");
262 bool clear = true;
263 for (size_t i = 0; i < e.nprogs; i++) clear &= !e.prog[i].has_from && !e.prog[i].has_to;
264 CHECK(clear && at_is(&e, "pipehash", (uint32_t[]){0}, 1));
265 mst_resumed none = mst_resume_moves(&e);
266 CHECK_U(none.n, 0);
267 mst_resumed_free(&none);
268 mst_outcome old[] = {out_of("pipesrc", 5, false), out_of("pipehash", 5, false), out_of("pipesink-w", 5, false)};
269 CHECK(!mst_mig_verdict(m, 8, true, old, 3, "", 100, 90000, &good, &why, &gap));
270 CHECK(mst_mig_verdict(m, 8, true, old, 3, "", 90001, 90000, &good, &why, &gap) && !good);
271 free(why);
272 CHECK(mst_mig_verdict(m, 8, false, ok, 3, "", 100, 90000, &good, &why, &gap) && !good);
273 free(why);
274 mfd_doc back = mst_end_move(&s, m, false, "back");
275 CHECK(at_is(&back, "pipehash", (uint32_t[]){2}, 1) && !mfd_prog_get(&back, "pipehash")->has_from);
276 mfd_doc_free(&back), mfd_doc_free(&e), mfd_doc_free(&s), mfd_doc_free(&s0);
277 mst_resumed_free(&r);
278 mst_move_free(&mv);
279}
280
281static void verdict_is_its_programs(void) {
282 mst_move mv;
283 mfd_doc s = moving(&mv);
284 mst_outcome o[] = {out_of("pipesrc", 8, true), out_of("pipehash", 8, true), out_of("pipesink-w", 8, true), out_of("other", 8, false)};
285 bool good;
286 char *why;
287 double gap;
288 CHECK(mst_mig_verdict(&mv, 8, true, o, 4, "load other: jit: boom", 100, 90000, &good, &why, &gap) && good);
289 free(why);
290 o[1].ok = false;
291 CHECK(mst_mig_verdict(&mv, 8, true, o, 4, "migrate pipehash from system 2: no view", 100, 90000, &good, &why, &gap) && !good);
292 CHECK(mt_contains(why, "pipehash") && !mt_contains(why, "pipesrc,"));
293 free(why);
294 CHECK(!mst_mig_verdict(&mv, 8, true, o, 2, "", 100, 90000, &good, &why, &gap));
295 mfd_doc_free(&s);
296 mst_move_free(&mv);
297}
298
299static void pin_never_splits(void) {
300 msm_member ms[4];
301 fleet4(ms);
302 mst_move mv;
303 mfd_doc s = moving(&mv);
304 mfd_doc ended = mst_end_move(&s, &mv, true, "done");
305 mfd_doc d = placed(ms, 4, &ended);
306 CHECK(at_is(&d, "pipehash", (uint32_t[]){0}, 1) && at_is(&d, "pipesrc", (uint32_t[]){0}, 1));
307 mfd_prog_get(&d, "pipesrc")->pin = ml_some((uint32_t[]){1}, 1);
308 mfd_doc p = placed(ms, 4, &d);
309 CHECK(at_is(&p, "pipesrc", (uint32_t[]){0}, 1));
310 mst_tracker *t = mst_new();
311 msm_perfs perf = {0};
312 msw_weights w = msw_default();
313 msp_input in = input(&p, ms, &perf, &w);
314 mst_move m;
315 CHECK(!mst_step(t, &in, 0, false, &m));
316 mst_free(t);
317 mfd_doc_free(&p), mfd_doc_free(&d), mfd_doc_free(&ended), mfd_doc_free(&s);
318 mst_move_free(&mv);
319 free_members(ms, 4);
320}
321
322/* A simulated fleet over 10 minutes: members leave and return, the load oscillates on one member; the
323 * placement stays valid at every step and programs do not flap. */
324/* sched_tests.rs simulation_is_stable_and_valid */
325static void simulation(void) {
326 msw_weights w = msw_default();
327 w.hold_ms = 6000, w.dwell_ms = 20000, w.per_min = 3, w.replace_ms = 3000;
328 msm_member ms[4];
329 fleet4(ms);
330 mfd_doc d = doc_new();
331 mesh_needs n;
332 n = needs("cuda", "metal", "wasm"), n.replicas = (mesh_replicas){false, 2}, mfd_prog_put(&d, "svc", prog_of(n));
333 n = needs("wasm", NULL, NULL), n.replicas = (mesh_replicas){false, 1}, n.cpu = 1.0f, mfd_prog_put(&d, "cpu1", prog_of(n));
334 n = needs("wasm", NULL, NULL), n.replicas = (mesh_replicas){false, 1}, n.cpu = 1.0f, mfd_prog_put(&d, "cpu2", prog_of(n));
335 n = needs("metal", NULL, NULL), n.replicas = (mesh_replicas){false, 1}, mfd_prog_put(&d, "mac", prog_of(n));
336 n = needs("cuda", "metal", NULL), n.replicas = (mesh_replicas){false, 1}, mfd_prog_put(&d, "hash", prog_of(n));
337 n = needs("wasm", NULL, NULL), n.colocate_with = mt_strdup("hash"), msn_strs_push(&n.feeds, &n.nfeeds, "hash"), mfd_prog_put(&d, "src", prog_of(n));
338 n = needs("wasm", NULL, NULL), mfd_prog_put(&d, "every", prog_of(n));
339 msm_perfs perf = {0};
340 mst_tracker *t = mst_new();
341 ms_map moves = {0}; /* program -> "t1,t2,..." */
342 uint64_t down_at[4] = {0};
343 bool down[4] = {0};
344 uint64_t rng = 12345;
345#define RAND(n) (rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17, rng % (n))
346 bool inflight = false;
347 mst_move im = {0};
348 uint64_t it0 = 0;
349 size_t total = 0;
350 for (uint64_t step = 0; step < 600; step++) {
351 uint64_t now = step * 1000;
352 bool phase = (now / 20000) % 2 == 1;
353 ms[1].cpu_util = (phase ? 0.95f : 0.1f) + (float)RAND(10) / 200.0f;
354 ms[1].psi_cpu = mof32_some(phase ? 0.4f : 0.01f);
355 for (int k = 0; k < 4; k++)
356 if (k != 1) ms[k].cpu_util = 0.1f + (float)RAND(20) / 200.0f;
357 if (step == 120) ms[2].up = false, down[2] = true, down_at[2] = now;
358 if (step == 200) ms[2].up = true, down[2] = false;
359 for (int k = 0; k < 4; k++) ms[k].down_ms = down[k] ? now - down_at[k] : 0;
360 mst_owner(t, ms, 4, now, &w);
361 msp_input in = input(&d, ms, &perf, &w);
362 mt_strs ch;
363 msp_explains ex;
364 mfd_doc nd = msp_place(&in, &ch, &ex);
365 mt_strs_free(&ch), msp_explains_free(&ex), mfd_doc_free(&d);
366 d = nd;
367 if (inflight && now - it0 >= 2000) {
368 mfd_doc e = mst_end_move(&d, &im, true, "done");
369 mfd_doc_free(&d);
370 d = e;
371 inflight = false;
372 mst_move_free(&im);
373 }
374 in = input(&d, ms, &perf, &w);
375 mst_move m;
376 if (mst_step(t, &in, now, inflight, &m)) {
377 mst_moved(t, &m, now);
378 for (size_t i = 0; i < m.programs.n; i++) {
379 const char *prev = ms_get(&moves, m.programs.v[i]);
380 char *v = prev ? mt_fmt("%s,%llu", prev, (unsigned long long)now) : mt_fmt("%llu", (unsigned long long)now);
381 ms_set(&moves, m.programs.v[i], v, true);
382 free(v);
383 total++;
384 }
385 mfd_doc s = mst_start_move(&d, &m, d.version + 1);
386 mfd_doc_free(&d);
387 d = s;
388 im = m;
389 it0 = now;
390 inflight = true;
391 }
392 if (phase && now % 20000 == 19000)
393 for (size_t i = 0; i < d.nprogs; i++)
394 if (!d.prog[i].needs.replicas.all && d.prog[i].at.some && ml_has(&d.prog[i].at.l, 1)) {
395 fprintf(stderr, "%s still on the loaded member at %llu\n", d.pname[i], (unsigned long long)now);
396 CHECK(0);
397 }
398 for (size_t i = 0; i < d.nprogs; i++) {
399 const mfd_prog *p = &d.prog[i];
400 ml_list at = msp_members_of_at(&p->at, ms, 4, &p->needs);
401 if (!p->needs.replicas.all && !p->needs.colocate_with) CHECK(at.n <= p->needs.replicas.n);
402 for (size_t j = 0; j < at.n; j++) {
403 const msm_member *mm = &ms[at.v[j]];
404 CHECK(msm_exec_for(mm, p->needs.backends, p->needs.nbackends) != NULL);
405 CHECK(mm->up || mm->down_ms < w.replace_ms || p->has_from);
406 }
407 if (!strcmp(d.pname[i], "mac")) CHECK(at.n == 1 && at.v[0] == 3);
408 if (!strcmp(d.pname[i], "src") && !p->has_from) CHECK(ml_opt_eq(&p->at, &mfd_prog_get(&d, "hash")->at));
409 ml_free(&at);
410 }
411 }
412 CHECK(total <= 30);
413 CHECK(total > 0);
414 for (size_t i = 0; i < moves.n; i++) {
415 mt_strs ts = mt_split(moves.v[i], ',');
416 CHECK(ts.n <= 12);
417 for (size_t j = 1; j < ts.n; j++) CHECK(strtoull(ts.v[j], NULL, 10) - strtoull(ts.v[j - 1], NULL, 10) >= w.dwell_ms);
418 mt_strs_free(&ts);
419 }
420 fprintf(stderr, "10 simulated minutes: %zu moves\n", total);
421 if (inflight) mst_move_free(&im);
422 ms_free(&moves);
423 mst_free(t);
424 mfd_doc_free(&d);
425 free_members(ms, 4);
426}
427
428int main(void) {
429 colocation_and_pins();
430 hysteresis_and_budget();
431 owner_policy();
432 groups_proposed();
433 new_leader_takes_over();
434 verdict_is_its_programs();
435 pin_never_splits();
436 simulation();
437 return mesh_check_done();
438}