1/* @ajax/mesh-ftz: the cluster float mode `Z` (mesh-ref ftz.rs).
2 *
3 * A Z value is a plain float; what makes it Z is that every operation goes through these functions,
4 * each the PTX op it stands for: add/sub/mul/div .rn.ftz, abs/neg .ftz, setp .ftz, cvt .ftz. A
5 * subnormal input is flushed to the zero of its sign, the IEEE result is correctly rounded, and a
6 * subnormal result is flushed the same way. Loads, stores and moves never flush (a float variable
7 * keeps its bits); only operations do.
8 *
9 * On wasm32 every float operation of Z is an exported, never-inlined function `mesh_ftz_*`
10 * (defined in ftz.c), and nothing else here uses a float instruction: flushes and NaN tests are
11 * integer bit tests. meshd's load-time float guard refuses an FTZ wasm image with a float
12 * instruction anywhere else. Elsewhere they are static inline.
13 *
14 * Freestanding: only <stdint.h>, <stdbool.h>, <float.h> and <stdatomic.h>. Code using Z must be
15 * compiled with -ffp-contract=off (no fused multiply-add) on a target with FLT_EVAL_METHOD 0. */
16#ifndef MESH_FTZ_H
17#define MESH_FTZ_H
1819#include <float.h>
20#include <stdatomic.h>
21#include <stdbool.h>
22#include <stdint.h>
2324_Static_assert(FLT_EVAL_METHOD == 0, "Z needs float arithmetic in float precision (no x87)");
2526/* Subnormals flushed by Z operations since the process started (inputs and results). */
27extern _Atomic uint64_t mesh_ftz_flush_count;
28uint64_t mesh_ftz_flushes(void);
2930typedef union {
31 float f;
32 uint32_t u;
33} mesh_ftz_pun;
3435static inline uint32_t mesh_ftz_bits(float x) {
36 mesh_ftz_pun p;
37 p.f = x;
38 return p.u;
39}
4041static inline float mesh_ftz_from_bits(uint32_t b) {
42 mesh_ftz_pun p;
43 p.u = b;
44 return p.f;
45}
4647/* A subnormal to the zero of its sign; anything else unchanged. A bit test, no float instruction. */
48static inline float mesh_ftz_flush(float x) {
49 uint32_t b = mesh_ftz_bits(x);
50 if ((b & 0x7f800000u) == 0 && (b & 0x007fffffu) != 0) {
51 atomic_fetch_add_explicit(&mesh_ftz_flush_count, 1, memory_order_relaxed);
52 return mesh_ftz_from_bits(b & 0x80000000u);
53 }
54 return x;
55}
5657/* setp.nan.ftz.f32 against itself: a bit test. */
58static inline bool mesh_ftz_is_nan(float x) { return (mesh_ftz_bits(x) & 0x7fffffffu) > 0x7f800000u; }
5960/* cvt.rn.f32.u32 and cvt.rn.f32.s32: an integer is never subnormal, so nothing to flush. */
61static inline float mesh_ftz_from_u32(uint32_t x) { return (float)x; }
62static inline float mesh_ftz_from_i32(int32_t x) { return (float)x; }
6364/* A NaN operand's result: x86's rule with a as the first operand (the NaN of a, else of b, quieted).
65 * C, like Rust, leaves the operand order of a commutative + or * to the compiler, so without this the
66 * payload of NaN + NaN would depend on how the code happened to compile. */
67static inline float mesh_ftz_nan2(float a, float b) {
68 uint32_t x = mesh_ftz_bits(a);
69 if ((x & 0x7fffffffu) <= 0x7f800000u) x = mesh_ftz_bits(b);
70 return mesh_ftz_from_bits(x | 0x00400000u);
71}
7273#define MESH_FTZ_BINOP(a, b, op) \
74 do { \
75 a = mesh_ftz_flush(a); \
76 b = mesh_ftz_flush(b); \
77 if (mesh_ftz_is_nan(a) || mesh_ftz_is_nan(b)) return mesh_ftz_nan2(a, b); \
78 return mesh_ftz_flush(a op b); \
79 } while (0)
8081#define MESH_FTZ_LESS (-1)
82#define MESH_FTZ_EQUAL 0
83#define MESH_FTZ_GREATER 1
84#define MESH_FTZ_UNORDERED 2
8586#if defined(__wasm32__)
87/* The only places an FTZ wasm image may hold float instructions (ftz.c). */
88float mesh_ftz_add(float a, float b);
89float mesh_ftz_sub(float a, float b);
90float mesh_ftz_mul(float a, float b);
91float mesh_ftz_div(float a, float b);
92float mesh_ftz_neg(float a);
93float mesh_ftz_abs(float a);
94double mesh_ftz_to_f64(float a);
95float mesh_ftz_from_f64(double a);
96int32_t mesh_ftz_cmp(float a, float b);
97#else
98static inline float mesh_ftz_add(float a, float b) { MESH_FTZ_BINOP(a, b, +); }
99static inline float mesh_ftz_sub(float a, float b) { MESH_FTZ_BINOP(a, b, -); }
100static inline float mesh_ftz_mul(float a, float b) { MESH_FTZ_BINOP(a, b, *); }
101static inline float mesh_ftz_div(float a, float b) { MESH_FTZ_BINOP(a, b, /); }
102/* neg.ftz: the sign bit of the flushed input flipped (a NaN keeps its payload). */
103static inline float mesh_ftz_neg(float a) { return mesh_ftz_from_bits(mesh_ftz_bits(mesh_ftz_flush(a)) ^ 0x80000000u); }
104/* abs.ftz: the sign bit of the flushed input cleared. */
105static inline float mesh_ftz_abs(float a) { return mesh_ftz_from_bits(mesh_ftz_bits(mesh_ftz_flush(a)) & 0x7fffffffu); }
106/* cvt.ftz.f64.f32 and cvt.rn.ftz.f32.f64. */
107static inline double mesh_ftz_to_f64(float a) { return (double)mesh_ftz_flush(a); }
108static inline float mesh_ftz_from_f64(double a) { return mesh_ftz_flush((float)a); }
109/* setp.ftz: MESH_FTZ_LESS, _EQUAL, _GREATER, or _UNORDERED (a NaN), of the flushed inputs. */
110static inline int32_t mesh_ftz_cmp(float a, float b) {
111 a = mesh_ftz_flush(a);
112 b = mesh_ftz_flush(b);
113 if (a < b) return MESH_FTZ_LESS;
114 if (a > b) return MESH_FTZ_GREATER;
115 if (a == b) return MESH_FTZ_EQUAL;
116 return MESH_FTZ_UNORDERED;
117}
118#endif
119120/* The compares of setp.{eq,ne,lt,le,gt,ge}.ftz.f32: false whenever a NaN is involved, except ne. */
121static inline bool mesh_ftz_eq(float a, float b) { return mesh_ftz_cmp(a, b) == MESH_FTZ_EQUAL; }
122static inline bool mesh_ftz_ne(float a, float b) { return mesh_ftz_cmp(a, b) != MESH_FTZ_EQUAL; }
123static inline bool mesh_ftz_lt(float a, float b) { return mesh_ftz_cmp(a, b) == MESH_FTZ_LESS; }
124static inline bool mesh_ftz_gt(float a, float b) { return mesh_ftz_cmp(a, b) == MESH_FTZ_GREATER; }
125static inline bool mesh_ftz_le(float a, float b) {
126 int32_t c = mesh_ftz_cmp(a, b);
127 return c == MESH_FTZ_LESS || c == MESH_FTZ_EQUAL;
128}
129static inline bool mesh_ftz_ge(float a, float b) {
130 int32_t c = mesh_ftz_cmp(a, b);
131 return c == MESH_FTZ_GREATER || c == MESH_FTZ_EQUAL;
132}
133134#endif