somatize_health/
lib.rs

1//! Whether what happened is healthy. **An opinion, and it says so.**
2//!
3//! The third of the three things observability splits into: the declaration
4//! drawn, the record of what happened, and a judgement **about** that record —
5//! with thresholds somebody chose, which is what makes it not a fact.
6//!
7//! > A diagnosis has to be reproducible from the stored record, without training
8//! > again.
9//!
10//! Which is why this crate has no dependencies and touches nothing: numbers in,
11//! [`Flag`]s out. The statistics are measured where torch is, cross as numbers,
12//! and are judged here — so changing a threshold costs a scan, and an alarm you
13//! cannot re-ask is one you cannot argue with.
14//!
15//! [`verdict`] asks whether a network is **learning**; [`leaning`] asks whether
16//! it is learning **what you think**, which no gradient will ever say.
17//!
18//! It does not measure, and it does not decide when to look or how often: a
19//! verdict that quietly needed a particular cadence would be a threshold hiding
20//! in a schedule.
21//!
22//! **`Dead` and `Saturated` read the maximum and not the mean.** A layer that
23//! dies one step in four is dead, and the average is what hides it. It is the
24//! original's finding, and obvious only once somebody has been bitten.
25
26#![forbid(unsafe_code)]
27#![warn(missing_docs)]
28
29mod flag;
30mod leaning;
31mod seen;
32mod thresholds;
33
34pub use flag::Flag;
35pub use leaning::{Contribution, leaning, shares};
36pub use seen::Seen;
37pub use thresholds::Thresholds;
38
39/// Everything wrong with what was seen, in the order it is worth reading. Empty
40/// means nothing tripped, which is not the same as healthy — a metric nobody
41/// measured cannot flag, and [`Seen`] says which those were. What stops a run
42/// first comes first: a `NaN` makes every number below it meaningless.
43pub fn verdict(seen: &Seen, thresholds: &Thresholds) -> Vec<Flag> {
44    let mut flags = Vec::new();
45    if seen.nan {
46        flags.push(Flag::Nan);
47    }
48    if seen.inf {
49        flags.push(Flag::Inf);
50    }
51    if let Some(norm) = seen.grad_norm {
52        if norm < thresholds.grad_low {
53            flags.push(Flag::Vanishing);
54        } else if norm > thresholds.grad_high {
55            flags.push(Flag::Exploding);
56        }
57    }
58    // One-sided: growing is the half that was measured to separate. See
59    // `Thresholds::gain_drift`, and `health/tests/normalisation.py` for why
60    // there is no bound underneath it.
61    if seen
62        .signal_gain
63        .is_some_and(|gain| gain > thresholds.gain_drift)
64    {
65        flags.push(Flag::MissingNormalisation);
66    }
67    // The maximum over the window, never the mean.
68    if seen.zero_frac_max.is_some_and(|f| f > thresholds.dead_frac) {
69        flags.push(Flag::Dead);
70    }
71    if seen
72        .sat_frac_max
73        .is_some_and(|f| f > thresholds.saturated_frac)
74    {
75        flags.push(Flag::Saturated);
76    }
77    if let Some(ratio) = seen.update_ratio {
78        if ratio < thresholds.update_low {
79            flags.push(Flag::Stalled);
80        } else if ratio > thresholds.update_high {
81            flags.push(Flag::Overstepping);
82        }
83    }
84    if seen.dead_channels > 0 {
85        flags.push(Flag::DeadChannels(seen.dead_channels));
86    }
87    if seen.ignored_channels > 0 {
88        flags.push(Flag::IgnoredChannels(seen.ignored_channels));
89    }
90    if seen
91        .group_cka
92        .is_some_and(|cka| cka > thresholds.leakage_cka)
93    {
94        flags.push(Flag::Leakage);
95    }
96    if narrowing(seen, thresholds) {
97        flags.push(Flag::Narrowing);
98    }
99    if losing_plasticity(seen, thresholds) {
100        flags.push(Flag::LosingPlasticity);
101    }
102    flags
103}
104
105/// Whether the update has collapsed into a few directions **relative to what
106/// this run was doing before**.
107///
108/// Huang et al. (2026) monitor the spectrum of `dW = W_t - W_{t-d}` and find it
109/// collapses thousands of steps before the loss does. Their certificate is the
110/// deviation from a healthy baseline run, which nobody watching one run has, so
111/// this compares against its own recent median instead — a substitution that was
112/// measured and does **not** hold. Hence
113/// [`Thresholds::narrowing_of_usual`] is `0.0` and this never fires: the metric
114/// is recorded and drawn, which is a weaker claim than an alarm.
115fn narrowing(seen: &Seen, thresholds: &Thresholds) -> bool {
116    if thresholds.narrowing_of_usual <= 0.0 {
117        return false;
118    }
119    let (Some(rank), Some(usual)) = (seen.update_rank, seen.update_rank_usual) else {
120        return false;
121    };
122    usual > 0.0 && rank / usual < thresholds.narrowing_of_usual
123}
124
125/// Whether the network is losing its ability to learn anything new. A
126/// conjunction, and that is the point: Dohare et al. (2024) tie plasticity loss
127/// to parameter norms rising, units going dormant and the rank falling **at
128/// once**. Any one alone is a network that is training.
129fn losing_plasticity(seen: &Seen, thresholds: &Thresholds) -> bool {
130    let (Some(weights), Some(rank), Some(dormant)) = (
131        seen.param_norm_slope,
132        seen.eff_rank_slope,
133        seen.dormancy_frac,
134    ) else {
135        return false;
136    };
137    weights > thresholds.plasticity_growth
138        && rank < -thresholds.plasticity_growth
139        && dormant > thresholds.dormant_frac
140}
somatize_health/lib.rs

somatize_health/
lib.rs
