somatize_health/flag.rs
1//! What can be wrong. The vocabulary of a diagnosis.
2
3use std::fmt;
4
5/// One thing that looks wrong with a node.
6///
7/// An enum because the set is closed and named: a diagnosis is only useful if
8/// two runs of it say the same word for the same thing. Every variant is an
9/// **opinion at a threshold** and none of them is a fact — the facts are in the
10/// record; these are what somebody thinks of them.
11#[derive(Debug, Clone, PartialEq, Eq)]
12pub enum Flag {
13 /// A number stopped being a number. Nothing below it means anything.
14 Nan,
15 /// Or stopped being finite.
16 Inf,
17 /// The parameter gradients are so small this node is not being trained. The
18 /// classic depth pathology, and a **profile over depth** rather than a
19 /// property of a network: the early layers go quiet while the last learns.
20 Vanishing,
21 /// The parameter gradients are so large the next step will not be a step.
22 Exploding,
23 /// The signal has grown over a stretch nobody is normalising.
24 ///
25 /// A conjunction, and both halves are load-bearing: the structural half is
26 /// baked into the measurement, which counts the gain from the last
27 /// normalisation upstream. A badly-initialised stack *with* a norm layer
28 /// drifts under 3x and trains.
29 ///
30 /// One-sided, and measured rather than assumed: it says nothing about a
31 /// signal that shrank, because Adam is scale-invariant per parameter. See
32 /// `health/tests/normalisation.py`.
33 MissingNormalisation,
34 /// Most of what this node outputs is zero, on at least one step.
35 ///
36 /// Read off the **maximum** over the window: a layer that dies one step in
37 /// four is dead, and the mean is exactly what hides it.
38 Dead,
39 /// Most of what it outputs is pinned at the far end of its non-linearity,
40 /// where the derivative is nothing. Also read off the maximum.
41 Saturated,
42 /// It is moving, but by so little relative to its own weights that it will
43 /// not arrive. The ratio of update to weight, which practice puts near
44 /// `1e-3` — the cheapest signal there is.
45 Stalled,
46 /// It is moving so much relative to its own weights that each step throws
47 /// away where it was.
48 Overstepping,
49 /// How many channels are dead — output near zero across the window. Separate
50 /// from [`Flag::Dead`]: a layer can be alive with a quarter of its width
51 /// doing nothing, which is a width problem and not a layer problem.
52 DeadChannels(usize),
53 /// How many channels are **alive and never asked for**: they compute
54 /// something and no gradient comes back. Gradient starvation, and a dormant
55 /// channel is not the same thing — it is computing nothing to be ignored.
56 IgnoredChannels(usize),
57 /// Two groups of channels the architecture means to keep apart are carrying
58 /// the same information, by linear CKA.
59 Leakage,
60 /// The update has collapsed into a few directions compared with what this
61 /// run was doing before. The earliest warning there is: it moves thousands
62 /// of steps before the loss does.
63 Narrowing,
64 /// An input the model is not using: taking it away costs nothing. A network
65 /// with a perfectly healthy gradient can be ignoring an input all afternoon
66 /// without a single other flag firing.
67 IgnoredInput(String),
68 /// One input carries everything, and nothing else would take over.
69 ///
70 /// Not a failure and not always wrong: sometimes one channel really is the
71 /// signal. It is worth knowing before the day that channel is missing.
72 SoleReliance(String),
73 /// The weights keep growing, the representation keeps narrowing and the
74 /// units keep going quiet — **all three at once**, which is what tells a
75 /// network that has stopped being able to learn from one that is training.
76 LosingPlasticity,
77}
78
79impl Flag {
80 /// The word this flag is written down as, which is what a record keeps and
81 /// what somebody greps for.
82 pub fn name(&self) -> &'static str {
83 match self {
84 Self::Nan => "NAN",
85 Self::Inf => "INF",
86 Self::Vanishing => "VANISHING",
87 Self::Exploding => "EXPLODING",
88 Self::Dead => "DEAD",
89 Self::Saturated => "SATURATED",
90 Self::Stalled => "STALLED",
91 Self::Overstepping => "OVERSTEPPING",
92 Self::DeadChannels(_) => "DEAD_CHANNELS",
93 Self::IgnoredChannels(_) => "IGNORED_CHANNELS",
94 Self::IgnoredInput(_) => "IGNORED_INPUT",
95 Self::SoleReliance(_) => "SOLE_RELIANCE",
96 Self::MissingNormalisation => "MISSING_NORMALISATION",
97 Self::Leakage => "LEAKAGE",
98 Self::Narrowing => "NARROWING",
99 Self::LosingPlasticity => "LOSING_PLASTICITY",
100 }
101 }
102
103 /// Which family of trouble this is. A closed set, so a figure can give each
104 /// family a colour instead of painting everything one red. By **what to do
105 /// about them** and not by what was measured: `VANISHING` and `EXPLODING`
106 /// are both answered by looking at depth and initialisation.
107 pub fn family(&self) -> &'static str {
108 match self {
109 Self::Nan | Self::Inf => "numeric",
110 Self::Vanishing | Self::Exploding | Self::MissingNormalisation => "signal",
111 Self::Dead | Self::Saturated => "activation",
112 Self::Stalled | Self::Overstepping => "step",
113 Self::DeadChannels(_)
114 | Self::IgnoredChannels(_)
115 | Self::Leakage
116 | Self::Narrowing
117 | Self::LosingPlasticity => "capacity",
118 Self::IgnoredInput(_) | Self::SoleReliance(_) => "data",
119 }
120 }
121
122 /// What to do about it, in one line. Part of the flag and not of whoever
123 /// draws it: the thresholds and the advice are the same opinion.
124 pub fn about(&self) -> &'static str {
125 match self {
126 Self::Nan => "a number stopped being one; every metric below this is meaningless",
127 Self::Inf => "something overflowed; look at the step before this one",
128 Self::Vanishing => {
129 "this node is barely being trained — look at the depth profile, not at this node \
130 alone: it is the early layers that go quiet first"
131 }
132 Self::Exploding => "the next step will not be a step; clip, or lower the rate",
133 Self::Dead => {
134 "most of the output is zero on at least one step; the non-linearity or \
135 the init is cutting everything off"
136 }
137 Self::Saturated => "most of the output is pinned where the derivative is nothing",
138 Self::Stalled => {
139 "the update is tiny next to the weights; the rate is too low for \
140 this node, or nothing is reaching it"
141 }
142 Self::Overstepping => "each step throws away where it was; the rate is too high",
143 Self::DeadChannels(_) => {
144 "part of the width is doing nothing — a width problem, not a \
145 layer problem"
146 }
147 Self::IgnoredChannels(_) => {
148 "these channels compute something nobody asks for; the \
149 gradient never comes back for them"
150 }
151 Self::IgnoredInput(_) => {
152 "the model is not using this input: taking it away costs nothing. If this is \
153 the channel the work is about, nothing in the network is the problem"
154 }
155 Self::SoleReliance(_) => {
156 "one input carries everything and nothing else would take over if it went"
157 }
158 Self::MissingNormalisation => {
159 "the signal grows over a stretch with nothing normalising it; the first \
160 step will be taken on numbers this size"
161 }
162 Self::Leakage => "two groups meant to stay apart carry the same information",
163 Self::Narrowing => {
164 "the update has collapsed into a few directions; this moves long \
165 before the loss does"
166 }
167 Self::LosingPlasticity => {
168 "weights growing, rank falling, units going quiet — it is \
169 losing the ability to learn anything new"
170 }
171 }
172 }
173}
174
175impl fmt::Display for Flag {
176 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
177 match self {
178 Self::DeadChannels(n) | Self::IgnoredChannels(n) => write!(f, "{}({n})", self.name()),
179 Self::IgnoredInput(what) | Self::SoleReliance(what) => {
180 write!(f, "{}({what})", self.name())
181 }
182 _ => f.write_str(self.name()),
183 }
184 }
185}