somatize_health/
leaning.rs

1//! What a model is **leaning on** — not the same question as whether it is
2//! healthy. A network can pass every check in [`verdict`](crate::verdict) and be
3//! learning the wrong thing.
4//!
5//! It comes from a real project: symptom channels for detecting a mental-health
6//! condition, months spent on the architecture, and the predictive signal was in
7//! the **self-disclosure** and not in the presence of symptoms. No amount of
8//! looking at gradients was ever going to say so. What says so is cheap: take
9//! one input away and score it again.
10//!
11//! A contribution is the score with an input **shuffled** minus the score with
12//! it intact, as a share of what all of them are worth. Shuffled and not zeroed,
13//! because a zero is a value — often an unusually informative one.
14//!
15//! It is a **ranking** and not an attribution. Two inputs carrying the same
16//! signal both look unimportant, because removing either leaves the other, and
17//! that is a true thing about the data rather than a flaw in the method.
18
19use crate::{Flag, Thresholds};
20
21/// What one input turned out to be worth.
22#[derive(Debug, Clone, PartialEq)]
23pub struct Contribution {
24    /// What it is called — the key of the input, or the node that reads it.
25    pub name: String,
26    /// How much worse the score gets without it, as a share of the total drop
27    /// across every input. Between `0.0` and `1.0` when the drops are positive;
28    /// a **negative** one is real and means the model does better without that
29    /// input, which is worth seeing rather than clamping away.
30    pub share: f64,
31    /// And the raw difference, in whatever the score was measured in.
32    pub drop: f64,
33}
34
35/// What is wrong with what a model is leaning on. Empty is not a clean bill:
36/// with one input there is nothing to compare.
37///
38/// The two findings are opposite ends of one worry. An input that costs nothing
39/// to remove is one the model is not using; an input that carries everything is
40/// a model with one leg.
41pub fn leaning(shares: &[Contribution], thresholds: &Thresholds) -> Vec<Flag> {
42    if shares.len() < 2 {
43        return Vec::new();
44    }
45    let mut flags = Vec::new();
46    for one in shares {
47        if one.share < thresholds.ignored_input {
48            flags.push(Flag::IgnoredInput(one.name.clone()));
49        }
50    }
51    if let Some(most) = shares.iter().max_by(|a, b| {
52        a.share
53            .partial_cmp(&b.share)
54            .unwrap_or(std::cmp::Ordering::Equal)
55    }) && most.share > thresholds.sole_reliance
56    {
57        flags.push(Flag::SoleReliance(most.name.clone()));
58    }
59    flags
60}
61
62/// Turns raw drops into shares, which is the only arithmetic here. Divided by
63/// the **total** and not the largest, so they add up to one. When nothing
64/// matters, every share is zero rather than a division by nothing, and
65/// `IGNORED_INPUT` fires on all of them — which is right.
66pub fn shares(drops: &[(String, f64)]) -> Vec<Contribution> {
67    let total: f64 = drops.iter().map(|(_, drop)| drop.max(0.0)).sum();
68    drops
69        .iter()
70        .map(|(name, drop)| Contribution {
71            name: name.clone(),
72            share: if total > 0.0 { drop / total } else { 0.0 },
73            drop: *drop,
74        })
75        .collect()
76}
somatize_health/leaning.rs

somatize_health/
leaning.rs
