somatize_core/
summary.rs

1//! What a run says about itself — the data half.
2//!
3//! [`RunSummary`] and [`RunConclusion`] are pure, serializable facts:
4//! how a run ended, what it cost, what it measured, what it flagged.
5//! They live in `soma-core` (not next to the code that computes them)
6//! for the same reason [`GraphOverlay`](crate::viz::GraphOverlay) does:
7//! the producer is `soma-runtime`'s `summarize`, which reads a run
8//! directory, but the *consumers* — the experiment journal, the MCP
9//! tools, any front-end — must be able to name these types without
10//! taking a dependency on the execution engine.
11//!
12//! [`RunConclusion::render_headline`] is a deterministic template: same
13//! facts, same string, no model in the loop. That is what makes a
14//! headline safe to hash, snapshot-test and index for retrieval.
15
16use crate::fingerprint::ArchitectureFingerprint;
17use crate::tracking::GitInfo;
18use chrono::{DateTime, Utc};
19use serde::{Deserialize, Serialize};
20use std::collections::BTreeMap;
21use std::fmt::Write as _;
22
23/// How a run ended, independent of whether its results were any good.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
25#[serde(rename_all = "snake_case")]
26#[non_exhaustive]
27pub enum RunOutcome {
28    /// Finished successfully.
29    Completed,
30    /// Finished with an error.
31    Failed,
32    /// Marked running, but the heartbeat went stale: the process died.
33    Crashed,
34    /// Still going.
35    Running,
36}
37
38impl RunOutcome {
39    /// Map a [`RunInfo`]-style state string onto an outcome. Anything
40    /// unrecognized is treated as still running, never as success.
41    ///
42    /// [`RunInfo`]: https://docs.rs/somatize-runtime
43    pub fn from_state(state: &str) -> Self {
44        match state {
45            "completed" => Self::Completed,
46            "failed" => Self::Failed,
47            "crashed" => Self::Crashed,
48            _ => Self::Running,
49        }
50    }
51
52    /// Past-tense verb used to open a headline.
53    pub fn verb(&self) -> &'static str {
54        match self {
55            Self::Completed => "completed",
56            Self::Failed => "failed",
57            Self::Crashed => "crashed",
58            Self::Running => "running",
59        }
60    }
61}
62
63/// The node that dominated wall time, and by how much.
64#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
65pub struct NodeCost {
66    /// The dominant node's id.
67    pub node_id: String,
68    /// Its total wall time in milliseconds.
69    pub duration_ms: u64,
70    /// This node's share of all node compute time, in `[0, 1]`.
71    pub share: f64,
72}
73
74/// One flag family and where it fired.
75#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
76pub struct FlagCount {
77    /// Flag family name (e.g. `LEAKAGE`).
78    pub flag: String,
79    /// How many times it fired, duplicates included.
80    pub count: usize,
81    /// Node (or `node/module.path`) ids that raised it, sorted, deduped.
82    pub nodes: Vec<String>,
83}
84
85impl FlagCount {
86    /// Group one flag's occurrences: `count` is how many times it fired,
87    /// `nodes` the distinct places it fired in.
88    pub fn group(flag: impl Into<String>, mut nodes: Vec<String>) -> Self {
89        let count = nodes.len();
90        nodes.sort();
91        nodes.dedup();
92        Self {
93            flag: flag.into(),
94            count,
95            nodes,
96        }
97    }
98
99    /// Union of two flag groupings, summing counts and merging nodes.
100    pub fn merge_all(a: &[FlagCount], b: &[FlagCount]) -> Vec<FlagCount> {
101        let mut grouped: BTreeMap<&str, Vec<String>> = BTreeMap::new();
102        for flag in a.iter().chain(b) {
103            grouped
104                .entry(flag.flag.as_str())
105                .or_default()
106                .extend(flag.nodes.iter().cloned());
107        }
108        grouped
109            .into_iter()
110            .map(|(flag, nodes)| FlagCount::group(flag, nodes))
111            .collect()
112    }
113}
114
115/// What a run's agent steps cost, absent for runs that had none.
116///
117/// Totals across every step node (spawned instances included). Token
118/// counts are what the providers reported; a mock provider reports
119/// whatever it likes, so zeros here mean "nothing reported", not free.
120#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
121pub struct AgentCost {
122    /// Total step turns taken.
123    pub turns: u64,
124    /// Prompt tokens, as reported by the providers.
125    pub input_tokens: u64,
126    /// Completion tokens, as reported by the providers.
127    pub output_tokens: u64,
128    /// Tool invocations across all steps.
129    pub tool_calls: u64,
130    /// Steps that ended in an error — their cost is included above.
131    pub steps_failed: u64,
132    /// Times a step suspended for external input.
133    pub suspensions: u64,
134}
135
136/// Trial statistics for a study run.
137#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
138pub struct TrialSummary {
139    /// Trials the study ran, in any terminal state.
140    pub total: usize,
141    /// Trials that ran to completion.
142    pub completed: usize,
143    /// Trials a pruner stopped early — control flow, not failure.
144    pub pruned: usize,
145    /// Trials that ended in an error.
146    pub failed: usize,
147    /// Id of the best-scoring trial, `None` when none scored.
148    pub best_trial_id: Option<String>,
149    /// Objective value of the best trial.
150    pub best_value: Option<f64>,
151    /// Metric the study optimized, when it declares one.
152    pub objective: Option<String>,
153}
154
155/// What happened in a run, in fields and in one line.
156#[derive(Debug, Clone, Default, Serialize, Deserialize)]
157pub struct RunConclusion {
158    /// Deterministic one-line rendering of everything below.
159    #[serde(default)]
160    pub headline: String,
161    /// How the run ended, `None` when the status file was unreadable.
162    #[serde(default)]
163    pub outcome: Option<RunOutcome>,
164    /// The slowest node, when node timings exist.
165    #[serde(default)]
166    pub dominant_cost: Option<NodeCost>,
167    /// `hits / (hits + misses)`, `None` when the run touched no cache.
168    #[serde(default)]
169    pub cache_hit_ratio: Option<f64>,
170    /// `HealthFlag` events, grouped by flag.
171    #[serde(default)]
172    pub health_flags: Vec<FlagCount>,
173    /// Flags from `diagnostics/report.json`, grouped by flag. Overlaps
174    /// `health_flags` by construction (the audit both emits events and
175    /// writes the report); kept separate because the report survives a
176    /// truncated event log and carries the audited filter ids.
177    #[serde(default)]
178    pub audit_flags: Vec<FlagCount>,
179    /// Trial statistics, `None` for non-study runs.
180    #[serde(default)]
181    pub trials: Option<TrialSummary>,
182    /// What the run's agent steps cost, `None` for runs with none.
183    #[serde(default)]
184    pub agent_cost: Option<AgentCost>,
185    /// What the summarizer could not read, in the summarizer's words.
186    /// Never an error — a half-written run is still worth recording.
187    #[serde(default)]
188    pub warnings: Vec<String>,
189}
190
191/// Number of metrics named in a headline before it says "+N more".
192const HEADLINE_METRICS: usize = 3;
193
194impl RunConclusion {
195    /// Whether the conclusion carries any fact at all.
196    pub fn is_empty(&self) -> bool {
197        self.outcome.is_none()
198            && self.dominant_cost.is_none()
199            && self.cache_hit_ratio.is_none()
200            && self.health_flags.is_empty()
201            && self.audit_flags.is_empty()
202            && self.trials.is_none()
203            && self.agent_cost.is_none()
204    }
205
206    /// Render this conclusion as one deterministic line.
207    ///
208    /// Section order is fixed — outcome, error, trials, metrics, cost,
209    /// cache, flags — so two runs' headlines are comparable by eye and
210    /// diffable in tests. `error` is the run's first node error, which
211    /// for a failed run is the single most useful thing it can say.
212    pub fn render_headline(
213        &self,
214        duration_ms: Option<u64>,
215        metrics: &BTreeMap<String, f64>,
216        error: Option<&str>,
217    ) -> String {
218        let mut parts: Vec<String> = Vec::new();
219
220        let outcome = self.outcome.unwrap_or(RunOutcome::Running);
221        let preposition = match outcome {
222            RunOutcome::Completed => "in",
223            RunOutcome::Running => "for",
224            _ => "after",
225        };
226        parts.push(match duration_ms {
227            Some(ms) => format!("{} {preposition} {}", outcome.verb(), human_duration(ms)),
228            None => outcome.verb().to_string(),
229        });
230
231        if let Some(error) = error {
232            parts.push(format!("error: {}", one_line(error, 120)));
233        }
234
235        if let Some(trials) = &self.trials {
236            let mut line = format!("{} trials", trials.total);
237            let mut lost = Vec::new();
238            if trials.pruned > 0 {
239                lost.push(format!("{} pruned", trials.pruned));
240            }
241            if trials.failed > 0 {
242                lost.push(format!("{} failed", trials.failed));
243            }
244            if !lost.is_empty() {
245                let _ = write!(line, " ({})", lost.join(", "));
246            }
247            match (&trials.objective, trials.best_value) {
248                (Some(objective), Some(best)) => {
249                    let _ = write!(line, ", best {objective}={}", round4(best));
250                }
251                // A study that produced no scorable trial is a dead end
252                // worth remembering, so say so instead of staying quiet.
253                _ if trials.total > 0 => line.push_str(", no scorable trial"),
254                _ => {}
255            }
256            parts.push(line);
257        }
258
259        if !metrics.is_empty() {
260            let mut named: Vec<String> = metrics
261                .iter()
262                .take(HEADLINE_METRICS)
263                .map(|(name, value)| format!("{name}={}", round4(*value)))
264                .collect();
265            if metrics.len() > HEADLINE_METRICS {
266                named.push(format!("+{} more", metrics.len() - HEADLINE_METRICS));
267            }
268            parts.push(named.join(" "));
269        }
270
271        if let Some(cost) = &self.dominant_cost {
272            parts.push(format!(
273                "slowest {} ({}, {}% of compute)",
274                cost.node_id,
275                human_duration(cost.duration_ms),
276                (cost.share * 100.0).round() as i64
277            ));
278        }
279
280        if let Some(ratio) = self.cache_hit_ratio {
281            parts.push(format!("cache {}% hits", (ratio * 100.0).round() as i64));
282        }
283
284        if let Some(agent) = &self.agent_cost {
285            let mut line = format!("agent {} turns", agent.turns);
286            if agent.input_tokens + agent.output_tokens > 0 {
287                let _ = write!(
288                    line,
289                    ", {}→{} tokens",
290                    human_count(agent.input_tokens),
291                    human_count(agent.output_tokens)
292                );
293            }
294            if agent.tool_calls > 0 {
295                let _ = write!(line, ", {} tool calls", agent.tool_calls);
296            }
297            if agent.steps_failed > 0 {
298                let _ = write!(line, ", {} steps failed", agent.steps_failed);
299            }
300            if agent.suspensions > 0 {
301                let _ = write!(line, ", {} suspended", agent.suspensions);
302            }
303            parts.push(line);
304        }
305
306        let flags = FlagCount::merge_all(&self.health_flags, &self.audit_flags);
307        if !flags.is_empty() {
308            let rendered: Vec<String> = flags
309                .iter()
310                .map(|f| {
311                    if f.count > 1 {
312                        format!("{}×{}", f.flag, f.count)
313                    } else {
314                        f.flag.clone()
315                    }
316                })
317                .collect();
318            parts.push(format!("flags: {}", rendered.join(", ")));
319        }
320
321        parts.join(" · ")
322    }
323}
324
325/// Everything one run directory says about itself.
326///
327/// Produced by `somatize_runtime::tracking::summarize`; consumed by the
328/// experiment journal and anything that wants run facts without opening
329/// five files.
330#[derive(Debug, Clone, Serialize, Deserialize)]
331pub struct RunSummary {
332    /// The run's unique identifier.
333    pub run_id: String,
334    /// Absolute path, so a consumer can go read raw artifacts.
335    pub run_dir: String,
336    /// Human-readable run name, from the manifest.
337    pub name: String,
338    /// Manifest kind as its snake_case string (`fit`, `train`, `study`).
339    pub kind: String,
340    /// When the run started.
341    pub created_at: DateTime<Utc>,
342    /// When the run reached a terminal state, `None` while running.
343    #[serde(default)]
344    pub finished_at: Option<DateTime<Utc>>,
345    /// Wall time from start to finish, when both ends are known.
346    #[serde(default)]
347    pub duration_ms: Option<u64>,
348    /// Free-form labels, from the manifest.
349    #[serde(default)]
350    pub tags: Vec<String>,
351    /// Git context captured at run start.
352    #[serde(default)]
353    pub git: GitInfo,
354    /// Named seeds declared at run start (e.g. `{"torch": 42}`).
355    #[serde(default)]
356    pub seeds: BTreeMap<String, i64>,
357    /// Hyperparameters declared at run start (`manifest.params`).
358    #[serde(default)]
359    pub params: BTreeMap<String, serde_json::Value>,
360    /// What the run was expected to show, declared before it ran.
361    #[serde(default)]
362    pub hypothesis: Option<String>,
363    /// Run this one derives from, as recorded in the manifest.
364    #[serde(default)]
365    pub parent_run_id: Option<String>,
366    /// From `fingerprint.json`, absent for runs started before it
367    /// existed or for graph-less runs (a study run has no graph).
368    #[serde(default)]
369    pub architecture: Option<ArchitectureFingerprint>,
370    /// Human topology line, empty when the run has no `graph.json`.
371    #[serde(default)]
372    pub pipeline_summary: String,
373    /// Last recorded value per metric name.
374    #[serde(default)]
375    pub metrics: BTreeMap<String, f64>,
376    /// The derived facts and their [`RunConclusion::headline`].
377    #[serde(default)]
378    pub conclusion: RunConclusion,
379}
380
381/// Compact count rendering: `842`, `12.3k`, `1.2M`.
382pub fn human_count(n: u64) -> String {
383    if n < 1_000 {
384        return n.to_string();
385    }
386    if n < 1_000_000 {
387        return format!("{:.1}k", n as f64 / 1_000.0);
388    }
389    format!("{:.1}M", n as f64 / 1_000_000.0)
390}
391
392/// Compact wall-clock rendering: `840ms`, `2.4s`, `3m 07s`, `1h 12m`.
393pub fn human_duration(ms: u64) -> String {
394    if ms < 1_000 {
395        return format!("{ms}ms");
396    }
397    let secs = ms as f64 / 1000.0;
398    if secs < 60.0 {
399        return format!("{secs:.1}s");
400    }
401    let total = ms / 1000;
402    let (h, m, s) = (total / 3600, (total % 3600) / 60, total % 60);
403    if h > 0 {
404        format!("{h}h {m:02}m")
405    } else {
406        format!("{m}m {s:02}s")
407    }
408}
409
410/// Four decimals, trailing zeros trimmed — stable across platforms.
411pub fn round4(value: f64) -> String {
412    if !value.is_finite() {
413        return format!("{value}");
414    }
415    let text = format!("{value:.4}");
416    let trimmed = text.trim_end_matches('0').trim_end_matches('.');
417    if trimmed.is_empty() { "0" } else { trimmed }.to_string()
418}
419
420/// Collapse to a single line and cap the length — a headline is one
421/// line by contract, and an error message is not.
422pub fn one_line(text: &str, max: usize) -> String {
423    let text = text.replace('\n', " ");
424    if text.chars().count() <= max {
425        return text;
426    }
427    let head: String = text.chars().take(max).collect();
428    format!("{head}…")
429}
430
431#[cfg(test)]
432mod tests {
433    use super::*;
434
435    fn metrics(pairs: &[(&str, f64)]) -> BTreeMap<String, f64> {
436        pairs.iter().map(|(k, v)| ((*k).to_string(), *v)).collect()
437    }
438
439    #[test]
440    fn headline_sections_appear_in_a_fixed_order() {
441        let conclusion = RunConclusion {
442            outcome: Some(RunOutcome::Completed),
443            dominant_cost: Some(NodeCost {
444                node_id: "encoder".into(),
445                duration_ms: 9_000,
446                share: 0.75,
447            }),
448            cache_hit_ratio: Some(0.5),
449            health_flags: vec![FlagCount::group("LEAKAGE", vec!["a".into()])],
450            audit_flags: vec![FlagCount::group("LEAKAGE", vec!["b".into()])],
451            trials: Some(TrialSummary {
452                total: 12,
453                pruned: 4,
454                objective: Some("val_f1".into()),
455                best_value: Some(0.9),
456                ..TrialSummary::default()
457            }),
458            ..RunConclusion::default()
459        };
460        let headline = conclusion.render_headline(Some(12_000), &metrics(&[("loss", 0.25)]), None);
461        assert_eq!(
462            headline,
463            "completed in 12.0s · 12 trials (4 pruned), best val_f1=0.9 · loss=0.25 · \
464             slowest encoder (9.0s, 75% of compute) · cache 50% hits · flags: LEAKAGE×2"
465        );
466    }
467
468    #[test]
469    fn headline_is_stable_across_renderings() {
470        let conclusion = RunConclusion {
471            outcome: Some(RunOutcome::Completed),
472            ..RunConclusion::default()
473        };
474        let m = metrics(&[("b", 1.0), ("a", 2.0), ("d", 3.0), ("c", 4.0)]);
475        let first = conclusion.render_headline(Some(1_000), &m, None);
476        for _ in 0..5 {
477            assert_eq!(conclusion.render_headline(Some(1_000), &m, None), first);
478        }
479        // Alphabetical, capped, with an honest count of the remainder.
480        assert!(first.contains("a=2 b=1 c=4 +1 more"), "{first}");
481    }
482
483    #[test]
484    fn an_error_never_breaks_the_single_line_contract() {
485        let conclusion = RunConclusion {
486            outcome: Some(RunOutcome::Failed),
487            ..RunConclusion::default()
488        };
489        let headline = conclusion.render_headline(
490            Some(500),
491            &BTreeMap::new(),
492            Some("shape mismatch\nexpected [32, 8]\ngot [32, 16]"),
493        );
494        assert_eq!(
495            headline,
496            "failed after 500ms · error: shape mismatch expected [32, 8] got [32, 16]"
497        );
498        assert!(!headline.contains('\n'));
499    }
500
501    #[test]
502    fn a_headline_without_a_duration_still_names_the_outcome() {
503        let conclusion = RunConclusion {
504            outcome: Some(RunOutcome::Running),
505            ..RunConclusion::default()
506        };
507        assert_eq!(
508            conclusion.render_headline(None, &BTreeMap::new(), None),
509            "running"
510        );
511        assert_eq!(
512            conclusion.render_headline(Some(30_000), &BTreeMap::new(), None),
513            "running for 30.0s"
514        );
515    }
516
517    #[test]
518    fn flag_grouping_counts_occurrences_and_dedupes_places() {
519        let flag = FlagCount::group("DEAD_CHANNELS", vec!["b".into(), "a".into(), "a".into()]);
520        assert_eq!(flag.count, 3);
521        assert_eq!(flag.nodes, vec!["a", "b"]);
522
523        let merged = FlagCount::merge_all(
524            &[FlagCount::group("X", vec!["n1".into()])],
525            &[
526                FlagCount::group("X", vec!["n2".into()]),
527                FlagCount::group("A", vec!["n3".into()]),
528            ],
529        );
530        assert_eq!(merged.len(), 2);
531        assert_eq!(merged[0].flag, "A", "merged flags sort by name");
532        assert_eq!(merged[1].flag, "X");
533        assert_eq!(merged[1].count, 2);
534        assert_eq!(merged[1].nodes, vec!["n1", "n2"]);
535    }
536
537    #[test]
538    fn conclusion_emptiness_ignores_the_headline() {
539        assert!(RunConclusion::default().is_empty());
540        let only_text = RunConclusion {
541            headline: "something".into(),
542            ..RunConclusion::default()
543        };
544        assert!(only_text.is_empty(), "prose alone is not a fact");
545        let with_outcome = RunConclusion {
546            outcome: Some(RunOutcome::Failed),
547            ..RunConclusion::default()
548        };
549        assert!(!with_outcome.is_empty());
550    }
551
552    #[test]
553    fn summary_roundtrips_and_tolerates_a_minimal_record() {
554        let summary = RunSummary {
555            run_id: "r1".into(),
556            run_dir: "/tmp/r1".into(),
557            name: "baseline".into(),
558            kind: "train".into(),
559            created_at: Utc::now(),
560            finished_at: None,
561            duration_ms: Some(10),
562            tags: vec!["mos".into()],
563            git: GitInfo::default(),
564            seeds: BTreeMap::from([("torch".into(), 42)]),
565            params: BTreeMap::from([("lr".into(), serde_json::json!(0.01))]),
566            hypothesis: Some("wider is better".into()),
567            parent_run_id: Some("r0".into()),
568            architecture: None,
569            pipeline_summary: "a → b".into(),
570            metrics: metrics(&[("f1", 0.5)]),
571            conclusion: RunConclusion::default(),
572        };
573        let json = serde_json::to_string(&summary).unwrap();
574        let back: RunSummary = serde_json::from_str(&json).unwrap();
575        assert_eq!(back.run_id, "r1");
576        assert_eq!(back.seeds["torch"], 42);
577        assert_eq!(back.params["lr"], serde_json::json!(0.01));
578        assert_eq!(back.hypothesis.as_deref(), Some("wider is better"));
579
580        let minimal = serde_json::json!({
581            "run_id": "r", "run_dir": "/tmp/r", "name": "n", "kind": "fit",
582            "created_at": "2026-07-30T10:00:00Z",
583        });
584        let back: RunSummary = serde_json::from_value(minimal).unwrap();
585        assert!(back.metrics.is_empty());
586        assert!(back.params.is_empty());
587        assert!(back.conclusion.is_empty());
588    }
589
590    #[test]
591    fn unknown_outcome_reads_as_running_not_success() {
592        assert_eq!(RunOutcome::from_state("completed"), RunOutcome::Completed);
593        assert_eq!(RunOutcome::from_state("crashed"), RunOutcome::Crashed);
594        assert_eq!(RunOutcome::from_state("teleported"), RunOutcome::Running);
595    }
596
597    #[test]
598    fn human_duration_scales() {
599        assert_eq!(human_duration(0), "0ms");
600        assert_eq!(human_duration(840), "840ms");
601        assert_eq!(human_duration(2_400), "2.4s");
602        assert_eq!(human_duration(59_900), "59.9s");
603        assert_eq!(human_duration(187_000), "3m 07s");
604        assert_eq!(human_duration(4_320_000), "1h 12m");
605    }
606
607    #[test]
608    fn round4_trims_without_losing_precision() {
609        assert_eq!(round4(1.0), "1");
610        assert_eq!(round4(0.9125), "0.9125");
611        assert_eq!(round4(0.912_549), "0.9125");
612        assert_eq!(round4(-0.5), "-0.5");
613        assert_eq!(round4(f64::NAN), "NaN");
614    }
615
616    #[test]
617    fn one_line_truncates_on_characters_not_bytes() {
618        assert_eq!(one_line("abc", 10), "abc");
619        assert_eq!(one_line("a\nb", 10), "a b");
620        assert_eq!(one_line("ααααα", 3), "ααα…");
621    }
622}
somatize_core/summary.rs

somatize_core/
summary.rs
