somatize_tree/trials.rs
1//! What was run with a version: the trials, their states and their curves.
2//!
3//! A commit is the version and does not change. What gets tried with it grows
4//! without end — a hundred trials, three analyses, a report — and none of that
5//! can touch a commit's hash, so it is **associated** with the version rather
6//! than versioned. soma writes it from the machine running the study; here it
7//! is only read, because whoever claimed a trial is its only writer and a
8//! second one would invent a race that does not exist today.
9//!
10//! The name is the whole of the coupling. soma binds each trial to
11//! `<study>/trial/<n>/<attempt>`, and a study's name is any string:
12//!
13//! ```text
14//! exp/<tree>/<commit> ← that version's study
15//! exp/<tree>/<commit>/trial/3/0 ← its fourth trial, first attempt
16//! exp/<tree>/<commit>/said/2 ← what somebody said about that commit
17//! ```
18//!
19//! A commit's study **is** the prefix its journal already lives under, so the
20//! trials land beneath it with no line of soma changing: no correspondence
21//! record, no index to keep. And the store's cost rule holds — state, point and
22//! score are in the **record** — so counting forty commits' trials is one scan
23//! and only the curve is paid for when somebody asks to see it.
24//!
25//! What cannot be said from here is **which is best**: whether `0.0837` is good
26//! depends on a direction that lives in the `Goal` handed to a sampler and is
27//! written in no record. Guessing it would be the quiet lie this tool exists
28//! not to let past, so either `soma-tree.toml` declares it or it is not said,
29//! and the range — true without knowing the direction — is shown instead.
30
31use serde::{Deserialize, Serialize};
32use somatize_store::{Digest, Store};
33use std::collections::BTreeMap;
34use std::fmt;
35
36/// Which way is better. Not in the store: declared.
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "lowercase")]
39pub enum Goal {
40 /// Less is better: a loss, an error, a time.
41 Min,
42 /// More is better: an accuracy, an F1, a reward.
43 Max,
44}
45
46impl Goal {
47 pub fn read(said: &str) -> Option<Self> {
48 match said {
49 "min" | "minimize" => Some(Self::Min),
50 "max" | "maximize" => Some(Self::Max),
51 _ => None,
52 }
53 }
54
55 /// The best of a few, in the declared direction.
56 pub fn best_of(&self, values: impl IntoIterator<Item = f64>) -> Option<f64> {
57 values
58 .into_iter()
59 .filter(|one| !one.is_nan())
60 .reduce(|best, one| match self {
61 Self::Min => best.min(one),
62 Self::Max => best.max(one),
63 })
64 }
65}
66
67/// A trial, as it comes back from a scan.
68///
69/// The states are soma's and not this side's — `running`, `done`, `pruned`,
70/// `failed` — and travel as text, so a growing vocabulary is not two places to
71/// migrate.
72#[derive(Debug, Clone, Serialize)]
73pub struct Trial {
74 pub trial: u32,
75 /// Which attempt. The highest wins: claiming is a link, so a trial whose
76 /// machine died is rescued by claiming the next one.
77 pub attempt: u32,
78 pub state: Option<String>,
79 /// The configuration that ran, as `str(point)` wrote it.
80 pub point: Option<String>,
81 /// Absent while running. Present on a `pruned`, and **not comparable**
82 /// with a `done`'s: it was measured after fewer epochs.
83 pub score: Option<f64>,
84 pub who: Option<String>,
85 pub when: u64,
86 /// Where the curve is. Not the curve: that is a fetch and this is not.
87 #[serde(skip)]
88 pub kept: Digest,
89}
90
91impl Trial {
92 /// Whether its score can be compared with another `done`'s.
93 pub fn comparable(&self) -> bool {
94 self.state.as_deref() == Some("done") && self.score.is_some()
95 }
96}
97
98/// A trial's curve, which is what costs a fetch.
99#[derive(Debug, Clone, Deserialize, Serialize)]
100pub struct Curve {
101 #[serde(default)]
102 pub point: String,
103 #[serde(default)]
104 pub reports: Vec<f64>,
105 #[serde(default)]
106 pub state: Option<String>,
107 /// Why it stopped. What a `pruned` has and a list of numbers does not.
108 #[serde(default)]
109 pub because: Option<String>,
110 #[serde(default)]
111 pub took: Option<f64>,
112}
113
114/// What is seen of a commit's trials without reading a single blob.
115#[derive(Debug, Clone, Default, Serialize)]
116pub struct Tally {
117 pub trials: u32,
118 pub running: u32,
119 pub done: u32,
120 pub pruned: u32,
121 pub failed: u32,
122 /// The range of what is comparable, true without knowing the direction.
123 pub lowest: Option<f64>,
124 pub highest: Option<f64>,
125 /// The best **only if somebody declared which way is better**. `None`
126 /// otherwise, and then the range is shown in its place.
127 pub best: Option<f64>,
128}
129
130/// The trials of one investigation, kept in a store.
131pub struct Trials<'a> {
132 kept: &'a dyn Store,
133 tree: String,
134 goal: Option<Goal>,
135}
136
137impl<'a> Trials<'a> {
138 pub fn of(tree: impl Into<String>, kept: &'a dyn Store) -> Self {
139 Self {
140 kept,
141 tree: tree.into(),
142 goal: None,
143 }
144 }
145
146 /// With the declared direction, if there is one.
147 pub fn towards(mut self, goal: Option<Goal>) -> Self {
148 self.goal = goal;
149 self
150 }
151
152 /// The name of a commit's study, which is the whole of the link.
153 pub fn study(&self, commit: &str) -> String {
154 format!("exp/{}/{commit}", self.tree)
155 }
156
157 /// A commit's trials, the highest attempt of each, in order.
158 ///
159 /// One scan and no fetches.
160 pub fn of_commit(&self, commit: &str) -> Result<Vec<Trial>, Trouble> {
161 let mut best: BTreeMap<u32, Trial> = BTreeMap::new();
162 let under = format!("{}/trial/", self.study(commit));
163 for bound in self.kept.bound().map_err(Trouble::Store)? {
164 let Some((trial, attempt)) = numbered(&bound.name, &under) else {
165 continue;
166 };
167 match best.get(&trial) {
168 Some(had) if had.attempt >= attempt => {}
169 _ => {
170 best.insert(
171 trial,
172 Trial {
173 trial,
174 attempt,
175 state: beside(&bound.meta, "state").map(str::to_string),
176 point: beside(&bound.meta, "point").map(str::to_string),
177 // Python's `repr(float(score))`, which Rust reads
178 // the same. A trial with no score is preferable to
179 // one with an invented score.
180 score: beside(&bound.meta, "score").and_then(|one| one.parse().ok()),
181 who: beside(&bound.meta, "who").map(str::to_string),
182 when: bound.when,
183 kept: bound.digest,
184 },
185 );
186 }
187 }
188 }
189 Ok(best.into_values().collect())
190 }
191
192 /// How many trials each commit has and how they are going, in **one scan**.
193 ///
194 /// Asking commit by commit would be forty walks of the store to draw a list
195 /// of forty rows.
196 pub fn counted(&self) -> Result<BTreeMap<String, Tally>, Trouble> {
197 let under = format!("exp/{}/", self.tree);
198 // The highest attempt of each `(commit, trial)` before counting:
199 // counting the records would count a rescued trial twice.
200 let mut best: BTreeMap<(String, u32), Highest> = BTreeMap::new();
201 for bound in self.kept.bound().map_err(Trouble::Store)? {
202 let Some(rest) = bound.name.strip_prefix(&under) else {
203 continue;
204 };
205 let Some((commit, numbers)) = rest.split_once("/trial/") else {
206 continue;
207 };
208 let Some((trial, attempt)) = numbered(numbers, "") else {
209 continue;
210 };
211 let mine = (commit.to_string(), trial);
212 match best.get(&mine) {
213 Some(had) if had.attempt >= attempt => {}
214 _ => {
215 best.insert(
216 mine,
217 Highest {
218 attempt,
219 state: beside(&bound.meta, "state").map(str::to_string),
220 score: beside(&bound.meta, "score").and_then(|one| one.parse().ok()),
221 },
222 );
223 }
224 }
225 }
226
227 let mut counted: BTreeMap<String, Tally> = BTreeMap::new();
228 let mut comparable: BTreeMap<String, Vec<f64>> = BTreeMap::new();
229 for ((commit, _), one) in best {
230 let tally = counted.entry(commit.clone()).or_default();
231 tally.trials += 1;
232 match one.state.as_deref() {
233 Some("running") => tally.running += 1,
234 Some("done") => tally.done += 1,
235 Some("pruned") => tally.pruned += 1,
236 Some("failed") => tally.failed += 1,
237 _ => {}
238 }
239 // Only a `done`'s enters the range: a `pruned`'s is real and not
240 // comparable — measured after fewer epochs — and it would make the
241 // range wider than anything anybody measured.
242 if let (Some("done"), Some(score)) = (one.state.as_deref(), one.score) {
243 comparable.entry(commit).or_default().push(score);
244 }
245 }
246 for (commit, scores) in comparable {
247 let Some(tally) = counted.get_mut(&commit) else {
248 continue;
249 };
250 tally.lowest = Goal::Min.best_of(scores.iter().copied());
251 tally.highest = Goal::Max.best_of(scores.iter().copied());
252 tally.best = self.goal.and_then(|goal| goal.best_of(scores));
253 }
254 Ok(counted)
255 }
256
257 /// A trial's curve. **This one is a fetch**, which is why it is apart.
258 pub fn curve(&self, of: &Trial) -> Result<Option<Curve>, Trouble> {
259 let Some(bytes) = self.kept.get(&of.kept).map_err(Trouble::Store)? else {
260 return Ok(None);
261 };
262 serde_json::from_slice(&bytes)
263 .map(Some)
264 .map_err(|why| Trouble::Garbled(why.to_string()))
265 }
266}
267
268/// The highest attempt seen of a trial, while scanning.
269struct Highest {
270 attempt: u32,
271 state: Option<String>,
272 score: Option<f64>,
273}
274
275/// The `(trial, attempt)` that name is, or `None` if it is not one.
276///
277/// A question and not an assumption, as in soma and for the same reason: a
278/// store holds whatever anybody put in it.
279fn numbered(name: &str, under: &str) -> Option<(u32, u32)> {
280 let rest = name.strip_prefix(under)?;
281 let (trial, attempt) = rest.split_once('/')?;
282 Some((trial.parse().ok()?, attempt.parse().ok()?))
283}
284
285fn beside<'a>(meta: &'a somatize_store::Meta, what: &str) -> Option<&'a str> {
286 meta.iter()
287 .find(|(said, _)| said == what)
288 .map(|(_, value)| value.as_str())
289}
290
291#[derive(Debug)]
292pub enum Trouble {
293 Store(somatize_store::StoreError),
294 Garbled(String),
295}
296
297impl fmt::Display for Trouble {
298 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
299 match self {
300 Self::Store(why) => write!(f, "the trials could not be reached: {why}"),
301 Self::Garbled(why) => write!(f, "a curve could not be read: {why}"),
302 }
303 }
304}
305
306impl std::error::Error for Trouble {}