sound_to_patch/notes/
stem.rs

1//! The notes step of a stem: one model run and one f0 track, the decode
2//! settings, the decode and the cleanup, in that order.
3
4use std::error::Error;
5use std::fmt;
6
7use basic_pitch::NoteParams;
8use serde::Serialize;
9
10use super::{
11    DecodeSettings, Detection, MODEL_SHA256, ModelRun, NoteList, OctaveCheck, PitchTrack, Profile,
12    QUIET_GATE, StemLevels, TrackError, TranscribeError, Transcriber, Voicing, check_octaves,
13    detect, drop_quiet_with, join_splits_with, merge_overlaps, split_at_attacks,
14};
15use crate::audio::SampleRate;
16use crate::units::Bpm;
17
18/// The version of the transcription crate, pinned in `Cargo.toml`.
19pub const TRANSCRIBER_VERSION: &str = "0.1.0";
20
21/// The note counts of the cleanup, one per step.
22#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
23pub struct CleanupSteps {
24    /// Notes from the decoder.
25    pub decoded: usize,
26    /// What the octave check did; it keeps the count.
27    pub octaves: OctaveCheck,
28    /// Notes after octave ghosts merged into their notes.
29    pub merged: usize,
30    /// Notes after notes split at attacks inside them.
31    pub split: usize,
32    /// Notes after split notes joined.
33    pub joined: usize,
34    /// Notes the quiet-note guard dropped.
35    pub quiet: usize,
36    /// Notes kept.
37    pub kept: usize,
38}
39
40/// Cleans decoded notes in five steps, and counts the notes after each.
41///
42/// The steps are the octave check (a monophonic part only, since the
43/// track reads a chord as one low period), the merge of notes of one key
44/// that overlap, the split of notes at attacks inside them, the join of
45/// split notes, and the quiet-note guard at
46/// [`QUIET_GATE`]. `mono` is the stem at `rate`, as the guard reads it.
47#[must_use]
48pub fn clean_up(
49    decoded: &NoteList,
50    run: &ModelRun,
51    track: &PitchTrack,
52    voicing: Voicing,
53    mono: &[f64],
54    rate: SampleRate,
55) -> (NoteList, CleanupSteps) {
56    let (checked, octaves) = match voicing {
57        Voicing::Monophonic => check_octaves(decoded, track),
58        Voicing::Polyphonic => (
59            decoded.clone(),
60            OctaveCheck {
61                unchecked: decoded.notes().len(),
62                ..OctaveCheck::default()
63            },
64        ),
65    };
66    let merged = merge_overlaps(&checked);
67    let levels = StemLevels::new(mono, rate);
68    let split = split_at_attacks(&merged, &levels);
69    let joined = join_splits_with(&split, run, &levels);
70    let (kept, quiet) = drop_quiet_with(&joined, &levels, QUIET_GATE);
71    let steps = CleanupSteps {
72        decoded: decoded.notes().len(),
73        octaves,
74        merged: merged.notes().len(),
75        split: split.notes().len(),
76        joined: joined.notes().len(),
77        quiet,
78        kept: kept.notes().len(),
79    };
80    (kept, steps)
81}
82
83/// The transcriber's model and settings, as the `transcription` group of
84/// `layers.json` records them.
85#[derive(Clone, Debug, PartialEq, Serialize)]
86pub struct TranscriptionRecord {
87    /// The SHA-256 of the model weights, 64 lowercase hex digits.
88    pub model_sha256: &'static str,
89    /// The version of the transcription crate.
90    pub crate_version: &'static str,
91    /// The profile name, or `detected` when the stem chose the settings.
92    pub profile: String,
93    /// Every decoder parameter, by the crate's own names.
94    pub note_params: NoteParams,
95}
96
97impl TranscriptionRecord {
98    /// The record of a decode with `settings`, from a profile or, with
99    /// `None`, from detection.
100    #[must_use]
101    pub fn new(profile: Option<Profile>, settings: &DecodeSettings) -> Self {
102        Self {
103            model_sha256: MODEL_SHA256,
104            crate_version: TRANSCRIBER_VERSION,
105            profile: profile.map_or("detected", Profile::name).to_owned(),
106            note_params: settings.note_params(),
107        }
108    }
109}
110
111/// Everything the notes step of a stem found.
112#[derive(Clone, Debug)]
113pub struct Transcription {
114    /// The model run, with its posteriorgrams.
115    pub run: ModelRun,
116    /// The f0 track.
117    pub track: PitchTrack,
118    /// What detection found. It runs with a profile as well.
119    pub detection: Detection,
120    /// The profile that overrode detection, if any.
121    pub profile: Option<Profile>,
122    /// The voicing the cleanup used: the profile's (chords for `pad`) or
123    /// the detected one.
124    pub voicing: Voicing,
125    /// The decode settings used.
126    pub settings: DecodeSettings,
127    /// The notes as the decoder gave them.
128    pub decoded: NoteList,
129    /// The counts of each cleanup step.
130    pub steps: CleanupSteps,
131    /// The clean notes, with their f0 contours.
132    pub notes: NoteList,
133    /// The record of the model and the settings.
134    pub record: TranscriptionRecord,
135}
136
137/// Runs the notes step on a mono stem at any rate: one model run, one f0
138/// track, detection, the decode with the detected settings or those of
139/// `profile`, and [`clean_up`].
140///
141/// # Errors
142///
143/// Returns a [`NotesError`] when the model run, a decode or the f0 track
144/// fails.
145pub fn transcribe_stem(
146    transcriber: &Transcriber,
147    mono: &[f64],
148    rate: SampleRate,
149    bpm: Bpm,
150    profile: Option<Profile>,
151) -> Result<Transcription, NotesError> {
152    let run = transcriber.run(mono, rate)?;
153    let track = transcriber.track(mono, rate)?;
154    let detection = detect(&run, &track, bpm)?;
155    let settings = profile.map_or(detection.settings, |profile| profile.settings(bpm));
156    let voicing = profile.map_or(detection.voicing, Profile::voicing);
157    let decoded = run.decode(&settings)?;
158    let (kept, steps) = clean_up(&decoded, &run, &track, voicing, mono, rate);
159    let notes = track.with_contours(&kept);
160    Ok(Transcription {
161        record: TranscriptionRecord::new(profile, &settings),
162        run,
163        track,
164        detection,
165        profile,
166        voicing,
167        settings,
168        decoded,
169        steps,
170        notes,
171    })
172}
173
174/// A notes step that failed.
175#[derive(Debug)]
176pub enum NotesError {
177    /// The model run or a decode failed.
178    Transcribe(TranscribeError),
179    /// The f0 track failed.
180    Track(TrackError),
181}
182
183impl fmt::Display for NotesError {
184    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185        match self {
186            Self::Transcribe(error) => write!(f, "{error}"),
187            Self::Track(error) => write!(f, "{error}"),
188        }
189    }
190}
191
192impl Error for NotesError {
193    fn source(&self) -> Option<&(dyn Error + 'static)> {
194        match self {
195            Self::Transcribe(error) => Some(error),
196            Self::Track(error) => Some(error),
197        }
198    }
199}
200
201impl From<TranscribeError> for NotesError {
202    fn from(error: TranscribeError) -> Self {
203        Self::Transcribe(error)
204    }
205}
206
207impl From<TrackError> for NotesError {
208    fn from(error: TrackError) -> Self {
209        Self::Track(error)
210    }
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216    use crate::notes::{Note, TrackFrame, Velocity};
217    use crate::units::{Decibels, Hz, MidiPitch, Seconds};
218    use basic_pitch::{Posteriorgram, Posteriorgrams};
219
220    fn heard(onset: f64, offset: f64, pitch: f64, velocity: u8) -> Note {
221        Note::new(
222            Seconds::new(onset).unwrap(),
223            Seconds::new(offset).unwrap(),
224            MidiPitch::new(pitch).unwrap(),
225        )
226        .unwrap()
227        .with_velocity(Velocity::new(velocity).unwrap())
228    }
229
230    #[test]
231    fn errors_name_their_cause_and_keep_their_source() {
232        let model = TranscribeError::Model(basic_pitch::Error::NonFiniteAudio {
233            index: 3,
234            value: f32::NAN,
235        });
236        let message = "the transcription model failed: audio sample 3 is not finite (NaN)";
237        assert_eq!(model.to_string(), message);
238        assert!(model.source().is_some());
239        let error = NotesError::from(model);
240        assert_eq!(error.to_string(), message);
241        assert!(error.source().is_some());
242        let track = NotesError::Track(TrackError::NotFinite(7));
243        assert_eq!(track.to_string(), TrackError::NotFinite(7).to_string());
244        assert!(track.source().is_some());
245    }
246
247    /// A chord part skips the octave check: its notes stay where the
248    /// decoder put them, every one unchecked.
249    #[test]
250    fn a_chord_part_keeps_its_octaves_unchecked() {
251        let run = ModelRun::from_posteriorgrams(
252            Posteriorgrams::new(
253                Posteriorgram::zeros(100, 88),
254                Posteriorgram::zeros(100, 88),
255                Posteriorgram::zeros(100, 264),
256            )
257            .unwrap(),
258        );
259        let track = PitchTrack::from_frames(
260            (0..200)
261                .map(|i| TrackFrame {
262                    time: Seconds::new(f64::from(i) * 0.005).unwrap(),
263                    f0: Some(Hz::from(MidiPitch::new(45.0).unwrap())),
264                    level: Decibels::new(-3.0).unwrap(),
265                })
266                .collect(),
267        );
268        let decoded = NoteList::new(vec![heard(0.2, 0.8, 45.0, 90), heard(0.2, 0.3, 57.0, 37)]);
269        let rate = SampleRate::new(22_050).unwrap();
270        let mono = vec![0.5; 22_050];
271        for (voicing, pitches, unchecked) in [
272            (Voicing::Polyphonic, vec![45.0, 57.0], 2),
273            (Voicing::Monophonic, vec![45.0], 0),
274        ] {
275            let (kept, steps) = clean_up(&decoded, &run, &track, voicing, &mono, rate);
276            let found: Vec<f64> = kept.notes().iter().map(|n| n.pitch().get()).collect();
277            assert_eq!(found, pitches, "{voicing:?}");
278            assert_eq!(steps.octaves.unchecked, unchecked);
279            assert_eq!((steps.decoded, steps.kept), (2, pitches.len()));
280        }
281    }
282
283    #[test]
284    fn the_transcriber_version_matches_the_pin() {
285        let manifest = include_str!("../../Cargo.toml");
286        assert!(
287            manifest.contains(&format!(
288                "basic-pitch-rs = {{ version = \"={TRANSCRIBER_VERSION}\""
289            )),
290            "Cargo.toml pins another basic-pitch-rs"
291        );
292    }
293}
sound_to_patch/notes/stem.rs

sound_to_patch/notes/
stem.rs
