sound_to_patch/
notes.rs

1//! The note list that every layer reads, the transcriber, the f0 track and
2//! the MIDI files that produce and store it, and the harmonic masks built
3//! from it.
4//!
5//! A note has an onset, an offset, a pitch and, once a pitch track has run,
6//! an f0 contour. Three more fields stay unset until a later step fills
7//! them: the octave flag, the velocity and the overlap flag. The
8//! transcriber and the MIDI reader both produce this type, and every later
9//! step consumes it.
10//!
11//! The transcriber runs the model once per stem ([`Transcriber::run`]) and
12//! decodes that run with [`DecodeSettings`], as many times as needed
13//! ([`ModelRun::decode`]). A [`Profile`] gives fixed settings per kind of
14//! stem; [`detect()`] derives them from the stem instead, from a probe decode
15//! and the [`PitchTrack`], the stem's f0 track, which also fills the note
16//! contours. [`drop_quiet`] drops notes that sound only far below the
17//! stem's loudest passage. [`write_midi`] stores a list as a format 0
18//! file, and [`finish()`] rewrites a list with the layer report into the
19//! final `notes.mid`.
20
21use crate::units::{Hz, MidiPitch, Seconds};
22
23mod cleanup;
24mod detect;
25mod finish;
26mod mask;
27mod midi;
28mod profile;
29mod pyin;
30mod quiet;
31mod stem;
32mod track;
33mod transcribe;
34mod write;
35pub(crate) use write::ticks_per_second;
36
37pub use cleanup::{
38    ATTACK_AFTER, ATTACK_BEFORE, ATTACK_DIP, COPY_SLACK, FUNDAMENTAL_DIP, HARMONIC_DIP,
39    HOLD_BEFORE, HOLD_RATIO, HOLD_SEARCH, HOLD_SPAN, OCTAVE_AGREEMENT, OCTAVE_FIX_CEILING,
40    OCTAVE_MIN_FRAMES, OCTAVE_TOLERANCE, OctaveCheck, RISE_LATEST, RISE_READ, RISE_SLACK,
41    RISE_SPAN, RISE_STEP, RISE_WINDOW, SPLIT_GAP, SPLIT_LOWEST, SPLIT_ONSET, check_octaves,
42    join_splits, join_splits_with, merge_overlaps, split_at_attacks,
43};
44pub use detect::{
45    CHORD_OVERLAP, CHORD_SHARE, Detection, HOLD_WINDOW, REFERENCE_PERCENTILE, REGISTER_MARGIN,
46    REGISTER_PERCENTILES, REGISTER_WINDOW, RegisterSource, Voicing, detect, detect_with_window,
47    probe_settings, voicing_of_notes,
48};
49pub use finish::{
50    BEND_MIN, BEND_RANGE, DROP_SLACK, DropSpan, EndRule, FinalNote, FinalNotes, FinishInput,
51    GLIDE_OVERLAP, GLIDE_SLACK, GlideSpan, LevelReader, NO_KNEE_CUT, NoteFacts, PAD_FALL,
52    VELOCITY_LAW_DB, VibratoSpan, finish,
53};
54pub use mask::{FrameMask, MaskParams, Masker, out_of_mask_fraction};
55pub use midi::{
56    BPM_TOLERANCE, DRUM_CHANNEL, MidiError, read_midi, read_midi_bytes, read_midi_near,
57};
58pub use profile::{
59    DEFAULT_MIN_NOTE_FRAMES, DecodeSettings, HIGHEST_MODEL_PITCH, LOWEST_MODEL_PITCH, PitchRange,
60    Profile, RangeError, min_note_frames, min_note_length,
61};
62pub use quiet::{HARMONICS, QUIET_FRAME, QUIET_GATE, StemLevels, drop_quiet, drop_quiet_with};
63#[doc(inline)]
64pub use sound_to_patch_core::notes::{NoteError, Velocity};
65pub use stem::{
66    CleanupSteps, NotesError, TRANSCRIBER_VERSION, Transcription, TranscriptionRecord, clean_up,
67    transcribe_stem,
68};
69pub use track::{
70    PitchTrack, TRACK_FMAX, TRACK_FMIN, TRACK_FRAME, TRACK_HOP, TrackError, TrackFrame,
71};
72pub use transcribe::{
73    Backend, MODEL_RATE, MODEL_SHA256, ModelRun, Tracker, TranscribeError, Transcriber,
74};
75pub use write::{
76    DEFAULT_VELOCITY, MIN_NOTE_LENGTH, MidiWriteError, TICKS_PER_BEAT, Velocities, midi_track_name,
77    write_midi, write_named_midi,
78};
79
80/// One point of an f0 contour.
81#[derive(Clone, Copy, Debug, PartialEq)]
82pub struct ContourPoint {
83    /// The time from the start of the stem.
84    pub time: Seconds,
85    /// The fundamental at that time.
86    pub f0: Hz,
87}
88
89/// The f0 of a note over time, so masks follow bends and glides. Points
90/// are sorted by time; between two points the pitch moves linearly in
91/// log-frequency, and outside them it holds the end value.
92#[derive(Clone, Debug, PartialEq)]
93pub struct Contour {
94    points: Vec<ContourPoint>,
95}
96
97impl Contour {
98    /// Sorts the points by time and checks them.
99    ///
100    /// # Errors
101    ///
102    /// Returns a [`NoteError`] for no point, a zero f0 or two points at the
103    /// same time.
104    pub fn new(mut points: Vec<ContourPoint>) -> Result<Self, NoteError> {
105        if points.is_empty() {
106            return Err(NoteError::EmptyContour);
107        }
108        if points.iter().any(|point| point.f0.get() <= 0.0) {
109            return Err(NoteError::ZeroF0);
110        }
111        points.sort_by(|a, b| a.time.get().total_cmp(&b.time.get()));
112        if points.windows(2).any(|pair| pair[0].time == pair[1].time) {
113            return Err(NoteError::DuplicateTime);
114        }
115        Ok(Self { points })
116    }
117
118    /// Returns the points in time order.
119    #[must_use]
120    pub fn points(&self) -> &[ContourPoint] {
121        &self.points
122    }
123
124    /// Returns the f0 at a time.
125    #[must_use]
126    pub fn f0_at(&self, time: Seconds) -> Hz {
127        let t = time.get();
128        let after = self.points.partition_point(|point| point.time.get() <= t);
129        match (
130            after.checked_sub(1).and_then(|i| self.points.get(i)),
131            self.points.get(after),
132        ) {
133            (Some(before), Some(next)) if before.time.get() < t => {
134                let span = next.time.get() - before.time.get();
135                let fraction = (t - before.time.get()) / span;
136                let log = before.f0.get().ln() * (1.0 - fraction) + next.f0.get().ln() * fraction;
137                Hz::clamped(log.exp())
138            }
139            (Some(only), _) | (None, Some(only)) => only.f0,
140            (None, None) => Hz::clamped(0.0),
141        }
142    }
143}
144
145/// One note: onset, offset, pitch, and the fields later steps fill.
146#[derive(Clone, Debug, PartialEq)]
147pub struct Note {
148    onset: Seconds,
149    offset: Seconds,
150    pitch: MidiPitch,
151    contour: Option<Contour>,
152    octave_uncertain: Option<bool>,
153    velocity: Option<Velocity>,
154    overlap: Option<bool>,
155}
156
157impl Note {
158    /// Builds a note with every optional field unset.
159    ///
160    /// # Errors
161    ///
162    /// Returns a [`NoteError`] when the offset is not after the onset.
163    pub fn new(onset: Seconds, offset: Seconds, pitch: MidiPitch) -> Result<Self, NoteError> {
164        if offset <= onset {
165            return Err(NoteError::OffsetNotAfterOnset { onset, offset });
166        }
167        Ok(Self {
168            onset,
169            offset,
170            pitch,
171            contour: None,
172            octave_uncertain: None,
173            velocity: None,
174            overlap: None,
175        })
176    }
177
178    /// Sets the f0 contour.
179    #[must_use]
180    pub fn with_contour(self, contour: Contour) -> Self {
181        Self {
182            contour: Some(contour),
183            ..self
184        }
185    }
186
187    /// Sets whether the pitch may be an octave off.
188    #[must_use]
189    pub fn with_octave_uncertain(self, flag: bool) -> Self {
190        Self {
191            octave_uncertain: Some(flag),
192            ..self
193        }
194    }
195
196    /// Sets the velocity.
197    #[must_use]
198    pub fn with_velocity(self, velocity: Velocity) -> Self {
199        Self {
200            velocity: Some(velocity),
201            ..self
202        }
203    }
204
205    /// Sets whether the note overlaps another.
206    #[must_use]
207    pub fn with_overlap(self, flag: bool) -> Self {
208        Self {
209            overlap: Some(flag),
210            ..self
211        }
212    }
213
214    /// Returns the onset.
215    #[must_use]
216    pub fn onset(&self) -> Seconds {
217        self.onset
218    }
219
220    /// Returns the offset.
221    #[must_use]
222    pub fn offset(&self) -> Seconds {
223        self.offset
224    }
225
226    /// Returns the pitch.
227    #[must_use]
228    pub fn pitch(&self) -> MidiPitch {
229        self.pitch
230    }
231
232    /// Returns the f0 contour, if a pitch track filled it.
233    #[must_use]
234    pub fn contour(&self) -> Option<&Contour> {
235        self.contour.as_ref()
236    }
237
238    /// Returns the octave flag, if set.
239    #[must_use]
240    pub fn octave_uncertain(&self) -> Option<bool> {
241        self.octave_uncertain
242    }
243
244    /// Returns the velocity, if set.
245    #[must_use]
246    pub fn velocity(&self) -> Option<Velocity> {
247        self.velocity
248    }
249
250    /// Returns the overlap flag, if set.
251    #[must_use]
252    pub fn overlap(&self) -> Option<bool> {
253        self.overlap
254    }
255
256    /// Returns whether the note sounds at a time: onset inclusive, offset
257    /// exclusive.
258    #[must_use]
259    pub fn sounds_at(&self, time: Seconds) -> bool {
260        self.onset <= time && time < self.offset
261    }
262
263    /// Returns the fundamental at a time: the contour when set, else the
264    /// pitch as a constant.
265    #[must_use]
266    pub fn f0_at(&self, time: Seconds) -> Hz {
267        self.contour
268            .as_ref()
269            .map_or_else(|| Hz::from(self.pitch), |contour| contour.f0_at(time))
270    }
271}
272
273/// The notes of a stem, sorted by onset.
274#[derive(Clone, Debug, Default, PartialEq)]
275pub struct NoteList {
276    notes: Vec<Note>,
277}
278
279impl NoteList {
280    /// Sorts the notes by onset, then by pitch.
281    #[must_use]
282    pub fn new(mut notes: Vec<Note>) -> Self {
283        notes.sort_by(|a, b| {
284            a.onset
285                .get()
286                .total_cmp(&b.onset.get())
287                .then(a.pitch.get().total_cmp(&b.pitch.get()))
288        });
289        Self { notes }
290    }
291
292    /// Returns the notes in order.
293    #[must_use]
294    pub fn notes(&self) -> &[Note] {
295        &self.notes
296    }
297
298    /// Returns whether the list holds no note.
299    #[must_use]
300    pub fn is_empty(&self) -> bool {
301        self.notes.is_empty()
302    }
303
304    /// Sets the overlap flag of every note: true when another note sounds
305    /// during part of it.
306    #[must_use]
307    pub fn with_overlaps(self) -> Self {
308        // The notes are sorted by onset, so an earlier note overlaps when its
309        // offset passes this onset, and a later one when the next onset comes
310        // before this offset.
311        let mut latest_offset = f64::NEG_INFINITY;
312        let flags: Vec<bool> = self
313            .notes
314            .iter()
315            .enumerate()
316            .map(|(i, note)| {
317                let before = latest_offset > note.onset.get();
318                let after = self
319                    .notes
320                    .get(i + 1)
321                    .is_some_and(|next| next.onset < note.offset);
322                latest_offset = latest_offset.max(note.offset.get());
323                before || after
324            })
325            .collect();
326        Self {
327            notes: self
328                .notes
329                .into_iter()
330                .zip(flags)
331                .map(|(note, flag)| note.with_overlap(flag))
332                .collect(),
333        }
334    }
335}
336
337#[cfg(test)]
338mod tests {
339    use super::*;
340
341    fn s(value: f64) -> Seconds {
342        Seconds::new(value).unwrap()
343    }
344
345    fn note(onset: f64, offset: f64, pitch: f64) -> Note {
346        Note::new(s(onset), s(offset), MidiPitch::new(pitch).unwrap()).unwrap()
347    }
348
349    proptest::proptest! {
350        #[test]
351        fn overlap_flags_match_every_pair(
352            spans in proptest::collection::vec((0u8..40, 1u8..10), 0..30)
353        ) {
354            let list = NoteList::new(
355                spans
356                    .iter()
357                    .map(|&(start, len)| {
358                        let onset = f64::from(start) / 4.0;
359                        note(onset, onset + f64::from(len) / 4.0, 60.0)
360                    })
361                    .collect(),
362            )
363            .with_overlaps();
364            let notes = list.notes();
365            for (i, a) in notes.iter().enumerate() {
366                let expected = notes.iter().enumerate().any(|(j, b)| {
367                    i != j && b.onset() < a.offset() && a.onset() < b.offset()
368                });
369                proptest::prop_assert_eq!(a.overlap(), Some(expected));
370            }
371        }
372    }
373
374    #[test]
375    fn a_new_note_reads_its_optional_fields_as_unset() {
376        let note = note(0.0, 1.0, 45.0);
377        assert_eq!(note.octave_uncertain(), None);
378        assert_eq!(note.velocity(), None);
379        assert_eq!(note.overlap(), None);
380        assert_eq!(note.contour(), None);
381        let set = note
382            .with_octave_uncertain(true)
383            .with_velocity(Velocity::new(100).unwrap())
384            .with_overlap(false);
385        assert_eq!(set.octave_uncertain(), Some(true));
386        assert_eq!(set.velocity().map(Velocity::get), Some(100));
387        assert_eq!(set.overlap(), Some(false));
388    }
389
390    #[test]
391    fn a_note_needs_its_offset_after_its_onset() {
392        let error = Note::new(s(1.0), s(1.0), MidiPitch::new(60.0).unwrap()).unwrap_err();
393        assert_eq!(
394            error.to_string(),
395            "a note ends at 1 s but starts at 1 s; the offset must come after the onset"
396        );
397        assert!(Velocity::new(0).is_err());
398        assert!(Velocity::new(128).is_err());
399    }
400
401    #[test]
402    fn without_a_contour_the_f0_is_the_pitch() {
403        let a2 = note(0.0, 1.0, 45.0);
404        assert!((a2.f0_at(s(0.5)).get() - 110.0).abs() < 1e-9);
405    }
406
407    #[test]
408    fn a_contour_moves_in_log_frequency_and_holds_its_ends() {
409        let point = |time, f0| ContourPoint {
410            time: s(time),
411            f0: Hz::new(f0).unwrap(),
412        };
413        let contour = Contour::new(vec![point(1.0, 220.0), point(0.0, 110.0)]).unwrap();
414        assert!((contour.f0_at(s(0.5)).get() - 110.0 * 2.0_f64.sqrt()).abs() < 1e-9);
415        assert_eq!(contour.f0_at(s(2.0)).get(), 220.0);
416        assert_eq!(contour.f0_at(s(0.0)).get(), 110.0);
417        let later = Contour::new(vec![point(1.0, 110.0), point(3.0, 220.0)]).unwrap();
418        assert!((later.f0_at(s(1.5)).get() - 110.0 * 2.0_f64.powf(0.25)).abs() < 1e-9);
419        assert!(Contour::new(vec![]).is_err());
420        assert!(Contour::new(vec![point(0.0, 110.0), point(0.0, 220.0)]).is_err());
421    }
422
423    #[test]
424    fn overlaps_mark_notes_that_share_time() {
425        let list = NoteList::new(vec![
426            note(1.0, 2.0, 60.0),
427            note(0.0, 1.5, 48.0),
428            note(3.0, 4.0, 50.0),
429        ])
430        .with_overlaps();
431        let flags: Vec<_> = list.notes().iter().map(Note::overlap).collect();
432        assert_eq!(flags, vec![Some(true), Some(true), Some(false)]);
433        assert_eq!(list.notes()[0].onset(), s(0.0));
434    }
435}
sound_to_patch/notes.rs

sound_to_patch/
notes.rs
