sound_to_patch/notes/
track.rs

1//! The f0 track of a stem: one probabilistic YIN (pYIN) run over the whole
2//! stem, frame by frame, with a voiced flag per frame.
3//!
4//! The track follows the fundamental, so it serves the register detection,
5//! the octave check of the notes and the f0 contour of each note. It runs
6//! at the model rate with frames of 2048 samples, a hop of 128 samples
7//! (5.8 ms) and pitch bins of a tenth of a semitone, over the whole stem
8//! at once. The tracker is the in-repo port of pYIN in `pyin.rs`; its cost
9//! grows with the length of the stem.
10
11use std::error::Error;
12use std::fmt;
13
14use super::pyin::{Pyin, PyinError, Settings};
15use super::{Contour, ContourPoint, MODEL_RATE, Note, NoteList};
16use crate::audio::{ResampleError, SampleRate, resample};
17use crate::cast;
18use crate::dsp::{StftParams, frame_rms};
19use crate::units::{Decibels, Hz, MidiPitch, Seconds};
20
21/// The lowest f0 the track looks for, below B0 (30.87 Hz).
22pub const TRACK_FMIN: Hz = Hz::from_const(25.0);
23/// The highest f0 the track looks for, just above C8 (4186 Hz), the top of
24/// the transcription model. A lower ceiling reads every tone above it as a
25/// subharmonic.
26pub const TRACK_FMAX: Hz = Hz::from_const(4200.0);
27/// Samples per analysis frame at the model rate.
28pub const TRACK_FRAME: usize = 2048;
29/// Samples between frames at the model rate.
30pub const TRACK_HOP: usize = 128;
31/// Pitch bins per semitone.
32const BINS_PER_SEMITONE: usize = 10;
33/// The level of a frame of digital silence.
34const SILENT_LEVEL: Decibels = Decibels::from_const(-200.0);
35
36/// One frame of the track.
37#[derive(Clone, Copy, Debug, PartialEq)]
38pub struct TrackFrame {
39    /// The centre of the frame from the start of the stem.
40    pub time: Seconds,
41    /// The f0, or `None` when the frame reads as unvoiced.
42    pub f0: Option<Hz>,
43    /// The RMS of the frame relative to the loudest frame of the stem:
44    /// 0 dB or less, and -200 dB for digital silence.
45    pub level: Decibels,
46}
47
48/// The f0 track of a stem.
49#[derive(Clone, Debug, Default, PartialEq)]
50pub struct PitchTrack {
51    frames: Vec<TrackFrame>,
52}
53
54impl PitchTrack {
55    /// Runs the tracker over a mono stem at any rate. The stem is
56    /// resampled to the model rate, and the copy the tracker reads is
57    /// scaled to a peak of 1, because the tracker treats a frame below a
58    /// fixed energy as silence. A silent or empty stem gives a track with
59    /// no voiced frame.
60    ///
61    /// # Errors
62    ///
63    /// Returns a [`TrackError`] when resampling fails, a sample is not
64    /// finite or the tracker cannot run: it refuses the settings, a
65    /// transform fails or a candidate has no pitch bin. The settings are
66    /// constants, so the last three point to a bug.
67    pub fn of(mono: &[f64], rate: SampleRate) -> Result<Self, TrackError> {
68        let mut samples = resample(mono, rate, MODEL_RATE).map_err(TrackError::Resample)?;
69        if let Some(index) = samples.iter().position(|x| !x.is_finite()) {
70            return Err(TrackError::NotFinite(index));
71        }
72        let peak = samples.iter().fold(0.0_f64, |peak, x| peak.max(x.abs()));
73        if samples.is_empty() || peak == 0.0 {
74            return Ok(Self::default());
75        }
76        for x in &mut samples {
77            *x /= peak;
78        }
79        let tracker = Pyin::new(Settings {
80            fmin: TRACK_FMIN.get(),
81            fmax: TRACK_FMAX.get(),
82            rate: MODEL_RATE.get(),
83            frame: TRACK_FRAME,
84            hop: TRACK_HOP,
85            bins_per_semitone: BINS_PER_SEMITONE,
86        })?;
87        let f0s = tracker.run(&samples)?;
88        // The same centred frames as the tracker's, so frame m lines up.
89        let rms = frame_rms(
90            &samples,
91            // A hop above zero never fails; the test of the settings pins it.
92            StftParams::fixed(TRACK_FRAME)
93                .with_hop(TRACK_HOP)
94                .unwrap_or(StftParams::fixed(TRACK_FRAME)),
95        );
96        let loudest = rms.iter().copied().fold(0.0, f64::max);
97        let frames = f0s
98            .into_iter()
99            .enumerate()
100            .map(|(m, f0)| TrackFrame {
101                time: Seconds::clamped(
102                    cast::usize_to_f64(m) * cast::usize_to_f64(TRACK_HOP)
103                        / f64::from(MODEL_RATE.get()),
104                ),
105                f0: f0.filter(|hz| hz.is_finite() && *hz > 0.0).map(Hz::clamped),
106                level: rms
107                    .get(m)
108                    .and_then(|&value| Decibels::from_amplitude_gain(value / loudest).ok())
109                    .map_or(SILENT_LEVEL, |level| {
110                        Decibels::clamped(level.get().max(SILENT_LEVEL.get()))
111                    }),
112            })
113            .collect();
114        Ok(Self { frames })
115    }
116
117    /// Builds a track from frames, such as a test's.
118    #[must_use]
119    pub fn from_frames(frames: Vec<TrackFrame>) -> Self {
120        Self { frames }
121    }
122
123    /// Returns every frame in time order.
124    #[must_use]
125    pub fn frames(&self) -> &[TrackFrame] {
126        &self.frames
127    }
128
129    /// Returns the voiced frames from `onset` (inclusive) to `offset`
130    /// (exclusive) as `(time, f0)`.
131    pub fn voiced_between(
132        &self,
133        onset: Seconds,
134        offset: Seconds,
135    ) -> impl Iterator<Item = (Seconds, Hz)> + '_ {
136        let start = self.frames.partition_point(|frame| frame.time < onset);
137        self.frames[start..]
138            .iter()
139            .take_while(move |frame| frame.time < offset)
140            .filter_map(|frame| frame.f0.map(|f0| (frame.time, f0)))
141    }
142
143    /// Returns the pitches of every voiced frame, as fractional MIDI
144    /// pitches.
145    #[must_use]
146    pub fn voiced_pitches(&self) -> Vec<MidiPitch> {
147        self.frames
148            .iter()
149            .filter_map(|frame| frame.f0)
150            .filter_map(|f0| MidiPitch::try_from(f0).ok())
151            .collect()
152    }
153
154    /// Returns the f0 contour of a span: the voiced frames inside it, or
155    /// `None` when none is voiced.
156    #[must_use]
157    pub fn contour(&self, onset: Seconds, offset: Seconds) -> Option<Contour> {
158        let points: Vec<ContourPoint> = self
159            .voiced_between(onset, offset)
160            .map(|(time, f0)| ContourPoint { time, f0 })
161            .collect();
162        Contour::new(points).ok()
163    }
164
165    /// Sets the f0 contour of every note that has a voiced frame inside it.
166    #[must_use]
167    pub fn with_contours(&self, notes: &NoteList) -> NoteList {
168        NoteList::new(
169            notes
170                .notes()
171                .iter()
172                .map(|note| match self.contour(note.onset(), note.offset()) {
173                    Some(contour) => note.clone().with_contour(contour),
174                    None => note.clone(),
175                })
176                .collect::<Vec<Note>>(),
177        )
178    }
179}
180
181/// An f0 track that could not be built.
182#[derive(Debug)]
183pub enum TrackError {
184    /// The stem could not be resampled to the model rate.
185    Resample(ResampleError),
186    /// A sample at this index is NaN or infinite.
187    NotFinite(usize),
188    /// The tracker could not run, for the reason given.
189    Tracker(&'static str),
190}
191
192impl fmt::Display for TrackError {
193    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
194        match self {
195            Self::Resample(source) => {
196                write!(f, "cannot prepare the stem for the f0 track: {source}")
197            }
198            Self::NotFinite(index) => write!(f, "cannot track f0: sample {index} is not finite"),
199            Self::Tracker(reason) => write!(f, "cannot track f0: {reason}"),
200        }
201    }
202}
203
204impl From<PyinError> for TrackError {
205    fn from(PyinError(reason): PyinError) -> Self {
206        Self::Tracker(reason)
207    }
208}
209
210impl Error for TrackError {
211    fn source(&self) -> Option<&(dyn Error + 'static)> {
212        match self {
213            Self::Resample(source) => Some(source),
214            Self::NotFinite(_) | Self::Tracker(_) => None,
215        }
216    }
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222    use crate::cast;
223
224    fn frame(time: f64, f0: Option<f64>) -> TrackFrame {
225        TrackFrame {
226            time: Seconds::new(time).unwrap(),
227            f0: f0.map(|hz| Hz::new(hz).unwrap()),
228            level: Decibels::new(0.0).unwrap(),
229        }
230    }
231
232    #[test]
233    fn voiced_frames_run_from_the_onset_up_to_the_offset() {
234        let track = PitchTrack::from_frames(vec![
235            frame(0.1, Some(110.0)),
236            frame(0.2, None),
237            frame(0.3, Some(220.0)),
238            frame(0.4, Some(440.0)),
239        ]);
240        let s = |value| Seconds::new(value).unwrap();
241        let times: Vec<f64> = track
242            .voiced_between(s(0.1), s(0.4))
243            .map(|(time, _)| time.get())
244            .collect();
245        assert_eq!(times, [0.1, 0.3]);
246        let pitches: Vec<f64> = track
247            .voiced_pitches()
248            .iter()
249            .map(|p| p.get().round())
250            .collect();
251        assert_eq!(pitches, [45.0, 57.0, 69.0]);
252    }
253
254    #[test]
255    fn errors_name_their_cause_and_keep_their_source() {
256        let not_finite = TrackError::NotFinite(7);
257        assert_eq!(
258            not_finite.to_string(),
259            "cannot track f0: sample 7 is not finite"
260        );
261        assert!(not_finite.source().is_none());
262        let native = SampleRate::new(48_000).unwrap();
263        let resample = TrackError::Resample(resample(&[f64::NAN], native, MODEL_RATE).unwrap_err());
264        assert!(
265            resample
266                .to_string()
267                .starts_with("cannot prepare the stem for the f0 track: ")
268        );
269        assert!(resample.source().is_some());
270        let tracker = TrackError::from(PyinError("the frame is too short"));
271        assert_eq!(
272            tracker.to_string(),
273            "cannot track f0: the frame is too short"
274        );
275        assert!(tracker.source().is_none());
276    }
277
278    /// The tracker refuses the settings unless these hold, and the f0 track
279    /// then fails with [`TrackError::Tracker`]. The settings are constants,
280    /// so this test is their only check.
281    #[test]
282    fn the_settings_meet_every_rule_of_the_tracker() {
283        let rate = f64::from(MODEL_RATE.get());
284        let (fmin, fmax) = (TRACK_FMIN.get(), TRACK_FMAX.get());
285        assert!(0.0 < fmin && fmin < fmax && fmax <= rate / 2.0);
286        let window = TRACK_FRAME / 2;
287        // The longest period, ceil(22050 / 25) = 882 lags, fits the frame,
288        // so the frame does not raise fmin silently.
289        let longest = cast::ceil_to_usize(rate / fmin).unwrap();
290        assert!(longest < TRACK_FRAME - window - 1);
291        let shortest = cast::floor_to_usize(rate / fmax).unwrap().max(1);
292        assert!(longest - shortest >= 2);
293        // Pitch bins against the widest transition of 35.92 octaves per
294        // second.
295        let bins = cast::floor_to_usize(
296            12.0 * cast::usize_to_f64(BINS_PER_SEMITONE) * (fmax / fmin).log2(),
297        )
298        .unwrap()
299            + 1;
300        let per_frame =
301            cast::round_to_usize(35.92 * 12.0 * cast::usize_to_f64(TRACK_HOP) / rate).unwrap();
302        assert!(bins > per_frame * 10);
303        assert!(TRACK_HOP > 0 && window > 0);
304    }
305}
sound_to_patch/notes/track.rs

sound_to_patch/notes/
track.rs
