sound_to_patch/estimate/
voices.rs

1//! The voices layer: how many notes sound at once, and how a mono line joins
2//! its notes.
3//!
4//! The count comes from the note list and each note's held part. The join
5//! statistics come from the mono mix around each join. The method page is
6//! `docs/src/analysis/voices.md`.
7
8use std::error::Error;
9use std::fmt;
10
11use crate::audio::SampleRate;
12use crate::cast;
13use crate::confidence::{Confidence, ConfidenceError, Level, NoSpread, Reason, ReasonCode, Signal};
14use crate::dsp::{Stft, StftError, StftParams, frame_descriptors, quantile};
15use crate::estimate::envelope::{
16    EnvelopeLayer, KNEE_END_GAP, LevelTrack, RMS_PERIODS, SILENCE_DB, rms_window,
17};
18use crate::notes::{Note, NoteList};
19use crate::units::{Cents, Decibels, Hz, Seconds};
20
21/// Notes this close in pitch count as one voice: seven detuned copies of one
22/// pitch form one cluster.
23pub const CLUSTER_CENTS: Cents = Cents::from_const(60.0);
24/// Two notes count as two voices when their held parts overlap for this long or
25/// more, and a later note starting this far before an earlier note's held end
26/// stacks on it.
27pub const OVERLAP_MIN: Seconds = Seconds::from_const(0.05);
28/// Consecutive notes join when the gap between them is under this.
29pub const JOIN_GAP_MAX: Seconds = Seconds::from_const(0.03);
30/// The dip of a join is read this far either side of the onset.
31pub const JOIN_SPAN: Seconds = Seconds::from_const(0.025);
32/// Each note's level beside a join is read over at most this long.
33pub const SUSTAIN_SPAN: Seconds = Seconds::from_const(0.1);
34/// Each neighbour's level is this quantile of its track: a window of one period
35/// of the lower note still ripples on the higher one.
36pub const SUSTAIN_QUANTILE: f64 = 0.1;
37/// The centroid is read over this long on each side of a join.
38pub const CENTROID_SPAN: Seconds = Seconds::from_const(0.02);
39/// A join that dips this far under the quieter neighbour is a retrigger.
40///
41/// Between a 0 dB join and a 12 dB dip that rises over 5 ms at 110 Hz, which
42/// reads 1.35 dB through a window of one period. On a sampled set of generated
43/// joins, 1 dB keeps 91 % of legato joins and finds 86 % of retriggers of 12 dB
44/// or more with attacks of 5 ms or more.
45pub const DIP_RETRIGGER: Decibels = Decibels::from_const(1.0);
46/// An attack longer than this hides a retrigger, so a join reads low.
47pub const ATTACK_HIDES_JOIN: Seconds = Seconds::from_const(0.1);
48/// The share of frames that agree for a high count.
49const AGREE_HIGH: f64 = 0.9;
50/// The share of frames that agree for a medium count.
51const AGREE_MEDIUM: f64 = 0.7;
52
53/// The convention of the count.
54pub const VOICES_CONVENTION: &str = "each note counts from its onset to its note-off knee, or to \
5525 ms before its end without one, so a release tail holds no voice, and a held part stops at the stem end; pitches within 60 cents form \
56one voice; max_polyphony is the most voices that have started by any onset and are held 50 ms after \
57it, mono when that is 1; mean_polyphony is the mean count over the time any note is held, so short \
58overlaps can lift it above max_polyphony; null without notes, or when no note holds a sounding part";
59
60/// The convention of a join.
61pub const JOIN_CONVENTION: &str = "consecutive notes of a mono line whose gap, the later onset \
62minus the earlier end in the note list, is under 30 ms, and whose later onset lies less than 50 ms \
63inside the earlier note's held part; dip_db is the 10th percentile of the \
64level of the quieter note over up to 100 ms beside the join minus the lowest level from 25 ms before \
65the onset, or from the earlier held end when that comes first, to 25 ms after it, on an RMS over one period of the lower note; centroid_change is the centroid over f0 \
66after the join over the same before it, minus 1, over 20 ms or two periods each side, and the \
67verdict does not read it, and it is null when either side holds no signal or runs past either end of the stem; \
68legato when dip_db is under 1; dip_db and legato are null with no estimate when neither note has a \
69level beside the join; confidence is low when the verdict is legato, as a retrigger with an instant \
70attack keeps its level too, when no attack was read, when half a period before the earlier level window or one period before the dip read falls before the stem start, when one period past the 25 ms after the onset falls past the stem end, or when the median attack is over 100 ms, else \
71medium";
72
73/// What the voices layer reads.
74#[derive(Clone, Copy, Debug)]
75pub struct VoicesInput<'a> {
76    /// The mono mix at the native rate.
77    pub mono: &'a [f64],
78    /// The native rate.
79    pub rate: SampleRate,
80    /// The note list, transcribed or given.
81    pub notes: &'a NoteList,
82    /// The envelope layer of the same notes, for their knees and attack.
83    pub envelope: &'a EnvelopeLayer,
84}
85
86/// The join between two consecutive notes of a mono line.
87#[derive(Clone, Debug, PartialEq)]
88pub struct Join {
89    /// The later note's onset.
90    pub time: Seconds,
91    /// The later onset minus the earlier note's end in the list; negative
92    /// when they overlap there.
93    pub gap: f64,
94    /// How far the level falls at the join under the quieter neighbour; `None`
95    /// when neither note has a level beside the join.
96    pub dip: Option<Decibels>,
97    /// The centroid over f0 after the join over the same before it, minus 1;
98    /// `None` when either side holds no signal or runs past either end of the
99    /// stem.
100    pub centroid_change: Option<f64>,
101    /// The join shows no new attack; `None` without a dip.
102    pub legato: Option<bool>,
103    /// How far the layer trusts the verdict.
104    pub confidence: Confidence<NoSpread>,
105}
106
107/// The voices layer of a stem.
108#[derive(Clone, Debug, PartialEq)]
109pub struct VoicesLayer {
110    /// One voice at a time, by [`VoicesLayer::max_polyphony`].
111    pub mono: bool,
112    /// The most voices at once.
113    pub max_polyphony: usize,
114    /// The mean count over the time that any note is held. An overlap under
115    /// [`OVERLAP_MIN`] counts here and not in `max_polyphony`, so this can exceed
116    /// it.
117    pub mean_polyphony: f64,
118    /// The joins of a mono line, in time order; empty on any other line.
119    pub joins: Vec<Join>,
120    /// The confidence of the count; no estimate without notes or when no note
121    /// holds a sounding part in the stem.
122    pub confidence: Confidence<NoSpread>,
123}
124
125/// Why the voices layer failed.
126#[derive(Debug)]
127pub enum VoicesError {
128    /// A confidence could not be built.
129    Confidence(ConfidenceError),
130    /// A spectrum could not be read.
131    Stft(StftError),
132}
133
134impl fmt::Display for VoicesError {
135    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
136        match self {
137            Self::Confidence(error) => write!(f, "voices confidence: {error}"),
138            Self::Stft(error) => write!(f, "voices spectrum: {error}"),
139        }
140    }
141}
142
143impl Error for VoicesError {}
144
145impl From<ConfidenceError> for VoicesError {
146    fn from(error: ConfidenceError) -> Self {
147        Self::Confidence(error)
148    }
149}
150
151impl From<StftError> for VoicesError {
152    fn from(error: StftError) -> Self {
153        Self::Stft(error)
154    }
155}
156
157/// A note's onset, the end of its held part and its pitch in hertz.
158#[derive(Clone, Copy, Debug)]
159struct Held {
160    onset: f64,
161    end: f64,
162    hz: f64,
163}
164
165impl VoicesLayer {
166    /// Reads the voices layer of a stem.
167    ///
168    /// # Errors
169    ///
170    /// Returns a [`VoicesError`] when a confidence or a spectrum cannot be
171    /// built.
172    pub fn of(input: &VoicesInput<'_>) -> Result<Self, VoicesError> {
173        let notes = input.notes.notes();
174        if notes.is_empty() {
175            return Self::no_count(Reason::new(ReasonCode::NoNoteList, Signal::Flag(true))?);
176        }
177        let held = held_parts(notes, input.envelope, stem_end(input));
178        if held.is_empty() {
179            return Self::no_count(Reason::new(ReasonCode::TooFewNotes, Signal::Count(0))?);
180        }
181        let max_polyphony = held
182            .iter()
183            .map(|h| voices_at(&held, h.onset, h.onset + OVERLAP_MIN.get()).max(1))
184            .max()
185            .unwrap_or(1);
186        let (mean_polyphony, agree) = frame_counts(&held, max_polyphony);
187        let level = if agree >= AGREE_HIGH {
188            Level::High
189        } else if agree >= AGREE_MEDIUM {
190            Level::Medium
191        } else {
192            Level::Low
193        };
194        let mono = max_polyphony == 1;
195        Ok(Self {
196            mono,
197            max_polyphony,
198            mean_polyphony,
199            joins: if mono {
200                joins(input, notes)?
201            } else {
202                Vec::new()
203            },
204            confidence: Confidence::estimate(
205                level,
206                None,
207                vec![Reason::new(
208                    ReasonCode::CountAgreement,
209                    Signal::Number(agree),
210                )?],
211            )?,
212        })
213    }
214
215    fn no_count(reason: Reason) -> Result<Self, VoicesError> {
216        Ok(Self {
217            mono: false,
218            max_polyphony: 0,
219            mean_polyphony: 0.0,
220            joins: Vec::new(),
221            confidence: Confidence::no_estimate(vec![reason])?,
222        })
223    }
224
225    /// The lines the command prints.
226    #[must_use]
227    pub fn summary(&self) -> Vec<String> {
228        if !self.confidence.is_estimate() {
229            let no_list = self
230                .confidence
231                .reasons()
232                .iter()
233                .any(|r| r.code() == ReasonCode::NoNoteList);
234            return vec![
235                if no_list {
236                    "voices: no notes, so no count"
237                } else {
238                    "voices: no note holds a sounding part, so no count"
239                }
240                .to_owned(),
241            ];
242        }
243        let legato = self.joins.iter().filter(|j| j.legato == Some(true)).count();
244        vec![format!(
245            "voices: {}, up to {}, mean {:.2}; {} joins, {legato} legato",
246            if self.mono { "mono" } else { "poly" },
247            self.max_polyphony,
248            self.mean_polyphony,
249            self.joins.len(),
250        )]
251    }
252}
253
254fn hz(note: &Note) -> f64 {
255    Hz::from(note.pitch()).get()
256}
257
258/// The end of a note's held part: its knee, or [`KNEE_END_GAP`] before its end
259/// when it has none. A release tail holds no voice.
260fn held_end(note: &Note, envelope: &EnvelopeLayer) -> f64 {
261    let knee = envelope
262        .notes
263        .iter()
264        .find(|e| e.onset == note.onset() && e.pitch == note.pitch())
265        .and_then(|e| e.knee);
266    knee.map_or(note.offset().get() - KNEE_END_GAP.get(), Seconds::get)
267        .max(note.onset().get())
268}
269
270fn is_silent(note: &Note, envelope: &EnvelopeLayer) -> bool {
271    envelope.silent.contains(&(note.onset(), note.pitch()))
272}
273
274/// The length of the stem.
275fn stem_end(input: &VoicesInput<'_>) -> f64 {
276    cast::usize_to_f64(input.mono.len()) / input.rate.hz().get()
277}
278
279/// Each note's held part, up to the stem end. A note that does not sound holds
280/// none, nor does one that ends within [`KNEE_END_GAP`] of its onset or starts
281/// past the stem.
282fn held_parts(notes: &[Note], envelope: &EnvelopeLayer, stem_end: f64) -> Vec<Held> {
283    notes
284        .iter()
285        .filter(|n| !is_silent(n, envelope))
286        .map(|n| Held {
287            onset: n.onset().get(),
288            end: held_end(n, envelope).min(stem_end),
289            hz: hz(n),
290        })
291        .filter(|h| h.end > h.onset)
292        .collect()
293}
294
295/// The clusters of the pitches that started by `started` and are held at `t`.
296fn voices_at(held: &[Held], started: f64, t: f64) -> usize {
297    let mut pitches: Vec<f64> = held
298        .iter()
299        .filter(|h| h.onset <= started && t <= h.end)
300        .map(|h| h.hz)
301        .collect();
302    pitches.sort_by(f64::total_cmp);
303    let mut count = usize::from(!pitches.is_empty());
304    for pair in pitches.windows(2) {
305        if 1200.0 * (pair[1] / pair[0]).log2() > CLUSTER_CENTS.get() {
306            count += 1;
307        }
308    }
309    count
310}
311
312/// The mean count over the time any note is held, and the share of that time
313/// that agrees with `max`: one voice for a mono line, two or more otherwise.
314fn frame_counts(held: &[Held], max: usize) -> (f64, f64) {
315    let mut edges: Vec<f64> = held.iter().flat_map(|h| [h.onset, h.end]).collect();
316    edges.sort_by(f64::total_cmp);
317    let (mut total, mut time, mut agree) = (0.0, 0.0, 0.0);
318    for pair in edges.windows(2) {
319        let span = pair[1] - pair[0];
320        let middle = f64::midpoint(pair[0], pair[1]);
321        let count = voices_at(held, middle, middle);
322        if span > 0.0 && count > 0 {
323            total += cast::usize_to_f64(count) * span;
324            time += span;
325            if (max == 1) == (count == 1) {
326                agree += span;
327            }
328        }
329    }
330    if time > 0.0 {
331        (total / time, agree / time)
332    } else {
333        (1.0, 1.0)
334    }
335}
336
337/// The joins of a mono line. A note that starts well inside the earlier one's
338/// held part stacks on it and joins nothing. It leaves the earlier note in place
339/// unless that note ends sooner. A silent note leaves it in place too.
340fn joins(input: &VoicesInput<'_>, notes: &[Note]) -> Result<Vec<Join>, VoicesError> {
341    let attack = input.envelope.attack.value().map(|s| s.median().get());
342    let mut read = Vec::new();
343    let mut earlier: Option<&Note> = None;
344    for note in notes.iter().filter(|n| !is_silent(n, input.envelope)) {
345        let start = note.onset().get();
346        if let Some(before) = earlier {
347            let stacked = start <= before.onset().get()
348                || start + OVERLAP_MIN.get()
349                    < held_end(before, input.envelope).min(stem_end(input));
350            if stacked {
351                if note.offset().get() > before.offset().get() {
352                    earlier = Some(note);
353                }
354                continue;
355            }
356            let gap = start - before.offset().get();
357            if gap < JOIN_GAP_MAX.get() {
358                read.push(join(input, before, note, gap, attack)?);
359            }
360        }
361        earlier = Some(note);
362    }
363    Ok(read)
364}
365
366fn join(
367    input: &VoicesInput<'_>,
368    before: &Note,
369    after: &Note,
370    gap: f64,
371    attack: Option<f64>,
372) -> Result<Join, VoicesError> {
373    let at = after.onset().get();
374    let span = JOIN_SPAN.get();
375    // A transcribed onset can land after the silence it follows, so the dip reads
376    // from the earlier note's held end at the latest.
377    let dip_from = (at - span).min(held_end(before, input.envelope));
378    let from = (dip_from - SUSTAIN_SPAN.get()).max(before.onset().get());
379    let to = (at + span + SUSTAIN_SPAN.get())
380        .min(after.offset().get())
381        .min(stem_end(input));
382    // Shorter than a period, a window ripples with the waveform.
383    let lower = Hz::clamped(hz(before).min(hz(after)));
384    let fs = input.rate.hz().get();
385    let margin = rms_window(lower, RMS_PERIODS).get();
386    let first = cast::floor_to_usize((from - margin).max(0.0) * fs)
387        .unwrap_or(0)
388        .min(input.mono.len());
389    let last = cast::ceil_to_usize((to + margin) * fs)
390        .unwrap_or(first)
391        .clamp(first, input.mono.len());
392    let track = LevelTrack::rms(
393        &input.mono[first..last],
394        first,
395        input.rate,
396        (Seconds::clamped(from), Seconds::clamped(to)),
397        lower,
398        RMS_PERIODS,
399    );
400    let level = |a: f64, b: f64| -> &[f64] {
401        let (lo, hi) = (track.index(a), track.index(b));
402        &track.db()[lo.min(hi)..hi]
403    };
404    let low_level = |a: f64, b: f64| {
405        let mut sorted = level(a, b).to_vec();
406        sorted.sort_by(f64::total_cmp);
407        quantile(&sorted, SUSTAIN_QUANTILE).filter(|v| *v > SILENCE_DB.get())
408    };
409    let left = low_level(from, dip_from);
410    let right = low_level(at + span, to);
411    let low = level(dip_from, at + span)
412        .iter()
413        .copied()
414        .fold(f64::INFINITY, f64::min);
415    let quieter = match (left, right) {
416        (Some(l), Some(r)) => Some(l.min(r)),
417        (Some(v), None) | (None, Some(v)) => Some(v),
418        (None, None) => None,
419    };
420    let dip = quieter
421        .filter(|_| low.is_finite())
422        .map(|q| (q - low).max(0.0));
423    let centroid_change = centroid_change(input, before, after)?;
424    let legato = dip.map(|d| d < DIP_RETRIGGER.get());
425    let confidence = match (dip, attack) {
426        (None, _) => Confidence::no_estimate(vec![Reason::new(
427            ReasonCode::TooFewFrames,
428            Signal::Count(0),
429        )?])?,
430        (Some(_), None) => Confidence::estimate(
431            Level::Low,
432            None,
433            vec![Reason::new(ReasonCode::NoAttackRead, Signal::Flag(true))?],
434        )?,
435        (Some(_), Some(attack)) => {
436            let hides = attack > ATTACK_HIDES_JOIN.get();
437            let kept = legato == Some(true);
438            let cut = from - 0.5 * margin < 0.0
439                || dip_from - margin < 0.0
440                || at + span + margin > stem_end(input);
441            Confidence::estimate(
442                if hides || kept || cut {
443                    Level::Low
444                } else {
445                    Level::Medium
446                },
447                None,
448                vec![
449                    Reason::new(ReasonCode::AttackHidesJoin, Signal::Flag(hides))?,
450                    Reason::new(ReasonCode::LevelKeptAtJoin, Signal::Flag(kept))?,
451                    Reason::new(ReasonCode::StemEdgeAtJoin, Signal::Flag(cut))?,
452                ],
453            )?
454        }
455    };
456    Ok(Join {
457        time: after.onset(),
458        gap,
459        dip: dip.map(Decibels::clamped),
460        centroid_change,
461        legato,
462        confidence,
463    })
464}
465
466/// The centroid over f0 over [`CENTROID_SPAN`] after the join over the same
467/// before it, minus 1; `None` when either side holds no signal or runs past
468/// either end of the stem.
469fn centroid_change(
470    input: &VoicesInput<'_>,
471    before: &Note,
472    after: &Note,
473) -> Result<Option<f64>, VoicesError> {
474    let fs = input.rate.hz().get();
475    let at = cast::round_to_usize(after.onset().get() * fs).unwrap_or(0);
476    let lower = hz(before).min(hz(after));
477    let len = cast::round_to_usize(f64::max(CENTROID_SPAN.get(), 2.0 / lower) * fs)
478        .unwrap_or(0)
479        .max(4);
480    if at < len {
481        return Ok(None);
482    }
483    let side = |range: std::ops::Range<usize>, f0: f64| -> Result<Option<f64>, VoicesError> {
484        let Some(samples) = input.mono.get(range) else {
485            return Ok(None);
486        };
487        let params = StftParams::new(samples.len())?;
488        let mut stft = Stft::new(params);
489        // Frame 2 of a hop of a quarter is centred on the samples.
490        let magnitudes: Vec<f64> = stft.frame(samples, 2).iter().map(|c| c.norm()).collect();
491        Ok(frame_descriptors(&magnitudes, params.bin_hz(1, input.rate))
492            .map(|d| d.centroid.get() / f0))
493    };
494    let early = side(at - len..at, hz(before))?;
495    let late = side(at..at + len, hz(after))?;
496    Ok(match (early, late) {
497        (Some(e), Some(l)) if e > 0.0 => Some(l / e - 1.0),
498        _ => None,
499    })
500}
sound_to_patch/estimate/voices.rs

sound_to_patch/estimate/
voices.rs
