1use super::{
12 DEFAULT_MIN_NOTE_FRAMES, DecodeSettings, HIGHEST_MODEL_PITCH, LOWEST_MODEL_PITCH, ModelRun,
13 NoteList, PitchRange, PitchTrack, TranscribeError, min_note_frames,
14};
15use crate::cast;
16use crate::dsp::quantile;
17use crate::units::{Bpm, Decibels, MidiPitch, Semitones};
18
19pub const REGISTER_PERCENTILES: (f64, f64) = (1.0, 99.0);
22pub const REGISTER_WINDOW: Decibels = Decibels::from_const(20.0);
25pub const HOLD_WINDOW: Decibels = Decibels::from_const(10.0);
29pub const REFERENCE_PERCENTILE: f64 = 95.0;
32pub const REGISTER_MARGIN: Semitones = Semitones::from_const(2.0);
34pub const CHORD_SHARE: f64 = 0.68;
38pub const CHORD_OVERLAP: f64 = 0.5;
41
42#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
44pub enum Voicing {
45 Monophonic,
47 Polyphonic,
49}
50
51impl Voicing {
52 #[must_use]
54 pub fn name(self) -> &'static str {
55 match self {
56 Self::Monophonic => "monophonic",
57 Self::Polyphonic => "polyphonic",
58 }
59 }
60}
61
62#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
64pub enum RegisterSource {
65 F0Track,
67 ProbeNotes,
69 Both,
71 None,
73}
74
75impl RegisterSource {
76 #[must_use]
78 pub fn name(self) -> &'static str {
79 match self {
80 Self::F0Track => "f0 track",
81 Self::ProbeNotes => "probe notes",
82 Self::Both => "f0 track and held probe notes",
83 Self::None => "whole model range",
84 }
85 }
86}
87
88#[derive(Clone, Copy, Debug, PartialEq)]
90pub struct Detection {
91 pub settings: DecodeSettings,
93 pub voicing: Voicing,
95 pub chord_share: f64,
98 pub window: Decibels,
100 pub source: RegisterSource,
102 pub probe_held: Option<f64>,
105 pub f0_register: Option<(MidiPitch, MidiPitch)>,
108 pub probe_register: Option<(MidiPitch, MidiPitch)>,
110 pub probe_notes: usize,
112}
113
114#[must_use]
117pub fn probe_settings(bpm: Bpm) -> DecodeSettings {
118 DecodeSettings {
119 range: full_range(),
120 min_note_frames: min_note_frames(bpm),
121 melodia: true,
122 }
123}
124
125fn full_range() -> PitchRange {
126 PitchRange::new(
127 MidiPitch::clamped(f64::from(LOWEST_MODEL_PITCH)),
128 MidiPitch::clamped(f64::from(HIGHEST_MODEL_PITCH)),
129 )
130 .unwrap_or(super::Profile::Lead.range())
131}
132
133pub fn detect(run: &ModelRun, track: &PitchTrack, bpm: Bpm) -> Result<Detection, TranscribeError> {
147 detect_with_window(run, track, bpm, REGISTER_WINDOW)
148}
149
150pub fn detect_with_window(
157 run: &ModelRun,
158 track: &PitchTrack,
159 bpm: Bpm,
160 window: Decibels,
161) -> Result<Detection, TranscribeError> {
162 let probe = run.decode(&probe_settings(bpm))?;
163 let chord_share = chord_share(&probe);
164 let voicing = voicing_of(chord_share);
165 let f0_register = f0_register(track, window);
166 let probe_register = probe_register(&probe);
167 let probe_pitches = hold_candidates(&probe, track);
168 let (range, source, probe_held) =
169 choose_range(voicing, f0_register, probe_register, &probe_pitches);
170 Ok(Detection {
171 settings: DecodeSettings {
172 range,
173 min_note_frames: match voicing {
174 Voicing::Monophonic => min_note_frames(bpm),
175 Voicing::Polyphonic => DEFAULT_MIN_NOTE_FRAMES,
176 },
177 melodia: true,
178 },
179 voicing,
180 chord_share,
181 window,
182 source,
183 probe_held,
184 f0_register,
185 probe_register,
186 probe_notes: probe.notes().len(),
187 })
188}
189
190fn voicing_of(chord_share: f64) -> Voicing {
191 if chord_share >= CHORD_SHARE {
192 Voicing::Polyphonic
193 } else {
194 Voicing::Monophonic
195 }
196}
197
198type Register = Option<(MidiPitch, MidiPitch)>;
199
200fn choose_range(
207 voicing: Voicing,
208 f0: Register,
209 probe: Register,
210 probe_pitches: &[f64],
211) -> (PitchRange, RegisterSource, Option<f64>) {
212 let span = |(low, high): (MidiPitch, MidiPitch)| range_of(low.get(), high.get());
213 let held = f0.filter(|_| !probe_pitches.is_empty()).map(|register| {
214 let range = span(register);
215 let (low, high) = (range.lowest().get(), range.highest().get());
216 let inside = probe_pitches
217 .iter()
218 .filter(|&&p| (low..=high).contains(&p))
219 .count();
220 cast::usize_to_f64(inside) / cast::usize_to_f64(probe_pitches.len())
221 });
222 match (voicing, f0, probe) {
223 (Voicing::Polyphonic, _, Some(probe)) | (_, None, Some(probe)) => {
224 (span(probe), RegisterSource::ProbeNotes, held)
225 }
226 (_, Some(f0), _) if held.is_some_and(|h| h < 1.0) => {
227 let low = probe_pitches.iter().copied().fold(f0.0.get(), f64::min);
228 let high = probe_pitches.iter().copied().fold(f0.1.get(), f64::max);
229 (range_of(low, high), RegisterSource::Both, held)
230 }
231 (_, Some(f0), _) => (span(f0), RegisterSource::F0Track, held),
232 (_, None, None) => (full_range(), RegisterSource::None, held),
233 }
234}
235
236fn range_of(low: f64, high: f64) -> PitchRange {
239 let margin = REGISTER_MARGIN.get();
240 let clamp = |pitch: f64| {
241 pitch.clamp(
242 f64::from(LOWEST_MODEL_PITCH),
243 f64::from(HIGHEST_MODEL_PITCH),
244 )
245 };
246 PitchRange::new(
247 MidiPitch::clamped(clamp((low - margin).floor())),
248 MidiPitch::clamped(clamp((high + margin).ceil())),
249 )
250 .unwrap_or_else(|_| full_range())
251}
252
253#[must_use]
259pub fn voicing_of_notes(notes: &NoteList) -> Voicing {
260 voicing_of(chord_share(notes))
261}
262
263fn chord_share(notes: &NoteList) -> f64 {
272 let notes = notes.notes();
273 let class = |pitch: MidiPitch| cast::round_to_usize(pitch.get()).unwrap_or(0) % 12;
274 let struck = notes
275 .iter()
276 .filter(|note| {
277 let span = note.offset().get() - note.onset().get();
278 notes.iter().any(|other| {
279 let shared = other.offset().get().min(note.offset().get())
280 - other.onset().get().max(note.onset().get());
281 class(other.pitch()) != class(note.pitch()) && shared >= CHORD_OVERLAP * span
282 })
283 })
284 .count();
285 if notes.is_empty() {
286 0.0
287 } else {
288 cast::usize_to_f64(struck) / cast::usize_to_f64(notes.len())
289 }
290}
291
292fn hold_candidates(probe: &NoteList, track: &PitchTrack) -> Vec<f64> {
303 let Some(reference) = reference_level(track) else {
304 return Vec::new();
305 };
306 let notes = probe.notes();
307 notes
308 .iter()
309 .filter(|note| {
310 let mut levels: Vec<f64> = track
311 .frames()
312 .iter()
313 .filter(|frame| note.sounds_at(frame.time))
314 .map(|frame| frame.level.get())
315 .collect();
316 levels.sort_by(f64::total_cmp);
317 quantile(&levels, 0.5).is_some_and(|level| level >= reference - HOLD_WINDOW.get())
318 })
319 .filter(|note| {
320 let (mut frames, mut shared) = (0_usize, 0_usize);
321 for frame in track.frames().iter().filter(|f| note.sounds_at(f.time)) {
322 frames += 1;
323 shared += usize::from(notes.iter().any(|other| {
324 let gap = (other.pitch().get() - note.pitch().get()).abs();
325 let weaker_twin = (gap - 12.0).abs() < 0.5
326 && other
327 .velocity()
328 .zip(note.velocity())
329 .is_some_and(|(twin, this)| twin < this);
330 gap >= 0.5 && !weaker_twin && other.sounds_at(frame.time)
331 }));
332 }
333 shared * 2 <= frames
334 })
335 .map(|note| note.pitch().get())
336 .collect()
337}
338
339fn reference_level(track: &PitchTrack) -> Option<f64> {
341 let mut levels: Vec<f64> = track
342 .frames()
343 .iter()
344 .filter(|frame| frame.f0.is_some())
345 .map(|frame| frame.level.get())
346 .collect();
347 levels.sort_by(f64::total_cmp);
348 quantile(&levels, REFERENCE_PERCENTILE / 100.0)
349}
350
351fn f0_register(track: &PitchTrack, window: Decibels) -> Option<(MidiPitch, MidiPitch)> {
356 let reference = reference_level(track)?;
357 let voiced: Vec<_> = track
358 .frames()
359 .iter()
360 .filter(|frame| frame.f0.is_some())
361 .collect();
362 let mut pitches: Vec<f64> = voiced
363 .iter()
364 .filter(|frame| (frame.level.get() - reference).abs() <= window.get().abs())
365 .filter_map(|frame| frame.f0)
366 .filter_map(|f0| MidiPitch::try_from(f0).ok())
367 .map(MidiPitch::get)
368 .collect();
369 pitches.sort_by(f64::total_cmp);
370 let (low, high) = REGISTER_PERCENTILES;
371 Some((
372 MidiPitch::clamped(quantile(&pitches, low / 100.0)?),
373 MidiPitch::clamped(quantile(&pitches, high / 100.0)?),
374 ))
375}
376
377fn probe_register(notes: &NoteList) -> Option<(MidiPitch, MidiPitch)> {
378 let pitches = notes.notes().iter().map(|note| note.pitch().get());
379 let low = pitches.clone().reduce(f64::min)?;
380 let high = pitches.reduce(f64::max)?;
381 Some((MidiPitch::clamped(low), MidiPitch::clamped(high)))
382}
383
384#[cfg(test)]
385mod tests {
386 use super::*;
387 use crate::notes::{Note, TrackFrame};
388 use crate::units::{Hz, Seconds};
389
390 fn s(value: f64) -> Seconds {
391 Seconds::new(value).unwrap()
392 }
393
394 fn note(onset: f64, offset: f64, pitch: f64) -> Note {
395 Note::new(s(onset), s(offset), MidiPitch::new(pitch).unwrap()).unwrap()
396 }
397
398 fn track_of(pitch: impl Fn(usize) -> Option<f64>) -> PitchTrack {
401 PitchTrack::from_frames(
402 (0..1000)
403 .map(|i| TrackFrame {
404 time: s(cast::usize_to_f64(i) * 0.005),
405 f0: pitch(i).map(|p| Hz::from(MidiPitch::new(p).unwrap())),
406 level: Decibels::new(0.0).unwrap(),
407 })
408 .collect(),
409 )
410 }
411
412 #[test]
413 fn each_register_source_has_its_words() {
414 let sources = [
415 RegisterSource::F0Track,
416 RegisterSource::ProbeNotes,
417 RegisterSource::Both,
418 RegisterSource::None,
419 ];
420 assert_eq!(
421 sources.map(RegisterSource::name),
422 [
423 "f0 track",
424 "probe notes",
425 "f0 track and held probe notes",
426 "whole model range"
427 ]
428 );
429 }
430
431 #[test]
432 fn the_probe_register_spans_the_lowest_and_highest_notes() {
433 let notes = NoteList::new(vec![
434 note(0.0, 1.0, 57.0),
435 note(0.0, 1.0, 52.0),
436 note(1.0, 2.0, 64.0),
437 ]);
438 let (low, high) = probe_register(¬es).unwrap();
439 assert_eq!((low.get(), high.get()), (52.0, 64.0));
440 assert_eq!(probe_register(&NoteList::new(Vec::new())), None);
441 }
442
443 #[test]
444 fn frames_outside_the_window_leave_the_register_alone() {
445 let frames = (0..900)
448 .map(|i| TrackFrame {
449 time: s(cast::usize_to_f64(i) * 0.005),
450 f0: Some(Hz::from(
451 MidiPitch::new(if i % 3 == 0 { 72.0 } else { 40.0 }).unwrap(),
452 )),
453 level: Decibels::new(if i % 3 == 0 { -30.0 } else { -3.0 }).unwrap(),
454 })
455 .collect();
456 let track = PitchTrack::from_frames(frames);
457 let (low, high) = f0_register(&track, REGISTER_WINDOW).unwrap();
458 assert!((low.get() - 40.0).abs() < 1e-9 && (high.get() - 40.0).abs() < 1e-9);
459 let (_, wide) = f0_register(&track, Decibels::new(40.0).unwrap()).unwrap();
460 assert!((wide.get() - 72.0).abs() < 1e-9);
461 }
462
463 #[test]
464 fn the_register_is_the_1st_to_99th_percentile_of_voiced_frames() {
465 let track = track_of(|i| (i % 3 != 0).then(|| 40.0 + cast::usize_to_f64(i) / 100.0));
467 let (low, high) = f0_register(&track, REGISTER_WINDOW).unwrap();
468 let mut voiced: Vec<f64> = (0..1000)
469 .filter(|i| i % 3 != 0)
470 .map(|i| 40.0 + cast::usize_to_f64(i) / 100.0)
471 .collect();
472 voiced.sort_by(f64::total_cmp);
473 assert!((low.get() - quantile(&voiced, 0.01).unwrap()).abs() < 1e-9);
474 assert!((high.get() - quantile(&voiced, 0.99).unwrap()).abs() < 1e-9);
475 assert!((low.get() - 40.105).abs() < 0.01, "{}", low.get());
476 assert!((high.get() - 49.885).abs() < 0.01, "{}", high.get());
477 assert_eq!(f0_register(&track_of(|_| None), REGISTER_WINDOW), None);
478 }
479
480 #[test]
481 fn the_range_reaches_two_semitones_beyond_the_register() {
482 let ends = |range: PitchRange| (range.lowest().get(), range.highest().get());
483 assert_eq!(ends(range_of(40.105, 49.885)), (38.0, 52.0));
484 assert_eq!(ends(range_of(45.0, 45.0)), (43.0, 47.0));
485 assert_eq!(ends(range_of(22.0, 107.5)), (21.0, 108.0));
486 }
487
488 fn pitch(value: f64) -> MidiPitch {
489 MidiPitch::new(value).unwrap()
490 }
491
492 fn bass_with(level: f64, count: usize, transient: Option<f64>) -> PitchTrack {
496 let line = [36.0, 38.0, 40.0, 41.0, 43.0, 45.0, 47.0, 48.0];
497 let mut frames: Vec<TrackFrame> = (0..800)
498 .map(|i| TrackFrame {
499 time: s(cast::usize_to_f64(i) * 0.005),
500 f0: Some(Hz::from(pitch(line[i / 100]))),
501 level: Decibels::new(level).unwrap(),
502 })
503 .collect();
504 frames.extend((0..count).map(|i| TrackFrame {
505 time: s(4.0 + cast::usize_to_f64(i) * 0.005),
506 f0: transient.map(|p| Hz::from(pitch(p))),
507 level: Decibels::new(0.0).unwrap(),
508 }));
509 PitchTrack::from_frames(frames)
510 }
511
512 fn line_range(track: &PitchTrack) -> (f64, f64) {
513 let (low, high) = f0_register(track, REGISTER_WINDOW).unwrap();
514 let range = range_of(low.get(), high.get());
515 (range.lowest().get(), range.highest().get())
516 }
517
518 #[test]
519 fn a_loud_burst_ring_or_thump_leaves_the_register_alone() {
520 assert_eq!(line_range(&bass_with(-34.0, 20, Some(19.0))), (34.0, 50.0));
522 assert_eq!(line_range(&bass_with(-30.0, 12, Some(90.5))), (34.0, 50.0));
524 assert_eq!(line_range(&bass_with(-25.0, 27, Some(34.6))), (34.0, 50.0));
526 assert_eq!(line_range(&bass_with(-40.0, 20, None)), (34.0, 50.0));
528 }
529
530 #[test]
531 fn the_register_choice_takes_each_source_in_its_case() {
532 let line: Vec<f64> = vec![36.0, 38.0, 40.0, 41.0, 43.0, 45.0, 47.0, 48.0];
533 let ends = |(range, source, _): (PitchRange, RegisterSource, Option<f64>)| {
534 (range.lowest().get(), range.highest().get(), source)
535 };
536 let f0 = Some((pitch(36.0), pitch(48.0)));
537 let probe = Some((pitch(36.0), pitch(60.0)));
538 assert_eq!(
539 ends(choose_range(Voicing::Polyphonic, f0, probe, &line)),
540 (34.0, 62.0, RegisterSource::ProbeNotes)
541 );
542 assert_eq!(
543 ends(choose_range(Voicing::Monophonic, f0, probe, &line)),
544 (34.0, 50.0, RegisterSource::F0Track)
545 );
546 let misread = Some((pitch(19.0), pitch(19.0)));
549 let (range, source, held) = choose_range(Voicing::Monophonic, misread, probe, &line);
550 assert_eq!(
551 (range.lowest().get(), range.highest().get(), source),
552 (21.0, 50.0, RegisterSource::Both)
553 );
554 assert_eq!(held, Some(0.0));
555 assert_eq!(
556 ends(choose_range(Voicing::Monophonic, None, probe, &line)),
557 (34.0, 62.0, RegisterSource::ProbeNotes)
558 );
559 assert_eq!(
560 ends(choose_range(Voicing::Monophonic, None, None, &[])),
561 (21.0, 108.0, RegisterSource::None)
562 );
563 }
564
565 #[test]
566 fn a_hold_candidate_outside_the_f0_range_widens_it() {
567 let f0 = Some((pitch(45.0), pitch(52.0)));
568 let probe = Some((pitch(45.0), pitch(70.0)));
569 let (range, source, held) =
570 choose_range(Voicing::Monophonic, f0, probe, &[45.0, 47.0, 50.0, 52.0]);
571 assert_eq!(
572 (range.highest().get(), source, held),
573 (54.0, RegisterSource::F0Track, Some(1.0))
574 );
575 let (range, source, held) = choose_range(
578 Voicing::Monophonic,
579 f0,
580 probe,
581 &[45.0, 47.0, 50.0, 52.0, 57.0],
582 );
583 assert_eq!(
584 (range.lowest().get(), range.highest().get(), source, held),
585 (43.0, 59.0, RegisterSource::Both, Some(0.8))
586 );
587 }
588
589 #[test]
590 fn a_held_double_stop_above_a_line_falls_outside_the_range() {
591 let track = PitchTrack::from_frames(
595 (0..400)
596 .map(|i| TrackFrame {
597 time: s(cast::usize_to_f64(i) * 0.005),
598 f0: Some(Hz::from(pitch(48.0))),
599 level: Decibels::new(-3.0).unwrap(),
600 })
601 .collect(),
602 );
603 let probe = NoteList::new(vec![
604 note(0.0, 0.5, 45.0),
605 note(0.5, 1.0, 47.0),
606 note(1.0, 1.5, 50.0),
607 note(1.5, 2.0, 52.0),
608 note(0.5, 1.5, 64.0),
609 ]);
610 let candidates = hold_candidates(&probe, &track);
611 assert_eq!(candidates, vec![45.0, 52.0]);
613 let (range, source, _) = choose_range(
614 Voicing::Monophonic,
615 Some((pitch(45.0), pitch(52.0))),
616 Some((pitch(45.0), pitch(64.0))),
617 &candidates,
618 );
619 assert_eq!(source, RegisterSource::F0Track);
620 assert!(range.highest().get() < 64.0, "{}", range.highest().get());
621 }
622
623 #[test]
624 fn ghosts_bleed_and_quiet_notes_do_not_test_the_f0_range() {
625 let track = PitchTrack::from_frames(
630 (0..1000)
631 .map(|i| TrackFrame {
632 time: s(cast::usize_to_f64(i) * 0.005),
633 f0: Some(Hz::from(pitch(45.0))),
634 level: Decibels::new(if i < 800 && !(600..700).contains(&i) {
635 -3.0
636 } else {
637 -30.0
638 })
639 .unwrap(),
640 })
641 .collect(),
642 );
643 let probe = NoteList::new(vec![
644 note(0.0, 1.0, 45.0),
645 note(0.1, 0.4, 57.0),
646 note(0.2, 0.5, 69.0),
647 note(1.0, 2.0, 47.0),
648 note(2.0, 3.0, 48.0),
649 note(2.1, 2.4, 72.0),
650 note(2.95, 3.55, 64.0),
651 note(4.2, 4.8, 72.0),
652 ]);
653 assert_eq!(hold_candidates(&probe, &track), vec![45.0, 47.0, 48.0]);
654 }
655
656 #[test]
657 fn a_weaker_octave_twin_leaves_a_note_alone() {
658 let track = PitchTrack::from_frames(
659 (0..200)
660 .map(|i| TrackFrame {
661 time: s(cast::usize_to_f64(i) * 0.005),
662 f0: Some(Hz::from(pitch(45.0))),
663 level: Decibels::new(-3.0).unwrap(),
664 })
665 .collect(),
666 );
667 let heard = |onset, offset, key, velocity| {
668 note(onset, offset, key).with_velocity(crate::notes::Velocity::new(velocity).unwrap())
669 };
670 let probe = NoteList::new(vec![heard(0.2, 0.4, 57.0, 98), heard(0.19, 0.4, 45.0, 48)]);
673 assert_eq!(hold_candidates(&probe, &track), vec![57.0]);
674 let probe = NoteList::new(vec![heard(0.2, 0.4, 57.0, 90), heard(0.2, 0.4, 45.0, 90)]);
676 assert_eq!(hold_candidates(&probe, &track), Vec::<f64>::new());
677 }
678
679 #[test]
680 fn voicing_turns_polyphonic_at_the_chord_share() {
681 assert_eq!(voicing_of(CHORD_SHARE), Voicing::Polyphonic);
682 assert_eq!(voicing_of(CHORD_SHARE - 1e-9), Voicing::Monophonic);
683 assert_eq!(voicing_of(0.0), Voicing::Monophonic);
684 }
685
686 #[test]
687 fn quantiles_interpolate_linearly() {
688 let values = [1.0, 2.0, 3.0, 4.0, 5.0];
689 assert_eq!(quantile(&values, 0.0), Some(1.0));
690 assert_eq!(quantile(&values, 0.5), Some(3.0));
691 assert_eq!(quantile(&values, 0.99), Some(4.96));
692 assert_eq!(quantile(&values, 1.0), Some(5.0));
693 assert_eq!(quantile(&[], 0.5), None);
694 }
695
696 #[test]
697 fn notes_held_together_are_chords_and_ghosts_or_late_ends_are_not() {
698 let bass = NoteList::new(vec![
700 note(0.0, 1.2, 45.0),
701 note(0.02, 0.3, 57.0),
702 note(1.0, 2.0, 47.0),
703 ]);
704 assert_eq!(chord_share(&bass), 0.0);
705 let chords = NoteList::new(vec![
708 note(0.0, 2.0, 57.0),
709 note(0.03, 2.0, 61.0),
710 note(0.5, 2.1, 64.0),
711 note(1.0, 1.9, 59.0),
712 note(1.9, 4.0, 62.0),
713 ]);
714 assert!((chord_share(&chords) - 0.8).abs() < 1e-12);
715 let pair = |end: f64| {
717 chord_share(&NoteList::new(vec![
718 note(0.0, 1.0, 57.0),
719 note(0.5, end, 60.0),
720 ]))
721 };
722 assert!((pair(1.0) - 1.0).abs() < 1e-12);
723 assert!((pair(0.999) - 0.5).abs() < 1e-12);
724 assert_eq!(chord_share(&NoteList::default()), 0.0);
725 }
726}