sound_to_patch/render/
check.rs

1//! The load check: a render counts only when the plug-in kept the preset.
2//!
3//! A host call reports success even when the plug-in kept its old sound, so
4//! the check looks at the result three ways:
5//!
6//! 1. The clip must sound: a clip whose every sample stays below the tail
7//!    floor is silent.
8//! 2. The state the plug-in saves after the render must hold the sent
9//!    preset: the name, the tuning and every control. When it holds the
10//!    init patch instead, the plug-in fell back to init.
11//! 3. The clip is compared with the plug-in's own init patch. Each clip is
12//!    reduced to its level in octave bands, and the distance is the
13//!    largest band difference in dB, over the bands above a floor of mean
14//!    bin power and within 30 dB of the loudest. A clip within three
15//!    times the distance of two init renders is init-like.
16//!
17//! The first two are faults. The third is a warning: mean band levels miss
18//! a change of envelope or modulation, so a preset that loaded can still
19//! lie within the margin. The margin stands in for a spread measured over
20//! many renders, which a later release stores.
21
22use std::fmt;
23
24use serde::{Deserialize, Serialize};
25use serde_json::{Map, Value};
26
27use super::state::{Mismatch, TEMPO_PATH, readback_mismatches, saved_preset};
28use super::{PresetState, RenderError, RenderJob, Rendered, Renderer};
29use crate::audio::{Audio, SampleRate};
30use crate::cast;
31use crate::dsp::{Padding, Stft, StftParams, Window};
32use crate::preset::registry::registry_for;
33use crate::synth::Synth;
34
35/// The STFT frame of the band levels: 4096 samples, about 93 ms at
36/// 44.1 kHz, so the lowest band holds several bins.
37const FRAME: usize = 4096;
38/// The lower edge of the lowest octave band, in Hz. The bands run up to the
39/// Nyquist frequency.
40const LOWEST_EDGE_HZ: f64 = 44.0;
41/// A band quieter than this is left out of the distance, so noise in
42/// empty bands does not count.
43///
44/// The unit is that of [`BandLevels`], the mean bin power of the band, not
45/// dB below full scale. A full-scale sine reads `10 log10(3 / 32 / bins)`
46/// dB in a band of `bins` bins. At 44.1 kHz that is about -40 dB in the
47/// top band of 1001 bins and -16 dB in the band from 44 to 88 Hz of 4
48/// bins, so the floor passes a sine above about -50 dBFS in the top band
49/// and above about -74 dBFS in the lowest one.
50const BAND_FLOOR_DB: f64 = -90.0;
51/// A band more than this many dB below the loudest band of either clip is
52/// left out too. Below a note's fundamental a band holds only window
53/// leakage, whose level moves by several dB between two renders of one
54/// patch.
55const BAND_RANGE_DB: f64 = 30.0;
56/// The distance between two init renders is scaled by this factor to give
57/// the margin: a clip within the margin of the init patch is init-like.
58/// Two renders give no distribution, so the factor is a stopgap.
59const SPREAD_FACTOR: f64 = 3.0;
60/// The smallest margin in dB, for a plug-in whose init renders match
61/// almost exactly.
62const MIN_MARGIN_DB: f64 = 0.1;
63/// How many mismatches a message names.
64const SHOWN: usize = 5;
65
66/// The level of a clip in octave bands, in dB of mean bin power.
67///
68/// For each band: the mean over its bins of the Hann-windowed bin power
69/// over the frame length squared, averaged over the frames. A band of many
70/// bins reads lower than a band of few for one tone: a full-scale sine
71/// reads `10 log10(3 / 32 / bins)` dB in a band of `bins` bins.
72#[derive(Clone, Debug, PartialEq)]
73pub struct BandLevels(Vec<f64>);
74
75impl BandLevels {
76    /// Measures the mono mix of `audio`.
77    #[must_use]
78    pub fn of(audio: &Audio) -> Self {
79        let mono = audio.mono();
80        let rate = audio.sample_rate();
81        let params = StftParams::new(FRAME)
82            .map(|params| params.with_window(Window::Hann).with_padding(Padding::Zero))
83            .ok();
84        let Some(params) = params else {
85            return Self(Vec::new());
86        };
87        let mut stft = Stft::new(params);
88        let bins = params.bins();
89        let mut power = vec![0.0; bins];
90        let frames = params.frame_count(mono.len()).max(1);
91        for frame in 0..frames {
92            for (sum, value) in power.iter_mut().zip(stft.frame(&mono, frame)) {
93                *sum += value.norm_sqr();
94            }
95        }
96        let scale = 1.0 / cast::usize_to_f64(frames * FRAME * FRAME);
97        let bin_hz = params.bin_width(rate).get();
98        Self(
99            band_edges(rate)
100                .windows(2)
101                .map(|edge| {
102                    let in_band = power.iter().enumerate().filter(|(bin, _)| {
103                        let hz = cast::usize_to_f64(*bin) * bin_hz;
104                        (edge[0]..edge[1]).contains(&hz)
105                    });
106                    let (sum, count) =
107                        in_band.fold((0.0, 0_usize), |(s, n), (_, p)| (s + p, n + 1));
108                    let mean = if count == 0 {
109                        0.0
110                    } else {
111                        sum * scale / cast::usize_to_f64(count)
112                    };
113                    10.0 * mean.max(1e-30).log10()
114                })
115                .collect(),
116        )
117    }
118
119    /// The largest level difference in a band where either clip is above
120    /// the band floor and within 30 dB of the loudest band.
121    /// Two clips with no such band are 0 dB apart.
122    #[must_use]
123    pub fn distance(&self, other: &Self) -> f64 {
124        let loudest = self
125            .0
126            .iter()
127            .chain(&other.0)
128            .fold(f64::NEG_INFINITY, |a, b| a.max(*b));
129        let floor = BAND_FLOOR_DB.max(loudest - BAND_RANGE_DB);
130        self.0
131            .iter()
132            .zip(&other.0)
133            .filter(|(a, b)| a.max(**b) > floor)
134            .map(|(a, b)| (a - b).abs())
135            .fold(0.0, f64::max)
136    }
137}
138
139/// The octave band edges from [`LOWEST_EDGE_HZ`] up to the Nyquist
140/// frequency.
141fn band_edges(rate: SampleRate) -> Vec<f64> {
142    let nyquist = rate.nyquist().get();
143    std::iter::successors(Some(LOWEST_EDGE_HZ), |edge| Some(edge * 2.0))
144        .take_while(|edge| *edge < nyquist)
145        .chain(std::iter::once(nyquist))
146        .collect()
147}
148
149/// What the load check measured for a render that passed.
150#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
151pub struct LoadCheck {
152    /// The distance of the clip from the init patch, in dB.
153    pub init_distance_db: f64,
154    /// The distance between two init renders, in dB.
155    pub init_spread_db: f64,
156    /// The margin the distance had to exceed, in dB.
157    pub margin_db: f64,
158    /// Whether the clip lies within the margin of the init patch. The
159    /// render still counts, because the read-back matched.
160    pub init_like: bool,
161}
162
163impl LoadCheck {
164    /// The warning of an init-like render, if this one is.
165    #[must_use]
166    pub fn warning(&self) -> Option<String> {
167        self.init_like.then(|| {
168            format!(
169                "the render lies {:.2} dB from the init patch, within the margin of {:.2} dB. The saved state matches, so the preset loaded; octave-band levels miss changes of envelope and modulation. Listen to the clip to check that it differs from the init patch",
170                self.init_distance_db, self.margin_db
171            )
172        })
173    }
174}
175
176/// Why a render does not count.
177#[derive(Debug)]
178pub enum LoadFault {
179    /// The plug-in or its host failed.
180    Render(RenderError),
181    /// The clip is silent.
182    Silent {
183        /// What in the preset can explain the silence. Empty when nothing
184        /// known does.
185        reasons: Vec<String>,
186    },
187    /// The plug-in fell back to its init patch: the state it saved is the
188    /// init state.
189    InitFallback,
190    /// The state the plug-in saved differs from the sent one.
191    Readback(Vec<Mismatch>),
192}
193
194impl fmt::Display for LoadFault {
195    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
196        match self {
197            Self::Render(error) => write!(f, "{error}"),
198            Self::Silent { reasons } if reasons.is_empty() => write!(
199                f,
200                "the render is silent: every sample stays below -96 dBFS, and nothing known in the preset explains it. Open the preset in the synth and check that it sounds"
201            ),
202            Self::Silent { reasons } => write!(
203                f,
204                "the render is silent: {}. Fix the preset so that it sounds",
205                reasons.join("; ")
206            ),
207            Self::InitFallback => write!(
208                f,
209                "the plug-in did not load the preset: after the render it saves its init patch. Check that the preset loads in the synth and that its build is not newer than the plug-in"
210            ),
211            Self::Readback(mismatches) => {
212                let shown: Vec<String> = mismatches
213                    .iter()
214                    .take(SHOWN)
215                    .map(ToString::to_string)
216                    .collect();
217                let more = mismatches.len().saturating_sub(SHOWN);
218                let tempo = if mismatches.iter().any(|m| m.path == TEMPO_PATH) {
219                    ". The plug-in sets settings.beats_per_minute from the host tempo, --bpm / 60 in beats per second, so an edit of the preset cannot fix that value; report it with the tempo"
220                } else {
221                    ""
222                };
223                write!(
224                    f,
225                    "the plug-in did not keep the preset: {}{}{tempo}. Check that the preset loads in the synth with these values",
226                    shown.join("; "),
227                    if more > 0 {
228                        format!(" and {more} more")
229                    } else {
230                        String::new()
231                    }
232                )
233            }
234        }
235    }
236}
237
238impl std::error::Error for LoadFault {}
239
240impl From<RenderError> for LoadFault {
241    fn from(error: RenderError) -> Self {
242        Self::Render(error)
243    }
244}
245
246/// Renders `job` and checks that the plug-in loaded its preset.
247///
248/// The renderer must not have loaded a state yet: the check renders the
249/// init patch twice first, and keeps only the band levels of each clip.
250///
251/// # Errors
252///
253/// Returns the [`LoadFault`] that explains why the render does not count.
254pub fn render_checked(
255    renderer: &mut dyn Renderer,
256    job: &RenderJob,
257) -> Result<(Rendered, LoadCheck), LoadFault> {
258    let init = BandLevels::of(&renderer.render_init(job)?.audio);
259    let spread = init.distance(&BandLevels::of(&renderer.render_init(job)?.audio));
260    let init_state = renderer.save_state()?;
261    let output = renderer.render(job)?;
262    let saved = renderer.save_state()?;
263    let floor = job.recipe().tail_floor_gain();
264    let silent = output
265        .audio
266        .channels()
267        .iter()
268        .all(|channel| channel.iter().all(|sample| sample.abs() < floor));
269    if silent {
270        return Err(LoadFault::Silent {
271            reasons: silence_reasons(job.state()),
272        });
273    }
274    let mismatches = readback_mismatches(job.state(), &saved, job.bpm())?;
275    if !mismatches.is_empty() {
276        let fell_back = matches!(
277            (saved_preset(&saved), saved_preset(&init_state)),
278            (Some(saved), Some(init)) if saved == init
279        );
280        return Err(if fell_back {
281            LoadFault::InitFallback
282        } else {
283            LoadFault::Readback(mismatches)
284        });
285    }
286    let distance = BandLevels::of(&output.audio).distance(&init);
287    let margin = (SPREAD_FACTOR * spread).max(MIN_MARGIN_DB);
288    Ok((
289        output,
290        LoadCheck {
291            init_distance_db: distance,
292            init_spread_db: spread,
293            margin_db: margin,
294            init_like: distance <= margin,
295        },
296    ))
297}
298
299/// What in a preset can make it render silent.
300#[must_use]
301pub fn silence_reasons(state: &PresetState) -> Vec<String> {
302    match state.synth() {
303        Synth::Vital => saved_preset(state.bytes())
304            .and_then(|root| root.get("settings").and_then(Value::as_object).cloned())
305            .map(|settings| vital_silence_reasons(&settings))
306            .unwrap_or_default(),
307        Synth::Serum2 => Vec::new(),
308    }
309}
310
311fn vital_silence_reasons(settings: &Map<String, Value>) -> Vec<String> {
312    let registry = registry_for(Synth::Vital).ok();
313    let value = |key: &str| {
314        settings
315            .get(key)
316            .and_then(Value::as_f64)
317            .or_else(|| registry.and_then(|r| r.get(key)).map(|spec| spec.default))
318    };
319    let mut reasons = Vec::new();
320    let sources = ["osc_1_on", "osc_2_on", "osc_3_on", "sample_on"];
321    if sources.iter().all(|key| value(key) == Some(0.0)) {
322        reasons.push("no oscillator and no sample is on".to_owned());
323    }
324    if value("volume") == Some(0.0) {
325        reasons.push("the master volume is at its minimum".to_owned());
326    }
327    reasons
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    #[test]
335    fn a_tempo_mismatch_says_the_preset_cannot_fix_it() {
336        let tempo = Mismatch {
337            path: TEMPO_PATH.to_owned(),
338            sent: 2.0.into(),
339            saved: Some(2.5.into()),
340        };
341        let message = LoadFault::Readback(vec![tempo]).to_string();
342        assert!(
343            message.contains("from the host tempo, --bpm / 60 in beats per second"),
344            "{message}"
345        );
346        assert!(message.contains("cannot fix that value"), "{message}");
347        let other = Mismatch {
348            path: "settings.volume".to_owned(),
349            sent: 1.0.into(),
350            saved: None,
351        };
352        let message = LoadFault::Readback(vec![other]).to_string();
353        assert!(!message.contains("host tempo"), "{message}");
354    }
355
356    fn tone(hz: f64, gain: f64) -> Audio {
357        let rate = SampleRate::new(44_100).unwrap();
358        let samples: Vec<f64> = (0..44_100)
359            .map(|n| gain * (std::f64::consts::TAU * hz * cast::usize_to_f64(n) / 44_100.0).sin())
360            .collect();
361        Audio::new(rate, vec![samples.clone(), samples]).unwrap()
362    }
363
364    #[test]
365    fn the_bands_run_in_octaves_up_to_nyquist() {
366        let edges = band_edges(SampleRate::new(44_100).unwrap());
367        assert_eq!(edges.first(), Some(&44.0));
368        assert_eq!(edges.last(), Some(&22_050.0));
369        assert_eq!(edges.len(), 10);
370    }
371
372    #[test]
373    fn a_level_change_shows_in_its_band() {
374        let a = BandLevels::of(&tone(440.0, 0.5));
375        let b = BandLevels::of(&tone(440.0, 0.25));
376        assert!((a.distance(&b) - 6.02).abs() < 0.05, "{}", a.distance(&b));
377        assert!(a.distance(&a) < 1e-9);
378        let c = BandLevels::of(&tone(3_000.0, 0.5));
379        assert!(a.distance(&c) > 40.0, "{}", a.distance(&c));
380    }
381
382    /// A sine of amplitude A at a bin centre puts A N / 4 into its bin and
383    /// A N / 8 into each neighbour under a Hann window of N samples, so its
384    /// band reads the mean bin power `3 A² / 32 / bins`, not its level in
385    /// dBFS. Ten seconds of tone keep the zero-padded edge frames below
386    /// 0.1 dB of the mean.
387    #[test]
388    fn a_band_level_is_the_mean_bin_power_of_the_band() {
389        let rate = 44_100.0;
390        let bin_hz = rate / cast::usize_to_f64(FRAME);
391        let full_scale = |bin: usize| {
392            let hz = cast::usize_to_f64(bin) * bin_hz;
393            let samples: Vec<f64> = (0..441_000)
394                .map(|n| (std::f64::consts::TAU * hz * cast::usize_to_f64(n) / rate).sin())
395                .collect();
396            let audio = Audio::new(
397                SampleRate::new(44_100).unwrap(),
398                vec![samples.clone(), samples],
399            );
400            BandLevels::of(&audio.unwrap()).0
401        };
402        let edges = band_edges(SampleRate::new(44_100).unwrap());
403        let bins_in = |band: usize| {
404            (0..=FRAME / 2)
405                .filter(|bin| {
406                    (edges[band]..edges[band + 1]).contains(&(cast::usize_to_f64(*bin) * bin_hz))
407                })
408                .count()
409        };
410        let expected =
411            |band: usize| 10.0 * (3.0 / 32.0 / cast::usize_to_f64(bins_in(band))).log10();
412        let top = edges.len() - 2;
413        // Bin 1500 lies at 16.15 kHz, bin 6 at 64.6 Hz.
414        let high = full_scale(1500)[top];
415        let low = full_scale(6)[0];
416        assert!(
417            (high - expected(top)).abs() < 0.1,
418            "{high} {}",
419            expected(top)
420        );
421        assert!((low - expected(0)).abs() < 0.1, "{low} {}", expected(0));
422        // A full-scale sine reads about -40 dB in the top band at 44.1 kHz,
423        // so the floor of -90 dB passes a sine of about -50 dBFS there.
424        assert!((high + 40.3).abs() < 0.1, "{high}");
425        assert!((low + 16.3).abs() < 0.1, "{low}");
426    }
427
428    #[test]
429    fn leakage_far_below_the_loudest_band_does_not_count() {
430        let a = BandLevels(vec![-80.0, -40.0, -55.0]);
431        let b = BandLevels(vec![-72.0, -40.0, -56.0]);
432        assert!((a.distance(&b) - 1.0).abs() < 1e-9, "{}", a.distance(&b));
433    }
434
435    #[test]
436    fn silence_is_explained_by_the_controls() {
437        let state = |settings: Value| {
438            let root = serde_json::json!({ "synth_version": "1.0.6", "settings": settings });
439            PresetState::new(Synth::Vital, serde_json::to_vec(&root).unwrap())
440        };
441        let off = state(serde_json::json!({ "osc_1_on": 0.0, "volume": 0.0 }));
442        assert_eq!(
443            silence_reasons(&off),
444            [
445                "no oscillator and no sample is on",
446                "the master volume is at its minimum"
447            ]
448        );
449        // Oscillator 1 is on by default.
450        assert_eq!(
451            silence_reasons(&state(serde_json::json!({}))),
452            Vec::<String>::new()
453        );
454    }
455}
sound_to_patch/render/check.rs

sound_to_patch/render/
check.rs
