sound_to_patch/
render.rs

1//! Rendering a preset and its notes to audio through a synth.
2//!
3//! A [`Renderer`] plays a [`RenderJob`]: the plug-in state of a preset, the
4//! notes and the [`Recipe`]. It returns the audio and a [`RenderRecord`]
5//! that names everything the audio depends on, so a later comparison can
6//! cite it. Calibration, compare, refine, the bench and the dataset all
7//! render through this trait and add no recipe field of their own.
8//!
9//! Renders of one preset are not bit-identical: a synth starts its unison
10//! voices at random phases. Compare two renders within the repeat-render
11//! spread, never sample by sample.
12//!
13//! [`fake::FakeRenderer`] is a seeded additive synth, so the render code runs
14//! in CI with no plug-in. [`plugin::PluginRenderer`] renders through a real
15//! plug-in in the `sound-to-patch-host` process, with the state that
16//! [`state::preset_state`] builds.
17
18use std::error::Error;
19use std::fmt;
20use std::path::Path;
21
22use serde::{Deserialize, Serialize};
23
24use crate::audio::AudioError;
25use crate::cast;
26use crate::notes::{MidiError, NoteList, Velocity, read_midi_bytes};
27use crate::preset::format::FormatError;
28use crate::synth::{Synth, SynthError};
29use crate::units::{Bpm, Seconds};
30
31pub mod check;
32pub mod fake;
33pub mod file;
34pub mod plugin;
35pub mod session;
36pub mod spread;
37pub mod state;
38pub mod timing;
39pub mod vita;
40
41#[doc(inline)]
42pub use sound_to_patch_core::render::{
43    ClipEnd, ControlKind, EventKind, MAX_CONTROLLER, MAX_KEY, MAX_NOTE_END, MAX_PITCH_BEND,
44    Material, NoteEvent, Oversampling, PITCH_BEND_CENTRE, PlayError, PreRoll, ProcessMode,
45    RECIPE_VERSION, Recipe, RenderNote, TailWatch, frame_of, note_events, recipe, wire,
46};
47
48/// How far a pitch from a note list may lie from a whole key. The MIDI
49/// reader gives whole keys; a transcribed pitch with a bend is no key.
50const KEY_TOLERANCE: f64 = 1e-9;
51
52/// The render notes of a note list, such as the one the MIDI reader
53/// gives. A note without a velocity takes `default`.
54///
55/// # Errors
56///
57/// Returns a [`RenderError`] for a pitch that is not a whole key.
58pub fn notes_of_list(list: &NoteList, default: Velocity) -> Result<Vec<RenderNote>, RenderError> {
59    list.notes()
60        .iter()
61        .map(|note| {
62            let pitch = note.pitch().get();
63            // A MIDI pitch lies in 0 to 127, so only a fractional pitch
64            // fails here.
65            let key = cast::round_to_usize(pitch)
66                .and_then(|key| u8::try_from(key).ok())
67                .filter(|key| (pitch - f64::from(*key)).abs() <= KEY_TOLERANCE)
68                .ok_or_else(|| {
69                    RenderError::Play(PlayError::Note(format!(
70                        "pitch {pitch} at {} is not a whole MIDI key, so the list carries a transcribed or bent pitch; pass the notes of a MIDI file",
71                        note.onset()
72                    )))
73                })?;
74            Ok(RenderNote::new(
75                key,
76                note.onset(),
77                note.offset(),
78                note.velocity().unwrap_or(default),
79            )?)
80        })
81        .collect()
82}
83
84/// The SHA-256 of `bytes` in lower-case hex.
85pub(crate) fn sha256_hex(bytes: &[u8]) -> String {
86    use sha2::{Digest, Sha256};
87    use std::fmt::Write as _;
88    Sha256::digest(bytes)
89        .iter()
90        .fold(String::with_capacity(64), |mut hex, byte| {
91            let _ = write!(hex, "{byte:02x}");
92            hex
93        })
94}
95
96/// A controller change and its time.
97#[derive(Clone, Copy, Debug, PartialEq)]
98pub struct Control {
99    /// When the change happens, from the start of the clip.
100    pub time: Seconds,
101    /// The change.
102    pub kind: ControlKind,
103}
104
105/// The plug-in state of a preset: the bytes a synth loads as its state, and
106/// the synth they belong to. Each synth's render state adapter builds it.
107#[derive(Clone, Debug, PartialEq, Eq)]
108pub struct PresetState {
109    synth: Synth,
110    bytes: Vec<u8>,
111}
112
113impl PresetState {
114    /// Wraps the state bytes of `synth`.
115    #[must_use]
116    pub fn new(synth: Synth, bytes: Vec<u8>) -> Self {
117        Self { synth, bytes }
118    }
119
120    /// Returns the synth.
121    #[must_use]
122    pub fn synth(&self) -> Synth {
123        self.synth
124    }
125
126    /// Returns the state bytes.
127    #[must_use]
128    pub fn bytes(&self) -> &[u8] {
129        &self.bytes
130    }
131}
132
133/// One render: the preset state, the notes, the tempo and the recipe.
134#[derive(Clone, Debug, PartialEq)]
135pub struct RenderJob {
136    state: PresetState,
137    notes: Vec<RenderNote>,
138    controls: Vec<Control>,
139    bpm: Bpm,
140    recipe: Recipe,
141    midi_sha256: Option<String>,
142}
143
144impl RenderJob {
145    /// Builds a job with the current recipe.
146    ///
147    /// # Errors
148    ///
149    /// Returns [`RenderError::NoNotes`] for an empty note list.
150    pub fn new(state: PresetState, notes: Vec<RenderNote>, bpm: Bpm) -> Result<Self, RenderError> {
151        if notes.is_empty() {
152            return Err(RenderError::NoNotes);
153        }
154        Ok(Self {
155            state,
156            notes,
157            controls: Vec::new(),
158            bpm,
159            recipe: Recipe::current(),
160            midi_sha256: None,
161        })
162    }
163
164    /// Builds a job that plays the notes of a MIDI file. The file is read
165    /// as `analyze --notes` reads it, so every tempo in it must match
166    /// `bpm`. Notes without a velocity take the recipe's velocity. The
167    /// record carries the SHA-256 of the file.
168    ///
169    /// # Errors
170    ///
171    /// Returns [`RenderError::Midi`] for a file the MIDI reader refuses, and
172    /// the errors of [`notes_of_list`] and [`RenderJob::new`].
173    pub fn from_midi(state: PresetState, path: &Path, bpm: Bpm) -> Result<Self, RenderError> {
174        let (list, bytes) = read_midi_bytes(path, bpm).map_err(RenderError::Midi)?;
175        let notes = notes_of_list(&list, Recipe::current().velocity())?;
176        let mut job = Self::new(state, notes, bpm)?;
177        job.midi_sha256 = Some(sha256_hex(&bytes));
178        Ok(job)
179    }
180
181    /// Builds a job that plays the recipe's notes for `material`.
182    #[must_use]
183    pub fn for_material(state: PresetState, material: Material, bpm: Bpm) -> Self {
184        let recipe = Recipe::current();
185        Self {
186            state,
187            notes: recipe.notes(material),
188            controls: Vec::new(),
189            bpm,
190            recipe,
191            midi_sha256: None,
192        }
193    }
194
195    /// Returns the preset state.
196    #[must_use]
197    pub fn state(&self) -> &PresetState {
198        &self.state
199    }
200
201    /// Returns the notes.
202    #[must_use]
203    pub fn notes(&self) -> &[RenderNote] {
204        &self.notes
205    }
206
207    /// Returns the tempo, which synced LFOs and delays follow.
208    #[must_use]
209    pub fn bpm(&self) -> Bpm {
210        self.bpm
211    }
212
213    /// Returns the recipe.
214    #[must_use]
215    pub fn recipe(&self) -> &Recipe {
216        &self.recipe
217    }
218
219    /// Adds controller changes. The clip still ends by the notes and the
220    /// tail, so a change after the clip's end has no effect.
221    ///
222    /// # Errors
223    ///
224    /// Returns [`RenderError::Play`] for a value that does not fit its
225    /// MIDI message, as [`ControlKind::checked`] does.
226    pub fn with_controls(mut self, controls: Vec<Control>) -> Result<Self, RenderError> {
227        for control in &controls {
228            control.kind.checked()?;
229        }
230        self.controls = controls;
231        Ok(self)
232    }
233
234    /// Returns the controller changes.
235    #[must_use]
236    pub fn controls(&self) -> &[Control] {
237        &self.controls
238    }
239
240    /// Returns the SHA-256 of the MIDI file the notes came from, in hex.
241    #[must_use]
242    pub fn midi_sha256(&self) -> Option<&str> {
243        self.midi_sha256.as_deref()
244    }
245
246    /// Returns the end of the last note.
247    #[must_use]
248    pub fn last_note_off(&self) -> Seconds {
249        self.notes
250            .iter()
251            .map(|note| note.offset())
252            .fold(Seconds::from_const(0.0), |a, b| if b > a { b } else { a })
253    }
254}
255
256/// What renders: the synth and its build, the host and the architecture of
257/// the process that loads the plug-in.
258#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
259pub struct HostIdentity {
260    /// The synth.
261    pub synth: Synth,
262    /// The synth's build, such as `1.0.7`.
263    pub build: String,
264    /// The host and its version, such as `vst3-host 0.10.0`.
265    pub host: String,
266    /// The architecture of the host process, such as `x86_64`.
267    pub arch: String,
268}
269
270/// Everything a render depends on, so a consumer can cite it and analyze
271/// the audio with the same notes.
272#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
273pub struct RenderRecord {
274    /// What rendered.
275    pub identity: HostIdentity,
276    /// The recipe version.
277    pub recipe_version: u32,
278    /// The sample rate in samples per second.
279    pub sample_rate: u32,
280    /// The block size in samples.
281    pub block_size: u32,
282    /// How the host told the plug-in it processes.
283    pub process_mode: ProcessMode,
284    /// The settle time rendered in silence after the state load.
285    pub settle: Seconds,
286    /// The oversampling of a Vital render.
287    pub oversampling: Oversampling,
288    /// The tempo.
289    pub bpm: Bpm,
290    /// The end of the last note.
291    pub last_note_off: Seconds,
292    /// The clip length.
293    pub length: Seconds,
294    /// Whether the tail reached its cap before it went quiet.
295    pub tail_capped: bool,
296    /// The SHA-256 of the MIDI file the notes came from, in hex. Absent for
297    /// notes that came from no file, such as the recipe's notes.
298    #[serde(default, skip_serializing_if = "Option::is_none")]
299    pub midi_sha256: Option<String>,
300    /// The load check of the render, when it ran.
301    #[serde(default, skip_serializing_if = "Option::is_none")]
302    pub load_check: Option<check::LoadCheck>,
303}
304
305impl RenderRecord {
306    /// Fills the fields that come from the job and the recipe.
307    #[must_use]
308    pub fn new(identity: HostIdentity, job: &RenderJob, end: ClipEnd) -> Self {
309        let recipe = job.recipe();
310        let rate = recipe.sample_rate();
311        Self {
312            identity,
313            recipe_version: recipe.version(),
314            sample_rate: rate.get(),
315            block_size: recipe.block_size(),
316            process_mode: recipe.process_mode(),
317            settle: recipe.settle(),
318            oversampling: recipe.vital_oversampling(),
319            bpm: job.bpm(),
320            last_note_off: job.last_note_off(),
321            length: rate.seconds(end.frames),
322            tail_capped: end.capped,
323            midi_sha256: job.midi_sha256.clone(),
324            load_check: None,
325        }
326    }
327}
328
329/// The audio of a render and its record.
330#[derive(Clone, Debug, PartialEq)]
331pub struct Rendered {
332    /// Stereo audio at the recipe's rate.
333    pub audio: crate::audio::Audio,
334    /// What the audio depends on.
335    pub record: RenderRecord,
336}
337
338/// Plays a [`RenderJob`] through one synth and returns the audio.
339///
340/// A backend loads the state, plays the recipe's pre-roll (settle, warm-up
341/// note, settle) and throws it away, then renders the clip. The clip ends
342/// once the [`TailWatch`] says the tail is over.
343///
344/// Events go to the synth in the order of [`note_events`]. A note-on of a
345/// key that already sounds is sent as a second note-on, and each note-off
346/// of a key ends the oldest sounding note of that key, first in, first out,
347/// as the MIDI reader pairs them. How the synth voices a repeated key is
348/// the synth's own rule.
349pub trait Renderer {
350    /// Returns what renders.
351    fn identity(&self) -> HostIdentity;
352
353    /// Renders one job.
354    ///
355    /// # Errors
356    ///
357    /// Returns a [`RenderError`] when the state does not load, the plug-in
358    /// fails or the audio is not valid.
359    fn render(&mut self, job: &RenderJob) -> Result<Rendered, RenderError>;
360
361    /// Renders the job's notes with the plug-in's own init patch: the job's
362    /// state is not loaded. Only a renderer that has loaded no state yet
363    /// holds the init patch, so the load check calls this first.
364    ///
365    /// # Errors
366    ///
367    /// Returns [`RenderError::Host`] once a state has loaded, or
368    /// [`RenderError::Engine`] for the open engine, and the errors of
369    /// [`Renderer::render`].
370    fn render_init(&mut self, job: &RenderJob) -> Result<Rendered, RenderError>;
371
372    /// Returns the state the plug-in saves now.
373    ///
374    /// # Errors
375    ///
376    /// Returns a [`RenderError`] when the plug-in fails.
377    fn save_state(&mut self) -> Result<Vec<u8>, RenderError>;
378
379    /// Returns how long the state load of the last render took, when the
380    /// backend measures it. A render of the init patch loads no state.
381    fn last_state_load(&self) -> Option<Seconds> {
382        None
383    }
384}
385
386/// A render that failed.
387#[derive(Debug)]
388pub enum RenderError {
389    /// A note or a controller change cannot be played.
390    Play(PlayError),
391    /// The job holds no note.
392    NoNotes,
393    /// The synth did not take the state.
394    State(String),
395    /// The state belongs to another synth than the renderer's.
396    WrongSynth {
397        /// The synth of the state.
398        state: Synth,
399        /// The synth of the renderer.
400        renderer: Synth,
401    },
402    /// The process that hosts the plug-in stopped.
403    Crashed(String),
404    /// The process that hosts the plug-in stopped answering.
405    TimedOut {
406        /// How long the renderer waited.
407        after: Seconds,
408        /// The host's last output.
409        output: String,
410    },
411    /// The plug-in gave audio that is not valid.
412    Audio(AudioError),
413    /// The preset file is not a valid preset.
414    Preset(FormatError),
415    /// The synth has no renderer yet.
416    Synth(SynthError),
417    /// The MIDI file cannot be read.
418    Midi(MidiError),
419    /// The plug-in host cannot start, cannot load the plug-in or refused a
420    /// request.
421    Host {
422        /// Why.
423        cause: String,
424        /// How to fix it.
425        fix: String,
426    },
427    /// The open-engine script of `tools/vita` cannot start, failed or gave
428    /// a reply this program cannot read.
429    Engine {
430        /// Why.
431        cause: String,
432        /// How to fix it.
433        fix: String,
434    },
435    /// A fault of this program, not of the preset, the synth or the host:
436    /// a state it built that holds no preset, or a renderer that panicked.
437    Internal(String),
438}
439
440impl fmt::Display for RenderError {
441    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442        match self {
443            Self::Play(source) => write!(f, "{source}"),
444            Self::NoNotes => write!(
445                f,
446                "cannot render: the job has no notes. Pass a MIDI file with at least one note"
447            ),
448            Self::State(cause) => write!(
449                f,
450                "the synth did not load the preset: {cause}. Check that the preset loads in the synth"
451            ),
452            Self::WrongSynth { state, renderer } => write!(
453                f,
454                "the preset is for {state}, but the renderer runs {renderer}. Render it with --synth {state}"
455            ),
456            Self::Crashed(cause) => write!(
457                f,
458                "the plug-in host stopped: {cause}. Render the preset again; if it stops again, load the preset in the synth and report it with this message"
459            ),
460            Self::TimedOut { after, output } => write!(
461                f,
462                "the plug-in host stopped answering after {after}; its last output: {output}. Render the preset again; if it stops again, load the preset in the synth and report it with this message"
463            ),
464            Self::Audio(source) => write!(
465                f,
466                "the plug-in gave audio that is not valid: {source}. Report it with the preset"
467            ),
468            Self::Preset(source) => write!(f, "{source}"),
469            Self::Synth(source) => write!(f, "{source}"),
470            Self::Midi(source) => write!(f, "{source}"),
471            Self::Host { cause, fix } => {
472                write!(f, "cannot render through the plug-in host: {cause}. {fix}")
473            }
474            Self::Engine { cause, fix } => {
475                write!(f, "cannot render through the open engine: {cause}. {fix}")
476            }
477            Self::Internal(cause) => write!(
478                f,
479                "cannot render: {cause}. This is a fault of this program, not of the preset or the synth; report it with this message"
480            ),
481        }
482    }
483}
484
485impl Error for RenderError {
486    fn source(&self) -> Option<&(dyn Error + 'static)> {
487        match self {
488            Self::Play(source) => source.source(),
489            Self::Audio(source) => Some(source),
490            Self::Preset(source) => Some(source),
491            Self::Synth(source) => Some(source),
492            Self::Midi(source) => Some(source),
493            _ => None,
494        }
495    }
496}
497
498impl From<PlayError> for RenderError {
499    fn from(source: PlayError) -> Self {
500        Self::Play(source)
501    }
502}
503
504impl From<AudioError> for RenderError {
505    fn from(source: AudioError) -> Self {
506        Self::Audio(source)
507    }
508}
509
510#[cfg(test)]
511mod tests {
512    use super::*;
513    use crate::notes::Note;
514    use crate::units::MidiPitch;
515
516    #[test]
517    fn a_play_error_prints_the_same_text_as_a_render_error() {
518        for error in [
519            PlayError::Note("key 128".to_owned()),
520            PlayError::Control("bend 20000".to_owned()),
521        ] {
522            let text = error.to_string();
523            let render = RenderError::from(error);
524            assert_eq!(render.to_string(), text);
525            assert!(Error::source(&render).is_none());
526        }
527    }
528
529    fn s(value: f64) -> Seconds {
530        Seconds::new(value).unwrap()
531    }
532
533    fn v(value: u8) -> Velocity {
534        Velocity::new(value).unwrap()
535    }
536
537    #[test]
538    fn a_controller_value_must_fit_its_midi_message() {
539        let job = || {
540            RenderJob::new(
541                PresetState::new(Synth::Vital, Vec::new()),
542                vec![RenderNote::new(60, s(0.0), s(1.0), v(100)).unwrap()],
543                Bpm::new(120.0).unwrap(),
544            )
545            .unwrap()
546        };
547        let at = |kind| Control { time: s(0.5), kind };
548        let error = job()
549            .with_controls(vec![at(ControlKind::PitchBend(MAX_PITCH_BEND + 1))])
550            .unwrap_err();
551        assert!(error.to_string().contains("pass 0 to 16383"), "{error}");
552        let error = job()
553            .with_controls(vec![at(ControlKind::ModWheel(128))])
554            .unwrap_err();
555        assert!(error.to_string().contains("pass 0 to 127"), "{error}");
556        let edges = vec![
557            at(ControlKind::PitchBend(MAX_PITCH_BEND)),
558            at(ControlKind::ModWheel(MAX_CONTROLLER)),
559        ];
560        assert_eq!(job().with_controls(edges).unwrap().controls().len(), 2);
561    }
562
563    #[test]
564    fn the_wrong_synth_error_names_the_flag_to_pass() {
565        let error = RenderError::WrongSynth {
566            state: Synth::Vital,
567            renderer: Synth::Serum2,
568        };
569        assert!(error.to_string().ends_with("Render it with --synth vital"));
570    }
571
572    #[test]
573    fn a_note_list_converts_with_its_velocity_or_the_default() {
574        let pitch = |p: f64| MidiPitch::new(p).unwrap();
575        let list = NoteList::new(vec![
576            Note::new(s(0.0), s(1.0), pitch(36.0)).unwrap(),
577            Note::new(s(1.0), s(2.0), pitch(48.0))
578                .unwrap()
579                .with_velocity(v(64)),
580        ]);
581        let notes = notes_of_list(&list, v(100)).unwrap();
582        let keys: Vec<(u8, u8)> = notes
583            .iter()
584            .map(|n| (n.key(), n.velocity().get()))
585            .collect();
586        assert_eq!(keys, [(36, 100), (48, 64)]);
587        let bent = NoteList::new(vec![Note::new(s(0.0), s(1.0), pitch(36.4)).unwrap()]);
588        assert!(notes_of_list(&bent, v(100)).is_err());
589    }
590
591    #[test]
592    fn a_job_needs_a_note_and_knows_its_last_note_off() {
593        let state = PresetState::new(Synth::Vital, Vec::new());
594        let bpm = Bpm::new(120.0).unwrap();
595        assert!(matches!(
596            RenderJob::new(state.clone(), Vec::new(), bpm),
597            Err(RenderError::NoNotes)
598        ));
599        let job = RenderJob::for_material(state, Material::Pad, bpm);
600        assert_eq!(job.notes().len(), 3);
601        assert_eq!(job.last_note_off(), s(3.0));
602    }
603}
sound_to_patch/render.rs

sound_to_patch/
render.rs
