1use crate::defaults::{FloatType, IntegerType};
21use crate::duration::Duration;
22use crate::error::{Error, Result};
23use crate::metadata::Metadata;
24use crate::note::Note;
25use crate::pitch::Pitch;
26use crate::rest::Rest;
27use crate::scale::{Scale, ScaleType};
28use crate::stream::{Stream, StreamElement, StreamKind};
29
30pub const AUDIO_CHUNK_LENGTH: usize = 1024;
33
34pub const RECORD_SAMPLE_RATE: FloatType = 44100.0;
37
38const REST_FREQUENCY: FloatType = 10.0;
40
41fn audio_error(message: impl Into<String>) -> Error {
42 Error::Analysis(message.into())
43}
44
45fn default_scale(scale: Option<&Scale>) -> Result<Scale> {
46 match scale {
47 Some(scale) => Ok(scale.clone()),
48 None => Ok(Scale::new(ScaleType::Major, Pitch::from_name("C4")?)),
49 }
50}
51
52pub fn histogram(data: &[FloatType], bins: usize) -> Result<(Vec<usize>, Vec<FloatType>)> {
69 let most = data
70 .iter()
71 .copied()
72 .reduce(FloatType::max)
73 .ok_or_else(|| audio_error("max() arg is an empty sequence"))?;
74 let least = data.iter().copied().fold(most, FloatType::min);
75 let width = (most - least) / bins as FloatType;
76 let mut counts = vec![0; bins];
77 for value in data {
78 let mut count = 1;
79 while *value > least + count as FloatType * width {
80 count += 1;
81 }
82 *counts
83 .get_mut(count - 1)
84 .ok_or_else(|| audio_error("list index out of range"))? += 1;
85 }
86 let edges = (0..=bins)
87 .map(|index| least + index as FloatType * width)
88 .collect();
89 Ok((counts, edges))
90}
91
92pub fn interpolation(correlation: &[FloatType], peak: usize) -> FloatType {
96 if peak == 0 || peak + 1 == correlation.len() {
97 return peak as FloatType;
98 }
99 let (before, at, after) = (
100 correlation[peak - 1],
101 correlation[peak],
102 correlation[peak + 1],
103 );
104 (before - after) / (before - 2.0 * at + after) * 0.5 + peak as FloatType
105}
106
107pub fn autocorrelation_frequency(
135 signal: &[FloatType],
136 sample_rate: FloatType,
137) -> Result<FloatType> {
138 if signal.is_empty() {
139 return Err(audio_error("v cannot be empty"));
140 }
141 let length = signal.len();
142 let correlation: Vec<FloatType> = (0..length)
143 .map(|lag| {
144 signal[..length - lag]
145 .iter()
146 .zip(&signal[lag..])
147 .map(|(a, b)| a * b)
148 .sum()
149 })
150 .collect();
151 let Some(beginning) = correlation
152 .windows(2)
153 .position(|pair| pair[1] - pair[0] > 0.0)
154 else {
155 return Ok(REST_FREQUENCY);
156 };
157 let peak = correlation[beginning..]
159 .iter()
160 .enumerate()
161 .fold((0, FloatType::NEG_INFINITY), |best, (index, value)| {
162 if *value > best.1 {
163 (index, *value)
164 } else {
165 best
166 }
167 })
168 .0
169 + beginning;
170 Ok(sample_rate / interpolation(&correlation, peak))
171}
172
173pub fn prepare_thresholds(scale: Option<&Scale>) -> Result<(Vec<FloatType>, Vec<Pitch>)> {
181 let scale = match scale {
182 Some(scale) => scale.clone(),
183 None => Scale::new(ScaleType::Chromatic, Pitch::from_name("C4")?),
184 };
185 let pitches = scale.pitches()?;
186 let mut remainders: Vec<FloatType> = pitches
187 .iter()
188 .map(|pitch| pitch.frequency_hz().log2().fract())
189 .collect();
190 if let Some(last) = remainders.last_mut() {
191 *last += 1.0;
192 }
193 let thresholds = remainders
194 .windows(2)
195 .map(|pair| (pair[0] + pair[1]) / 2.0)
196 .collect();
197 Ok((thresholds, pitches))
198}
199
200pub fn normalize_input_frequency(
209 frequency: FloatType,
210 thresholds: &[FloatType],
211 pitches: &[Pitch],
212) -> Result<(FloatType, Pitch)> {
213 let log = frequency.log2();
214 if !log.is_finite() {
215 return Err(audio_error(format!(
216 "cannot place a frequency of {frequency}"
217 )));
218 }
219 let (remainder, octave) = (log.fract(), log.trunc() as IntegerType);
220 let (index, shift) = match thresholds
221 .iter()
222 .position(|threshold| remainder < *threshold)
223 {
224 Some(index) => (index, 4),
225 None => (pitches.len().wrapping_sub(1), 3),
226 };
227 let scale_pitch = pitches
228 .get(index)
229 .ok_or_else(|| audio_error("list index out of range"))?;
230 let mut placed = scale_pitch.clone();
231 placed.set_octave(Some(octave - shift));
232 Ok((scale_pitch.frequency_hz(), placed))
233}
234
235pub fn detect_pitch_frequencies(
243 frequencies: &[FloatType],
244 scale: Option<&Scale>,
245) -> Result<Vec<FloatType>> {
246 let (thresholds, pitches) = prepare_thresholds(Some(&default_scale(scale)?))?;
247 frequencies
248 .iter()
249 .map(|frequency| {
250 Ok(
251 normalize_input_frequency(*frequency, &thresholds, &pitches)?
252 .1
253 .frequency_hz(),
254 )
255 })
256 .collect()
257}
258
259pub fn smooth_frequencies(frequencies: &[FloatType], smooth_levels: usize) -> Result<Vec<i64>> {
278 if smooth_levels < 1 {
279 return Err(audio_error("smoothLevels must be >= 1"));
280 }
281 let count = frequencies.len();
282 if smooth_levels > count {
283 return Err(audio_error(format!(
284 "There cannot be more smoothLevels ({smooth_levels}) than inputPitches ({count})"
285 )));
286 }
287 let mut smoothed = frequencies.to_vec();
288 let levels = smooth_levels as FloatType;
289 let beginning = smoothed[..smooth_levels].iter().sum::<FloatType>() / levels;
290 let ends = (0..smooth_levels)
291 .map(|i| smoothed[count - 1 - i])
292 .sum::<FloatType>()
293 / levels;
294 let half_down = smooth_levels / 2;
295 let half_up = smooth_levels.div_ceil(2);
296 for i in 0..count {
297 smoothed[i] = if i < half_down {
298 beginning.trunc()
299 } else if (i as i64) > count as i64 - half_up as i64 - 1 {
300 ends.trunc()
301 } else {
302 (0..smooth_levels)
303 .map(|j| smoothed[i + j - half_down])
304 .sum::<FloatType>()
305 / levels
306 };
307 }
308 Ok(smoothed
309 .into_iter()
310 .map(|value| value.round_ties_even() as i64)
311 .collect())
312}
313
314pub fn pitch_frequencies_to_objects(
327 frequencies: &[FloatType],
328 scale: Option<&Scale>,
329) -> Result<(Vec<Pitch>, Vec<FloatType>)> {
330 let (thresholds, pitches) = prepare_thresholds(Some(&default_scale(scale)?))?;
331 let mut placed: Vec<Pitch> = frequencies
332 .iter()
333 .map(|frequency| Ok(normalize_input_frequency(*frequency, &thresholds, &pitches)?.1))
334 .collect::<Result<_>>()?;
335 let count = placed.len();
336 let mut plotted = Vec::new();
337 let mut i = 0;
338 while i + 1 < count {
339 let name = placed[i].name();
340 let hold = i;
341 let mut total: IntegerType = 0;
342 while i + 1 < count && placed[i].name() == name {
343 total += placed[i].octave().unwrap_or(4);
344 i += 1;
345 }
346 let octave =
347 (total as FloatType / (i - hold) as FloatType).round_ties_even() as IntegerType;
348 for j in 0..i - hold {
349 let index = (hold + j + count - 1) % count;
351 placed[index].set_octave(Some(octave));
352 plotted.push(placed[index].frequency_hz());
353 }
354 }
355 Ok((placed, plotted))
356}
357
358fn pitch_at_frequency(hertz: FloatType) -> Result<Pitch> {
360 let space = 12.0 * ((hertz / 440.0).ln() / (2.0 as FloatType).ln()) + 69.0;
361 Pitch::from_pitch_space((space * 1e6).round_ties_even() / 1e6)
362}
363
364pub fn join_consecutive_identical_pitches(
374 mut pitches: Vec<Pitch>,
375) -> Result<(Vec<StreamElement>, Vec<usize>)> {
376 let first = pitches
377 .first_mut()
378 .ok_or_else(|| audio_error("list index out of range"))?;
379 *first = pitch_at_frequency(REST_FREQUENCY)?;
380 let mut notes: Vec<StreamElement> = Vec::new();
381 let mut lengths = Vec::new();
382 let (mut good, mut bad) = (0, 0);
383 let mut j = 0;
384 while j < pitches.len() {
385 let frequency = pitches[j].frequency_hz();
386 let mut valid = false;
387 while j < pitches.len() && pitches[j].frequency_hz() == frequency {
388 good += 1;
389 if good >= 6 {
390 valid = true;
391 if bad >= 15 {
392 lengths.push(bad);
393 notes.push(Rest::new(Duration::quarter()).into());
394 }
395 bad = 0;
396 }
397 j += 1;
398 }
399 if valid {
400 lengths.push(good);
401 notes.push(Note::from_pitch(pitches[j - 1].clone()).into());
402 } else {
403 bad += good;
404 }
405 good = 0;
406 j += 1;
407 }
408 Ok((notes, lengths))
409}
410
411pub fn quantize_duration(length: FloatType) -> FloatType {
414 let length = length * 100.0;
415 let typical = [25.0, 50.0, 100.0, 150.0, 200.0, 400.0];
416 let mut chosen = typical[0];
417 for (index, pair) in typical.windows(2).enumerate() {
418 if length > (pair[0] + pair[1]) / 2.0 {
419 chosen = typical[index + 1];
420 }
421 }
422 chosen / 100.0
423}
424
425pub fn quarter_length_estimation(
435 lengths: &[usize],
436 most_repeated_quarter_length: FloatType,
437) -> Result<FloatType> {
438 let mut data: Vec<FloatType> = lengths.iter().map(|length| *length as FloatType).collect();
439 data.push(0.0);
440 let (counts, edges) = histogram(&data, 8)?;
441 let fullest = counts.iter().copied().max().unwrap_or(0);
442 let index = counts
443 .iter()
444 .rposition(|count| *count == fullest)
445 .unwrap_or(0);
446 let estimate = (edges[index] + edges[index + 1]) / 2.0;
447 let most_repeated = if most_repeated_quarter_length == 0.0 {
448 1.0
449 } else {
450 most_repeated_quarter_length
451 };
452 Ok(estimate * (2.0 as FloatType).powf(0.0 - most_repeated.log2()))
453}
454
455pub fn notes_and_durations_to_stream(
467 notes: &[StreamElement],
468 lengths: &[usize],
469 score_notes: Option<&Stream>,
470 remove_rests_at_beginning: bool,
471) -> Result<(Stream, FloatType)> {
472 let most_common = match score_notes {
473 Some(score) => crate::features::vector_by_id(score, "QL2")?
474 .and_then(|vector| vector.first().map(|value| value.as_float()))
475 .unwrap_or(1.0),
476 None => 1.0,
477 };
478 let estimate = quarter_length_estimation(lengths, most_common)?;
479 let mut part = Stream::with_kind(StreamKind::Part);
480 let mut skipping = remove_rests_at_beginning;
481 for (index, length) in lengths.iter().enumerate() {
482 let mut element = notes
483 .get(index)
484 .cloned()
485 .ok_or_else(|| audio_error("list index out of range"))?;
486 let duration = Duration::new(quantize_duration(*length as FloatType / estimate))?;
487 let rest = match &mut element {
488 StreamElement::Note(note) => {
489 note.set_duration(duration);
490 false
491 }
492 StreamElement::Rest(rest) => {
493 rest.set_duration(duration);
494 true
495 }
496 _ => false,
497 };
498 if !(skipping && rest) {
499 part.push(element);
500 skipping = false;
501 }
502 }
503 let mut score = Stream::with_kind(StreamKind::Score);
504 let mut metadata = Metadata::default();
505 metadata.add_text("title", "Automatic Music21 Transcription");
506 score.set_metadata(Some(metadata));
507 score.insert(0.0, part);
508 Ok((score, estimate))
509}
510
511pub fn decision_process(
525 parts: &[(usize, FloatType)],
526 note_prediction: FloatType,
527 beginning_data: &[FloatType],
528 last_note_position: FloatType,
529 countdown: usize,
530 page: Option<(FloatType, FloatType)>,
531) -> Result<(usize, usize)> {
532 let begins = |part: usize| -> Result<FloatType> {
533 let id = parts
534 .get(part)
535 .ok_or_else(|| audio_error("list index out of range"))?
536 .0;
537 beginning_data
538 .get(id)
539 .copied()
540 .ok_or_else(|| audio_error("list index out of range"))
541 };
542 let mut position = 0;
543 let mut i = 0;
544 while i < parts.len() && begins(i)? < note_prediction {
545 i += 1;
546 position = i;
547 }
548 if parts.len() == 1 {
549 position = 0;
550 }
551 let first = beginning_data
552 .first()
553 .copied()
554 .ok_or_else(|| audio_error("list index out of range"))?;
555 let mut distance = (first - note_prediction).abs();
556 let best = parts
557 .first()
558 .ok_or_else(|| audio_error("list index out of range"))?
559 .1;
560 for (index, (_, probability)) in parts.iter().enumerate() {
561 let start = begins(index)?;
562 if *probability >= 0.9 * best
563 && start > last_note_position
564 && (start - note_prediction).abs() < distance
565 {
566 distance = (start - note_prediction).abs();
567 position = index;
568 }
569 }
570 let start = begins(position)?;
571 let probability = parts[position].1;
572 let doubtful = probability < 0.6
573 || parts.len() == 1
574 || (distance > 20.0 && countdown == 0)
575 || (distance > 30.0 && countdown == 1)
576 || page.is_some_and(|(first, last)| (start < first || start > last) && countdown < 2);
577 Ok((position, if doubtful { countdown + 1 } else { 0 }))
578}
579
580fn wav_samples(bytes: &[u8]) -> Result<(Vec<u8>, usize, usize)> {
583 let word = |at: usize| -> Result<u32> {
584 bytes
585 .get(at..at + 4)
586 .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]))
587 .ok_or_else(|| audio_error("file does not start with RIFF id"))
588 };
589 if bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WAVE") {
590 return Err(audio_error("file does not start with RIFF id"));
591 }
592 let mut at = 12;
593 let mut frame_size = None;
594 while at + 8 <= bytes.len() {
595 let id = &bytes[at..at + 4];
596 let size = word(at + 4)? as usize;
597 let body = at + 8;
598 let end = (body + size).min(bytes.len());
599 match id {
600 b"fmt " => {
601 let field = |offset: usize| -> Result<u16> {
602 bytes
603 .get(body + offset..body + offset + 2)
604 .map(|b| u16::from_le_bytes([b[0], b[1]]))
605 .ok_or_else(|| audio_error("fmt chunk too short"))
606 };
607 let format = field(0)?;
608 if format != 1 && format != 0xFFFE {
609 return Err(audio_error(format!("unknown format: {format}")));
610 }
611 let channels = usize::from(field(2)?);
612 let width = usize::from(field(14)?).div_ceil(8);
613 if channels == 0 || width == 0 {
614 return Err(audio_error("bad # of channels or sample width"));
615 }
616 frame_size = Some(channels * width);
617 }
618 b"data" => {
619 let frame_size =
620 frame_size.ok_or_else(|| audio_error("data chunk before fmt chunk"))?;
621 let data = bytes[body..end].to_vec();
622 let frames = size / frame_size;
623 return Ok((data, frames, frame_size));
624 }
625 _ => {}
626 }
627 at = body + size + (size & 1);
628 }
629 Err(audio_error("fmt chunk and/or data chunk missing"))
630}
631
632pub fn frequencies_from_wav(bytes: &[u8]) -> Result<Vec<FloatType>> {
642 let (data, frames, frame_size) = wav_samples(bytes)?;
643 let chunk_bytes = AUDIO_CHUNK_LENGTH * frame_size;
644 (0..frames / AUDIO_CHUNK_LENGTH)
645 .map(|chunk| {
646 let samples: Vec<FloatType> = data[chunk * chunk_bytes..(chunk + 1) * chunk_bytes]
647 .as_chunks::<2>()
648 .0
649 .iter()
650 .map(|pair| FloatType::from(i16::from_le_bytes(*pair)))
651 .collect();
652 autocorrelation_frequency(&samples, RECORD_SAMPLE_RATE)
653 })
654 .collect()
655}
656
657pub fn monophonic_stream_from_wav(bytes: &[u8], scale: Option<&Scale>) -> Result<Stream> {
670 let frequencies = frequencies_from_wav(bytes)?;
671 let detected = detect_pitch_frequencies(&frequencies, scale)?;
672 let smoothed: Vec<FloatType> = smooth_frequencies(&detected, 7)?
673 .into_iter()
674 .map(|value| value as FloatType)
675 .collect();
676 let (pitches, _) = pitch_frequencies_to_objects(&smoothed, scale)?;
677 let (notes, lengths) = join_consecutive_identical_pitches(pitches)?;
678 let (score, _) = notes_and_durations_to_stream(¬es, &lengths, None, true)?;
679 score
680 .parts()
681 .first()
682 .map(|part| (*part).clone())
683 .ok_or_else(|| audio_error("no part"))
684}