music21/features/
language.rs

1//! Which language a text is in, by the letters that follow each pair of
2//! letters: music21's `text.LanguageDetector` and `text.Trigram`.
3//!
4//! Each language is known by a long text in it, the excerpts music21
5//! carries from Project Gutenberg (in `src/features/language/`, as music21
6//! has them). A text is compared with each by how alike their counts of
7//! letter-after-two-letters are, and is in the language it is most like.
8//!
9//! A [`Trigram`] needs none of that and is always here; the excerpts, and
10//! what reads a text against them, come with the `language-detection`
11//! feature.
12
13use std::collections::HashMap;
14#[cfg(feature = "language-detection")]
15use std::sync::OnceLock;
16
17use crate::defaults::FloatType;
18#[cfg(feature = "language-detection")]
19use crate::error::{Error, Result};
20
21/// The languages known, in music21's order: English, French, Italian,
22/// German, Chinese, Latin and Dutch.
23pub const LANGUAGE_CODES: [&str; 7] = ["en", "fr", "it", "de", "cn", "la", "nl"];
24
25/// The name of each language music21 knows, by its code.
26pub fn language_name(code: &str) -> Option<&'static str> {
27    Some(match code {
28        "en" => "English",
29        "fr" => "French",
30        "it" => "Italian",
31        "de" => "German",
32        "cn" => "Chinese",
33        "la" => "Latin",
34        "nl" => "Dutch",
35        _ => return None,
36    })
37}
38
39#[cfg(feature = "language-detection")]
40fn excerpt(code: &str) -> &'static str {
41    match code {
42        "en" => include_str!("language/en.txt"),
43        "fr" => include_str!("language/fr.txt"),
44        "it" => include_str!("language/it.txt"),
45        "de" => include_str!("language/de.txt"),
46        "cn" => include_str!("language/cn.txt"),
47        "la" => include_str!("language/la.txt"),
48        _ => include_str!("language/nl.txt"),
49    }
50}
51
52/// How often each letter follows each pair of letters in a text: music21's
53/// `Trigram`.
54#[derive(Clone, Debug, Default, PartialEq)]
55pub struct Trigram {
56    counts: HashMap<(char, char), HashMap<char, u64>>,
57    length: FloatType,
58}
59
60impl Trigram {
61    /// The trigrams of a text, letter by letter, starting after two spaces.
62    pub fn from_text(text: &str) -> Self {
63        Self::counted(text.chars())
64    }
65
66    /// The trigrams of a list of words, each stripped and followed by a
67    /// space: how music21 reads its excerpts.
68    pub fn from_words<'a>(words: impl IntoIterator<Item = &'a str>) -> Self {
69        Self::counted(
70            words
71                .into_iter()
72                .flat_map(|word| word.trim().chars().chain(std::iter::once(' '))),
73        )
74    }
75
76    fn counted(letters: impl Iterator<Item = char>) -> Self {
77        let mut trigram = Self::default();
78        trigram.count(letters);
79        trigram
80    }
81
82    /// Counts a text's trigrams into these, starting after two spaces
83    /// again: music21's `parseExcerpt` on a string.
84    pub fn add_text(&mut self, text: &str) {
85        self.count(text.chars());
86    }
87
88    /// Counts a list of words into these, each stripped and followed by a
89    /// space: music21's `parseExcerpt` on a list.
90    pub fn add_words<'a>(&mut self, words: impl IntoIterator<Item = &'a str>) {
91        self.count(
92            words
93                .into_iter()
94                .flat_map(|word| word.trim().chars().chain(std::iter::once(' '))),
95        );
96    }
97
98    fn count(&mut self, letters: impl Iterator<Item = char>) {
99        let mut pair = (' ', ' ');
100        for letter in letters {
101            *self
102                .counts
103                .entry(pair)
104                .or_default()
105                .entry(letter)
106                .or_insert(0) += 1;
107            pair = (pair.1, letter);
108        }
109        let total: u64 = self
110            .counts
111            .values()
112            .flat_map(HashMap::values)
113            .map(|count| count * count)
114            .sum();
115        // Python's `total ** 0.5`.
116        self.length = (total as FloatType).powf(0.5);
117    }
118
119    /// The square root of the sum of every count squared: music21's
120    /// `length`.
121    pub fn length(&self) -> FloatType {
122        self.length
123    }
124
125    /// How alike two texts' trigrams are, from nought to one: the cosine of
126    /// their counts, music21's `similarity`.
127    pub fn similarity(&self, other: &Trigram) -> FloatType {
128        let mut total: u64 = 0;
129        for (pair, ours) in &self.counts {
130            if let Some(theirs) = other.counts.get(pair) {
131                for (letter, count) in ours {
132                    if let Some(their) = theirs.get(letter) {
133                        total += count * their;
134                    }
135                }
136            }
137        }
138        total as FloatType / (self.length * other.length)
139    }
140
141    /// One less [`Self::similarity`]: music21's `-`.
142    pub fn distance(&self, other: &Trigram) -> FloatType {
143        1.0 - self.similarity(other)
144    }
145}
146
147/// The trigrams of each language's excerpt, read once.
148#[cfg(feature = "language-detection")]
149fn known() -> &'static [Trigram; 7] {
150    static KNOWN: OnceLock<[Trigram; 7]> = OnceLock::new();
151    KNOWN.get_or_init(|| {
152        LANGUAGE_CODES.map(|code| Trigram::from_words(excerpt(code).split_whitespace()))
153    })
154}
155
156/// The trigrams of a language's excerpt, by its code: what music21's
157/// `LanguageDetector.trigrams` holds.
158///
159/// ```
160/// use music21::features::language::known_trigram;
161///
162/// let french = known_trigram("fr").expect("music21 knows French");
163/// let italian = known_trigram("it").expect("music21 knows Italian");
164/// assert!((0.50..0.55).contains(&french.distance(italian)));
165/// assert!(known_trigram("xx").is_none());
166/// ```
167#[cfg(feature = "language-detection")]
168pub fn known_trigram(code: &str) -> Option<&'static Trigram> {
169    let at = LANGUAGE_CODES.iter().position(|known| *known == code)?;
170    Some(&known()[at])
171}
172
173/// The language a text is most like, by its code, or nothing for an empty
174/// text or one like none of them: music21's
175/// `LanguageDetector.mostLikelyLanguage`. Where two are as alike, the first
176/// of [`LANGUAGE_CODES`] wins.
177///
178/// ```
179/// use music21::features::language::most_likely_language;
180///
181/// let text = "Ich habe ein Buch, und ich gehe nach Hause mit meinem Bruder.";
182/// assert_eq!(most_likely_language(text), Some("de"));
183/// assert_eq!(most_likely_language(""), None);
184/// ```
185#[cfg(feature = "language-detection")]
186pub fn most_likely_language(text: &str) -> Option<&'static str> {
187    if text.is_empty() {
188        return None;
189    }
190    let trigram = Trigram::from_text(text);
191    let mut best = None;
192    let mut least = 1.0;
193    for (code, known) in LANGUAGE_CODES.iter().zip(known()) {
194        let distance = known.distance(&trigram);
195        if distance < least {
196            best = Some(*code);
197            least = distance;
198        }
199    }
200    best
201}
202
203/// The language a text is most like, as one more than its place in
204/// [`LANGUAGE_CODES`], or nought for an empty text: music21's
205/// `mostLikelyLanguageNumeric`.
206///
207/// # Errors
208///
209/// A text like none of the languages, which music21 refuses.
210#[cfg(feature = "language-detection")]
211pub fn most_likely_language_numeric(text: &str) -> Result<usize> {
212    if text.is_empty() {
213        return Ok(0);
214    }
215    let code = most_likely_language(text).ok_or_else(|| {
216        Error::Text("got a language that was not in the codes; should not happen".to_string())
217    })?;
218    Ok(LANGUAGE_CODES
219        .iter()
220        .position(|known| *known == code)
221        .map_or(0, |at| at + 1))
222}
223
224#[cfg(test)]
225mod tests {
226    use super::Trigram;
227
228    /// music21's `parseExcerpt` counts into what is there, starting after
229    /// two spaces again.
230    #[test]
231    fn a_text_counted_twice_counts_double() {
232        let once = Trigram::from_text("abc");
233        let mut twice = Trigram::from_text("abc");
234        twice.add_text("abc");
235        assert_eq!(twice.length(), 2.0 * once.length());
236        assert!((twice.similarity(&once) - 1.0).abs() < 1e-12);
237
238        let mut words = Trigram::default();
239        words.add_words([" ab ", "c"]);
240        assert_eq!(words, Trigram::from_words(["ab", "c"]));
241    }
242}
music21/features/language.rs

music21/features/
language.rs
