music21_rs/features/
language.rs

1//! Which language a text is in, by the letters that follow each pair of
2//! letters: music21's `text.LanguageDetector` and `text.Trigram`.
3//!
4//! Each language is known by a long text in it, the excerpts music21
5//! carries from Project Gutenberg (in `src/features/language/`, as music21
6//! has them). A text is compared with each by how alike their counts of
7//! letter-after-two-letters are, and is in the language it is most like.
8
9use std::collections::HashMap;
10use std::sync::OnceLock;
11
12use crate::defaults::FloatType;
13use crate::error::{Error, Result};
14
15/// The languages known, in music21's order: English, French, Italian,
16/// German, Chinese, Latin and Dutch.
17pub const LANGUAGE_CODES: [&str; 7] = ["en", "fr", "it", "de", "cn", "la", "nl"];
18
19/// The name of each language music21 knows, by its code.
20pub fn language_name(code: &str) -> Option<&'static str> {
21    Some(match code {
22        "en" => "English",
23        "fr" => "French",
24        "it" => "Italian",
25        "de" => "German",
26        "cn" => "Chinese",
27        "la" => "Latin",
28        "nl" => "Dutch",
29        _ => return None,
30    })
31}
32
33fn excerpt(code: &str) -> &'static str {
34    match code {
35        "en" => include_str!("language/en.txt"),
36        "fr" => include_str!("language/fr.txt"),
37        "it" => include_str!("language/it.txt"),
38        "de" => include_str!("language/de.txt"),
39        "cn" => include_str!("language/cn.txt"),
40        "la" => include_str!("language/la.txt"),
41        _ => include_str!("language/nl.txt"),
42    }
43}
44
45/// How often each letter follows each pair of letters in a text: music21's
46/// `Trigram`.
47#[derive(Clone, Debug, Default, PartialEq)]
48pub struct Trigram {
49    counts: HashMap<(char, char), HashMap<char, u64>>,
50    length: FloatType,
51}
52
53impl Trigram {
54    /// The trigrams of a text, letter by letter, starting after two spaces.
55    pub fn from_text(text: &str) -> Self {
56        Self::counted(text.chars())
57    }
58
59    /// The trigrams of a list of words, each stripped and followed by a
60    /// space: how music21 reads its excerpts.
61    pub fn from_words<'a>(words: impl IntoIterator<Item = &'a str>) -> Self {
62        Self::counted(
63            words
64                .into_iter()
65                .flat_map(|word| word.trim().chars().chain(std::iter::once(' '))),
66        )
67    }
68
69    fn counted(letters: impl Iterator<Item = char>) -> Self {
70        let mut counts: HashMap<(char, char), HashMap<char, u64>> = HashMap::new();
71        let mut pair = (' ', ' ');
72        for letter in letters {
73            *counts.entry(pair).or_default().entry(letter).or_insert(0) += 1;
74            pair = (pair.1, letter);
75        }
76        let total: u64 = counts
77            .values()
78            .flat_map(HashMap::values)
79            .map(|count| count * count)
80            .sum();
81        Self {
82            counts,
83            // Python's `total ** 0.5`.
84            length: (total as FloatType).powf(0.5),
85        }
86    }
87
88    /// The square root of the sum of every count squared: music21's
89    /// `length`.
90    pub fn length(&self) -> FloatType {
91        self.length
92    }
93
94    /// How alike two texts' trigrams are, from nought to one: the cosine of
95    /// their counts, music21's `similarity`.
96    pub fn similarity(&self, other: &Trigram) -> FloatType {
97        let mut total: u64 = 0;
98        for (pair, ours) in &self.counts {
99            if let Some(theirs) = other.counts.get(pair) {
100                for (letter, count) in ours {
101                    if let Some(their) = theirs.get(letter) {
102                        total += count * their;
103                    }
104                }
105            }
106        }
107        total as FloatType / (self.length * other.length)
108    }
109
110    /// One less [`Self::similarity`]: music21's `-`.
111    pub fn distance(&self, other: &Trigram) -> FloatType {
112        1.0 - self.similarity(other)
113    }
114}
115
116/// The trigrams of each language's excerpt, read once.
117fn known() -> &'static [Trigram; 7] {
118    static KNOWN: OnceLock<[Trigram; 7]> = OnceLock::new();
119    KNOWN.get_or_init(|| {
120        LANGUAGE_CODES.map(|code| Trigram::from_words(excerpt(code).split_whitespace()))
121    })
122}
123
124/// The language a text is most like, by its code, or nothing for an empty
125/// text or one like none of them: music21's
126/// `LanguageDetector.mostLikelyLanguage`. Where two are as alike, the first
127/// of [`LANGUAGE_CODES`] wins.
128///
129/// ```
130/// use music21_rs::features::language::most_likely_language;
131///
132/// let text = "Ich habe ein Buch, und ich gehe nach Hause mit meinem Bruder.";
133/// assert_eq!(most_likely_language(text), Some("de"));
134/// assert_eq!(most_likely_language(""), None);
135/// ```
136pub fn most_likely_language(text: &str) -> Option<&'static str> {
137    if text.is_empty() {
138        return None;
139    }
140    let trigram = Trigram::from_text(text);
141    let mut best = None;
142    let mut least = 1.0;
143    for (code, known) in LANGUAGE_CODES.iter().zip(known()) {
144        let distance = known.distance(&trigram);
145        if distance < least {
146            best = Some(*code);
147            least = distance;
148        }
149    }
150    best
151}
152
153/// The language a text is most like, as one more than its place in
154/// [`LANGUAGE_CODES`], or nought for an empty text: music21's
155/// `mostLikelyLanguageNumeric`.
156///
157/// # Errors
158///
159/// A text like none of the languages, which music21 refuses.
160pub fn most_likely_language_numeric(text: &str) -> Result<usize> {
161    if text.is_empty() {
162        return Ok(0);
163    }
164    let code = most_likely_language(text).ok_or_else(|| {
165        Error::Text("got a language that was not in the codes; should not happen".to_string())
166    })?;
167    Ok(LANGUAGE_CODES
168        .iter()
169        .position(|known| *known == code)
170        .map_or(0, |at| at + 1))
171}
music21_rs/features/language.rs

music21_rs/features/
language.rs
