music21_rs/features/
dataset.rs

1//! Features of many pieces in one table, each row labelled with its piece's
2//! class, written for machine learning: music21's `DataSet` and its output
3//! formats, and the ways music21 finds extractors by id and by name.
4
5use super::{DataInstance, Extractor, Feature, Value, jsymbolic, native};
6use crate::{
7    error::{Error, Result},
8    metadata::Metadata,
9    stream::Stream,
10};
11
12/// Where an extractor comes from: music21's jSymbolic ports or its own.
13#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
15pub enum Library {
16    /// [`jsymbolic::JSYMBOLIC`].
17    JSymbolic,
18    /// [`native::NATIVE`].
19    Native,
20}
21
22impl Library {
23    /// Both, jSymbolic first, as music21 looks through them.
24    pub const ALL: [Library; 2] = [Library::JSymbolic, Library::Native];
25
26    /// The library's extractors, in music21's order.
27    pub fn extractors(self) -> &'static [Extractor] {
28        match self {
29            Self::JSymbolic => jsymbolic::JSYMBOLIC,
30            Self::Native => native::NATIVE,
31        }
32    }
33
34    /// The library's name as music21's `getIndex` reports it.
35    pub fn name(self) -> &'static str {
36        match self {
37            Self::JSymbolic => "jsymbolic",
38            Self::Native => "native",
39        }
40    }
41}
42
43/// The extractors with any of these ids, from these libraries, in the
44/// libraries' own order: music21's `extractorsById`. An id is read without
45/// regard to case, hyphens or spaces, so `p-20` finds `P20`; the id `all`,
46/// first, finds every extractor. Both libraries have a `P22`, so asking for
47/// it in both finds two.
48pub fn extractors_by_id(ids: &[&str], libraries: &[Library]) -> Vec<&'static Extractor> {
49    let wanted: Vec<String> = ids
50        .iter()
51        .map(|id| id.trim().to_lowercase().replace(['-', ' '], ""))
52        .collect();
53    let Some(first) = wanted.first() else {
54        return Vec::new();
55    };
56    let everything = first == "all";
57    libraries
58        .iter()
59        .flat_map(|library| library.extractors())
60        .filter(|extractor| everything || wanted.contains(&extractor.id().to_lowercase()))
61        .collect()
62}
63
64/// The first extractor [`extractors_by_id`] finds for an id: music21's
65/// `extractorById`.
66pub fn extractor_by_id(id: &str, libraries: &[Library]) -> Option<&'static Extractor> {
67    extractors_by_id(&[id], libraries).into_iter().next()
68}
69
70/// The values of the first extractor with an id, read from a piece:
71/// music21's `vectorById`. Nothing for an id no extractor has.
72///
73/// # Errors
74///
75/// A piece that cannot be prepared, or the feature cannot be read from.
76pub fn vector_by_id(stream: &Stream, id: &str) -> Result<Option<Vec<Value>>> {
77    let Some(extractor) = extractor_by_id(id, &Library::ALL) else {
78        return Ok(None);
79    };
80    let data = DataInstance::new(stream)?;
81    Ok(Some(extractor.extract(&data)?.values().to_vec()))
82}
83
84/// Where the extractor of a feature named so sits in its library, and
85/// which library, looking through jSymbolic first unless one is given:
86/// music21's `getIndex`, as `(61, JSymbolic)` for `Range`. Nothing for a
87/// name no extractor has.
88///
89/// music21's own native list ends with its language feature, which is not
90/// here, so its index is never answered.
91pub fn index_of(name: &str, library: Option<Library>) -> Option<(usize, Library)> {
92    let libraries: &[Library] = match library {
93        Some(Library::JSymbolic) => &[Library::JSymbolic],
94        Some(Library::Native) => &[Library::Native],
95        None => &Library::ALL,
96    };
97    libraries.iter().find_map(|library| {
98        library
99            .extractors()
100            .iter()
101            .position(|extractor| extractor.name() == name)
102            .map(|index| (index, *library))
103    })
104}
105
106/// Every feature of a piece, jSymbolic's and then music21's own, each a
107/// blank one where it cannot be read: music21's `allFeaturesAsList`, but
108/// for its language feature.
109///
110/// # Errors
111///
112/// A piece that cannot be prepared.
113pub fn all_features_as_list(stream: &Stream) -> Result<Vec<Vec<Value>>> {
114    let mut set = DataSet::new("");
115    set.add_extractors(Library::ALL.iter().flat_map(|library| library.extractors()));
116    set.add_data(stream, "", None)?;
117    set.process();
118    Ok(set.features()[0]
119        .iter()
120        .map(|feature| feature.values().to_vec())
121        .collect())
122}
123
124/// A file format a [`DataSet`] is written in: music21's `OutputFormat`.
125#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
126#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
127pub enum OutputFormat {
128    /// Tab-separated, with the two header rows Orange reads: music21's
129    /// `OutputTabOrange`.
130    Tab,
131    /// Comma-separated values: music21's `OutputCSV`.
132    Csv,
133    /// Weka's Attribute-Relation File Format: music21's `OutputARFF`.
134    Arff,
135}
136
137impl OutputFormat {
138    /// The format a name stands for, as music21 reads one: `tab`, `orange`
139    /// or `taborange`; `csv` or `comma`; `arff` or `attribute`; in any case.
140    pub fn from_name(name: &str) -> Option<Self> {
141        match name.to_lowercase().as_str() {
142            "tab" | "orange" | "taborange" => Some(Self::Tab),
143            "csv" | "comma" => Some(Self::Csv),
144            "arff" | "attribute" => Some(Self::Arff),
145            _ => None,
146        }
147    }
148
149    /// The format a file's extension names, `data.csv` for
150    /// [`OutputFormat::Csv`].
151    pub fn from_path(path: &str) -> Option<Self> {
152        path.rsplit_once('.')
153            .and_then(|(_, extension)| Self::from_name(extension))
154    }
155
156    /// The extension music21 gives a file in this format, dot and all.
157    pub fn extension(self) -> &'static str {
158        match self {
159            Self::Tab => ".tab",
160            Self::Csv => ".csv",
161            Self::Arff => ".arff",
162        }
163    }
164}
165
166/// One cell of a [`DataSet`]'s table: a piece's id or class, or a value.
167#[derive(Clone, Debug, PartialEq)]
168#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
169pub enum Cell {
170    /// An id or a class value.
171    Text(String),
172    /// A feature's value.
173    Value(Value),
174}
175
176/// Written as music21 writes the cell: the text, or the value as Python
177/// writes the number.
178impl std::fmt::Display for Cell {
179    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
180        match self {
181            Self::Text(text) => f.write_str(text),
182            Self::Value(value) => write!(f, "{value}"),
183        }
184    }
185}
186
187/// A piece added to a data set.
188#[derive(Debug)]
189struct Entry {
190    data: DataInstance,
191    id: String,
192    class_value: String,
193}
194
195/// An extractor that could not read its feature from a piece, so that the
196/// data set stands a blank feature in for it.
197#[derive(Clone, Debug)]
198pub struct Failure {
199    /// Which piece, counted from nought in the order they were added.
200    pub row: usize,
201    /// The extractor's id.
202    pub extractor: &'static str,
203    /// Why.
204    pub error: Error,
205}
206
207/// Features of many pieces in one table: music21's `DataSet`. Each piece
208/// is a row, labelled with an id and a value of the class the set is about
209/// -- its composer, say -- and each extractor gives it as many columns as
210/// it has values. [`DataSet::process`] reads every feature of every piece,
211/// and [`DataSet::to_text`] writes the table for a machine-learning tool.
212///
213/// ```
214/// use music21_rs::features::{DataSet, Library, OutputFormat, extractors_by_id};
215/// use music21_rs::tinynotation::from_tiny_notation;
216///
217/// let mut set = DataSet::new("Composer");
218/// set.add_extractors(extractors_by_id(&["R31", "R35"], &[Library::JSymbolic]));
219/// set.add_data(&from_tiny_notation("4/4 c4 d e2")?, "Bach", Some("first piece"))?;
220/// set.process();
221/// assert_eq!(
222///     set.to_text(OutputFormat::Csv),
223///     "Identifier,Initial_Time_Signature_0,Initial_Time_Signature_1,Changes_of_Meter,Composer\n\
224///      first_piece,4,4,0,Bach"
225/// );
226/// # Ok::<(), music21_rs::Error>(())
227/// ```
228#[derive(Debug)]
229pub struct DataSet {
230    class_label: String,
231    extractors: Vec<&'static Extractor>,
232    entries: Vec<Entry>,
233    features: Vec<Vec<Feature>>,
234    failures: Vec<Failure>,
235}
236
237impl DataSet {
238    /// An empty data set about a class, named as its column is: music21's
239    /// `DataSet(classLabel=...)`.
240    pub fn new(class_label: impl Into<String>) -> Self {
241        Self {
242            class_label: class_label.into(),
243            extractors: Vec::new(),
244            entries: Vec::new(),
245            features: Vec::new(),
246            failures: Vec::new(),
247        }
248    }
249
250    /// The name of the class the set is about.
251    pub fn class_label(&self) -> &str {
252        &self.class_label
253    }
254
255    /// The extractors, in the order their columns come.
256    pub fn extractors(&self) -> &[&'static Extractor] {
257        &self.extractors
258    }
259
260    /// Adds extractors after those already there: music21's
261    /// `addFeatureExtractors`.
262    pub fn add_extractors(&mut self, extractors: impl IntoIterator<Item = &'static Extractor>) {
263        self.extractors.extend(extractors);
264    }
265
266    /// Adds a piece with its value of the class, prepared at once: music21's
267    /// `addData`. Its id is the one given or, without one, the title of its
268    /// metadata, or nothing; spaces in it become underscores, as music21's
269    /// `getId` writes them.
270    ///
271    /// # Errors
272    ///
273    /// A piece that cannot be prepared.
274    pub fn add_data(
275        &mut self,
276        stream: &Stream,
277        class_value: impl Into<String>,
278        id: Option<&str>,
279    ) -> Result<()> {
280        let id = match id {
281            Some(id) => id.to_string(),
282            None => stream
283                .metadata()
284                .and_then(Metadata::title)
285                .unwrap_or_default(),
286        };
287        self.entries.push(Entry {
288            data: DataInstance::new(stream)?,
289            id: id.replace(' ', "_"),
290            class_value: class_value.into(),
291        });
292        Ok(())
293    }
294
295    /// How many pieces have been added.
296    pub fn len(&self) -> usize {
297        self.entries.len()
298    }
299
300    /// Whether no piece has been added.
301    pub fn is_empty(&self) -> bool {
302        self.entries.is_empty()
303    }
304
305    /// Reads every feature of every piece, standing a blank feature in for
306    /// one that cannot be read and keeping why in [`Self::failures`]:
307    /// music21's `process`.
308    pub fn process(&mut self) {
309        self.features.clear();
310        self.failures.clear();
311        for (row, entry) in self.entries.iter().enumerate() {
312            let mut features = Vec::with_capacity(self.extractors.len());
313            for extractor in &self.extractors {
314                features.push(match extractor.extract(&entry.data) {
315                    Ok(feature) => feature,
316                    Err(error) => {
317                        self.failures.push(Failure {
318                            row,
319                            extractor: extractor.id(),
320                            error,
321                        });
322                        extractor.blank()
323                    }
324                });
325            }
326            self.features.push(features);
327        }
328    }
329
330    /// Reads every feature of every piece, stopping at the first that cannot
331    /// be read: music21's `process` with `failFast`.
332    ///
333    /// # Errors
334    ///
335    /// The first feature that cannot be read.
336    pub fn try_process(&mut self) -> Result<()> {
337        self.features.clear();
338        self.failures.clear();
339        for entry in &self.entries {
340            let features = self
341                .extractors
342                .iter()
343                .map(|extractor| extractor.extract(&entry.data))
344                .collect::<Result<Vec<_>>>()?;
345            self.features.push(features);
346        }
347        Ok(())
348    }
349
350    /// The features read, a row for each piece and a feature for each
351    /// extractor; empty until processed.
352    pub fn features(&self) -> &[Vec<Feature>] {
353        &self.features
354    }
355
356    /// The features [`Self::process`] could not read.
357    pub fn failures(&self) -> &[Failure] {
358        &self.failures
359    }
360
361    /// The column names: the id's, each extractor's attribute labels, and
362    /// the class's, spaces made underscores: music21's `getAttributeLabels`.
363    pub fn attribute_labels(&self, include_class_label: bool, include_id: bool) -> Vec<String> {
364        let mut labels = Vec::new();
365        if include_id {
366            labels.push("Identifier".to_string());
367        }
368        for extractor in &self.extractors {
369            labels.extend(extractor.attribute_labels());
370        }
371        if include_class_label {
372            labels.push(self.class_label.replace(' ', "_"));
373        }
374        labels
375    }
376
377    /// Whether each column is discrete, nothing for the id's and the class
378    /// always discrete: music21's `getDiscreteLabels`.
379    pub fn discrete_labels(
380        &self,
381        include_class_label: bool,
382        include_id: bool,
383    ) -> Vec<Option<bool>> {
384        let mut labels = Vec::new();
385        if include_id {
386            labels.push(None);
387        }
388        for extractor in &self.extractors {
389            labels.extend(std::iter::repeat_n(
390                Some(extractor.discrete()),
391                extractor.dimensions(),
392            ));
393        }
394        if include_class_label {
395            labels.push(Some(true));
396        }
397        labels
398    }
399
400    /// Whether each column is the class's, nothing for the id's: music21's
401    /// `getClassPositionLabels`.
402    pub fn class_position_labels(
403        &self,
404        include_id: bool,
405        include_class_label: bool,
406    ) -> Vec<Option<bool>> {
407        let mut labels = Vec::new();
408        if include_id {
409            labels.push(None);
410        }
411        for extractor in &self.extractors {
412            labels.extend(std::iter::repeat_n(Some(false), extractor.dimensions()));
413        }
414        if include_class_label {
415            labels.push(Some(true));
416        }
417        labels
418    }
419
420    /// The table, a row for each piece: its id, every value of every
421    /// feature, and its class value: music21's `getFeaturesAsList`.
422    pub fn features_as_list(&self, include_class_label: bool, include_id: bool) -> Vec<Vec<Cell>> {
423        self.entries
424            .iter()
425            .zip(&self.features)
426            .map(|(entry, features)| {
427                let mut row = Vec::new();
428                if include_id {
429                    row.push(Cell::Text(entry.id.clone()));
430                }
431                for feature in features {
432                    row.extend(feature.values().iter().copied().map(Cell::Value));
433                }
434                if include_class_label {
435                    row.push(Cell::Text(entry.class_value.clone()));
436                }
437                row
438            })
439            .collect()
440    }
441
442    /// Each class value once, in the order first met: music21's
443    /// `getUniqueClassValues`.
444    pub fn unique_class_values(&self) -> Vec<&str> {
445        let mut values: Vec<&str> = Vec::new();
446        for entry in &self.entries {
447            if !values.contains(&entry.class_value.as_str()) {
448                values.push(&entry.class_value);
449            }
450        }
451        values
452    }
453
454    /// The table written in a format, ids and classes included, lines
455    /// joined with newlines and none after the last: music21's `getString`.
456    pub fn to_text(&self, format: OutputFormat) -> String {
457        let rows = self.features_as_list(true, true);
458        let joined = |row: &[Cell], separator: &str| -> String {
459            row.iter()
460                .map(ToString::to_string)
461                .collect::<Vec<_>>()
462                .join(separator)
463        };
464        let mut lines: Vec<String> = Vec::new();
465        match format {
466            OutputFormat::Tab => {
467                lines.push(self.attribute_labels(true, true).join("\t"));
468                lines.push(
469                    self.discrete_labels(true, true)
470                        .into_iter()
471                        .map(|discrete| match discrete {
472                            None => "string",
473                            Some(true) => "discrete",
474                            Some(false) => "continuous",
475                        })
476                        .collect::<Vec<_>>()
477                        .join("\t"),
478                );
479                lines.push(
480                    self.class_position_labels(true, true)
481                        .into_iter()
482                        .map(|class| match class {
483                            None => "meta",
484                            Some(true) => "class",
485                            Some(false) => "",
486                        })
487                        .collect::<Vec<_>>()
488                        .join("\t"),
489                );
490                lines.extend(rows.iter().map(|row| joined(row, "\t")));
491            }
492            OutputFormat::Csv => {
493                lines.push(self.attribute_labels(true, true).join(","));
494                lines.extend(rows.iter().map(|row| joined(row, ",")));
495            }
496            OutputFormat::Arff => {
497                lines.push(format!("@RELATION {}", self.class_label));
498                let labels = self.attribute_labels(true, true);
499                let discrete = self.discrete_labels(true, true);
500                let classes = self.class_position_labels(true, true);
501                for ((label, discrete), class) in labels.iter().zip(discrete).zip(classes) {
502                    lines.push(match (class, discrete) {
503                        (Some(true), _) => {
504                            format!(
505                                "@ATTRIBUTE class {{{}}}",
506                                self.unique_class_values().join(",")
507                            )
508                        }
509                        (_, None) => format!("@ATTRIBUTE {label} STRING"),
510                        _ => format!("@ATTRIBUTE {label} NUMERIC"),
511                    });
512                }
513                lines.push("@DATA".to_string());
514                lines.extend(rows.iter().map(|row| joined(row, ",")));
515            }
516        }
517        lines.join("\n")
518    }
519}
520
521#[cfg(test)]
522mod tests {
523    use super::*;
524    use crate::tinynotation::from_tiny_notation;
525
526    fn two_pieces() -> Result<DataSet> {
527        let line = from_tiny_notation("4/4 c4 d e2")?;
528        let mut set = DataSet::new("Composer");
529        set.add_extractors(extractors_by_id(&["R31"], &[Library::JSymbolic]));
530        set.add_extractors(extractors_by_id(&["P22"], &[Library::Native]));
531        set.add_extractors(extractors_by_id(&["R35"], &[Library::JSymbolic]));
532        set.add_data(&line, "Bach", Some("my piece"))?;
533        set.add_data(&line, "Handel", None)?;
534        set.process();
535        Ok(set)
536    }
537
538    #[test]
539    fn a_data_set_is_written_as_music21_writes_one() -> Result<()> {
540        // Each read off music21's DataSet.getString for the same set.
541        let set = two_pieces()?;
542        assert_eq!(
543            set.to_text(OutputFormat::Tab),
544            "Identifier\tInitial_Time_Signature_0\tInitial_Time_Signature_1\tQuality\t\
545             Changes_of_Meter\tComposer\nstring\tdiscrete\tdiscrete\tdiscrete\tdiscrete\t\
546             discrete\nmeta\t\t\t\t\tclass\nmy_piece\t4\t4\t1\t0\tBach\n\t4\t4\t1\t0\tHandel"
547        );
548        assert_eq!(
549            set.to_text(OutputFormat::Csv),
550            "Identifier,Initial_Time_Signature_0,Initial_Time_Signature_1,Quality,\
551             Changes_of_Meter,Composer\nmy_piece,4,4,1,0,Bach\n,4,4,1,0,Handel"
552        );
553        assert_eq!(
554            set.to_text(OutputFormat::Arff),
555            "@RELATION Composer\n@ATTRIBUTE Identifier STRING\n\
556             @ATTRIBUTE Initial_Time_Signature_0 NUMERIC\n\
557             @ATTRIBUTE Initial_Time_Signature_1 NUMERIC\n@ATTRIBUTE Quality NUMERIC\n\
558             @ATTRIBUTE Changes_of_Meter NUMERIC\n@ATTRIBUTE class {Bach,Handel}\n@DATA\n\
559             my_piece,4,4,1,0,Bach\n,4,4,1,0,Handel"
560        );
561        Ok(())
562    }
563
564    #[test]
565    fn extractors_are_found_by_id_and_by_name() {
566        let ids = |found: Vec<&'static Extractor>| -> Vec<&str> {
567            found.iter().map(|extractor| extractor.id()).collect()
568        };
569        assert_eq!(
570            ids(extractors_by_id(&["p22", "ql1", "P-20"], &Library::ALL)),
571            ["P20", "P22", "P22", "QL1"]
572        );
573        assert_eq!(ids(extractors_by_id(&["p 21"], &Library::ALL)), ["P21"]);
574        let all = extractors_by_id(&["all"], &Library::ALL);
575        // music21 has 92: its language feature comes with `language-detection`.
576        let expected = if cfg!(feature = "language-detection") {
577            92
578        } else {
579            91
580        };
581        assert_eq!(all.len(), expected);
582        assert_eq!(index_of("Range", None), Some((61, Library::JSymbolic)));
583        assert_eq!(
584            index_of("Ends With Landini Melodic Contour", None),
585            Some((18, Library::Native))
586        );
587        assert_eq!(
588            index_of("Tonal Certainty", Some(Library::Native)),
589            Some((1, Library::Native))
590        );
591        assert_eq!(index_of("aBrandNewFeature!", None), None);
592        assert_eq!(OutputFormat::from_path("test.tab"), Some(OutputFormat::Tab));
593        assert_eq!(OutputFormat::from_path("junk"), None);
594    }
595
596    #[test]
597    fn every_feature_of_a_piece_is_listed() -> Result<()> {
598        // music21: features.allFeaturesAsList(s)[2:5] == [[2], [2], [1.0]]
599        let features = all_features_as_list(&from_tiny_notation("4/4 c4 d e2")?)?;
600        let expected = if cfg!(feature = "language-detection") {
601            92
602        } else {
603            91
604        };
605        assert_eq!(features.len(), expected);
606        assert_eq!(
607            features[2..5],
608            [
609                vec![Value::Integer(2)],
610                vec![Value::Integer(2)],
611                vec![Value::Float(1.0)]
612            ]
613        );
614        Ok(())
615    }
616}
music21_rs/features/dataset.rs

music21_rs/features/
dataset.rs
