Initial MessageFormat 2

Cargo.toml

@@ -23,6 +23,7 @@ members = [
     "utils/databake",
     "utils/databake/derive",
     "experimental/segmenter",
+    "experimental/message",
     "ffi/capi_cdylib",
     "ffi/diplomat",
     "ffi/capi_staticlib",

experimental/message/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "icu_message"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+smallvec = "1.6"
+intl-memoizer = "0.5"
+icu_locid = { path = "../../components/locid" }
+
+[dev-dependencies]
+iai = "0.1"
+criterion = "0.3.4"
+
+[[bench]]
+name = "parser_iai"
+harness = false
+
+[[bench]]
+name = "parser"
+harness = false
+
+[[bench]]
+name = "mf"
+harness = false

experimental/message/benches/mf.rs

@@ -0,0 +1,76 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use icu_message::parser::Parser;
+use icu_message::types::VariableType;
+use icu_message::MessageFormat;
+use std::collections::HashMap;
+
+fn overview_bench(c: &mut Criterion) {
+    let source = "{Hello World}";
+    c.bench_function("message/format/simple/format_from_source", |b| {
+        let mf = MessageFormat::<&str>::new();
+        b.iter(|| {
+            let _ = mf.format_from_source::<&str, &str>(black_box(source), None);
+        })
+    });
+
+    c.bench_function("message/format/simple/format_to_string", |b| {
+        let mf = MessageFormat::<&str>::new();
+        let parser = Parser::new(source);
+        let msg = parser.parse().unwrap();
+        b.iter(|| {
+            let _ = mf.format_to_string::<&str, &str>(black_box(&msg), None);
+        })
+    });
+
+    let source = "{Today is {$today}} a good day.";
+    let mut vars = HashMap::new();
+    vars.insert("today".to_string(), VariableType::String("January 25 2022"));
+
+    c.bench_function("message/format/placeholder/format_from_source", |b| {
+        let mf = MessageFormat::<&str>::new();
+        b.iter(|| {
+            let _ = mf.format_from_source::<&str, _>(black_box(source), Some(&vars));
+        })
+    });
+
+    c.bench_function("message/format/placeholder/format_to_string", |b| {
+        let mf = MessageFormat::<&str>::new();
+        let parser = Parser::new(source);
+        let msg = parser.parse().unwrap();
+        b.iter(|| {
+            let _ = mf.format_to_string::<&str, &str>(black_box(&msg), Some(&vars));
+        })
+    });
+}
+
+fn compare_bench(c: &mut Criterion) {
+    let mut sources = vec![];
+    for i in 0..99 {
+        let source = format!("{{Value {i}}}");
+        sources.push(source);
+    }
+
+    let messages: Vec<_> = sources
+        .iter()
+        .map(|s| {
+            let parser = Parser::new(s.as_str());
+            parser.parse().unwrap()
+        })
+        .collect();
+
+    c.bench_function("message/format/compare/simple", |b| {
+        let mf = MessageFormat::<&str>::new();
+        b.iter(|| {
+            for msg in &messages {
+                let _ = mf.format_to_string::<_, &str>(black_box(msg), None);
+            }
+        })
+    });
+}
+
+criterion_group!(benches, overview_bench, compare_bench);
+criterion_main!(benches);

experimental/message/benches/parser.rs

@@ -0,0 +1,44 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use icu_message::parser::Parser;
+
+fn overview_bench(c: &mut Criterion) {
+    let source = "{Hello World}";
+    c.bench_function("message/parse/simple", |b| {
+        b.iter(|| {
+            let parser = Parser::new(black_box(source));
+            let _ = parser.parse();
+        })
+    });
+
+    let source = "{Today is {$today}} a good day.";
+    c.bench_function("message/parse/placeholder", |b| {
+        b.iter(|| {
+            let parser = Parser::new(black_box(source));
+            let _ = parser.parse();
+        })
+    });
+}
+
+fn compare_bench(c: &mut Criterion) {
+    let mut messages = vec![];
+
+    for i in 0..99 {
+        messages.push(format!("{{Value {i}}}"));
+    }
+
+    c.bench_function("message/parse/compare/simple", |b| {
+        b.iter(|| {
+            for msg in &messages {
+                let parser = Parser::new(black_box(msg.as_str()));
+                let _ = parser.parse();
+            }
+        })
+    });
+}
+
+criterion_group!(benches, overview_bench, compare_bench);
+criterion_main!(benches);

experimental/message/benches/parser_iai.rs

@@ -0,0 +1,13 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use icu_message::parser::Parser;
+
+fn iai_parse_message() {
+    let source = "{Hello World}";
+    let parser = Parser::new(source);
+    let _ = parser.parse();
+}
+
+iai::main!(iai_parse_message,);

experimental/message/src/ast.rs

@@ -0,0 +1,98 @@
+use smallvec::SmallVec;
+
+#[derive(Debug, PartialEq)]
+pub struct Message<S> {
+    pub declarations: SmallVec<[Declaration<S>; 1]>,
+    pub value: MessageValue<S>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Declaration<S> {
+    pub variable: S,
+    pub expression: Expression<S>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum MessageValue<S> {
+    Pattern(Pattern<S>),
+    Select(Box<Select<S>>),
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Select<S> {
+    pub selector: SmallVec<[Expression<S>; 1]>,
+    pub variants: SmallVec<[Variant<S>; 3]>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Variant<S> {
+    pub key: SmallVec<[VariantKey<S>; 1]>,
+    pub pattern: Pattern<S>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Pattern<S> {
+    pub body: SmallVec<[PatternElement<S>; 3]>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum PatternElement<S> {
+    Text(S),
+    Placeholder(Placeholder<S>),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Placeholder<S> {
+    Markup {
+        name: S,
+        options: SmallVec<[Option<S>; 1]>,
+    },
+    MarkupEnd {
+        name: S,
+    },
+    Expression(Expression<S>),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Expression<S> {
+    Operand {
+        operand: Operand<S>,
+        annotation: std::option::Option<Annotation<S>>,
+    },
+    Annotation(Annotation<S>),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Operand<S> {
+    Literal(Literal<S>),
+    Variable(S),
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Annotation<S> {
+    pub function: S,
+    pub options: SmallVec<[Option<S>; 1]>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Literal<S> {
+    pub value: S,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum VariantKey<S> {
+    Literal(Literal<S>),
+    Asterisk,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Option<S> {
+    name: S,
+    value: OptionValue<S>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum OptionValue<S> {
+    Literal(Literal<S>),
+    Variable(S),
+}

experimental/message/src/functions/mod.rs

@@ -0,0 +1,9 @@
+use crate::types::VariableType;
+
+pub struct Number;
+
+impl Number {
+    pub fn format(input: &VariableType<String>) -> VariableType<String> {
+        VariableType::String("Hello from function".to_string())
+    }
+}