Refactor `fluent_embed`
Dependencies
- [2]
F5LG7WENEmit compilation errors from Fluent source code - [3]
V5S5K33AAdd basic error handling for invalid paths in proc_macro attribute - [4]
BMUMO42IAdd support for inline string and number literals - [5]
UOMQT7LTAdd support for cardinal CLDR plural selectors - [6]
XGNME3WRMove `Group::derive_enum` to new `crate::parse_macro` module - [7]
BANMRGROSwitch `wax` to temporary fork - [8]
5FIVUZYFUnify `fluent_embed` macro API as `localize()` - [9]
RLX6XPNZReturn an error when user provides an exact path - [10]
XEEXWJLGAdd simple end-to-end test for selectors - [11]
ROSR4HD5Parse captured glob as locale - [12]
D652S2N3Rename `parse` module to `parse_fluent` - [13]
2XQ6ZB4WStore multiple locales in a single `Group` - [14]
3C3CHSY5Implement `to_syn` for groups containing simple text messages - [15]
3WEPY3OXAdd `locale` parameter to derived `localize()` function - [16]
HJMYJDC7Simplify `fluent_embed::group` module - [17]
VNSHGQYNSupport using glob paths in `localize` macro - [18]
QSK7JRBAAdd simple `attribute_path` function - [19]
BQ6N55O7Refactor how `Group` stores messages - [20]
NO3PDO7PRefactor `fluent_embed` to support structs - [21]
O77KA6C4Create `fluent_embed` crate - [22]
5TEX4MNUSplit `fluent_embed` into `group` and `parse` modules - [23]
OCR4YRQ2Parse group from fluent file specified by macro attribute
Change contents
- edit in fluent_embed/src/lib.rs at line 1
use crate::group::Group; - edit in fluent_embed/src/lib.rs at line 8
use syn::DeriveInput; - replacement in fluent_embed/src/lib.rs at line 11[3.4389]→[3.3464:3475](∅→∅),[3.74]→[3.3464:3475](∅→∅),[3.3475]→[2.6660:6682](∅→∅),[2.6682]→[3.1608:1625](∅→∅),[3.912]→[3.1608:1625](∅→∅)
mod group;pub mod parse_fluent;mod parse_macro;mod derive;mod fluent; - replacement in fluent_embed/src/lib.rs at line 14
pub use parse_fluent::FluentError;pub use fluent::Error as FluentError; - replacement in fluent_embed/src/lib.rs at line 37
pub fn attribute_groups(path_literal: &syn::LitStr) -> Result<Group, AttributeError> {pub fn locales_from_attribute(attribute: &syn::LitStr,) -> Result<HashMap<Locale, PathBuf>, AttributeError> { - replacement in fluent_embed/src/lib.rs at line 42
let attribute_glob = path_literal.value();let attribute_glob = attribute.value(); - replacement in fluent_embed/src/lib.rs at line 44
let mut resources = HashMap::new();let mut paths = HashMap::new();let mut locales = HashMap::new(); - replacement in fluent_embed/src/lib.rs at line 64
// Parse the file into a `Group`let fluent_contents = std::fs::read_to_string(entry.path()).unwrap();let resource = fluent_syntax::parser::parse(fluent_contents).unwrap();resources.insert(locale.clone(), resource);paths.insert(locale, entry.to_candidate_path().to_string());// Insert this locale (and make sure it's unique!)let previous_value = locales.insert(locale, entry.into_path());assert!(previous_value.is_none()); - replacement in fluent_embed/src/lib.rs at line 69
Ok(Group::new(locale!("en-US"), resources, paths))Ok(locales) - replacement in fluent_embed/src/lib.rs at line 72[3.5475]→[2.7297:7398](∅→∅),[3.746]→[3.342:383](∅→∅),[2.7398]→[3.342:383](∅→∅),[3.342]→[3.342:383](∅→∅)
pub fn localize(path: &syn::LitStr, derive_input: &DeriveInput) -> Result<TokenStream, MacroError> {let group = attribute_groups(path)?;pub fn localize(attribute: &syn::LitStr,derive_input: &syn::DeriveInput,) -> Result<TokenStream, MacroError> {let locales = locales_from_attribute(&attribute)?;// let context = todo!();// TODO: user-controlled canonical localelet group = fluent::Group::new(locale!("en-US"), locales)?; - replacement in fluent_embed/src/lib.rs at line 83
parse_macro::derive_struct(group, &derive_input.ident, &struct_data.fields)derive::for_struct(group, &derive_input.ident, &struct_data.fields) - replacement in fluent_embed/src/lib.rs at line 85
syn::Data::Enum(enum_data) => parse_macro::derive_enum(group, &enum_data.variants),syn::Data::Enum(enum_data) => derive::for_enum(group, &enum_data.variants), - file addition: fluent[3.41]
- file addition: mod.rs[0.959]
use crate::derive::DeriveContext;use std::collections::HashMap;use std::path::PathBuf;use fluent_syntax::ast::{Entry, Message, Pattern, Resource};use miette::Diagnostic;use thiserror::Error;mod ast;mod group;pub use group::Group;#[derive(Diagnostic, Debug, Error)]#[diagnostic(transparent)]#[error(transparent)]pub enum Error {InvalidReference(#[from] ast::InvalidReference),}#[derive(Clone, Copy, Debug)]pub struct MessageContext<'context> {syntax_tree: &'context Resource<String>,pattern: &'context Pattern<String>,path: &'context str,derive_context: &'context DeriveContext,}#[derive(Clone, Debug)]struct SourceFile {path: PathBuf,syntax_tree: fluent_syntax::ast::Resource<String>,messages: HashMap<String, Message<String>>,}impl SourceFile {fn new(path: PathBuf) -> Result<Self, Error> {let file_contents = std::fs::read_to_string(&path).unwrap();// TODO: return an error instead of paniclet syntax_tree = fluent_syntax::parser::parse(file_contents).unwrap();// Keep track of all the messages in our Fluent filelet mut messages = HashMap::with_capacity(syntax_tree.body.len());for entry in &syntax_tree.body {match entry {Entry::Message(message_entry) => {let message_id = message_entry.id.name.to_owned();// Insert this message id (and make sure it's unique!)let previous_value = messages.insert(message_id, message_entry.clone());assert!(previous_value.is_none());}Entry::Term(_) => todo!(),Entry::Comment(_) => todo!(),Entry::GroupComment(_) => todo!(),Entry::ResourceComment(_) => todo!(),Entry::Junk { .. } => todo!(),}}Ok(Self {path,syntax_tree,messages,})}/// Unique message IDs, to compare with other localesfn message_ids(&self) -> impl Iterator<Item = &str> {self.messages.keys().map(String::as_str)}fn contains_expression(&self, id: &str) -> bool {self.messages.contains_key(id)}fn remove_expression(&mut self,id: &str,derive_context: &DeriveContext,) -> Result<syn::Expr, Error> {let message = self.messages.remove(id).unwrap();let message_context = MessageContext {syntax_tree: &self.syntax_tree,// Any message where this value is `None` shouldn't be accessed directly, see:// https://docs.rs/fluent-syntax/latest/fluent_syntax/ast/struct.Attribute.html#example// TODO: return an error instead of panicpattern: message.value.as_ref().unwrap(),path: &self.path.to_string_lossy(),derive_context,};ast::message_body(message_context)}} - file move: group.rs → group.rs
- replacement in fluent_embed/src/fluent/group.rs at line 1[3.3543]→[2.7551:7609](∅→∅),[3.6167]→[3.428:459](∅→∅),[2.7609]→[3.428:459](∅→∅),[3.3543]→[3.428:459](∅→∅)
use crate::parse_fluent::{self, FluentError, References};use std::collections::HashMap;use super::{Error, SourceFile};use crate::derive::DeriveContext;use std::collections::{HashMap, HashSet};use std::path::PathBuf; - edit in fluent_embed/src/fluent/group.rs at line 6
use fluent_syntax::ast::{Entry, Message, Resource}; - edit in fluent_embed/src/fluent/group.rs at line 9[3.486]→[3.0:87](∅→∅),[3.87]→[3.622:625](∅→∅),[3.622]→[3.622:625](∅→∅),[3.625]→[3.88:327](∅→∅),[3.327]→[3.834:835](∅→∅),[3.834]→[3.834:835](∅→∅),[3.835]→[3.328:659](∅→∅),[3.659]→[3.974:975](∅→∅),[3.974]→[3.974:975](∅→∅),[3.975]→[3.660:761](∅→∅),[3.761]→[3.1077:1101](∅→∅),[3.1077]→[3.1077:1101](∅→∅),[3.1101]→[3.762:797](∅→∅),[3.797]→[3.1101:1107](∅→∅),[3.1101]→[3.1101:1107](∅→∅),[3.1107]→[3.824:826](∅→∅),[3.824]→[3.824:826](∅→∅),[3.417]→[3.3690:3691](∅→∅),[3.826]→[3.3690:3691](∅→∅),[3.3690]→[3.3690:3691](∅→∅),[3.3691]→[3.867:891](∅→∅)
struct LocaleGroup {locale: Locale,messages: Box<[Option<Message<String>>]>,}impl LocaleGroup {fn new(locale: Locale,resource: Resource<String>,canonical_messages: &[Message<String>],) -> Self {let mut messages = vec![None; canonical_messages.len()].into_boxed_slice();for entry in resource.body {if let Entry::Message(message) = entry {let index = canonical_messages.iter().position(|canonical_message| canonical_message.id.name == message.id.name).expect("Message ID must be in canonical group");assert!(messages[index].is_none());messages[index] = Some(message);}}Self { locale, messages }}}#[derive(Clone, Debug)] - replacement in fluent_embed/src/fluent/group.rs at line 11
canonical_messages: Vec<Message<String>>,extra_locales: Vec<LocaleGroup>,paths: HashMap<Locale, String>,locales: HashMap<Locale, SourceFile>, - replacement in fluent_embed/src/fluent/group.rs at line 17
mut resources: HashMap<Locale, Resource<String>>,paths: HashMap<Locale, String>,) -> Self {let canonical_resource = resources.remove(&canonical_locale).unwrap();locale_paths: HashMap<Locale, PathBuf>,) -> Result<Self, Error> {let mut locales = HashMap::with_capacity(locale_paths.len()); - replacement in fluent_embed/src/fluent/group.rs at line 21
let mut canonical_messages = Vec::new();for (locale, path) in locale_paths {let resource = SourceFile::new(path)?; - replacement in fluent_embed/src/fluent/group.rs at line 24[3.4799]→[3.1139:1239](∅→∅),[3.1239]→[3.932:982](∅→∅),[3.982]→[3.1395:1409](∅→∅),[3.1395]→[3.1395:1409](∅→∅)
for entry in canonical_resource.body {if let Entry::Message(message) = entry {canonical_messages.push(message);}// Insert resource (and make sure it's unique!)let previous_value = locales.insert(locale, resource);assert!(previous_value.is_none()); - replacement in fluent_embed/src/fluent/group.rs at line 29
let extra_locales = resources.into_iter().map(|(locale, resource)| LocaleGroup::new(locale, resource, &canonical_messages)).collect::<Vec<_>>();// Collect all keys used in the Fluent source codelet keys: HashSet<&str> =HashSet::from_iter(locales.values().flat_map(|resource| resource.message_ids()));// Collect all keys from the canonical resource// TODO: return an error instead of paniclet canonical_resource = locales.get(&canonical_locale).unwrap();// TODO: return an error instead of panic// Make sure canonical resource contains all message keysassert_eq!(keys, HashSet::from_iter(canonical_resource.message_ids())); - replacement in fluent_embed/src/fluent/group.rs at line 39
Self {Ok(Self { - replacement in fluent_embed/src/fluent/group.rs at line 41[3.1864]→[3.1176:1235](∅→∅),[3.1235]→[2.7812:7831](∅→∅),[3.1235]→[3.1886:1896](∅→∅),[2.7831]→[3.1886:1896](∅→∅),[3.1886]→[3.1886:1896](∅→∅)
canonical_messages,extra_locales,paths,}locales,}) - replacement in fluent_embed/src/fluent/group.rs at line 49
pub fn canonical_message(&self,pub fn remove_canonical_message(&mut self, - replacement in fluent_embed/src/fluent/group.rs at line 52
references: &References,) -> Result<syn::Expr, FluentError> {let message = self.canonical_messages.iter().find(|message| message.id.name == id).expect("Message id must be valid");derive_context: &DeriveContext,) -> Result<syn::Expr, Error> {let canonical_resource = self.locales.get_mut(&self.canonical_locale).unwrap();let message = canonical_resource.remove_expression(id, derive_context)?; - replacement in fluent_embed/src/fluent/group.rs at line 57
let path = self.paths.get(&self.canonical_locale).unwrap();parse_fluent::message(message, references, path)Ok(message) - replacement in fluent_embed/src/fluent/group.rs at line 60
pub fn additional_messages(&self,pub fn remove_additional_messages(&mut self, - replacement in fluent_embed/src/fluent/group.rs at line 63
references: &References,) -> Result<Vec<(&Locale, syn::Expr)>, FluentError> {let mut messages = Vec::with_capacity(self.extra_locales.len());let message_column = self.canonical_messages.iter().position(|message| message.id.name == id).expect("Message id must be valid");derive_context: &DeriveContext,) -> Result<HashMap<&Locale, syn::Expr>, Error> {// Create a message for every locale *except* the canonoical localelet mut messages = HashMap::with_capacity(self.locales.len() - 1); - replacement in fluent_embed/src/fluent/group.rs at line 68
for locale_group in &self.extra_locales {for (locale, compiled_messages) in self.locales.iter_mut() {// Skip the canonical localeif locale == &self.canonical_locale {continue;} - replacement in fluent_embed/src/fluent/group.rs at line 75
if let Some(message) = &locale_group.messages[message_column] {let path = self.paths.get(&locale_group.locale).unwrap();let message_expr = parse_fluent::message(&message, references, path)?;messages.push((&locale_group.locale, message_expr))if compiled_messages.contains_expression(id) {let message = compiled_messages.remove_expression(id, derive_context)?;// Insert this message (and make sure it's unique!)let previous_value = messages.insert(locale, message);assert!(previous_value.is_none()); - file move: parse_fluent.rs → ast.rs
- replacement in fluent_embed/src/fluent/ast.rs at line 1
use std::collections::HashSet;use super::{Error, MessageContext};use crate::derive::ReferenceKind; - replacement in fluent_embed/src/fluent/ast.rs at line 4[2.2217]→[3.34:60](∅→∅),[3.33]→[3.34:60](∅→∅),[3.60]→[2.2218:2328](∅→∅),[3.218]→[3.140:143](∅→∅),[2.2328]→[3.140:143](∅→∅),[3.140]→[3.140:143](∅→∅)
use fluent_syntax::ast::{Entry, Expression, InlineExpression, Message, Pattern, PatternElement, Resource, Variant,VariantKey,};use fluent_syntax::ast::{Entry, Expression, InlineExpression, PatternElement, VariantKey}; - edit in fluent_embed/src/fluent/ast.rs at line 9
use thiserror::Error; - replacement in fluent_embed/src/fluent/ast.rs at line 21[3.239]→[2.2721:2880](∅→∅),[2.2880]→[3.2938:3028](∅→∅),[3.239]→[3.2938:3028](∅→∅),[3.3028]→[2.2881:3166](∅→∅)
#[derive(Diagnostic, Debug, Error)]#[diagnostic(transparent)]#[error(transparent)]pub enum FluentError {InvalidReference(#[from] InvalidReference),}#[derive(Clone, Copy, Debug)]pub enum ReferenceKind {EnumField,StructField,}#[derive(Clone, Debug)]pub struct References {pub kind: ReferenceKind,pub candidates: HashSet<String>,}pub(crate) fn pattern(message: &Message<String>,pattern: &Pattern<String>,references: &References,path: &str,) -> Result<syn::Expr, FluentError> {pub fn message_body(message_context: MessageContext) -> Result<syn::Expr, Error> { - replacement in fluent_embed/src/fluent/ast.rs at line 25
for element in &pattern.elements {for element in &message_context.pattern.elements { - replacement in fluent_embed/src/fluent/ast.rs at line 31
let target = inline_expression(message, selector, references, path)?;let arms: Vec<syn::Arm> = variants.iter().map(|item| variant(message, item, references, path)).collect::<Result<_, FluentError>>()?;let target = inline_expression(selector, message_context)?;let mut arms: Vec<syn::Arm> = Vec::with_capacity(variants.len());for variant in variants {let base_pattern: syn::Pat = match &variant.key {VariantKey::Identifier { name } => {let ident = format_ident!("{}", name.to_pascal_case());parse_quote!(::icu_plurals::PluralCategory::#ident)}VariantKey::NumberLiteral { .. } => todo!(),};// Create a new `MessageContext` for each variantlet variant_context = MessageContext {pattern: &variant.value,..message_context};let body = message_body(variant_context)?;// Default patterns match anything else// TODO: this can potentially generate unreachable patterns,// should be replaced with a more sophisticated implementationlet pattern = if variant.default {parse_quote!(#base_pattern | _)} else {base_pattern}; - edit in fluent_embed/src/fluent/ast.rs at line 58
arms.push(parse_quote!(#pattern => #body));} - replacement in fluent_embed/src/fluent/ast.rs at line 69
inline_expression(message, expression, references, path)?inline_expression(expression, message_context)? - edit in fluent_embed/src/fluent/ast.rs at line 86
message: &Message<String>, - replacement in fluent_embed/src/fluent/ast.rs at line 87
references: &References,path: &str,) -> Result<syn::Expr, FluentError> {message_context: MessageContext,) -> Result<syn::Expr, Error> { - replacement in fluent_embed/src/fluent/ast.rs at line 104
let ident = if let Some(variable) = references.candidates.get(&id.name.to_snake_case())let ident = if let Some(variable) = message_context.derive_context.valid_references.get(&id.name.to_snake_case()) - replacement in fluent_embed/src/fluent/ast.rs at line 111
// Create a fake `fluent_syntax::ast::Resource` to serialize into a Stringlet error_resource = Resource {body: vec![Entry::Message(message.to_owned())],};// Serialize the Fluent file AST back into a String - replacement in fluent_embed/src/fluent/ast.rs at line 113
&error_resource,&message_context.syntax_tree, - replacement in fluent_embed/src/fluent/ast.rs at line 122
return Err(FluentError::InvalidReference(InvalidReference {src: NamedSource::new(path, source_string.clone()),return Err(Error::InvalidReference(InvalidReference {src: NamedSource::new(message_context.path, source_string.clone()), - replacement in fluent_embed/src/fluent/ast.rs at line 127
references.candidatesmessage_context.derive_context.valid_references - replacement in fluent_embed/src/fluent/ast.rs at line 138
match references.kind {match message_context.derive_context.reference_kind { - edit in fluent_embed/src/fluent/ast.rs at line 148[2.6071]→[3.2590:2593](∅→∅),[3.2590]→[3.2590:2593](∅→∅),[3.2593]→[2.6072:6228](∅→∅),[3.485]→[3.696:746](∅→∅),[3.4034]→[3.696:746](∅→∅),[2.6228]→[3.696:746](∅→∅),[3.696]→[3.696:746](∅→∅),[3.746]→[2.6229:6297](∅→∅),[3.786]→[3.2775:3073](∅→∅),[3.4091]→[3.2775:3073](∅→∅),[2.6297]→[3.2775:3073](∅→∅),[3.2775]→[3.2775:3073](∅→∅),[3.3073]→[2.6298:6338](∅→∅),[2.6338]→[3.3109:3112](∅→∅),[3.3109]→[3.3109:3112](∅→∅),[3.3112]→[3.486:549](∅→∅),[3.549]→[3.869:893](∅→∅),[3.869]→[3.869:893](∅→∅),[3.893]→[3.3212:3399](∅→∅),[3.3212]→[3.3212:3399](∅→∅),[3.3399]→[2.6339:6392](∅→∅),[2.6392]→[3.3455:3463](∅→∅),[3.3455]→[3.3455:3463](∅→∅),[3.3463]→[3.550:551](∅→∅),[3.551]→[2.6393:6530](∅→∅),[3.4187]→[3.615:665](∅→∅),[2.6530]→[3.615:665](∅→∅),[3.615]→[3.615:665](∅→∅),[3.665]→[2.6531:6581](∅→∅),[3.4227]→[3.688:701](∅→∅),[2.6581]→[3.688:701](∅→∅),[3.688]→[3.688:701](∅→∅),[3.701]→[2.6582:6611](∅→∅),[2.6611]→[3.726:732](∅→∅),[3.726]→[3.726:732](∅→∅)
}fn variant(message: &Message<String>,variant: &Variant<String>,references: &References,path: &str,) -> Result<syn::Arm, FluentError> {let base_pattern = variant_key(&variant.key);let body = pattern(message, &variant.value, references, path)?;// Default patterns match anything else// TODO: this can potentially generate unreachable patterns,// should be replaced with a more sophisticated implementationlet pattern = if variant.default {parse_quote!(#base_pattern | _)} else {base_pattern};Ok(parse_quote!(#pattern => #body))}fn variant_key(variant_key: &VariantKey<String>) -> syn::Pat {match variant_key {VariantKey::Identifier { name } => {let ident = format_ident!("{}", name.to_pascal_case());parse_quote!(::icu_plurals::PluralCategory::#ident)}VariantKey::NumberLiteral { .. } => todo!(),}}pub(crate) fn message(message: &Message<String>,references: &References,path: &str,) -> Result<syn::Expr, FluentError> {if let Some(value) = message.value.as_ref() {pattern(message, value, references, path)} else {Ok(parse_quote!(()))} - file move: parse_macro.rs → derive.rs
- replacement in fluent_embed/src/derive.rs at line 1
use crate::group::Group;use crate::parse_fluent::{FluentError, ReferenceKind, References};use crate::fluent; - edit in fluent_embed/src/derive.rs at line 8
#[derive(Clone, Copy, Debug)]pub enum ReferenceKind {EnumField,StructField,} - edit in fluent_embed/src/derive.rs at line 15
#[derive(Clone, Debug)]pub struct DeriveContext {pub reference_kind: ReferenceKind,pub valid_references: HashSet<String>,} - replacement in fluent_embed/src/derive.rs at line 22
group: &Group,group: &mut fluent::Group, - replacement in fluent_embed/src/derive.rs at line 24
reference_kind: &References,) -> Result<TokenStream, FluentError> {reference_kind: &DeriveContext,) -> Result<TokenStream, fluent::Error> { - replacement in fluent_embed/src/derive.rs at line 27
let canonical_message = group.canonical_message(id, reference_kind)?;let canonical_message = group.remove_canonical_message(id, reference_kind)?; - replacement in fluent_embed/src/derive.rs at line 30
.additional_messages(id, reference_kind)?.remove_additional_messages(id, reference_kind)? - replacement in fluent_embed/src/derive.rs at line 70
pub fn derive_struct(group: Group,pub fn for_struct(mut group: fluent::Group, - replacement in fluent_embed/src/derive.rs at line 74
) -> Result<TokenStream, FluentError> {) -> Result<TokenStream, fluent::Error> { - replacement in fluent_embed/src/derive.rs at line 77
syn::Fields::Named(named_fields) => References {syn::Fields::Named(named_fields) => DeriveContext { - replacement in fluent_embed/src/derive.rs at line 79
kind: ReferenceKind::StructField,reference_kind: ReferenceKind::StructField, - replacement in fluent_embed/src/derive.rs at line 81
candidates: unique_named_fields(named_fields),valid_references: unique_named_fields(named_fields), - replacement in fluent_embed/src/derive.rs at line 88
expr_for_message(&group, &ident_kebab_case, &references)expr_for_message(&mut group, &ident_kebab_case, &references) - replacement in fluent_embed/src/derive.rs at line 91
pub fn derive_enum(group: Group,pub fn for_enum(mut group: fluent::Group, - replacement in fluent_embed/src/derive.rs at line 94
) -> Result<TokenStream, FluentError> {) -> Result<TokenStream, fluent::Error> { - replacement in fluent_embed/src/derive.rs at line 117
syn::Fields::Named(named_fields) => References {kind: ReferenceKind::EnumField,candidates: unique_named_fields(named_fields),syn::Fields::Named(named_fields) => DeriveContext {reference_kind: ReferenceKind::EnumField,valid_references: unique_named_fields(named_fields), - replacement in fluent_embed/src/derive.rs at line 125
let arm_body = expr_for_message(&group, &variant_kebab_case, &references)?;let arm_body = expr_for_message(&mut group, &variant_kebab_case, &references)?;