Refactor `fluent_embed` to support structs

finchie

Jul 3, 2024, 5:19 AM

NO3PDO7PY7J3WPADNCS5VD6HKFY63E23I3SDR4DHXNVQJTG27RAAC

Dependencies

• [2] WBI5HFOB Add simple wrapper for `libc::settext()` to query system locale
• [3] P6FW2GGO Remove unnecessary parameters in generated `localize()` function
• [4] 5FIVUZYF Unify `fluent_embed` macro API as `localize()`
• [5] O77KA6C4 Create `fluent_embed` crate
• [6] 2XQ6ZB4W Store multiple locales in a single `Group`
• [7] 4MRF5E76 Generate simple locale matching code in `localize()`
• [8] 3WEPY3OX Add `locale` parameter to derived `localize()` function
• [9] XEEXWJLG Add simple end-to-end test for selectors
• [10] OCR4YRQ2 Parse group from fluent file specified by macro attribute
• [11] UOMQT7LT Add support for cardinal CLDR plural selectors
• [12] BQ6N55O7 Refactor how `Group` stores messages
• [13] 5TEX4MNU Split `fluent_embed` into `group` and `parse` modules
• [14] XGNME3WR Move `Group::derive_enum` to new `crate::parse_macro` module
• [15] QSK7JRBA Add simple `attribute_path` function
• [16] HJMYJDC7 Simplify `fluent_embed::group` module
• [17] SHNZZSZG Create `cli_macros` shim crate
• [18] D652S2N3 Rename `parse` module to `parse_fluent`
• [19] VNSHGQYN Support using glob paths in `localize` macro
• [20] ROSR4HD5 Parse captured glob as locale
• [*] UKFEFT6L Create basic `Output` proc-macro

Change contents

• replacement in fluent_embed/src/parse_macro.rs at line 1
  [4.39]→[4.0:31](∅→∅)
  − use std::collections::HashMap;
  [4.39]

  [4.31]
  + use crate::{group::Group, parse_fluent::ReferenceKind};
• replacement in fluent_embed/src/parse_macro.rs at line 3
  [4.25]→[4.40:103](∅→∅),[4.32]→[4.40:103](∅→∅),[4.39]→[4.40:103](∅→∅),[4.103]→[4.33:66](∅→∅)
  − use crate::group::Group;

  − use heck::{ToKebabCase, ToSnakeCase};

  − use icu_locid::{locale, Locale};
  [4.32]

  [4.0]
  + use heck::ToKebabCase;
• replacement in fluent_embed/src/parse_macro.rs at line 5
  [4.30]→[4.103:137](∅→∅),[4.103]→[4.103:137](∅→∅)
  − use quote::{format_ident, quote};
  [4.30]

  [4.31]
  + use quote::quote;
• replacement in fluent_embed/src/parse_macro.rs at line 8
  [4.138]→[4.138:199](∅→∅),[4.199]→[4.65:200](∅→∅),[4.200]→[3.0:59](∅→∅)
  − pub fn derive_enum(

  − group: Group,

  − ident: syn::Ident,

  − variants: Punctuated<syn::Variant, syn::token::Comma>,

  − ) -> TokenStream {

  − let mut idents = Vec::with_capacity(variants.len());

  − let mut messages = Vec::with_capacity(variants.len());
  [4.138]

  [4.397]
  + fn expr_for_message(group: &Group, id: &str, reference_kind: ReferenceKind) -> TokenStream {

  + let canonical_locale = group.canonical_locale().id.to_string();

  + let canonical_message = group.canonical_message(id, reference_kind);
• replacement in fluent_embed/src/parse_macro.rs at line 12
  [4.398]→[4.260:290](∅→∅),[4.290]→[4.438:512](∅→∅),[4.438]→[4.438:512](∅→∅)
  − for variant in variants {

  − let kebab_case_ident = variant.ident.to_string().to_kebab_case();
  [4.398]

  [4.663]
  + let (additional_locales, additional_messages): (Vec<_>, Vec<_>) =

  + group.additional_messages(id, reference_kind).unzip();

  + let additional_locales = additional_locales

  + .iter()

  + .map(|locale| locale.id.to_string())

  + .collect::<Vec<_>>();
• edit in fluent_embed/src/parse_macro.rs at line 19
  [4.664]→[4.664:1306](∅→∅),[4.1306]→[4.1306:1307](∅→∅),[4.1307]→[3.60:117](∅→∅),[4.57]→[4.1348:1354](∅→∅),[3.117]→[4.1348:1354](∅→∅),[4.128]→[4.1348:1354](∅→∅),[4.191]→[4.1348:1354](∅→∅),[4.275]→[4.1348:1354](∅→∅),[4.1348]→[4.1348:1354](∅→∅),[4.1354]→[4.1354:1355](∅→∅),[4.1355]→[3.118:269](∅→∅),[3.269]→[3.269:270](∅→∅)
  − let variant_ident = variant.ident;

  − idents.push(match variant.fields {

  − syn::Fields::Named(fields) => {

  − // Get the name of each field for pattern-matching

  − let field_idents = fields

  − .named

  − .iter()

  − .map(|field| field.ident.as_ref().unwrap())

  − .map(|ident| format_ident!("{}", ident.to_string().to_snake_case()));

  −

  − quote!(#variant_ident { #(#field_idents),* })

  − }

  − syn::Fields::Unnamed(_) => todo!(),

  − syn::Fields::Unit => quote!(#variant_ident),

  − });

  −

  − messages.push(group.message(&kebab_case_ident));

  − }

  −

  − let extra_locales = group.extra_locales().map(|locale| locale.id.to_string());

  − let canonical_locale = group.canonical_locale().id.to_string();

  −
• replacement in fluent_embed/src/parse_macro.rs at line 20
  [4.1368]→[4.1368:1390](∅→∅),[4.1390]→[3.271:625](∅→∅)
  − impl #ident {

  − fn localize(&self) -> String {

  − // Find the appropriate locale to use

  − let extra_locales = [#(::icu_locid::langid!(#extra_locales)),*];

  − let canonical_locale = ::icu_locid::langid!(#canonical_locale);

  − let locale = ::locale_select::match_locales(&extra_locales, &canonical_locale);
  [4.1368]

  [3.625]
  + // Find the appropriate locale to use

  + let additional_locales = [#(::icu_locid::langid!(#additional_locales)),*];

  + let canonical_locale = ::icu_locid::langid!(#canonical_locale);

  + let locale = ::locale_select::match_locales(&additional_locales, &canonical_locale);
• replacement in fluent_embed/src/parse_macro.rs at line 25
  [3.626]→[3.626:994](∅→∅)
  − // TODO: this shouldn't be generated on every call

  − // TODO: handle different rule types according to fluent code (not just cardinal)

  − let plural_rule_type = ::icu_plurals::PluralRuleType::Cardinal;

  − let plural_rules = ::icu_plurals::PluralRules::try_new(&locale.clone().into(), plural_rule_type).unwrap();
  [3.626]

  [4.712]
  + // TODO: handle different rule types according to fluent code (not just cardinal)

  + let plural_rule_type = ::icu_plurals::PluralRuleType::Cardinal;

  + let plural_rules = ::icu_plurals::PluralRules::try_new(&locale.clone().into(), plural_rule_type).unwrap();
• replacement in fluent_embed/src/parse_macro.rs at line 29
  [4.713]→[3.995:1051](∅→∅),[4.174]→[4.1476:1505](∅→∅),[4.760]→[4.1476:1505](∅→∅),[3.1051]→[4.1476:1505](∅→∅),[4.1476]→[4.1476:1505](∅→∅),[4.1505]→[3.1052:1104](∅→∅),[4.823]→[4.1557:1589](∅→∅),[3.1104]→[4.1557:1589](∅→∅),[4.1557]→[4.1557:1589](∅→∅),[4.1589]→[4.26:36](∅→∅)
  − // Match the information to the message

  − match self {

  − #(Self::#idents => #messages),*

  − }

  − }

  − }
  [4.713]

  [4.36]
  + #(if locale.normalizing_eq(#additional_locales) { return #additional_messages })else*

  + #canonical_message
• replacement in fluent_embed/src/parse_macro.rs at line 34
  [4.45]→[4.192:254](∅→∅),[4.67]→[4.353:469](∅→∅),[4.254]→[4.353:469](∅→∅),[4.353]→[4.353:469](∅→∅),[4.469]→[4.68:115](∅→∅)
  − pub fn attribute_groups(path_literal: syn::LitStr) -> Group {

  − // Read the fluent file at the given path

  − let manifest_root = std::env::var("CARGO_MANIFEST_DIR").unwrap();

  − let attribute_glob = path_literal.value();
  [4.45]

  [4.115]
  + // TODO: check that the fields in fluent source reference fields that exist

  + pub fn derive_struct(group: Group, ident: &syn::Ident) -> TokenStream {

  + let ident_kebab_case = ident.to_string().to_kebab_case();

  + expr_for_message(&group, &ident_kebab_case, ReferenceKind::StructField)

  + }
• replacement in fluent_embed/src/parse_macro.rs at line 40
  [4.116]→[4.255:295](∅→∅)
  − let mut resources = HashMap::new();
  [4.116]

  [4.149]
  + pub fn derive_enum(

  + group: Group,

  + enum_variants: &Punctuated<syn::Variant, syn::token::Comma>,

  + ) -> TokenStream {

  + // let mut match_arms: HashMap<String, syn::ExprBlock> = HashMap::with_capacity(enum_variants.len());

  + let mut match_arms: Vec<TokenStream> = Vec::with_capacity(enum_variants.len());
• replacement in fluent_embed/src/parse_macro.rs at line 47
  [4.150]→[4.150:438](∅→∅)
  − let glob = wax::Glob::new(&attribute_glob).unwrap();

  − for potential_entry in glob.walk(&manifest_root) {

  − // TODO: this assumes that the locale is the first capture

  − let entry = potential_entry.unwrap();

  − let captured_locale = entry.matched().get(1).unwrap();
  [4.150]

  [4.24]
  + for enum_variant in enum_variants {

  + let variant_kebab_case = enum_variant.ident.to_string().to_kebab_case();

  + let variant_pascal_case = &enum_variant.ident;
• replacement in fluent_embed/src/parse_macro.rs at line 51
  [4.25]→[4.25:253](∅→∅)
  − // Captured directories may suffix with a `/`

  − let stripped_locale = captured_locale.strip_suffix('/').unwrap_or(captured_locale);

  − let locale = Locale::try_from_bytes(stripped_locale.as_bytes()).unwrap();
  [4.25]

  [4.82]
  + // Destructure fields of the enum variant

  + // E.g. for the variant:

  + // Emails { unread_emails: u64 }

  + // Create the expression:

  + // Self::Emails { unread_emails }

  + let destructuring_pattern = match &enum_variant.fields {

  + syn::Fields::Named(named_fields) => {

  + let named_field_idents = named_fields.named.iter().map(|field| &field.ident);

  + quote!(#variant_pascal_case { #(#named_field_idents),* })

  + }

  + syn::Fields::Unnamed(_) => todo!(),

  + syn::Fields::Unit => todo!(),

  + };
• replacement in fluent_embed/src/parse_macro.rs at line 65
  [4.83]→[4.470:511](∅→∅),[4.511]→[4.254:332](∅→∅),[4.332]→[4.594:673](∅→∅),[4.594]→[4.594:673](∅→∅),[4.673]→[4.296:340](∅→∅)
  − // Parse the file into a `Group`

  − let fluent_contents = std::fs::read_to_string(entry.path()).unwrap();

  − let resource = fluent_syntax::parser::parse(fluent_contents).unwrap();

  − resources.insert(locale, resource);
  [4.83]

  [4.754]
  + let arm_body = expr_for_message(&group, &variant_kebab_case, ReferenceKind::EnumField);

  + match_arms.push(quote!(Self::#destructuring_pattern => { #arm_body }));
• edit in fluent_embed/src/parse_macro.rs at line 68
  [4.760]→[4.368:369](∅→∅),[4.786]→[4.368:369](∅→∅),[4.368]→[4.368:369](∅→∅),[4.369]→[4.341:385](∅→∅),[4.385]→[4.797:799](∅→∅),[4.772]→[4.797:799](∅→∅),[4.797]→[4.797:799](∅→∅)
  −

  − Group::new(locale!("en-US"), resources)

  − }
• replacement in fluent_embed/src/parse_macro.rs at line 69
  [4.328]→[4.800:941](∅→∅),[4.941]→[4.773:814](∅→∅),[4.814]→[4.386:427](∅→∅)
  − pub fn localize(

  − path: syn::LitStr,

  − ident: syn::Ident,

  − variants: Punctuated<syn::Variant, syn::token::Comma>,

  − ) -> TokenStream {

  − let groups = attribute_groups(path);

  − derive_enum(groups, ident, variants)
  [4.328]

  [4.1605]
  + quote! {

  + match self {

  + #(#match_arms),*

  + }

  + }
• replacement in fluent_embed/src/parse_fluent.rs at line 8
  [4.239]→[4.219:283](∅→∅)
  − pub(crate) fn pattern(pattern: &Pattern<String>) -> syn::Expr {
  [4.239]

  [4.352]
  + #[derive(Clone, Copy, Debug)]

  + pub enum ReferenceKind {

  + EnumField,

  + StructField,

  + }

  +

  + pub(crate) fn pattern(pattern: &Pattern<String>, reference_kind: ReferenceKind) -> syn::Expr {
• replacement in fluent_embed/src/parse_fluent.rs at line 22
  [4.640]→[4.121:189](∅→∅)
  − let expression = placeable_expression(&expression);
  [4.640]

  [4.714]
  + let expression = placeable_expression(&expression, reference_kind);
• replacement in fluent_embed/src/parse_fluent.rs at line 34
  [4.1019]→[4.284:356](∅→∅)
  − fn placeable_expression(expression: &Expression<String>) -> syn::Expr {
  [4.1019]

  [4.1133]
  + fn placeable_expression(expression: &Expression<String>, reference_kind: ReferenceKind) -> syn::Expr {
• replacement in fluent_embed/src/parse_fluent.rs at line 37
  [4.1211]→[4.292:424](∅→∅)
  − let target = inline_expression(selector);

  − let arms: Vec<syn::Arm> = variants.iter().map(variant).collect();
  [4.1211]

  [4.1367]
  + let target = inline_expression(selector, reference_kind);

  + let arms: Vec<syn::Arm> = variants.iter().map(|item| variant(item, reference_kind)).collect();
• replacement in fluent_embed/src/parse_fluent.rs at line 41
  [4.1395]→[4.425:485](∅→∅)
  − match plural_rules.category_for(*#target) {
  [4.1395]

  [4.485]
  + match plural_rules.category_for(#target) {
• replacement in fluent_embed/src/parse_fluent.rs at line 47
  [4.1541]→[4.517:590](∅→∅)
  − Expression::Inline(expression) => inline_expression(expression),
  [4.1541]

  [4.1620]
  + Expression::Inline(expression) => inline_expression(expression, reference_kind),
• replacement in fluent_embed/src/parse_fluent.rs at line 51
  [4.1629]→[4.357:432](∅→∅)
  − fn inline_expression(expression: &InlineExpression<String>) -> syn::Expr {
  [4.1629]

  [4.1746]
  + fn inline_expression(expression: &InlineExpression<String>, reference_kind: ReferenceKind) -> syn::Expr {
• replacement in fluent_embed/src/parse_fluent.rs at line 66
  [4.2466]→[4.2466:2499](∅→∅)
  − parse_quote!(#ident)
  [4.2466]

  [4.2499]
  + match reference_kind {

  + ReferenceKind::EnumField => parse_quote!(*#ident),

  + ReferenceKind::StructField => parse_quote!(self.#ident),

  + }

  +
• replacement in fluent_embed/src/parse_fluent.rs at line 79
  [4.2593]→[4.433:485](∅→∅)
  − fn variant(variant: &Variant<String>) -> syn::Arm {
  [4.2593]

  [4.696]
  + fn variant(variant: &Variant<String>, reference_kind: ReferenceKind) -> syn::Arm {
• replacement in fluent_embed/src/parse_fluent.rs at line 81
  [4.746]→[4.746:786](∅→∅)
  − let body = pattern(&variant.value);
  [4.746]

  [4.2775]
  + let body = pattern(&variant.value, reference_kind);
• replacement in fluent_embed/src/parse_fluent.rs at line 105
  [4.551]→[4.551:615](∅→∅)
  − pub(crate) fn message(message: &Message<String>) -> syn::Expr {
  [4.551]

  [4.615]
  + pub(crate) fn message(message: &Message<String>, reference_kind: ReferenceKind) -> syn::Expr {
• replacement in fluent_embed/src/parse_fluent.rs at line 107
  [4.665]→[4.665:688](∅→∅)
  − pattern(value)
  [4.665]

  [4.688]
  + pattern(value, reference_kind)
• edit in fluent_embed/src/lib.rs at line 1
  [4.74]

  [4.3464]
  + use crate::group::Group;

  + use std::collections::HashMap;

  +

  + use icu_locid::{locale, Locale};

  + use proc_macro2::TokenStream;

  + use quote::quote;

  + use syn::DeriveInput;

  +
• replacement in fluent_embed/src/lib.rs at line 13
  [4.1387]→[4.1022:1053](∅→∅)
  − pub use parse_macro::localize;
  [4.1387]
  + pub fn attribute_groups(path_literal: &syn::LitStr) -> Group {

  + // Read the fluent file at the given path

  + let manifest_root = std::env::var("CARGO_MANIFEST_DIR").unwrap();

  + let attribute_glob = path_literal.value();

  +

  + let mut resources = HashMap::new();

  +

  + let glob = wax::Glob::new(&attribute_glob).unwrap();

  + for potential_entry in glob.walk(&manifest_root) {

  + // TODO: this assumes that the locale is the first capture

  + let entry = potential_entry.unwrap();

  + let captured_locale = entry.matched().get(1).unwrap();

  +

  + // Captured directories may have a suffix of `/`

  + let stripped_locale = captured_locale.strip_suffix('/').unwrap_or(captured_locale);

  + let locale = Locale::try_from_bytes(stripped_locale.as_bytes()).unwrap();

  +

  + // Parse the file into a `Group`

  + let fluent_contents = std::fs::read_to_string(entry.path()).unwrap();

  + let resource = fluent_syntax::parser::parse(fluent_contents).unwrap();

  + resources.insert(locale, resource);

  + }

  +

  + Group::new(locale!("en-US"), resources)

  + }

  +

  + pub fn localize(path: &syn::LitStr, derive_input: &DeriveInput) -> TokenStream {

  + let group = attribute_groups(path);

  +

  + let body = match &derive_input.data {

  + syn::Data::Struct(_struct_data) => parse_macro::derive_struct(group, &derive_input.ident),

  + syn::Data::Enum(enum_data) => parse_macro::derive_enum(group, &enum_data.variants),

  + syn::Data::Union(_) => todo!(),

  + };

  +

  + let ident = &derive_input.ident;

  + quote! {

  + impl #ident {

  + // TODO: most of this shouldn't be generated on every call

  + fn localize(&self) -> String {

  + #body

  + }

  + }

  + }

  + }
• edit in fluent_embed/src/group.rs at line 1
  [4.3543]

  [4.428]
  + use crate::parse_fluent::{self, ReferenceKind};
• edit in fluent_embed/src/group.rs at line 6
  [4.417]→[4.175:197](∅→∅)
  − use syn::parse_quote;
• replacement in fluent_embed/src/group.rs at line 68
  [4.1898]→[4.1236:1351](∅→∅)
  − /// Returns an iterator over the localized messages paired with the relevant locale

  − fn messages_in_column(
  [4.1898]

  [4.1351]
  + fn message_column(&self, id: &str) -> usize {

  + self.canonical_messages

  + .iter()

  + .position(|message| message.id.name == id)

  + .expect("Message id must be valid")

  + }

  +

  + pub fn canonical_locale(&self) -> &Locale {

  + &self.canonical_locale

  + }

  +

  + pub fn canonical_message(&self, id: &str, reference_kind: ReferenceKind) -> syn::Expr {

  + let message_column = self.message_column(id);

  + let message = &self.canonical_messages[message_column];

  + parse_fluent::message(message, reference_kind)

  + }

  +

  + pub fn additional_messages(
• replacement in fluent_embed/src/group.rs at line 87
  [4.1366]→[4.1366:1458](∅→∅)
  − message_column: usize,

  − ) -> impl Iterator<Item = (&Locale, &Message<String>)> {
  [4.1366]

  [4.1458]
  + id: &str,

  + reference_kind: ReferenceKind,

  + ) -> impl Iterator<Item = (&Locale, syn::Expr)> {

  + let message_column = self.message_column(id);

  +
• replacement in fluent_embed/src/group.rs at line 98
  [4.1665]→[4.1665:1729](∅→∅)
  − .map(|message| (&locale_group.locale, message))
  [4.1665]

  [4.1729]
  + .map(|message: &Message<String>| {

  + (

  + &locale_group.locale,

  + parse_fluent::message(message, reference_kind),

  + )

  + })
• edit in fluent_embed/src/group.rs at line 106
  [4.1114]→[4.1114:1171](∅→∅),[4.1893]→[4.1893:2083](∅→∅),[4.2199]→[4.2199:2622](∅→∅),[4.2622]→[3.1105:1210](∅→∅),[3.1210]→[4.2622:2757](∅→∅),[4.2622]→[4.2622:2757](∅→∅),[4.2757]→[3.1211:1280](∅→∅),[3.1280]→[4.2852:2863](∅→∅),[4.2852]→[4.2852:2863](∅→∅),[4.1291]→[4.2028:2034](∅→∅),[4.2863]→[4.2028:2034](∅→∅),[4.2028]→[4.2028:2034](∅→∅),[4.2034]→[4.824:825](∅→∅),[4.825]→[3.1281:1510](∅→∅),[3.1510]→[4.1155:1161](∅→∅),[4.1155]→[4.1155:1161](∅→∅)
  −

  − pub fn message(&self, id: &str) -> syn::ExprBlock {

  − let message_column = self

  − .canonical_messages

  − .iter()

  − .position(|message| message.id.name == id)

  − .expect("Message id must be valid");

  −

  − let additional_locale_names = self

  − .messages_in_column(message_column)

  − .map(|(locale, _message)| locale.to_string())

  − .map(|locale_string| syn::LitStr::new(&locale_string, proc_macro2::Span::call_site()));

  −

  − let additional_locale_messages = self

  − .messages_in_column(message_column)

  − .map(|(_locale, message)| crate::parse_fluent::message(message));

  −

  − let canonical_message = crate::parse_fluent::message(&self.canonical_messages[message_column]);

  −

  − parse_quote! {{

  − #(if locale.normalizing_eq(#additional_locale_names) { return #additional_locale_messages })else*

  −

  − else {

  − #canonical_message

  − }

  − }}

  − }

  −

  − pub fn canonical_locale(&self) -> &Locale {

  − &self.canonical_locale

  − }

  −

  − pub fn extra_locales(&self) -> impl Iterator<Item = &Locale> {

  − self.extra_locales.iter().map(| LocaleGroup { locale, .. }| locale)

  − }
• replacement in cli_macros/src/lib.rs at line 5
  [4.178]→[4.178:206](∅→∅)
  − use syn::parse_macro_input;
  [4.178]

  [4.206]
  + use syn::{parse_macro_input, DeriveInput};
• edit in cli_macros/src/lib.rs at line 9
  [4.307]

  [4.307]
  + let original_item = proc_macro2::TokenStream::from(item.clone());
• replacement in cli_macros/src/lib.rs at line 11
  [4.378]→[4.378:441](∅→∅)
  − let parsed_enum: syn::ItemEnum = parse_macro_input!(item);
  [4.378]

  [4.441]
  + let parsed_input = parse_macro_input!(item as DeriveInput);
• replacement in cli_macros/src/lib.rs at line 13
  [4.442]→[4.442:600](∅→∅)
  − let enum_impl = fluent_embed::localize(

  − parsed_attribute.clone(),

  − parsed_enum.ident.clone(),

  − parsed_enum.variants.clone(),

  − );
  [4.442]

  [4.600]
  + let implementation = fluent_embed::localize(&parsed_attribute, &parsed_input);
• replacement in cli_macros/src/lib.rs at line 16
  [4.614]→[4.614:635](∅→∅)
  − #parsed_enum
  [4.614]

  [4.635]
  + #original_item
• replacement in cli_macros/src/lib.rs at line 18
  [4.636]→[4.636:655](∅→∅)
  − #enum_impl
  [4.636]

  [4.655]
  + #implementation
• edit in cli_macros/Cargo.toml at line 14
  [4.888]

  [4.888]
  + proc-macro2 = "1.0.78"

  + quote = "1.0.35"
• edit in cli_macros/Cargo.toml at line 17
  [4.954]→[4.954:971](∅→∅)
  − quote = "1.0.35"
• edit in Cargo.lock at line 34
  [2.18]

  [4.1208]
  + "proc-macro2",