Support Typst subdirectories

finchie

Oct 26, 2023, 4:55 PM

GYTRFADRDO4SYXV6V3PEPGGFIRDHQH5YBTKEJCWFAIZ5CX4P46NAC

Dependencies

• [2] 4MMVEN5Y Move generated docs inside of parent `docs` module
• [3] BMG4FSHN Add basic `clap` support
• [4] C73UJ7ZY Create simple `xilem_html` demo
• [5] 2N3KOCP7 Create MVP Pandoc->Rust compiler
• [6] BSJYWOYS Implement MVP Typst embedding
• [7] BA5Y6VSE Output Rust code using `syn`
• [8] ZYNEMGAZ Use generated Typst code from Rust
• [9] HEUMSBES Ouput use statements in generated file

Change contents

• edit in Cargo.lock at line 61
  [3.1837]

  [3.1837]
  + version = "1.3.2"

  + source = "registry+https://github.com/rust-lang/crates.io-index"

  + checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

  +

  + [[package]]

  + name = "bitflags"
• edit in Cargo.lock at line 261
  [3.912]

  [3.912]
  + ]

  +

  + [[package]]

  + name = "patricia_tree"

  + version = "0.6.3"

  + source = "registry+https://github.com/rust-lang/crates.io-index"

  + checksum = "1c8e0b346244f1606d39ec7c47046286cbdcc6553dbdf25b3b9550d5e026f2ed"

  + dependencies = [

  + "bitflags 1.3.2",
• edit in Cargo.lock at line 437
  [3.1000]

  [3.0]
  + "patricia_tree",
• replacement in Cargo.lock at line 634
  [3.8074]→[3.8074:8087](∅→∅)
  − "bitflags",
  [3.8074]

  [3.8087]
  + "bitflags 2.4.1",
• replacement in build.rs at line 3
  [3.360]→[3.3616:3654](∅→∅)
  − use typst_rust_gen::transform_pandoc;
  [3.360]

  [3.360]
  + use typst_rust_gen::output_rust;
• replacement in build.rs at line 23
  [3.952]→[3.952:988](∅→∅)
  − // Set up the staging directory
  [3.952]

  [3.3764]
  + // Set up the output directory
• replacement in build.rs at line 27
  [3.1052]→[3.1052:1124](∅→∅),[3.1204]→[3.1204:1288](∅→∅),[3.1288]→[3.3881:4167](∅→∅)
  − // Build all the files

  − let prefix = PathBuf::from(DOCS_SOURCE);

  −

  − for file in typst_files {

  − let parent = file.path().parent().unwrap();

  −

  − // TODO: Handle nested directories correctly

  − // let remainder = if parent != prefix {

  − // assert!(parent.starts_with(&prefix));

  − // parent.strip_prefix(&prefix).unwrap().to_path_buf()

  − // } else {

  − // PathBuf::new()

  − // };
  [3.1052]

  [3.1550]
  + // Keep track of all the filenames to later construct modules

  + // Each folder is a module, each file is a function

  + // For example: docs/test/hello.typ is docs::test::hello()

  + let filenames: Vec<String> = typst_files

  + .map(|entry| entry.path().to_string_lossy().to_string())

  + .collect();
• replacement in build.rs at line 34
  [3.1551]→[3.4168:4288](∅→∅)
  − // Store everything inside one large autogenerated Rust file

  − let output_path = out_dir.join("docs.rs");
  [3.1551]

  [3.1819]
  + // Store everything inside one large autogenerated Rust file

  + let output_path = out_dir.join("docs.rs");
• replacement in build.rs at line 37
  [3.1820]→[3.1820:1893](∅→∅),[3.1893]→[3.4289:4485](∅→∅),[3.4485]→[3.2167:2173](∅→∅),[3.2167]→[3.2167:2173](∅→∅)
  − std::fs::create_dir_all(output_path.parent().unwrap()).unwrap();

  − // TODO: this will only work for a single file, in the top-level directory

  − let rust_code = transform_pandoc(file.path());

  − std::fs::write(&output_path, rust_code).unwrap();

  − }
  [3.1820]

  [3.2173]
  + let rust_code = output_rust(filenames);

  + std::fs::write(&output_path, rust_code).unwrap();
• replacement in crates/typst_rust_gen/src/lib.rs at line 1
  [3.1210]→[3.1211:1232](∅→∅)
  − use std::path::Path;
  [3.1210]

  [3.35]
  + use std::path::PathBuf;
• edit in crates/typst_rust_gen/src/lib.rs at line 4
  [3.62]

  [3.62]
  + use patricia_tree::StringPatriciaSet;
• edit in crates/typst_rust_gen/src/lib.rs at line 7
  [3.113]

  [3.1232]
  +

  + const DOCS_DIR: &str = "docs";
• edit in crates/typst_rust_gen/src/lib.rs at line 68
  [3.1622]

  [3.1622]
  + }

  + }

  +

  + pub fn output_rust(files: Vec<String>) -> String {

  + let file = syn::File {

  + shebang: None,

  + attrs: Vec::new(),

  + items: vec![files_to_modules(files)],

  + };

  +

  + let output = quote!(#file).to_string();

  + output

  + }

  +

  + fn nth_deepest_module(global_module: &mut syn::ItemMod, limit: usize) -> &mut syn::ItemMod {

  + dbg!(&global_module, limit);

  + let mut prev_deepest = global_module;

  + let mut counter = 0;

  +

  + while counter < limit {

  + prev_deepest = match prev_deepest.content.as_mut().unwrap().1.last_mut().unwrap() {

  + syn::Item::Mod(next) => next,

  + _ => unreachable!(),

  + };

  +

  + counter += 1;

  + }

  +

  + prev_deepest

  + }

  +

  + fn new_module(name: &str) -> syn::ItemMod {

  + syn::ItemMod {

  + attrs: Vec::new(),

  + vis: syn::Visibility::Public(syn::token::Pub::default()),

  + unsafety: None,

  + mod_token: syn::token::Mod::default(),

  + ident: syn::Ident::new(name, proc_macro2::Span::call_site()),

  + content: Some((

  + syn::token::Brace::default(),

  + vec![syn::Item::Use(syn::ItemUse {

  + attrs: Vec::new(),

  + vis: syn::Visibility::Inherited,

  + use_token: syn::token::Use::default(),

  + leading_colon: None,

  + tree: syn::UseTree::Path(syn::UsePath {

  + ident: syn::Ident::new("xilem_html", proc_macro2::Span::call_site()),

  + colon2_token: syn::token::PathSep::default(),

  + tree: Box::new(syn::UseTree::Group(syn::UseGroup {

  + brace_token: syn::token::Brace::default(),

  + items: Punctuated::<syn::UseTree, syn::token::Comma>::from_iter(

  + vec![

  + syn::UseTree::Name(syn::UseName {

  + ident: syn::Ident::new(

  + "elements",

  + proc_macro2::Span::call_site(),

  + ),

  + }),

  + syn::UseTree::Name(syn::UseName {

  + ident: syn::Ident::new(

  + "ViewSequence",

  + proc_macro2::Span::call_site(),

  + ),

  + }),

  + ]

  + .into_iter(),

  + ),

  + })),

  + }),

  + semi_token: syn::token::Semi::default(),

  + })],

  + )),

  + semi: None,
• edit in crates/typst_rust_gen/src/lib.rs at line 142
  [3.1628]

  [3.1628]
  + }

  +

  + fn fs_path_to_module_ident(path: &str) -> String {

  + let file_path = PathBuf::from(path);

  + file_path

  + .file_name()

  + .unwrap()

  + .to_string_lossy()

  + .split('.')

  + .next()

  + .unwrap()

  + .to_string()
• replacement in crates/typst_rust_gen/src/lib.rs at line 156
  [3.1631]→[3.1309:1422](∅→∅),[3.1309]→[3.1309:1422](∅→∅)
  − pub fn transform_pandoc(path: &Path) -> String {

  − let mut pandoc = pandoc::new();

  − pandoc.add_input(path);
  [3.1631]

  [3.1422]
  + fn files_to_modules(mut files: Vec<String>) -> syn::Item {

  + let paths = StringPatriciaSet::from_iter(files.iter());

  + // Sorting the list of filenames allows for easier module creation later

  + files.sort();

  +

  + let mut global = new_module(DOCS_DIR);

  +

  + let mut parent = &mut global;

  + let mut depth = 0;

  +

  + for file in &files {

  + // Get the longest shared prefix by getting the parent directory

  + let file_path = PathBuf::from(&file);

  + let file_parent = file_path.parent().unwrap().to_string_lossy();

  + dbg!(&file_parent);

  + let prefix = paths

  + .get_longest_common_prefix(&file_parent)

  + .unwrap_or(&file_parent);

  + dbg!(prefix);

  + let mut common_prefixed = paths.iter_prefix(prefix);

  + let first_common = &common_prefixed.next().unwrap();

  + let last_common = &common_prefixed.last().unwrap_or(file.clone());

  +

  + // Create Rust AST

  + let pandoc = typst_to_pandoc(&file).iter().map(pandoc_to_xilem).collect();

  + let function = blocks_to_function(&fs_path_to_module_ident(&file), pandoc);

  +

  + // Increase depth if first item in prefix

  + // Make sure not to immediately jump 1 level too deep if there's only 1 file

  + if first_common == file && prefix != DOCS_DIR {

  + depth += 1;

  + parent

  + .content

  + .as_mut()

  + .unwrap()

  + .1

  + .push(syn::Item::Mod(new_module(

  + &file_path

  + .parent()

  + .unwrap()

  + .file_name()

  + .unwrap()

  + .to_string_lossy(),

  + )));

  + parent = nth_deepest_module(&mut global, depth);

  + }

  +

  + // Get a mutable reference to the module children

  + let content = &mut parent.content.as_mut();

  + // Append the function corresponding to our Typst file

  + content

  + .get_or_insert(&mut ((syn::token::Brace::default()), Vec::new()))

  + .1

  + .push(function);

  +

  + // Decrease depth if last item in prefix

  + dbg!(&first_common, &last_common);

  + if last_common == file && depth > 0 {

  + depth -= 1;

  + parent = nth_deepest_module(&mut global, depth);

  + }

  + }

  +

  + syn::Item::Mod(global)

  + }
• edit in crates/typst_rust_gen/src/lib.rs at line 222
  [3.1423]

  [3.1632]
  + fn typst_to_pandoc(filename: &str) -> Vec<pandoc_ast::Block> {

  + let mut pandoc = pandoc::new();

  + pandoc.add_input(filename);
• replacement in crates/typst_rust_gen/src/lib.rs at line 233
  [3.1711]→[3.1711:1781](∅→∅),[3.1781]→[3.1703:1820](∅→∅)
  − let ast = pandoc_ast::Pandoc::from_json(&output);

  − dbg!(&ast);

  −

  − let blocks: Vec<syn::Expr> = ast.blocks.iter().map(transform_block).collect();

  − dbg!(&blocks, blocks.len());
  [3.1711]

  [3.1781]
  + pandoc_ast::Pandoc::from_json(&output).blocks

  + }
• replacement in crates/typst_rust_gen/src/lib.rs at line 236
  [3.1782]→[3.1821:1871](∅→∅)
  − let file = syn::File {

  − shebang: None,
  [3.1782]

  [3.1871]
  + fn blocks_to_function(name: &str, blocks: Vec<syn::Expr>) -> syn::Item {

  + syn::Item::Fn(syn::ItemFn {
• replacement in crates/typst_rust_gen/src/lib.rs at line 239
  [3.1898]→[3.0:21](∅→∅),[3.21]→[2.0:42](∅→∅),[2.42]→[3.63:147](∅→∅),[3.63]→[3.63:147](∅→∅),[3.147]→[2.43:1852](∅→∅)
  − items: vec![

  − syn::Item::Mod(syn::ItemMod {

  − attrs: Vec::new(),

  − vis: syn::Visibility::Inherited,

  − unsafety: None,

  − mod_token: syn::token::Mod::default(),

  − ident: syn::Ident::new("docs", proc_macro2::Span::call_site()),

  − content: Some((syn::token::Brace::default(), vec![

  − syn::Item::Use(syn::ItemUse {

  − attrs: Vec::new(),

  − vis: syn::Visibility::Inherited,

  − use_token: syn::token::Use::default(),

  − leading_colon: None,

  − tree: syn::UseTree::Path(syn::UsePath {

  − ident: syn::Ident::new("xilem_html", proc_macro2::Span::call_site()),

  − colon2_token: syn::token::PathSep::default(),

  − tree: Box::new(syn::UseTree::Group(syn::UseGroup {

  − brace_token: syn::token::Brace::default(),

  − items: Punctuated::<syn::UseTree, syn::token::Comma>::from_iter(

  − vec![

  − syn::UseTree::Name(syn::UseName {

  − ident: syn::Ident::new(

  − "elements",

  − proc_macro2::Span::call_site(),

  − ),

  − }),

  − syn::UseTree::Name(syn::UseName {

  − ident: syn::Ident::new(

  − "ViewSequence",

  − proc_macro2::Span::call_site(),

  − ),

  − }),

  − ]
  [3.1898]

  [2.1852]
  + vis: syn::Visibility::Public(syn::token::Pub::default()),

  + sig: syn::Signature {

  + constness: None,

  + asyncness: None,

  + unsafety: None,

  + abi: None,

  + fn_token: syn::token::Fn::default(),

  + ident: syn::Ident::new(name, proc_macro2::Span::call_site()),

  + generics: syn::Generics { lt_token: None, params: Punctuated::from_iter(vec![syn::GenericParam::Type(syn::TypeParam { attrs: Vec::new(), ident: syn::Ident::new("T", proc_macro2::Span::call_site()), colon_token: None, bounds: Punctuated::new(), eq_token: None, default: None })].into_iter()), gt_token: None, where_clause: None },

  + paren_token: syn::token::Paren::default(),

  + inputs: Punctuated::new(),

  + variadic: None,

  + output: syn::ReturnType::Type(

  + syn::token::RArrow::default(),

  + Box::new(syn::Type::ImplTrait(syn::TypeImplTrait {

  + impl_token: syn::token::Impl::default(),

  + bounds: Punctuated::from_iter(

  + vec![syn::TypeParamBound::Trait(syn::TraitBound {

  + paren_token: None,

  + modifier: syn::TraitBoundModifier::None,

  + lifetimes: None,

  + path: syn::Path {

  + leading_colon: None,

  + segments: Punctuated::from_iter(

  + vec![syn::PathSegment {

  + ident: syn::Ident::new(

  + "ViewSequence",

  + proc_macro2::Span::call_site(),

  + ),

  + arguments: syn::PathArguments::AngleBracketed(

  + syn::AngleBracketedGenericArguments {

  + colon2_token: None,

  + lt_token: syn::token::Lt::default(),

  + args: Punctuated::from_iter(

  + vec![syn::GenericArgument::Type(

  + syn::Type::Path(syn::TypePath {

  + qself: None,

  + path: syn::Path {

  + leading_colon: None,

  + segments: Punctuated::from_iter(vec![syn::PathSegment { ident: syn::Ident::new("T", proc_macro2::Span::call_site()), arguments: syn::PathArguments::None }].into_iter()),

  + },

  + }),

  + )]

  + .into_iter(),

  + ),

  + gt_token: syn::token::Gt::default(),

  + },

  + ),

  + }]
• replacement in crates/typst_rust_gen/src/lib.rs at line 290
  [2.1937]→[2.1937:2135](∅→∅)
  − })),

  − }),

  − semi_token: syn::token::Semi::default(),

  − }),

  − syn::Item::Fn(syn::ItemFn {
  [2.1937]

  [2.2135]
  + },

  + })]

  + .into_iter(),

  + ),

  + })),

  + ),

  + },

  + block: Box::new(syn::Block {

  + brace_token: syn::token::Brace::default(),

  + stmts: vec![syn::Stmt::Expr(

  + syn::Expr::Tuple(syn::ExprTuple {
• replacement in crates/typst_rust_gen/src/lib.rs at line 302
  [2.2174]→[2.2174:6487](∅→∅),[2.6487]→[3.522:589](∅→∅),[3.522]→[3.522:589](∅→∅),[3.589]→[2.6488:6981](∅→∅),[2.6981]→[3.1894:1910](∅→∅),[3.1894]→[3.1894:1910](∅→∅),[3.3059]→[3.3168:3179](∅→∅),[3.3179]→[3.3072:3080](∅→∅),[3.3072]→[3.3072:3080](∅→∅),[3.3080]→[3.3180:3254](∅→∅)
  − vis: syn::Visibility::Public(syn::token::Pub::default()),

  − sig: syn::Signature {

  − constness: None,

  − asyncness: None,

  − unsafety: None,

  − abi: None,

  − fn_token: syn::token::Fn::default(),

  − ident: syn::Ident::new("test", proc_macro2::Span::call_site()),

  − generics: syn::Generics { lt_token: None, params: Punctuated::from_iter(vec![syn::GenericParam::Type(syn::TypeParam { attrs: Vec::new(), ident: syn::Ident::new("T", proc_macro2::Span::call_site()), colon_token: None, bounds: Punctuated::new(), eq_token: None, default: None })].into_iter()), gt_token: None, where_clause: None },

  − paren_token: syn::token::Paren::default(),

  − inputs: Punctuated::new(),

  − variadic: None,

  − output: syn::ReturnType::Type(

  − syn::token::RArrow::default(),

  − Box::new(syn::Type::ImplTrait(syn::TypeImplTrait {

  − impl_token: syn::token::Impl::default(),

  − bounds: Punctuated::from_iter(

  − vec![syn::TypeParamBound::Trait(syn::TraitBound {

  − paren_token: None,

  − modifier: syn::TraitBoundModifier::None,

  − lifetimes: None,

  − path: syn::Path {

  − leading_colon: None,

  − segments: Punctuated::from_iter(

  − vec![syn::PathSegment {

  − ident: syn::Ident::new(

  − "ViewSequence",

  − proc_macro2::Span::call_site(),

  − ),

  − arguments: syn::PathArguments::AngleBracketed(

  − syn::AngleBracketedGenericArguments {

  − colon2_token: None,

  − lt_token: syn::token::Lt::default(),

  − args: Punctuated::from_iter(

  − vec![syn::GenericArgument::Type(

  − syn::Type::Path(syn::TypePath {

  − qself: None,

  − path: syn::Path {

  − leading_colon: None,

  − segments: Punctuated::from_iter(vec![syn::PathSegment { ident: syn::Ident::new("T", proc_macro2::Span::call_site()), arguments: syn::PathArguments::None }].into_iter()),

  − },

  − }),

  − )]

  − .into_iter(),

  − ),

  − gt_token: syn::token::Gt::default(),

  − },

  − ),

  − }]

  − .into_iter(),

  − ),

  − },

  − })]

  − .into_iter(),

  − ),

  − })),

  − ),

  − },

  − block: Box::new(syn::Block {

  − brace_token: syn::token::Brace::default(),

  − stmts: vec![syn::Stmt::Expr(

  − syn::Expr::Tuple(syn::ExprTuple {

  − attrs: Vec::new(),

  − paren_token: syn::token::Paren::default(),

  − elems: Punctuated::from_iter(blocks.into_iter()),

  − }),

  − None,

  − )],

  − }),

  − }),])),

  − semi: None,

  − }),

  − ],

  − };

  −

  − let output = quote!(#file).to_string();

  − dbg!(&output);

  − output
  [2.2174]

  [3.3110]
  + paren_token: syn::token::Paren::default(),

  + elems: Punctuated::from_iter(blocks.into_iter()),

  + }),

  + None,

  + )],

  + }),

  + })
• replacement in crates/typst_rust_gen/src/lib.rs at line 338
  [3.1962]→[3.3799:3860](∅→∅)
  − fn transform_block(block: &pandoc_ast::Block) -> syn::Expr {
  [3.1962]

  [3.3860]
  + fn pandoc_to_xilem(block: &pandoc_ast::Block) -> syn::Expr {
• edit in crates/typst_rust_gen/src/lib.rs at line 374
  [3.4700]→[3.4700:4725](∅→∅)
  − dbg!(&full_literal);
• replacement in crates/typst_rust_gen/examples/basic.rs at line 2
  [3.2864]→[3.2864:2902](∅→∅)
  − use typst_rust_gen::transform_pandoc;
  [3.2864]

  [3.2902]
  + use typst_rust_gen::output_rust;
• replacement in crates/typst_rust_gen/examples/basic.rs at line 5
  [3.2915]→[3.2915:2971](∅→∅)
  − dbg!(transform_pandoc(Path::new("docs/test.typ")));
  [3.2915]

  [3.2971]
  + dbg!(output_rust(Path::new("docs/test.typ")));
• edit in crates/typst_rust_gen/Cargo.toml at line 9
  [3.3134]

  [3.4971]
  + patricia_tree = "0.6.3"