finchie/new_manual - Change GYTRFADRDO4SYXV6V3PEPGGFIRDHQH5YBTKEJCWFAIZ5CX4P46NAC

Support Typst subdirectories

Does not include support for directories without direct children, but that should be fixable. Apart from that, mostly robustness work then it should be a good time to start adding support for more Typst functionality.

Created by finchie on October 26, 2023

GYTRFADRDO4SYXV6V3PEPGGFIRDHQH5YBTKEJCWFAIZ5CX4P46NAC

Dependencies

In channels

main

Change contents

Insertion in Cargo.lock at line 61 [3.1510]

[3.1837]

version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"

Insertion in Cargo.lock at line 261 [3.1510]

[4.912]

]
[[package]]
name = "patricia_tree"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c8e0b346244f1606d39ec7c47046286cbdcc6553dbdf25b3b9550d5e026f2ed"
dependencies = [
 "bitflags 1.3.2",

Insertion in Cargo.lock at line 437 [3.1510]
[4.1000]
[5.0]
```
 "patricia_tree",
```
Replacement in Cargo.lock at line 634 [3.1510]
B:BD[3.8074] → [3.8074:8087]
```
 "bitflags",
```
[3.8074]
[3.8087]
```
 "bitflags 2.4.1",
```
Replacement in build.rs at line 3 [6.302]
B:BD[6.360] → [7.3616:3654]
```
use typst_rust_gen::transform_pandoc;
```
[6.360]
[6.360]
```
use typst_rust_gen::output_rust;
```
Replacement in build.rs at line 23 [6.302]
B:BD[6.952] → [6.952:988]
```
    // Set up the staging directory
```
[6.952]
[7.3764]
```
    // Set up the output directory
```

Replacement in build.rs at line 27 [6.302]

B:BD[6.1052] → [6.1052:1124]

B:BD[6.1204] → [6.1204:1288]

B:BD[6.1288] → [7.3881:4167]

    // Build all the files
    let prefix = PathBuf::from(DOCS_SOURCE);
    for file in typst_files {
        let parent = file.path().parent().unwrap();
        // TODO: Handle nested directories correctly
        // let remainder = if parent != prefix {
        //     assert!(parent.starts_with(&prefix));
        //     parent.strip_prefix(&prefix).unwrap().to_path_buf()
        // } else {
        //     PathBuf::new()
        // };

[6.1052]

[6.1550]

    // Keep track of all the filenames to later construct modules
    // Each folder is a module, each file is a function
    // For example: docs/test/hello.typ is docs::test::hello()
    let filenames: Vec<String> = typst_files
        .map(|entry| entry.path().to_string_lossy().to_string())
        .collect();

Replacement in build.rs at line 34 [6.302]

B:BD[6.1551] → [7.4168:4288]

        // Store everything inside one large autogenerated Rust file
        let output_path = out_dir.join("docs.rs");

[6.1551]

[6.1819]

    // Store everything inside one large autogenerated Rust file
    let output_path = out_dir.join("docs.rs");

Replacement in build.rs at line 37 [6.302]

B:BD[6.1820] → [6.1820:1893]

B:BD[6.1893] → [7.4289:4485]

∅:D[7.4485] → [6.2167:2173]

B:BD[6.2167] → [6.2167:2173]

        std::fs::create_dir_all(output_path.parent().unwrap()).unwrap();
        // TODO: this will only work for a single file, in the top-level directory
        let rust_code = transform_pandoc(file.path());
        std::fs::write(&output_path, rust_code).unwrap();
    }

[6.1820]

[6.2173]

    let rust_code = output_rust(filenames);
    std::fs::write(&output_path, rust_code).unwrap();

Replacement in crates/typst_rust_gen/src/lib.rs at line 1 [4.1210]
B:BD[4.1210] → [4.1211:1232]
```
use std::path::Path;
```
[4.1210]
[5.35]
```
use std::path::PathBuf;
```
Insertion in crates/typst_rust_gen/src/lib.rs at line 4 [4.1210]
[5.62]
[5.62]
```
use patricia_tree::StringPatriciaSet;
```
Insertion in crates/typst_rust_gen/src/lib.rs at line 7 [4.1210]
[5.113]
[4.1232]
```
const DOCS_DIR: &str = "docs";
```

Insertion in crates/typst_rust_gen/src/lib.rs at line 68 [4.1210]

[5.1622]

    }
}
pub fn output_rust(files: Vec<String>) -> String {
    let file = syn::File {
        shebang: None,
        attrs: Vec::new(),
        items: vec![files_to_modules(files)],
    };
    let output = quote!(#file).to_string();
    output
}
fn nth_deepest_module(global_module: &mut syn::ItemMod, limit: usize) -> &mut syn::ItemMod {
    dbg!(&global_module, limit);
    let mut prev_deepest = global_module;
    let mut counter = 0;
    while counter < limit {
        prev_deepest = match prev_deepest.content.as_mut().unwrap().1.last_mut().unwrap() {
            syn::Item::Mod(next) => next,
            _ => unreachable!(),
        };
        counter += 1;
    }
    prev_deepest
}
fn new_module(name: &str) -> syn::ItemMod {
    syn::ItemMod {
        attrs: Vec::new(),
        vis: syn::Visibility::Public(syn::token::Pub::default()),
        unsafety: None,
        mod_token: syn::token::Mod::default(),
        ident: syn::Ident::new(name, proc_macro2::Span::call_site()),
        content: Some((
            syn::token::Brace::default(),
            vec![syn::Item::Use(syn::ItemUse {
                attrs: Vec::new(),
                vis: syn::Visibility::Inherited,
                use_token: syn::token::Use::default(),
                leading_colon: None,
                tree: syn::UseTree::Path(syn::UsePath {
                    ident: syn::Ident::new("xilem_html", proc_macro2::Span::call_site()),
                    colon2_token: syn::token::PathSep::default(),
                    tree: Box::new(syn::UseTree::Group(syn::UseGroup {
                        brace_token: syn::token::Brace::default(),
                        items: Punctuated::<syn::UseTree, syn::token::Comma>::from_iter(
                            vec![
                                syn::UseTree::Name(syn::UseName {
                                    ident: syn::Ident::new(
                                        "elements",
                                        proc_macro2::Span::call_site(),
                                    ),
                                }),
                                syn::UseTree::Name(syn::UseName {
                                    ident: syn::Ident::new(
                                        "ViewSequence",
                                        proc_macro2::Span::call_site(),
                                    ),
                                }),
                            ]
                            .into_iter(),
                        ),
                    })),
                }),
                semi_token: syn::token::Semi::default(),
            })],
        )),
        semi: None,

Insertion in crates/typst_rust_gen/src/lib.rs at line 142 [4.1210]

[5.1628]

}
fn fs_path_to_module_ident(path: &str) -> String {
    let file_path = PathBuf::from(path);
    file_path
        .file_name()
        .unwrap()
        .to_string_lossy()
        .split('.')
        .next()
        .unwrap()
        .to_string()

Replacement in crates/typst_rust_gen/src/lib.rs at line 156 [4.1210]

∅:D[5.1631] → [4.1309:1422]

B:BD[4.1309] → [4.1309:1422]

pub fn transform_pandoc(path: &Path) -> String {
    let mut pandoc = pandoc::new();
    pandoc.add_input(path);

[5.1631]

[4.1422]

fn files_to_modules(mut files: Vec<String>) -> syn::Item {
    let paths = StringPatriciaSet::from_iter(files.iter());
    // Sorting the list of filenames allows for easier module creation later
    files.sort();
    let mut global = new_module(DOCS_DIR);
    let mut parent = &mut global;
    let mut depth = 0;
    for file in &files {
        // Get the longest shared prefix by getting the parent directory
        let file_path = PathBuf::from(&file);
        let file_parent = file_path.parent().unwrap().to_string_lossy();
        dbg!(&file_parent);
        let prefix = paths
            .get_longest_common_prefix(&file_parent)
            .unwrap_or(&file_parent);
        dbg!(prefix);
        let mut common_prefixed = paths.iter_prefix(prefix);
        let first_common = &common_prefixed.next().unwrap();
        let last_common = &common_prefixed.last().unwrap_or(file.clone());
        // Create Rust AST
        let pandoc = typst_to_pandoc(&file).iter().map(pandoc_to_xilem).collect();
        let function = blocks_to_function(&fs_path_to_module_ident(&file), pandoc);
        // Increase depth if first item in prefix
        // Make sure not to immediately jump 1 level too deep if there's only 1 file
        if first_common == file && prefix != DOCS_DIR {
            depth += 1;
            parent
                .content
                .as_mut()
                .unwrap()
                .1
                .push(syn::Item::Mod(new_module(
                    &file_path
                        .parent()
                        .unwrap()
                        .file_name()
                        .unwrap()
                        .to_string_lossy(),
                )));
            parent = nth_deepest_module(&mut global, depth);
        }
        // Get a mutable reference to the module children
        let content = &mut parent.content.as_mut();
        // Append the function corresponding to our Typst file
        content
            .get_or_insert(&mut ((syn::token::Brace::default()), Vec::new()))
            .1
            .push(function);
        // Decrease depth if last item in prefix
        dbg!(&first_common, &last_common);
        if last_common == file && depth > 0 {
            depth -= 1;
            parent = nth_deepest_module(&mut global, depth);
        }
    }
    syn::Item::Mod(global)
}

Insertion in crates/typst_rust_gen/src/lib.rs at line 222 [4.1210]

[4.1423]

[5.1632]

fn typst_to_pandoc(filename: &str) -> Vec<pandoc_ast::Block> {
    let mut pandoc = pandoc::new();
    pandoc.add_input(filename);

Replacement in crates/typst_rust_gen/src/lib.rs at line 233 [4.1210]

B:BD[4.1711] → [4.1711:1781]

B:BD[4.1781] → [5.1703:1820]

    let ast = pandoc_ast::Pandoc::from_json(&output);
    dbg!(&ast);
    let blocks: Vec<syn::Expr> = ast.blocks.iter().map(transform_block).collect();
    dbg!(&blocks, blocks.len());

[4.1711]

[4.1781]

    pandoc_ast::Pandoc::from_json(&output).blocks
}

Replacement in crates/typst_rust_gen/src/lib.rs at line 236 [4.1210]

B:BD[4.1782] → [5.1821:1871]

    let file = syn::File {
        shebang: None,

[4.1782]

[5.1871]

fn blocks_to_function(name: &str, blocks: Vec<syn::Expr>) -> syn::Item {
    syn::Item::Fn(syn::ItemFn {

Replacement in crates/typst_rust_gen/src/lib.rs at line 239 [4.1210]

B:BD[5.1898] → [8.0:21]

B:BD[8.21] → [2.0:42]

∅:D[2.42] → [8.63:147]

B:BD[8.63] → [8.63:147]

B:BD[8.147] → [2.43:1852]

        items: vec![
            syn::Item::Mod(syn::ItemMod {
                attrs: Vec::new(),
                vis: syn::Visibility::Inherited,
                unsafety: None,
                mod_token: syn::token::Mod::default(),
                ident: syn::Ident::new("docs", proc_macro2::Span::call_site()),
                content: Some((syn::token::Brace::default(), vec![
                    syn::Item::Use(syn::ItemUse {
                        attrs: Vec::new(),
                        vis: syn::Visibility::Inherited,
                        use_token: syn::token::Use::default(),
                        leading_colon: None,
                        tree: syn::UseTree::Path(syn::UsePath {
                            ident: syn::Ident::new("xilem_html", proc_macro2::Span::call_site()),
                            colon2_token: syn::token::PathSep::default(),
                            tree: Box::new(syn::UseTree::Group(syn::UseGroup {
                                brace_token: syn::token::Brace::default(),
                                items: Punctuated::<syn::UseTree, syn::token::Comma>::from_iter(
                                    vec![
                                        syn::UseTree::Name(syn::UseName {
                                            ident: syn::Ident::new(
                                                "elements",
                                                proc_macro2::Span::call_site(),
                                            ),
                                        }),
                                        syn::UseTree::Name(syn::UseName {
                                            ident: syn::Ident::new(
                                                "ViewSequence",
                                                proc_macro2::Span::call_site(),
                                            ),
                                        }),
                                    ]

[5.1898]

[2.1852]

        vis: syn::Visibility::Public(syn::token::Pub::default()),
        sig: syn::Signature {
            constness: None,
            asyncness: None,
            unsafety: None,
            abi: None,
            fn_token: syn::token::Fn::default(),
            ident: syn::Ident::new(name, proc_macro2::Span::call_site()),
            generics: syn::Generics { lt_token: None, params: Punctuated::from_iter(vec![syn::GenericParam::Type(syn::TypeParam { attrs: Vec::new(), ident: syn::Ident::new("T", proc_macro2::Span::call_site()), colon_token: None, bounds: Punctuated::new(), eq_token: None, default: None })].into_iter()), gt_token: None, where_clause: None },
            paren_token: syn::token::Paren::default(),
            inputs: Punctuated::new(),
            variadic: None,
            output: syn::ReturnType::Type(
                syn::token::RArrow::default(),
                Box::new(syn::Type::ImplTrait(syn::TypeImplTrait {
                    impl_token: syn::token::Impl::default(),
                    bounds: Punctuated::from_iter(
                        vec![syn::TypeParamBound::Trait(syn::TraitBound {
                            paren_token: None,
                            modifier: syn::TraitBoundModifier::None,
                            lifetimes: None,
                            path: syn::Path {
                                leading_colon: None,
                                segments: Punctuated::from_iter(
                                    vec![syn::PathSegment {
                                        ident: syn::Ident::new(
                                            "ViewSequence",
                                            proc_macro2::Span::call_site(),
                                        ),
                                        arguments: syn::PathArguments::AngleBracketed(
                                            syn::AngleBracketedGenericArguments {
                                                colon2_token: None,
                                                lt_token: syn::token::Lt::default(),
                                                args: Punctuated::from_iter(
                                                    vec![syn::GenericArgument::Type(
                                                        syn::Type::Path(syn::TypePath {
                                                            qself: None,
                                                            path: syn::Path {
                                                                leading_colon: None,
                                                                segments: Punctuated::from_iter(vec![syn::PathSegment { ident: syn::Ident::new("T", proc_macro2::Span::call_site()), arguments: syn::PathArguments::None }].into_iter()),
                                                            },
                                                        }),
                                                    )]
                                                    .into_iter(),
                                                ),
                                                gt_token: syn::token::Gt::default(),
                                            },
                                        ),
                                    }]

Replacement in crates/typst_rust_gen/src/lib.rs at line 290 [4.1210]

B:BD[2.1937] → [2.1937:2135]

                            })),
                        }),
                        semi_token: syn::token::Semi::default(),
                    }),
                    syn::Item::Fn(syn::ItemFn {

[2.1937]

[2.2135]

                            },
                        })]
                        .into_iter(),
                    ),
                })),
            ),
        },
        block: Box::new(syn::Block {
            brace_token: syn::token::Brace::default(),
            stmts: vec![syn::Stmt::Expr(
                syn::Expr::Tuple(syn::ExprTuple {

Replacement in crates/typst_rust_gen/src/lib.rs at line 302 [4.1210]

B:BD[2.2174] → [2.2174:6487]

∅:D[2.6487] → [8.522:589]

B:BD[8.522] → [8.522:589]

B:BD[8.589] → [2.6488:6981]

∅:D[2.6981] → [8.1894:1910]

B:BD[8.1894] → [8.1894:1910]

B:BD[5.3059] → [8.3168:3179]

∅:D[8.3179] → [5.3072:3080]

B:BD[5.3072] → [5.3072:3080]

B:BD[5.3080] → [8.3180:3254]

                    vis: syn::Visibility::Public(syn::token::Pub::default()),
                    sig: syn::Signature {
                        constness: None,
                        asyncness: None,
                        unsafety: None,
                        abi: None,
                        fn_token: syn::token::Fn::default(),
                        ident: syn::Ident::new("test", proc_macro2::Span::call_site()),
                        generics: syn::Generics { lt_token: None, params: Punctuated::from_iter(vec![syn::GenericParam::Type(syn::TypeParam { attrs: Vec::new(), ident: syn::Ident::new("T", proc_macro2::Span::call_site()), colon_token: None, bounds: Punctuated::new(), eq_token: None, default: None })].into_iter()), gt_token: None, where_clause: None },
                        paren_token: syn::token::Paren::default(),
                        inputs: Punctuated::new(),
                        variadic: None,
                        output: syn::ReturnType::Type(
                            syn::token::RArrow::default(),
                            Box::new(syn::Type::ImplTrait(syn::TypeImplTrait {
                                impl_token: syn::token::Impl::default(),
                                bounds: Punctuated::from_iter(
                                    vec![syn::TypeParamBound::Trait(syn::TraitBound {
                                        paren_token: None,
                                        modifier: syn::TraitBoundModifier::None,
                                        lifetimes: None,
                                        path: syn::Path {
                                            leading_colon: None,
                                            segments: Punctuated::from_iter(
                                                vec![syn::PathSegment {
                                                    ident: syn::Ident::new(
                                                        "ViewSequence",
                                                        proc_macro2::Span::call_site(),
                                                    ),
                                                    arguments: syn::PathArguments::AngleBracketed(
                                                        syn::AngleBracketedGenericArguments {
                                                            colon2_token: None,
                                                            lt_token: syn::token::Lt::default(),
                                                            args: Punctuated::from_iter(
                                                                vec![syn::GenericArgument::Type(
                                                                    syn::Type::Path(syn::TypePath {
                                                                        qself: None,
                                                                        path: syn::Path {
                                                                            leading_colon: None,
                                                                            segments: Punctuated::from_iter(vec![syn::PathSegment { ident: syn::Ident::new("T", proc_macro2::Span::call_site()), arguments: syn::PathArguments::None }].into_iter()),
                                                                        },
                                                                    }),
                                                                )]
                                                                .into_iter(),
                                                            ),
                                                            gt_token: syn::token::Gt::default(),
                                                        },
                                                    ),
                                                }]
                                                .into_iter(),
                                            ),
                                        },
                                    })]
                                    .into_iter(),
                                ),
                            })),
                        ),
                    },
                    block: Box::new(syn::Block {
                        brace_token: syn::token::Brace::default(),
                        stmts: vec![syn::Stmt::Expr(
                            syn::Expr::Tuple(syn::ExprTuple {
                                attrs: Vec::new(),
                                paren_token: syn::token::Paren::default(),
                                elems: Punctuated::from_iter(blocks.into_iter()),
                            }),
                            None,
                        )],
                    }),
                }),])),
                semi: None,
            }),
        ],
    };
    let output = quote!(#file).to_string();
    dbg!(&output);
    output

[2.2174]

[5.3110]

                    paren_token: syn::token::Paren::default(),
                    elems: Punctuated::from_iter(blocks.into_iter()),
                }),
                None,
            )],
        }),
    })

Replacement in crates/typst_rust_gen/src/lib.rs at line 338 [4.1210]

B:BD[4.1962] → [5.3799:3860]

fn transform_block(block: &pandoc_ast::Block) -> syn::Expr {

[4.1962]

[5.3860]

fn pandoc_to_xilem(block: &pandoc_ast::Block) -> syn::Expr {

Deletion in crates/typst_rust_gen/src/lib.rs at line 374 [4.1210]
B:BD[5.4700] → [5.4700:4725]
```
    dbg!(&full_literal);
```
Replacement in crates/typst_rust_gen/examples/basic.rs at line 2 [4.2842]
B:BD[4.2864] → [4.2864:2902]
```
use typst_rust_gen::transform_pandoc;
```
[4.2864]
[4.2902]
```
use typst_rust_gen::output_rust;
```
Replacement in crates/typst_rust_gen/examples/basic.rs at line 5 [4.2842]
B:BD[4.2915] → [4.2915:2971]
```
    dbg!(transform_pandoc(Path::new("docs/test.typ")));
```
[4.2915]
[4.2971]
```
    dbg!(output_rust(Path::new("docs/test.typ")));
```
Insertion in crates/typst_rust_gen/Cargo.toml at line 9 [4.3009]
[4.3134]
[5.4971]
```
patricia_tree = "0.6.3"
```

Support Typst subdirectories

Dependencies

In channels

Change contents

Insertion in Cargo.lock at line 61 [3.1510]

Insertion in Cargo.lock at line 261 [3.1510]

Insertion in Cargo.lock at line 437 [3.1510]

Replacement in Cargo.lock at line 634 [3.1510]

Replacement in build.rs at line 3 [6.302]

Replacement in build.rs at line 23 [6.302]

Replacement in build.rs at line 27 [6.302]

Replacement in build.rs at line 34 [6.302]

Replacement in build.rs at line 37 [6.302]

Replacement in crates/typst_rust_gen/src/lib.rs at line 1 [4.1210]

Insertion in crates/typst_rust_gen/src/lib.rs at line 4 [4.1210]

Insertion in crates/typst_rust_gen/src/lib.rs at line 7 [4.1210]