Jengamon/magus - Change R4Z6MXFX7FJZGNZVCTO54ATZBCJVB4ERAFHRKXXKQEB5GQQ5DO6AC

feat: make progress on initial lexer

Created by Jengamon on February 28, 2024

R4Z6MXFX7FJZGNZVCTO54ATZBCJVB4ERAFHRKXXKQEB5GQQ5DO6AC

Dependencies

In channels

main

Change contents

Replacement in magus_repl/src/lib.rs at line 12 [2.969]

B:BD[2.1320] → [2.1320:1348]

                _ = buffer;

[2.1320]

[2.1348]

                let lexer = magus::lexer::lexer(&buffer);
                for (token, span) in lexer.spanned() {
                    match token {
                        Ok(tok) => {
                            println!(
                                "[{}:{}] {:?} `{}`",
                                span.start,
                                span.end,
                                tok,
                                &buffer[span.start..span.end]
                            )
                        }
                        Err(e) => eprintln!(
                            "[{}:{}] !!{}!! `{}`",
                            span.start,
                            span.end,
                            e,
                            &buffer[span.start..span.end]
                        ),
                    }
                }

Replacement in magus_core/src/lib.rs at line 5 [2.2078]

B:BD[2.2248] → [2.2248:2316]

pub fn add(left: usize, right: usize) -> usize {
    left + right
}

[2.2248]

[2.2316]

pub mod lexer {
    use logos::{Lexer, Logos};
    use std::num::ParseIntError;
    // Use tiny winnow parsers to parse intra-element stuff
    use winnow::{
        ascii::hex_digit1,
        combinator::{alt, delimited, preceded, repeat},
        token::{any, one_of},
        PResult, Parser,
    };
    #[derive(thiserror::Error, Clone, PartialEq, Default, Debug)]
    pub enum LexError {
        #[default]
        #[error("unknown")]
        Unknown,
    }
    fn symbol_element_parse(input: &mut &str) -> PResult<Vec<char>> {
        repeat(
            0..,
            alt((
                delimited(r"\x", hex_digit1, r";").try_map(|s: &str| {
                    Ok::<_, ParseIntError>(
                        char::from_u32(u32::from_str_radix(s, 16)?)
                            .unwrap_or(char::REPLACEMENT_CHARACTER),
                    )
                }),
                preceded(r"\", one_of(['a', 'b', 't', 'n', 'r'])).map(|c| match c {
                    'a' => 7 as char,
                    'b' => 8 as char,
                    't' => 9 as char,
                    'n' => 0xa as char,
                    'r' => 0xd as char,
                    _ => unreachable!(r"Scheme R7RS only handles \a, \b, \t, \n, and \r"),
                }),
                any,
            )),
        )
        .parse_next(input)
    }
    fn parse_to_symbol_elements(lex: &mut Lexer<Token>) -> Result<String, LexError> {
        let slice = lex.slice();
        let target = &slice[1..slice.len() - 1];
        symbol_element_parse
            .parse(target)
            .map(|vc| vc.into_iter().collect())
            .map_err(|e| {
                // TODO A proper lex error (for ident symbols!)
                dbg!(std::any::type_name_of_val(&e));
                LexError::Unknown
            })
    }
    fn string_element_parse(input: &mut &str) -> PResult<Vec<char>> {
        repeat(
            0..,
            alt((
                delimited(r"\x", hex_digit1, r";")
                    .try_map(|s: &str| {
                        Ok::<_, ParseIntError>(
                            char::from_u32(u32::from_str_radix(s, 16)?)
                                .unwrap_or(char::REPLACEMENT_CHARACTER),
                        )
                    })
                    .map(Some),
                preceded(r"\", one_of(['a', 'b', 't', 'n', 'r', '"', '\\', '|']))
                    .map(|c| match c {
                        'a' => 7 as char,
                        'b' => 8 as char,
                        't' => 9 as char,
                        'n' => 0xa as char,
                        'r' => 0xd as char,
                        '"' => 0x22 as char,
                        '\\' => 0x5c as char,
                        '|' => 0x7c as char,
                        _ => unreachable!(
                            r#"Scheme R7RS strings only handle \a, \b, \t, \n, \r, \", \\, and \|"#
                        ),
                    })
                    .map(Some),
                delimited(
                    repeat::<_, _, (), _, _>(0.., one_of([' ', '\t', '\r'])),
                    '\n',
                    repeat::<_, _, (), _, _>(0.., one_of([' ', '\t', '\r'])),
                )
                .void()
                .map(|_| None),
                any.map(Some),
            )),
        )
        .map(|vc: Vec<Option<char>>| vc.into_iter().flatten().collect())
        .parse_next(input)
    }
    fn parse_to_string(lex: &mut Lexer<Token>) -> Result<String, LexError> {
        let slice = lex.slice();
        let target = &slice[1..slice.len() - 1];
        string_element_parse
            .parse(target)
            .map(|vc| vc.into_iter().collect())
            .map_err(|e| {
                // TODO A proper lex error
                dbg!(std::any::type_name_of_val(&e));
                LexError::Unknown
            })
    }
    #[derive(Debug, Clone, PartialEq, Eq, Hash, Logos)]
    #[logos(error = LexError)]
    pub enum Token {
        #[token("(")]
        LParen,
        #[token(")")]
        RParen,
        #[regex(r"[ \t\r]+")]
        Whitespace,
        #[regex(r"[A-Za-z!$%&*+\-./:<=>?@^_-][A-Za-z0-9!$%&*+\-./:<=>?@^_-]*", |lex| lex.slice().to_string())]
        #[regex(r"\|(?:\\x[0-9a-fA-F]+;|\\[abtnr]|[^|])*\|", parse_to_symbol_elements)]
        Identifier(String),
        #[regex(
            r#""(?:\\x[0-9a-fA-F]+;|\\[abtnr"\\|]|\\[ \t\r]*\n[ \t\r]*|[^"])*""#,
            parse_to_string
        )]
        String(String),
        // TODO lexers for num, complex, real
        #[regex(r"[+-]?[0-9][0-9_]*", priority = 3)]
        #[regex(r"[+-]?[0-9][0-9_]*e[+-]?[0-9_]+")]
        Integer,
        #[regex(r"[+-]?[0-9][0-9_]*\.[0-9]?[0-9_]*")]
        #[regex(r"[+-]?[0-9][0-9_]*\.[0-9]?[0-9_]*e[+-]?[0-9_]+")]
        #[regex(r"[+-]?\.[0-9][0-9_]*")]
        #[regex(r"[+-]?\.[0-9][0-9_]*e[+-]?[0-9_]+")]
        Decimal,
        #[token("+inf.0")]
        InfinityPos,
        #[token("-inf.0")]
        InifinityNeg,
        #[token("+nan.0")]
        NotANumberPos,
        #[token("-nan.0")]
        NotANumberNeg,
        #[token("+i")]
        ImaginaryPos,
        #[token("-i")]
        ImaginaryNeg,
        #[token(".", priority = 3)]
        Dot,
        #[token("'")]
        QuoteSymbol,
        #[token("`")]
        QuasiquoteSymbol,
        #[token(",")]
        UnquoteSymbol,
        #[token(",@")]
        UnquoteSplicingSymbol,
        #[token(";")]
        LineCommentStart,
        #[regex(r"\n+")]
        LineEnd,
        #[token("#;")]
        DatumCommentStart,
        #[token("#|")]
        BlockCommentStart,
        #[token("|#")]
        BlockCommentEnd,
    }
    pub fn lexer(input: &str) -> Lexer<'_, Token> {
        Token::lexer(input)
    }
    #[cfg(test)]
    mod tests {
        use super::Token;
        use assert2::assert;
        use logos::Logos;
        #[test]
        fn literal_identfier_equivalence() {
            let source = r"|\t\t||\x9;\x9;|";
            let mut lexer = Token::lexer(source);
            let tok1 = lexer.next();
            let tok1_slice = lexer.slice();
            let tok2 = lexer.next();
            let tok2_slice = lexer.slice();
            assert!(tok1 == tok2);
            assert!(tok1_slice != tok2_slice);
        }
        #[test]
        fn lex_literal_identifier() {
            let source = r"|H\x65;llo|";
            let mut lexer = Token::lexer(source);
            assert!(lexer.next() == Some(Ok(Token::Identifier("Hello".to_string()))));
            assert!(lexer.span() == (0..source.len()));
            assert!(lexer.slice() == "|H\\x65;llo|");
        }
        #[test]
        fn lex_identifier_or_number() {
            let source = "3let -i";
            let mut lexer = Token::lexer(source);
            assert!(lexer.next() == Some(Ok(Token::Integer)));
            assert!(lexer.span() == (0..1));
            assert!(lexer.slice() == "3");
            assert!(lexer.next() == Some(Ok(Token::Identifier("let".to_string()))));
            assert!(lexer.span() == (1..4));
            assert!(lexer.slice() == "let");
            assert!(lexer.next() == Some(Ok(Token::Whitespace)));
            assert!(lexer.span() == (4..5));
            assert!(lexer.slice() == " ");

Replacement in magus_core/src/lib.rs at line 223 [2.2078]

B:BD[2.2317] → [2.2317:2360]

#[cfg(test)]
mod tests {
    use super::*;

[2.2317]

[2.2360]

            assert!(lexer.next() == Some(Ok(Token::ImaginaryNeg)));
            assert!(lexer.span() == (5..7));
            assert!(lexer.slice() == "-i");
        }

Replacement in magus_core/src/lib.rs at line 228 [2.2078]

B:BD[2.2361] → [2.2361:2456]

    #[test]
    fn it_works() {
        let result = add(2, 2);
        assert_eq!(result, 4);

[2.2361]

[2.2456]

        #[test]
        fn lex_identifier() {
            let source = "+soup+ $?bama";
            let mut lexer = Token::lexer(source);
            assert!(lexer.next() == Some(Ok(Token::Identifier("+soup+".to_string()))));
            assert!(lexer.span() == (0..6));
            assert!(lexer.slice() == "+soup+");
            assert!(lexer.next() == Some(Ok(Token::Whitespace)));
            assert!(lexer.span() == (6..7));
            assert!(lexer.slice() == " ");
            assert!(lexer.next() == Some(Ok(Token::Identifier("$?bama".to_string()))));
            assert!(lexer.span() == (7..13));
            assert!(lexer.slice() == "$?bama");
        }

Insertion in magus_core/Cargo.toml at line 15 [2.2500]
[2.2825]
[2.2825]
```
cstree = "0.12.0"
```
Insertion in magus_core/Cargo.toml at line 22 [2.2500]
[2.3030]
```
assert2 = "0.3.13"
```
Insertion in README.md at line 19 [3.300]
[3.1124]
[3.1124]
```
## Non-goals
```

Replacement in README.md at line 22 [3.300]

B:BD[3.1136] → [3.1136:1297]

  - having the runtime be multi-thread compatible is nice to have, but we'll live if we don't have it.
  - this should be locked behind the feature flag "send".

[3.1136]

[3.1297]

  - we don't need to support multi-threading, which should make gc *much* easier

Insertion in Cargo.lock at line 33 [2.3269]

[2.4218]

]
[[package]]
name = "assert2"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c2a406fdd319a14eccedbc8d1ef96b27e4d2101f19d997ac19fb307d919f39"
dependencies = [
 "assert2-macros",
 "diff",
 "is-terminal",
 "yansi 0.5.1",

Insertion in Cargo.lock at line 48 [2.3269]

[2.4233]

name = "assert2-macros"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d83ae6bec6fbfa955b8ff04be5e63ae26ed2d9d068979d993142f13a2133344"
dependencies = [
 "proc-macro2",
 "quote",
 "rustc_version",
 "syn 1.0.109",
]
[[package]]

Insertion in Cargo.lock at line 134 [2.3269]

[2.6192]

name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]

Insertion in Cargo.lock at line 236 [2.3269]

[2.8533]

name = "cstree"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81b0cb07675f2025784ff3964c5b8fdcf0cc86c1288ae2d798549b7866b66d96"
dependencies = [
 "fxhash",
 "indexmap",
 "parking_lot",
 "sptr",
 "text-size",
 "triomphe",
]
[[package]]
name = "diff"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]

Insertion in Cargo.lock at line 311 [2.3269]

[2.10032]

name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
 "byteorder",
]
[[package]]

Replacement in Cargo.lock at line 334 [2.3269]
B:BD[2.10468] → [2.10468:10478]
```
 "yansi",
```
[2.10468]
[2.10478]
```
 "yansi 1.0.0-rc.1",
```

Insertion in Cargo.lock at line 336 [2.3269]

[2.10480]


[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"

Insertion in Cargo.lock at line 350 [2.3269]

[2.10681]

name = "hermit-abi"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60"
[[package]]

Insertion in Cargo.lock at line 385 [2.3269]

[2.11445]

name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
 "autocfg",
 "hashbrown",
]
[[package]]

Insertion in Cargo.lock at line 410 [2.3269]

[2.11746]

name = "is-terminal"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
dependencies = [
 "hermit-abi",
 "libc",
 "windows-sys 0.52.0",
]
[[package]]

Replacement in Cargo.lock at line 505 [2.3269]
B:BD[2.14099] → [2.14099:14107]
```
 "syn",
```
[2.14099]
[2.14107]
```
 "syn 2.0.51",
```
Insertion in Cargo.lock at line 521 [2.3269]
[2.14406]
[2.14406]
```
 "assert2",
 "cstree",
```
Replacement in Cargo.lock at line 582 [2.3269]
B:BD[2.15533] → [2.15533:15541]
```
 "syn",
```
[2.15533]
[2.15541]
```
 "syn 2.0.51",
```

Insertion in Cargo.lock at line 755 [2.3269]

[2.19968]


[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
 "semver",
]

Insertion in Cargo.lock at line 791 [2.3269]

[2.20661]

name = "semver"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
[[package]]

Replacement in Cargo.lock at line 813 [2.3269]
B:BD[2.21116] → [2.21116:21124]
```
 "syn",
```
[2.21116]
[2.21124]
```
 "syn 2.0.51",
```

Insertion in Cargo.lock at line 886 [2.3269]

[2.22957]


[[package]]
name = "sptr"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"

Replacement in Cargo.lock at line 924 [2.3269]
B:BD[2.23641] → [2.23641:23649]
```
 "syn",
```
[2.23641]
[2.23649]
```
 "syn 2.0.51",
```

Insertion in Cargo.lock at line 950 [2.3269]

[2.24307]

version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
 "proc-macro2",
 "quote",
 "unicode-ident",
]
[[package]]
name = "syn"

Insertion in Cargo.lock at line 991 [2.3269]

[2.25052]


[[package]]
name = "text-size"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233"

Replacement in Cargo.lock at line 1026 [2.3269]
B:BD[2.25788] → [2.25788:25796]
```
 "syn",
```
[2.25788]
[2.25796]
```
 "syn 2.0.51",
```

Insertion in Cargo.lock at line 1081 [2.3269]

[2.27067]

name = "triomphe"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3"
dependencies = [
 "stable_deref_trait",
]
[[package]]

Replacement in Cargo.lock at line 1172 [2.3269]
B:BD[2.29492] → [2.29492:29500]
```
 "syn",
```
[2.29492]
[2.29500]
```
 "syn 2.0.51",
```
Replacement in Cargo.lock at line 1194 [2.3269]
B:BD[2.30043] → [2.30043:30051]
```
 "syn",
```
[2.30043]
[2.30051]
```
 "syn 2.0.51",
```

Insertion in Cargo.lock at line 1385 [2.3269]

[2.35916]


[[package]]
name = "yansi"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"

feat: make progress on initial lexer

Dependencies

In channels

Change contents

Replacement in magus_repl/src/lib.rs at line 12 [2.969]

Replacement in magus_core/src/lib.rs at line 5 [2.2078]

Replacement in magus_core/src/lib.rs at line 223 [2.2078]

Replacement in magus_core/src/lib.rs at line 228 [2.2078]

Insertion in magus_core/Cargo.toml at line 15 [2.2500]

Insertion in magus_core/Cargo.toml at line 22 [2.2500]

Insertion in README.md at line 19 [3.300]

Replacement in README.md at line 22 [3.300]

Insertion in Cargo.lock at line 33 [2.3269]

Insertion in Cargo.lock at line 48 [2.3269]

Insertion in Cargo.lock at line 134 [2.3269]

Insertion in Cargo.lock at line 236 [2.3269]

Insertion in Cargo.lock at line 311 [2.3269]

Replacement in Cargo.lock at line 334 [2.3269]

Insertion in Cargo.lock at line 336 [2.3269]

Insertion in Cargo.lock at line 350 [2.3269]

Insertion in Cargo.lock at line 385 [2.3269]

Insertion in Cargo.lock at line 410 [2.3269]

Replacement in Cargo.lock at line 505 [2.3269]

Insertion in Cargo.lock at line 521 [2.3269]

Replacement in Cargo.lock at line 582 [2.3269]

Insertion in Cargo.lock at line 755 [2.3269]

Insertion in Cargo.lock at line 791 [2.3269]

Replacement in Cargo.lock at line 813 [2.3269]

Insertion in Cargo.lock at line 886 [2.3269]

Replacement in Cargo.lock at line 924 [2.3269]

Insertion in Cargo.lock at line 950 [2.3269]

Insertion in Cargo.lock at line 991 [2.3269]

Replacement in Cargo.lock at line 1026 [2.3269]

Insertion in Cargo.lock at line 1081 [2.3269]

Replacement in Cargo.lock at line 1172 [2.3269]

Replacement in Cargo.lock at line 1194 [2.3269]

Insertion in Cargo.lock at line 1385 [2.3269]