{"header":{"message":"feat: add bytecode module","description":null,"timestamp":"2024-02-28T15:48:38.464138Z","authors":[{}]},"hash":"UEEGB37HSGJL4VCRJYIAJRVAEPREYZTDPIDG7BJIJ4DUUSRX7LOQC","authors":[{"login":"Jengamon","key":"AFEuLzv5ecUJzaewFwoWh4jeCDqWZztr7dswYwyG9vfU"}],"deps":{"hashes":[["UEEGB37HSGJL4VCRJYIAJRVAEPREYZTDPIDG7BJIJ4DUUSRX7LOQC","feat: add bytecode module"],["AA",null],["YOZBOIICKWWMCBXLBSQBACSKM6I4APCXRVOLFP7B46TXXHKBJRWQC","feat: add decimal lex/parse"],["4YAWXUVFNTMPVW4VQ3AVGYROWEQUDXZUFYNAVRFW72REV6QTCO7QC","fix: add decimal lexer test"],["4CWI65HNTD3X3OP25QJOAO5B5QMCMNQFQJ4BUMQKBDUVVXKRWEOQC","feat: initial project structure"],["HJAFCFNCO6CM326HWPH7BXLTPIFCYVTRSITBCGLYBXACJJOL3JCQC","feat: add integer lex/parse"],["C6KYXU274XZV4ZAT3K5DELTMHBMC42K5FDF4SN6LSCNFNCRVYSLAC","fix: store integer as (base, exp) rather than realize"],["R4Z6MXFX7FJZGNZVCTO54ATZBCJVB4ERAFHRKXXKQEB5GQQ5DO6AC","feat: make progress on initial lexer"]],"deps":["YOZBOIICKWWMCBXLBSQBACSKM6I4APCXRVOLFP7B46TXXHKBJRWQC","4YAWXUVFNTMPVW4VQ3AVGYROWEQUDXZUFYNAVRFW72REV6QTCO7QC"],"known":["4CWI65HNTD3X3OP25QJOAO5B5QMCMNQFQJ4BUMQKBDUVVXKRWEOQC","R4Z6MXFX7FJZGNZVCTO54ATZBCJVB4ERAFHRKXXKQEB5GQQ5DO6AC","HJAFCFNCO6CM326HWPH7BXLTPIFCYVTRSITBCGLYBXACJJOL3JCQC","C6KYXU274XZV4ZAT3K5DELTMHBMC42K5FDF4SN6LSCNFNCRVYSLAC"]},"hunks":[{"FileAdd":{"meta":{"basename":"bytecode.rs","metadata":0},"context":[{"change":4,"pos":2045}],"contents":[{"Add":"//! Bytecode and related structures"},{"Add":""},{"Add":"// all bytecode should fit within a u32, so"},{"Add":"// all instructions look like:"},{"Add":"// - INST (8bits) A (24bits) (shape A)"},{"Add":"// - INST (8bits) A (8bits) B (16bits) (shape B)"},{"Add":"// - INST (8bits) A (8bits) B (8bits) C (8bits) (shape C)"},{"Add":""},{"Add":"// NOTE Endianness is undefined for now as this is not meant to be serialized."},{"Add":""},{"Add":"#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]"},{"Add":"#[repr(u8)]"},{"Add":"pub enum OpCode {"},{"Add":" NoOp,"},{"Add":"}"},{"Add":""},{"Add":"impl From for u8 {"},{"Add":" fn from(value: OpCode) -> Self {"},{"Add":" match value {"},{"Add":" OpCode::NoOp => 0,"},{"Add":" }"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"impl TryFrom for OpCode {"},{"Add":" // There is only 1 possible error: the u8 doesn't represent a valid op-code"},{"Add":" type Error = ();"},{"Add":" fn try_from(value: u8) -> Result {"},{"Add":" match value {"},{"Add":" 0 => Ok(Self::NoOp),"},{"Add":" _ => Err(()),"},{"Add":" }"},{"Add":" }"},{"Add":"}"},{"Add":"#[derive(Debug, Clone, Copy, PartialEq, Eq)]"},{"Add":"pub enum OpArguments {"},{"Add":" A(u32), // first 8 bits are ignored"},{"Add":" B(u8, u16),"},{"Add":" C(u8, u8, u8),"},{"Add":"}"},{"Add":""},{"Add":"impl OpArguments {"},{"Add":" const ARGUMENT_MASK: u32 = 0x00ff_ffff;"},{"Add":" // Private cuz"},{"Add":" fn stitch(self) -> u32 {"},{"Add":" match self {"},{"Add":" Self::A(a) => a & Self::ARGUMENT_MASK,"},{"Add":" Self::B(a, b) => ((a as u32) << 16) | (b as u32),"},{"Add":" Self::C(a, b, c) => ((a as u32) << 16) | ((b as u32) << 8) | (c as u32),"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" fn parse_a(n: u32) -> Self {"},{"Add":" let masked = n & Self::ARGUMENT_MASK;"},{"Add":" Self::A(masked)"},{"Add":" }"},{"Add":""},{"Add":" fn parse_b(n: u32) -> Self {"},{"Add":" let masked = n & Self::ARGUMENT_MASK;"},{"Add":" Self::B((masked >> 16) as u8, masked as u16)"},{"Add":" }"},{"Add":""},{"Add":" fn parse_c(n: u32) -> Self {"},{"Add":" let masked = n & Self::ARGUMENT_MASK;"},{"Add":" Self::C((masked >> 16) as u8, (masked >> 8) as u8, masked as u8)"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"#[derive(Debug, Clone, Copy, PartialEq, Eq)]"},{"Add":"pub struct ByteCode {"},{"Add":" op_code: OpCode,"},{"Add":" arguments: OpArguments,"},{"Add":"}"},{"Add":""},{"Add":"impl ByteCode {"},{"Add":" /// Produces a single number representing this code"},{"Add":" pub fn stitch(self) -> u32 {"},{"Add":" ((self.op_code as u32) << 24) | self.arguments.stitch()"},{"Add":" }"},{"Add":""},{"Add":" pub fn parse(n: u32) -> Option {"},{"Add":" // Get the op code"},{"Add":" let op_code = (((n & !OpArguments::ARGUMENT_MASK) >> 24) as u8)"},{"Add":" .try_into()"},{"Add":" .ok()?;"},{"Add":" // The only fallible part is getting the opcode. All other operations are infallible"},{"Add":" Some(match op_code {"},{"Add":" OpCode::NoOp => Self {"},{"Add":" op_code,"},{"Add":" arguments: OpArguments::parse_a(n),"},{"Add":" },"},{"Add":" })"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"impl TryFrom for ByteCode {"},{"Add":" type Error = ();"},{"Add":" fn try_from(value: u32) -> Result {"},{"Add":" Self::parse(value).ok_or(())"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"#[cfg(test)]"},{"Add":"mod tests {"},{"Add":" use super::{ByteCode, OpArguments, OpCode};"},{"Add":" use assert2::assert;"},{"Add":""},{"Add":" #[test]"},{"Add":" fn bytecode_parse() {"},{"Add":" let code = 0x00debeef;"},{"Add":" let op = ByteCode::parse(code);"},{"Add":" assert!("},{"Add":" op == Some(ByteCode {"},{"Add":" op_code: OpCode::NoOp,"},{"Add":" arguments: OpArguments::A(0xdebeef)"},{"Add":" })"},{"Add":" );"},{"Add":""},{"Add":" let code = 0x01adbeef;"},{"Add":" let op = ByteCode::parse(code);"},{"Add":" assert!(op == None);"},{"Add":" }"},{"Add":""},{"Add":" #[test]"},{"Add":" fn bytecode_stitch() {"},{"Add":" let bc = ByteCode {"},{"Add":" op_code: OpCode::NoOp,"},{"Add":" arguments: OpArguments::C(0xca, 0xbe, 0xef),"},{"Add":" };"},{"Add":" assert!(bc.stitch() == 0x00cabeef);"},{"Add":" }"},{"Add":"}"}]}},{"Replacement":{"inode":{"change":4,"pos":2078},"path":"magus_core/src/lib.rs","line":4,"ins":{"up_context":[{"change":4,"pos":1311}],"down_context":[],"flag":"BLOCK","start":3430,"end":3448,"inode":{"change":4,"pos":2078}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1311},"to":{"change":4,"start":1311,"end":3957},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":3957},"to":{"change":4,"start":99,"end":132},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":132},"to":{"change":4,"start":3983,"end":4079},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":3983},"to":{"change":4,"start":3983,"end":4079},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":4103},"to":{"change":4,"start":4103,"end":4742},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":4742},"to":{"change":4,"start":0,"end":72},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":72},"to":{"change":4,"start":198,"end":1181},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":198},"to":{"change":4,"start":198,"end":1181},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1181},"to":{"change":4,"start":73,"end":130},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":130},"to":{"change":4,"start":1248,"end":1301},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1248},"to":{"change":4,"start":1248,"end":1301},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1301},"to":{"change":4,"start":131,"end":212},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":212},"to":{"change":4,"start":1375,"end":1599},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1375},"to":{"change":4,"start":1375,"end":1599},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1599},"to":{"change":2,"start":53,"end":278},"introduced_by":2},{"previous":"","flag":"DELETED","from":{"change":2,"pos":278},"to":{"change":4,"start":1599,"end":1606},"introduced_by":2},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1599},"to":{"change":4,"start":1599,"end":1606},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1606},"to":{"change":2,"start":279,"end":2416},"introduced_by":2},{"previous":"","flag":"DELETED","from":{"change":2,"pos":2416},"to":{"change":4,"start":4798,"end":5411},"introduced_by":2},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":4798},"to":{"change":4,"start":4798,"end":5411},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":5411},"to":{"change":4,"start":1607,"end":1762},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1762},"to":{"change":4,"start":213,"end":242},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":242},"to":{"change":2,"start":2417,"end":2795},"introduced_by":2},{"previous":"","flag":"DELETED","from":{"change":2,"pos":2795},"to":{"change":4,"start":5766,"end":6757},"introduced_by":2},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":5766},"to":{"change":4,"start":5766,"end":6757},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":6757},"to":{"change":3,"start":0,"end":923},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":3,"pos":923},"to":{"change":4,"start":6757,"end":7649},"introduced_by":3},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":6757},"to":{"change":4,"start":6757,"end":7649},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":7649},"to":{"change":4,"start":243,"end":314},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":314},"to":{"change":4,"start":7712,"end":8131},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":1922},"to":{"change":4,"start":7712,"end":8131},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":7712},"to":{"change":4,"start":7712,"end":8131},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":8131},"to":{"change":4,"start":2316,"end":2317},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2316},"to":{"change":4,"start":2316,"end":2317},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2317},"to":{"change":4,"start":8132,"end":8299},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":8299},"to":{"change":4,"start":2360,"end":2361},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2360},"to":{"change":4,"start":2360,"end":2361},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2361},"to":{"change":4,"start":8300,"end":8967},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":8967},"to":{"change":4,"start":2456,"end":2464},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2456},"to":{"change":4,"start":2456,"end":2464},"introduced_by":4}],"inode":{"change":4,"pos":2078}},"ins_contents":[{"Add":"pub mod bytecode;"}],"del_contents":[{"Del":" fn symbol_element_parse(input: &mut &str) -> PResult> {\n repeat(\n 0..,\n alt((\n delimited(r\"\\x\", hex_digit1, r\";\").try_map(|s: &str| {\n Ok::<_, ParseIntError>(\n char::from_u32(u32::from_str_radix(s, 16)?)\n .unwrap_or(char::REPLACEMENT_CHARACTER),\n )\n }),\n preceded(r\"\\\", one_of(['a', 'b', 't', 'n', 'r'])).map(|c| match c {\n 'a' => 7 as char,\n 'b' => 8 as char,\n 't' => 9 as char,\n 'n' => 0xa as char,\n 'r' => 0xd as char,\n _ => unreachable!(r\"Scheme R7RS only handles \\a, \\b, \\t, \\n, and \\r\"),\n }),\n any,\n )),\n )\n .parse_next(input)\n }\n\n fn parse_to_symbol_elements(lex: &mut Lexer) -> Result {\n let slice = lex.slice();\n let target = &slice[1..slice.len() - 1];\n symbol_element_parse\n .parse(target)\n .map(|vc| vc.into_iter().collect())\n .map_err(|e| {\n // TODO A proper lex error (for ident symbols!)\n dbg!(std::any::type_name_of_val(&e));\n LexError::Unknown\n })\n }\n\n fn string_element_parse(input: &mut &str) -> PResult> {\n repeat(\n 0..,\n alt((\n delimited(r\"\\x\", hex_digit1, r\";\")\n .try_map(|s: &str| {\n Ok::<_, ParseIntError>(\n char::from_u32(u32::from_str_radix(s, 16)?)\n .unwrap_or(char::REPLACEMENT_CHARACTER),\n )\n })\n .map(Some),\n preceded(r\"\\\", one_of(['a', 'b', 't', 'n', 'r', '\"', '\\\\', '|']))\n .map(|c| match c {\n 'a' => 7 as char,\n 'b' => 8 as char,\n 't' => 9 as char,\n 'n' => 0xa as char,\n 'r' => 0xd as char,\n '\"' => 0x22 as char,\n '\\\\' => 0x5c as char,\n '|' => 0x7c as char,\n _ => unreachable!(\n r#\"Scheme R7RS strings only handle \\a, \\b, \\t, \\n, \\r, \\\", \\\\, and \\|\"#\n ),\n })\n .map(Some),\n delimited(\n repeat::<_, _, (), _, _>(0.., one_of([' ', '\\t', '\\r'])),\n"},{"Del":" '\\n'.void(),\n"},{"Del":" repeat::<_, _, (), _, _>(0.., one_of([' ', '\\t', '\\r'])),\n )\n"},{"Del":" .map(|_| None),\n any.map(Some),\n )),\n )\n .map(|vc: Vec>| vc.into_iter().flatten().collect())\n .parse_next(input)\n }\n\n fn parse_to_string(lex: &mut Lexer) -> Result {\n let slice = lex.slice();\n let target = &slice[1..slice.len() - 1];\n string_element_parse\n .parse(target)\n .map(|vc| vc.into_iter().collect())\n .map_err(|e| {\n // TODO A proper lex error\n dbg!(std::any::type_name_of_val(&e));\n LexError::Unknown\n })\n }\n\n"},{"Del":" fn integer_number_parser(input: &mut &str) -> PResult<(i64, u32)> {\n"},{"Del":" let sign = opt(one_of(['-', '+']));\n let digits_with_underscores = || {\n repeat(1.., alt((digit1.map(Some), '_'.map(|_| None))))\n .map(|vd: Vec>| vd.into_iter().flatten().collect::>().join(\"\"))\n };\n let signed_digits_with_underscores =\n (sign, digits_with_underscores()).map(|(sign, digits): (_, String)| {\n if let Some(sign) = sign {\n format!(\"{sign}{digits}\")\n } else {\n digits\n }\n });\n let exponent = preceded(\"e\", (opt('+').void(), digits_with_underscores()));\n (signed_digits_with_underscores, opt(exponent))\n .try_map(|(base, maybe_exp)| {\n let base = base.parse::()?;\n let exponent = if let Some((_, exponent)) = maybe_exp {\n exponent.parse::()?\n } else {\n 0\n };\n"},{"Del":" Ok::<_, ParseIntError>((base, exponent))\n"},{"Del":" })\n .parse_next(input)\n }\n\n"},{"Del":" fn parse_integer(lexer: &mut Lexer) -> Result<(i64, u32), LexError> {\n"},{"Del":" let target = lexer.slice();\n integer_number_parser.parse(target).map_err(|e| {\n // TODO A proper lex error\n dbg!(std::any::type_name_of_val(&e));\n LexError::Unknown\n })\n"},{"Del":" }\n\n #[derive(thiserror::Error, Debug)]\n pub enum DecimalParseError {\n #[error(transparent)]\n FloatError(#[from] ParseFloatError),\n #[error(transparent)]\n IntError(#[from] ParseIntError),\n"},{"Del":" }\n\n"},{"Del":" fn decimal_number_parser(input: &mut &str) -> PResult<(f64, i32)> {\n let sign = || opt(one_of(['-', '+']));\n let digits_with_underscores = || {\n repeat(1.., alt((digit1.map(Some), '_'.map(|_| None))))\n .map(|vd: Vec>| vd.into_iter().flatten().collect::>().join(\"\"))\n };\n let float_digits = || {\n (\n opt(digits_with_underscores()),\n '.',\n digits_with_underscores(),\n )\n .map(|(maybe_predot, _, postdot)| {\n maybe_predot\n .map(|predot| format!(\"{predot}.{postdot}\"))\n .unwrap_or_else(|| format!(\".{postdot}\"))\n })\n };\n let signed_base = (sign(), alt((float_digits(), digits_with_underscores()))).map(\n |(sign, digits): (_, String)| {\n if let Some(sign) = sign {\n format!(\"{sign}{digits}\")\n } else {\n digits\n }\n },\n );\n let exponent = preceded(\"e\", (sign(), digits_with_underscores()));\n (signed_base, opt(exponent))\n .try_map(|(base, maybe_exp)| {\n let base = base.parse::()?;\n let exponent = if let Some((sign, exponent)) = maybe_exp {\n let signed = if let Some(sign) = sign {\n format!(\"{sign}{exponent}\")\n } else {\n exponent\n };\n signed.parse::()?\n } else {\n 0\n };\n Ok::<_, DecimalParseError>((base, exponent))\n })\n .parse_next(input)\n }\n\n fn parse_float(lexer: &mut Lexer) -> Result<(f64, i32), LexError> {\n let target = lexer.slice();\n decimal_number_parser.parse(target).map_err(|e| {\n // TODO A proper lex error\n dbg!(std::any::type_name_of_val(&e));\n LexError::Unknown\n })\n }\n\n #[derive(Debug, Clone, PartialEq, Logos)]\n"},{"Del":" #[logos(error = LexError)]\n pub enum Token {\n #[token(\"(\")]\n LParen,\n #[token(\")\")]\n RParen,\n #[regex(r\"[ \\t\\r]+\")]\n Whitespace,\n #[regex(r\"[A-Za-z!$%&*+\\-./:<=>?@^_-][A-Za-z0-9!$%&*+\\-./:<=>?@^_-]*\", |lex| lex.slice().to_string())]\n #[regex(r\"\\|(?:\\\\x[0-9a-fA-F]+;|\\\\[abtnr]|[^|])*\\|\", parse_to_symbol_elements)]\n Identifier(String),\n #[regex(\n r#\"\"(?:\\\\x[0-9a-fA-F]+;|\\\\[abtnr\"\\\\|]|\\\\[ \\t\\r]*\\n[ \\t\\r]*|[^\"])*\"\"#,\n parse_to_string\n )]\n String(String),\n // TODO lexers for num, complex, real\n"},{"Del":" #[regex(r\"[+-]?[0-9][0-9_]*\", priority = 3, callback = parse_integer)]\n #[regex(r\"[+-]?[0-9][0-9_]*e\\+?[0-9_]+\", callback = parse_integer)]\n"},{"Del":" Integer((i64, u32)),\n"},{"Del":" #[regex(r\"[+-]?[0-9][0-9_]*\\.[0-9][0-9_]*\", parse_float)]\n #[regex(r\"[+-]?[0-9][0-9_]*\\.[0-9][0-9_]*e[+-]?[0-9_]+\", parse_float)]\n #[regex(r\"[+-]?\\.[0-9][0-9_]*\", parse_float)]\n #[regex(r\"[+-]?\\.[0-9][0-9_]*e[+-]?[0-9_]+\", parse_float)]\n #[regex(r\"[+-]?[0-9][0-9_]*e\\-[0-9_]+\", parse_float)] // Negative exponent\n Decimal((f64, i32)),\n"},{"Del":" #[token(\"+inf.0\")]\n InfinityPos,\n #[token(\"-inf.0\")]\n InifinityNeg,\n #[token(\"+nan.0\")]\n NotANumberPos,\n #[token(\"-nan.0\")]\n NotANumberNeg,\n #[token(\"+i\")]\n ImaginaryPos,\n #[token(\"-i\")]\n ImaginaryNeg,\n #[token(\".\", priority = 3)]\n Dot,\n #[token(\"'\")]\n QuoteSymbol,\n #[token(\"`\")]\n QuasiquoteSymbol,\n #[token(\",\")]\n UnquoteSymbol,\n #[token(\",@\")]\n UnquoteSplicingSymbol,\n\n #[token(\";\")]\n LineCommentStart,\n #[regex(r\"\\n+\")]\n LineEnd,\n #[token(\"#;\")]\n DatumCommentStart,\n #[token(\"#|\")]\n BlockCommentStart,\n #[token(\"|#\")]\n BlockCommentEnd,\n }\n\n pub fn lexer(input: &str) -> Lexer<'_, Token> {\n Token::lexer(input)\n }\n\n #[cfg(test)]\n mod tests {\n\n use super::Token;\n use assert2::assert;\n use logos::Logos;\n\n #[test]\n"},{"Del":" fn lex_decimal() {\n let source = r\"0.0 -0.0e0 -.0 .0e0 0e-1\";\n let data = [\n ((0.0, 0), (0..3), \"0.0\"),\n ((-0.0, 0), (4..10), \"-0.0e0\"),\n ((-0.0, 0), (11..14), \"-.0\"),\n ((0.0, 0), (15..19), \".0e0\"),\n ((0.0, -1), (20..24), \"0e-1\"),\n ];\n let lexer = Token::lexer(source);\n for ((_, (tok, span)), (parsed, tspan, tslice)) in lexer\n .spanned()\n .enumerate()\n // Skip all odd-indexed tokens (it's whitespace)\n .filter(|(idx, _)| idx % 2 == 0)\n .zip(data.into_iter())\n {\n let slice = &source[span.clone()];\n assert!(tok == Ok(Token::Decimal(parsed)));\n assert!(span == tspan);\n assert!(slice == tslice);\n }\n }\n\n #[test]\n"},{"Del":" fn literal_identfier_equivalence() {\n let source = r\"|\\t\\t||\\x9;\\x9;|\";\n let mut lexer = Token::lexer(source);\n\n let tok1 = lexer.next();\n let tok1_slice = lexer.slice();\n let tok2 = lexer.next();\n let tok2_slice = lexer.slice();\n assert!(tok1 == tok2);\n assert!(tok1_slice != tok2_slice);\n }\n\n #[test]\n fn lex_literal_identifier() {\n let source = r\"|H\\x65;llo|\";\n let mut lexer = Token::lexer(source);\n assert!(lexer.next() == Some(Ok(Token::Identifier(\"Hello\".to_string()))));\n assert!(lexer.span() == (0..source.len()));\n assert!(lexer.slice() == \"|H\\\\x65;llo|\");\n }\n\n #[test]\n fn lex_identifier_or_number() {\n let source = \"3let -i\";\n let mut lexer = Token::lexer(source);\n"},{"Del":" assert!(lexer.next() == Some(Ok(Token::Integer((3, 0)))));\n"},{"Del":" assert!(lexer.span() == (0..1));\n assert!(lexer.slice() == \"3\");\n\n assert!(lexer.next() == Some(Ok(Token::Identifier(\"let\".to_string()))));\n assert!(lexer.span() == (1..4));\n assert!(lexer.slice() == \"let\");\n\n assert!(lexer.next() == Some(Ok(Token::Whitespace)));\n assert!(lexer.span() == (4..5));\n assert!(lexer.slice() == \" \");\n"},{"Del":"\n"},{"Del":" assert!(lexer.next() == Some(Ok(Token::ImaginaryNeg)));\n assert!(lexer.span() == (5..7));\n assert!(lexer.slice() == \"-i\");\n }\n"},{"Del":"\n"},{"Del":" #[test]\n fn lex_identifier() {\n let source = \"+soup+ $?bama\";\n let mut lexer = Token::lexer(source);\n assert!(lexer.next() == Some(Ok(Token::Identifier(\"+soup+\".to_string()))));\n assert!(lexer.span() == (0..6));\n assert!(lexer.slice() == \"+soup+\");\n\n assert!(lexer.next() == Some(Ok(Token::Whitespace)));\n assert!(lexer.span() == (6..7));\n assert!(lexer.slice() == \" \");\n\n assert!(lexer.next() == Some(Ok(Token::Identifier(\"$?bama\".to_string()))));\n assert!(lexer.span() == (7..13));\n assert!(lexer.slice() == \"$?bama\");\n }\n"},{"Del":" }\n}\n"}]}}],"owner":"Jengamon","repo":"magus","token":"Th/seUs7brk8pVxRTADhKBduEw5wKGkJQolu5PEQEkM=","login":null}

…

Dependencies

Change contents