Cleaning language

crates/forth/src/compiler.rs

@@ -8,8 +8,6 @@ enum Token {
     Float(f64, SourceSpan),
     Str(String, SourceSpan),
     Word(String, SourceSpan),
-    QuoteStart(usize),
-    QuoteEnd(usize),
 }
 
 pub(super) fn compile_script(input: &str, dict: &Dictionary) -> Result<Vec<Op>, String> {
@@ -44,25 +42,21 @@ fn tokenize(input: &str) -> Vec<Token> {
             continue;
         }
 
-        if c == '(' {
-            while let Some(&(_, ch)) = chars.peek() {
-                chars.next();
-                if ch == ')' {
-                    break;
+        if c == ';' {
+            chars.next(); // consume first ;
+            if let Some(&(_, ';')) = chars.peek() {
+                // ;; starts a comment to end of line
+                chars.next(); // consume second ;
+                while let Some(&(_, ch)) = chars.peek() {
+                    if ch == '\n' {
+                        break;
+                    }
+                    chars.next();
                 }
+                continue;
             }
-            continue;
-        }
-
-        if c == '{' {
-            chars.next();
-            tokens.push(Token::QuoteStart(pos));
-            continue;
-        }
-
-        if c == '}' {
-            chars.next();
-            tokens.push(Token::QuoteEnd(pos));
+            // single ; is a word, create token
+            tokens.push(Token::Word(";".to_string(), SourceSpan { start: pos, end: pos + 1 }));
             continue;
         }
 
@@ -70,7 +64,7 @@ fn tokenize(input: &str) -> Vec<Token> {
         let mut word = String::new();
         let mut end = start;
         while let Some(&(i, ch)) = chars.peek() {
-            if ch.is_whitespace() || ch == '{' || ch == '}' {
+            if ch.is_whitespace() {
                 break;
             }
             end = i + ch.len_utf8();
@@ -101,18 +95,16 @@ fn compile(tokens: &[Token], dict: &Dictionary) -> Result<Vec<Op>, String> {
             Token::Int(n, span) => ops.push(Op::PushInt(*n, Some(*span))),
             Token::Float(f, span) => ops.push(Op::PushFloat(*f, Some(*span))),
             Token::Str(s, span) => ops.push(Op::PushStr(s.clone(), Some(*span))),
-            Token::QuoteStart(start_pos) => {
-                let (quote_ops, consumed, end_pos) = compile_quotation(&tokens[i + 1..], dict)?;
-                i += consumed;
-                let body_span = SourceSpan { start: *start_pos, end: end_pos + 1 };
-                ops.push(Op::Quotation(quote_ops, Some(body_span)));
-            }
-            Token::QuoteEnd(_) => {
-                return Err("unexpected }".into());
-            }
             Token::Word(w, span) => {
                 let word = w.as_str();
-                if word == ":" {
+                if word == "{" {
+                    let (quote_ops, consumed, end_span) = compile_quotation(&tokens[i + 1..], dict)?;
+                    i += consumed;
+                    let body_span = SourceSpan { start: span.start, end: end_span.end };
+                    ops.push(Op::Quotation(quote_ops, Some(body_span)));
+                } else if word == "}" {
+                    return Err("unexpected }".into());
+                } else if word == ":" {
                     let (consumed, name, body) = compile_colon_def(&tokens[i + 1..], dict)?;
                     i += consumed;
                     dict.lock().unwrap().insert(name, body);
@@ -163,38 +155,38 @@ fn is_list_end(word: &str) -> bool {
     matches!(word, "]" | ">" | ">>")
 }
 
-fn compile_quotation(tokens: &[Token], dict: &Dictionary) -> Result<(Vec<Op>, usize, usize), String> {
+fn compile_quotation(tokens: &[Token], dict: &Dictionary) -> Result<(Vec<Op>, usize, SourceSpan), String> {
     let mut depth = 1;
     let mut end_idx = None;
 
     for (i, tok) in tokens.iter().enumerate() {
-        match tok {
-            Token::QuoteStart(_) => depth += 1,
-            Token::QuoteEnd(_) => {
-                depth -= 1;
-                if depth == 0 {
-                    end_idx = Some(i);
-                    break;
+        if let Token::Word(w, _) = tok {
+            match w.as_str() {
+                "{" => depth += 1,
+                "}" => {
+                    depth -= 1;
+                    if depth == 0 {
+                        end_idx = Some(i);
+                        break;
+                    }
                 }
+                _ => {}
             }
-            _ => {}
         }
     }
 
     let end_idx = end_idx.ok_or("missing }")?;
-    let byte_pos = match &tokens[end_idx] {
-        Token::QuoteEnd(pos) => *pos,
+    let end_span = match &tokens[end_idx] {
+        Token::Word(_, span) => *span,
         _ => unreachable!(),
     };
     let quote_ops = compile(&tokens[..end_idx], dict)?;
-    Ok((quote_ops, end_idx + 1, byte_pos))
+    Ok((quote_ops, end_idx + 1, end_span))
 }
 
 fn token_span(tok: &Token) -> Option<SourceSpan> {
     match tok {
         Token::Int(_, s) | Token::Float(_, s) | Token::Str(_, s) | Token::Word(_, s) => Some(*s),
-        Token::QuoteStart(p) => Some(SourceSpan { start: *p, end: *p + 1 }),
-        Token::QuoteEnd(p) => Some(SourceSpan { start: *p, end: *p + 1 }),
     }
 }
 
@@ -218,7 +210,6 @@ fn compile_colon_def(tokens: &[Token], dict: &Dictionary) -> Result<(usize, Stri
     let semi_pos = semi_pos.ok_or("missing ';' in word definition")?;
     let body_tokens = &tokens[1..semi_pos];
     let body_ops = compile(body_tokens, dict)?;
-    // consumed = name + body + semicolon
     Ok((semi_pos + 1, name, body_ops))
 }
 

crates/forth/src/vm.rs

@@ -559,7 +559,7 @@ impl Forth {
                 Op::Pick => {
                     let idx_i = stack.pop().ok_or("stack underflow")?.as_int()?;
                     if idx_i < 0 {
-                        return Err(format!("pick index must be >= 0, got {}", idx_i));
+                        return Err(format!("pick index must be >= 0, got {idx_i}"));
                     }
                     let idx = idx_i as usize;
                     let mut quots: Vec<Value> = Vec::new();

crates/forth/src/words.rs

@@ -2300,5 +2300,7 @@ pub(super) fn compile_word(
         return true;
     }
 
-    false
+    // Unrecognized token becomes a string
+    ops.push(Op::PushStr(name.to_string(), span));
+    true
 }