Cleaning language

crates/forth/src/compiler.rs

@@ -8,8 +8,6 @@ enum Token {
     Float(f64, SourceSpan),
     Str(String, SourceSpan),
     Word(String, SourceSpan),
-    QuoteStart(usize),
-    QuoteEnd(usize),
 }
 
 pub(super) fn compile_script(input: &str, dict: &Dictionary) -> Result<Vec<Op>, String> {
@@ -44,25 +42,21 @@ fn tokenize(input: &str) -> Vec<Token> {
             continue;
         }
 
-        if c == '(' {
+        if c == ';' {
+            chars.next(); // consume first ;
+            if let Some(&(_, ';')) = chars.peek() {
+                // ;; starts a comment to end of line
+                chars.next(); // consume second ;
                 while let Some(&(_, ch)) = chars.peek() {
-                chars.next();
+                    if ch == '\n' {
-                if ch == ')' {
                         break;
                     }
+                    chars.next();
                 }
                 continue;
             }
-
+            // single ; is a word, create token
-        if c == '{' {
+            tokens.push(Token::Word(";".to_string(), SourceSpan { start: pos, end: pos + 1 }));
-            chars.next();
-            tokens.push(Token::QuoteStart(pos));
-            continue;
-        }
-
-        if c == '}' {
-            chars.next();
-            tokens.push(Token::QuoteEnd(pos));
             continue;
         }
 
@@ -70,7 +64,7 @@ fn tokenize(input: &str) -> Vec<Token> {
         let mut word = String::new();
         let mut end = start;
         while let Some(&(i, ch)) = chars.peek() {
-            if ch.is_whitespace() || ch == '{' || ch == '}' {
+            if ch.is_whitespace() {
                 break;
             }
             end = i + ch.len_utf8();
@@ -101,18 +95,16 @@ fn compile(tokens: &[Token], dict: &Dictionary) -> Result<Vec<Op>, String> {
             Token::Int(n, span) => ops.push(Op::PushInt(*n, Some(*span))),
             Token::Float(f, span) => ops.push(Op::PushFloat(*f, Some(*span))),
             Token::Str(s, span) => ops.push(Op::PushStr(s.clone(), Some(*span))),
-            Token::QuoteStart(start_pos) => {
-                let (quote_ops, consumed, end_pos) = compile_quotation(&tokens[i + 1..], dict)?;
-                i += consumed;
-                let body_span = SourceSpan { start: *start_pos, end: end_pos + 1 };
-                ops.push(Op::Quotation(quote_ops, Some(body_span)));
-            }
-            Token::QuoteEnd(_) => {
-                return Err("unexpected }".into());
-            }
             Token::Word(w, span) => {
                 let word = w.as_str();
-                if word == ":" {
+                if word == "{" {
+                    let (quote_ops, consumed, end_span) = compile_quotation(&tokens[i + 1..], dict)?;
+                    i += consumed;
+                    let body_span = SourceSpan { start: span.start, end: end_span.end };
+                    ops.push(Op::Quotation(quote_ops, Some(body_span)));
+                } else if word == "}" {
+                    return Err("unexpected }".into());
+                } else if word == ":" {
                     let (consumed, name, body) = compile_colon_def(&tokens[i + 1..], dict)?;
                     i += consumed;
                     dict.lock().unwrap().insert(name, body);
@@ -163,14 +155,15 @@ fn is_list_end(word: &str) -> bool {
     matches!(word, "]" | ">" | ">>")
 }
 
-fn compile_quotation(tokens: &[Token], dict: &Dictionary) -> Result<(Vec<Op>, usize, usize), String> {
+fn compile_quotation(tokens: &[Token], dict: &Dictionary) -> Result<(Vec<Op>, usize, SourceSpan), String> {
     let mut depth = 1;
     let mut end_idx = None;
 
     for (i, tok) in tokens.iter().enumerate() {
-        match tok {
+        if let Token::Word(w, _) = tok {
-            Token::QuoteStart(_) => depth += 1,
+            match w.as_str() {
-            Token::QuoteEnd(_) => {
+                "{" => depth += 1,
+                "}" => {
                     depth -= 1;
                     if depth == 0 {
                         end_idx = Some(i);
@@ -180,21 +173,20 @@ fn compile_quotation(tokens: &[Token], dict: &Dictionary) -> Result<(Vec<Op>, us
                 _ => {}
             }
         }
+    }
 
     let end_idx = end_idx.ok_or("missing }")?;
-    let byte_pos = match &tokens[end_idx] {
+    let end_span = match &tokens[end_idx] {
-        Token::QuoteEnd(pos) => *pos,
+        Token::Word(_, span) => *span,
         _ => unreachable!(),
     };
     let quote_ops = compile(&tokens[..end_idx], dict)?;
-    Ok((quote_ops, end_idx + 1, byte_pos))
+    Ok((quote_ops, end_idx + 1, end_span))
 }
 
 fn token_span(tok: &Token) -> Option<SourceSpan> {
     match tok {
         Token::Int(_, s) | Token::Float(_, s) | Token::Str(_, s) | Token::Word(_, s) => Some(*s),
-        Token::QuoteStart(p) => Some(SourceSpan { start: *p, end: *p + 1 }),
-        Token::QuoteEnd(p) => Some(SourceSpan { start: *p, end: *p + 1 }),
     }
 }
 
@@ -218,7 +210,6 @@ fn compile_colon_def(tokens: &[Token], dict: &Dictionary) -> Result<(usize, Stri
     let semi_pos = semi_pos.ok_or("missing ';' in word definition")?;
     let body_tokens = &tokens[1..semi_pos];
     let body_ops = compile(body_tokens, dict)?;
-    // consumed = name + body + semicolon
     Ok((semi_pos + 1, name, body_ops))
 }
 

crates/forth/src/vm.rs

@@ -559,7 +559,7 @@ impl Forth {
                 Op::Pick => {
                     let idx_i = stack.pop().ok_or("stack underflow")?.as_int()?;
                     if idx_i < 0 {
-                        return Err(format!("pick index must be >= 0, got {}", idx_i));
+                        return Err(format!("pick index must be >= 0, got {idx_i}"));
                     }
                     let idx = idx_i as usize;
                     let mut quots: Vec<Value> = Vec::new();

crates/forth/src/words.rs

@@ -2300,5 +2300,7 @@ pub(super) fn compile_word(
         return true;
     }
 
-    false
+    // Unrecognized token becomes a string
+    ops.push(Op::PushStr(name.to_string(), span));
+    true
 }

src/engine/sequencer.rs

@@ -197,6 +197,7 @@ impl AudioState {
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn spawn_sequencer(
     link: Arc<LinkState>,
     playing: Arc<std::sync::atomic::AtomicBool>,

src/views/engine_view.rs

@@ -187,6 +187,7 @@ fn render_devices(frame: &mut Frame, app: &App, area: Rect) {
     );
 }
 
+#[allow(clippy::too_many_arguments)]
 fn render_device_column(
     frame: &mut Frame,
     area: Rect,

src/views/help_view.rs

@@ -242,7 +242,7 @@ fn parse_markdown(md: &str) -> Vec<RLine<'static>> {
         match line {
             Line::Normal(composite) if composite.style == CompositeStyle::Code => {
                 code_line_nr += 1;
-                let raw: String = composite.compounds.iter().map(|c| &*c.src).collect();
+                let raw: String = composite.compounds.iter().map(|c: &minimad::Compound| c.src).collect();
                 let mut spans = vec![
                     Span::styled(format!(" {code_line_nr:>2} "), code_border_style()),
                     Span::styled("│ ", code_border_style()),

src/views/highlight.rs

@@ -97,25 +97,15 @@ pub fn tokenize_line(line: &str) -> Vec<Token> {
             continue;
         }
 
-        if c == '(' {
+        if c == ';' && chars.peek().map(|(_, ch)| *ch) == Some(';') {
-            let end = line.len();
+            // ;; starts a comment to end of line
-            let comment_end = line[start..]
-                .find(')')
-                .map(|i| start + i + 1)
-                .unwrap_or(end);
             tokens.push(Token {
                 start,
-                end: comment_end,
+                end: line.len(),
                 kind: TokenKind::Comment,
             });
-            while let Some((i, _)) = chars.peek() {
-                if *i >= comment_end {
             break;
         }
-                chars.next();
-            }
-            continue;
-        }
 
         if c == '"' {
             let mut end = start + 1;

tests/forth/definitions.rs

@@ -46,8 +46,8 @@ fn word_defined_in_one_forth_available_in_same() {
 }
 
 #[test]
-fn unknown_word_errors() {
+fn unknown_word_becomes_string() {
-    expect_error("nosuchword", "unknown word");
+    expect_str("nosuchword", "nosuchword");
 }
 
 #[test]

tests/forth/errors.rs

@@ -11,8 +11,8 @@ fn whitespace_only() {
 }
 
 #[test]
-fn unknown_word() {
+fn unknown_word_becomes_string() {
-    expect_error("foobar", "unknown word");
+    expect_str("foobar", "foobar");
 }
 
 #[test]
@@ -22,12 +22,12 @@ fn string_not_number() {
 
 #[test]
 fn comment_ignored() {
-    expect_int("1 (this is a comment) 2 +", 3);
+    expect_int("1 ;; this is a comment\n2 +", 3);
 }
 
 #[test]
 fn multiline_comment() {
-    expect_int("1 (multi\nline\ncomment) 2 +", 3);
+    expect_int("1 ;; first comment\n;; entire line comment\n2 +", 3);
 }
 
 #[test]

tests/forth/harness.rs

@@ -88,6 +88,10 @@ pub fn expect_int(script: &str, expected: i64) {
     expect_stack(script, &[Value::Int(expected, None)]);
 }
 
+pub fn expect_str(script: &str, expected: &str) {
+    expect_stack(script, &[Value::Str(expected.to_string(), None)]);
+}
+
 pub fn expect_float(script: &str, expected: f64) {
     let f = run(script);
     let stack = f.stack();