Add digit separator support for binary, octal, and hexadecimal numbers

Implements the digit separator specification from:
https://www.complang.tuwien.ac.at/ulrich/iso-prolog/digit_separators

Changes:
- Add digit separator support (underscore with optional whitespace/comments)
  for hexadecimal (0x), octal (0o), and binary (0b) number literals
- Decimal numbers already had digit separator support
- Add helper functions: skip_underscore_in_hexadecimal/octal/binary
- Update hexadecimal_constant, octal_constant, and binary_constant functions

Tests:
- Add 16 Rust unit tests in src/tests/parse_tokens.rs
- Add 18 Prolog integration tests in src/tests/digit_separators.pl
- Add CLI test configuration in tests/scryer/cli/src_tests/digit_separators.toml
- All tests verify: basic underscores, multiple underscores, whitespace after
  underscores, and comments between digits

Examples:
  0xDE_AD, 0xFF_ 00, 0xDE_ /* comment */ AD
  0o7_6, 0o77_ 00, 0o1_ /* octal */ 23
  0b10_11, 0b1111_ 0000, 0b10_ /* binary */ 11

Reference: mthom#3132 (comment)

Author: jjtolton

src/parser/lexer.rs

@@ -489,20 +489,14 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         if hexadecimal_digit_char!(c) {
             let mut token = String::with_capacity(16);
 
-            loop {
-                if hexadecimal_digit_char!(c) {
-                    self.skip_char(c);
-                    token.push(c);
-                    c = match self.lookahead_char() {
-                        Ok(c) => c,
-                        Err(e) if e.is_unexpected_eof() => {
-                            break;
-                        }
-                        Err(e) => return Err(e),
-                    };
-                } else {
-                    break;
-                }
+            self.skip_char(c);
+            token.push(c);
+            c = try_nt!(token, self.skip_underscore_in_hexadecimal());
+
+            while hexadecimal_digit_char!(c) {
+                token.push(c);
+                self.skip_char(c);
+                c = try_nt!(token, self.skip_underscore_in_hexadecimal());
             }
 
             self.parse_integer_by_radix(&token, 16)
@@ -520,20 +514,14 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         if octal_digit_char!(c) {
             let mut token = String::with_capacity(16);
 
-            loop {
-                if octal_digit_char!(c) {
-                    self.skip_char(c);
-                    token.push(c);
-                    c = match self.lookahead_char() {
-                        Ok(c) => c,
-                        Err(e) if e.is_unexpected_eof() => {
-                            break;
-                        }
-                        Err(e) => return Err(e),
-                    };
-                } else {
-                    break;
-                }
+            self.skip_char(c);
+            token.push(c);
+            c = try_nt!(token, self.skip_underscore_in_octal());
+
+            while octal_digit_char!(c) {
+                token.push(c);
+                self.skip_char(c);
+                c = try_nt!(token, self.skip_underscore_in_octal());
             }
 
             self.parse_integer_by_radix(&token, 8)
@@ -551,20 +539,14 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         if binary_digit_char!(c) {
             let mut token = String::with_capacity(16);
 
-            loop {
-                if binary_digit_char!(c) {
-                    self.skip_char(c);
-                    token.push(c);
-                    c = match self.lookahead_char() {
-                        Ok(c) => c,
-                        Err(e) if e.is_unexpected_eof() => {
-                            break;
-                        }
-                        Err(e) => return Err(e),
-                    };
-                } else {
-                    break;
-                }
+            self.skip_char(c);
+            token.push(c);
+            c = try_nt!(token, self.skip_underscore_in_binary());
+
+            while binary_digit_char!(c) {
+                token.push(c);
+                self.skip_char(c);
+                c = try_nt!(token, self.skip_underscore_in_binary());
             }
 
             self.parse_integer_by_radix(&token, 2)
@@ -693,6 +675,60 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         }
     }
 
+    fn skip_underscore_in_hexadecimal(&mut self) -> Result<char, ParserError> {
+        let mut c = self.lookahead_char()?;
+
+        if c == '_' {
+            self.skip_char(c);
+            self.scan_for_layout()?;
+            c = self.lookahead_char()?;
+
+            if hexadecimal_digit_char!(c) {
+                Ok(c)
+            } else {
+                Err(ParserError::ParseBigInt(self.line_num, self.col_num))
+            }
+        } else {
+            Ok(c)
+        }
+    }
+
+    fn skip_underscore_in_octal(&mut self) -> Result<char, ParserError> {
+        let mut c = self.lookahead_char()?;
+
+        if c == '_' {
+            self.skip_char(c);
+            self.scan_for_layout()?;
+            c = self.lookahead_char()?;
+
+            if octal_digit_char!(c) {
+                Ok(c)
+            } else {
+                Err(ParserError::ParseBigInt(self.line_num, self.col_num))
+            }
+        } else {
+            Ok(c)
+        }
+    }
+
+    fn skip_underscore_in_binary(&mut self) -> Result<char, ParserError> {
+        let mut c = self.lookahead_char()?;
+
+        if c == '_' {
+            self.skip_char(c);
+            self.scan_for_layout()?;
+            c = self.lookahead_char()?;
+
+            if binary_digit_char!(c) {
+                Ok(c)
+            } else {
+                Err(ParserError::ParseBigInt(self.line_num, self.col_num))
+            }
+        } else {
+            Ok(c)
+        }
+    }
+
     fn parse_integer_by_radix(&mut self, token: &str, radix: u32) -> Result<GInteger, ParserError> {
         i64::from_str_radix(token, radix)
             .map(|n| {

src/tests/digit_separators.pl

@@ -0,0 +1,93 @@
+:- module(digit_separators_tests, []).
+:- use_module(test_framework).
+
+test("decimal with single underscore", (
+    X = 1_000,
+    X =:= 1000
+)).
+
+test("decimal with multiple underscores", (
+    X = 1_000_000,
+    X =:= 1000000
+)).
+
+test("decimal with underscore and whitespace", (
+    X = 123_ 456,
+    X =:= 123456
+)).
+
+test("decimal with underscore and comment", (
+    X = 123_ /* comment */ 456,
+    X =:= 123456
+)).
+
+test("hexadecimal with single underscore", (
+    X = 0xDE_AD,
+    X =:= 0xDEAD
+)).
+
+test("hexadecimal with multiple underscores", (
+    X = 0x1_2_3_4,
+    X =:= 0x1234
+)).
+
+test("hexadecimal with underscore and whitespace", (
+    X = 0xFF_ 00,
+    X =:= 0xFF00
+)).
+
+test("hexadecimal with underscore and comment", (
+    X = 0xDE_ /* test */ AD,
+    X =:= 0xDEAD
+)).
+
+test("octal with single underscore", (
+    X = 0o7_6,
+    X =:= 0o76
+)).
+
+test("octal with multiple underscores", (
+    X = 0o1_2_3,
+    X =:= 0o123
+)).
+
+test("octal with underscore and whitespace", (
+    X = 0o77_ 00,
+    X =:= 0o7700
+)).
+
+test("octal with underscore and comment", (
+    X = 0o1_ /* octal */ 23,
+    X =:= 0o123
+)).
+
+test("binary with single underscore", (
+    X = 0b10_11,
+    X =:= 0b1011
+)).
+
+test("binary with multiple underscores", (
+    X = 0b1_0_1_0,
+    X =:= 0b1010
+)).
+
+test("binary with underscore and whitespace", (
+    X = 0b1111_ 0000,
+    X =:= 0b11110000
+)).
+
+test("binary with underscore and comment", (
+    X = 0b10_ /* binary */ 11,
+    X =:= 0b1011
+)).
+
+test("large decimal number with separators", (
+    X = 999_999_999,
+    X =:= 999999999
+)).
+
+test("hexadecimal case insensitive", (
+    X1 = 0xAB_CD,
+    X2 = 0xab_cd,
+    X1 =:= X2
+)).

src/tests/parse_tokens.rs

@@ -107,3 +107,115 @@ fn comment_then_eof() -> Result<(), ParserError> {
     assert!(read_all_tokens("% only a comment").is_err());
     Ok(())
 }
+
+#[test]
+fn decimal_with_underscore() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("1_000")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(1000)))]);
+    Ok(())
+}
+
+#[test]
+fn decimal_with_multiple_underscores() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("1_000_000")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(1000000)))]);
+    Ok(())
+}
+
+#[test]
+fn decimal_with_underscore_and_whitespace() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("123_ 456")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(123456)))]);
+    Ok(())
+}
+
+#[test]
+fn decimal_with_underscore_and_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("123_ /* comment */ 456")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(123456)))]);
+    Ok(())
+}
+
+#[test]
+fn hexadecimal_with_underscore() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0xDE_AD")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0xDEAD)))]);
+    Ok(())
+}
+
+#[test]
+fn hexadecimal_with_multiple_underscores() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0x1_2_3_4")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0x1234)))]);
+    Ok(())
+}
+
+#[test]
+fn hexadecimal_with_underscore_and_whitespace() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0xFF_ 00")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0xFF00)))]);
+    Ok(())
+}
+
+#[test]
+fn hexadecimal_with_underscore_and_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0xDE_ /* test */ AD")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0xDEAD)))]);
+    Ok(())
+}
+
+#[test]
+fn octal_with_underscore() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0o7_6")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0o76)))]);
+    Ok(())
+}
+
+#[test]
+fn octal_with_multiple_underscores() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0o1_2_3")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0o123)))]);
+    Ok(())
+}
+
+#[test]
+fn octal_with_underscore_and_whitespace() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0o77_ 00")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0o7700)))]);
+    Ok(())
+}
+
+#[test]
+fn octal_with_underscore_and_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0o1_ /* octal */ 23")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0o123)))]);
+    Ok(())
+}
+
+#[test]
+fn binary_with_underscore() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0b10_11")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0b1011)))]);
+    Ok(())
+}
+
+#[test]
+fn binary_with_multiple_underscores() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0b1_0_1_0")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0b1010)))]);
+    Ok(())
+}
+
+#[test]
+fn binary_with_underscore_and_whitespace() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0b1111_ 0000")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0b11110000)))]);
+    Ok(())
+}
+
+#[test]
+fn binary_with_underscore_and_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("0b10_ /* binary */ 11")?;
+    assert_eq!(tokens, [Token::Literal(Literal::Fixnum(Fixnum::build_with(0b1011)))]);
+    Ok(())
+}

tests/scryer/cli/src_tests/digit_separators.toml

@@ -0,0 +1 @@
+args = ["-f", "--no-add-history", "src/tests/digit_separators.pl", "-f", "-g", "use_module(library(digit_separators_tests)), digit_separators_tests:main_quiet(digit_separators_tests)"]