implemented support for ISO radix

Author: raydac

src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java

@@ -91,6 +91,23 @@ public final TokenizerResult get() {
     this.tokenizerResultPool.fill();
   }
 
+  private static boolean isCharAllowedForRadix(final char chr, final int radix) {
+    if (radix == 10) {
+      return Character.isDigit(chr);
+    } else if (radix < 10) {
+      return chr >= '0' && chr < ('0' + radix);
+    } else {
+      if (chr >= '0' && chr <= '9') {
+        return true;
+      }
+      final int diff = radix - 10;
+      if (chr >= 'A' && chr < ('A' + diff)) {
+        return true;
+      }
+      return chr >= 'a' && chr < ('a' + diff);
+    }
+  }
+
   TokenizerResult getLastPushed() {
     return this.lastPushedTerm;
   }
@@ -542,12 +559,30 @@ TokenizerResult readNextToken() {
                       strBuffer.clear();
                     }
                   } else {
-                    push(chr);
-                    return this.tokenizerResultPool.find().setData(
-                        makeTermFromString(strBuffer.toString(), radix, quoting, state),
-                        TokenizerState.INTEGER,
-                        getLastTokenLine(),
-                        getLastTokenPos());
+                    boolean radixCharFound = false;
+                    if (radix == 10 && strBuffer.isSingleChar('0')) {
+                      if (chr == 'x') {
+                        radixCharFound = true;
+                        radix = 16;
+                        strBuffer.clear();
+                      } else if (chr == 'o') {
+                        radixCharFound = true;
+                        radix = 8;
+                        strBuffer.clear();
+                      } else if (chr == 'b') {
+                        radixCharFound = true;
+                        radix = 2;
+                        strBuffer.clear();
+                      }
+                    }
+                    if (!radixCharFound) {
+                      push(chr);
+                      return this.tokenizerResultPool.find().setData(
+                          makeTermFromString(strBuffer.toString(), radix, quoting, state),
+                          TokenizerState.INTEGER,
+                          getLastTokenLine(),
+                          getLastTokenPos());
+                    }
                   }
                 }
               }
@@ -849,23 +884,6 @@ state, getLastTokenLine(),
     }
   }
 
-  private static boolean isCharAllowedForRadix(final char chr, final int radix) {
-    if (radix == 10) {
-      return Character.isDigit(chr);
-    } else if (radix < 10) {
-      return chr >= '0' && chr < ('0' + radix);
-    } else {
-      if (chr >= '0' && chr <= '9') {
-        return true;
-      }
-      final int diff = radix - 10;
-      if (chr >= 'A' && chr < ('A' + diff)) {
-        return true;
-      }
-      return chr >= 'a' && chr < ('a' + diff);
-    }
-  }
-
   PrologTerm makeTermFromString(final String str, final int radix, final PrologTerm.QuotingType quotingType, final TokenizerState state) {
     PrologTerm result;
 

src/test/java/com/igormaznitsa/prologparser/IntegrationTest.java

@@ -896,7 +896,7 @@ private String parseSortAndJoin(final String text) {
 
   @Test
   public void testStringWithIsoControl() {
-    assertThrows(PrologParserException.class, ()-> parseEd("'hello\u0000world'.").next());
+    assertThrows(PrologParserException.class, () -> parseEd("'hello\u0000world'.").next());
     assertEquals("'hello\\nworld'", parseEd("'hello\\\nworld'.").next().toString());
     assertEquals("'hello\\nworld'", parseEd("'hello\\\r\nworld'.").next().toString());
   }

src/test/java/com/igormaznitsa/prologparser/tokenizer/TokenizerTest.java

@@ -247,8 +247,13 @@ public void testQuoting() {
   @Test
   public void testUnderscoreInNumbers_Normal() {
     assertEquals(6384, ((PrologInt) tokenizeOf("2'001_1000_1111_0000.").readNextToken().getResult()).getNumber().intValue());
+    assertEquals(6384, ((PrologInt) tokenizeOf("0b001_1000_1111_0000.").readNextToken().getResult()).getNumber().intValue());
     assertEquals(255, ((PrologInt) tokenizeOf("16'F_F.").readNextToken().getResult()).getNumber().intValue());
+    assertEquals(255, ((PrologInt) tokenizeOf("0xF_F.").readNextToken().getResult()).getNumber().intValue());
     assertEquals(255, ((PrologInt) tokenizeOf("16'f_f.").readNextToken().getResult()).getNumber().intValue());
+    assertEquals(255, ((PrologInt) tokenizeOf("0xf_f.").readNextToken().getResult()).getNumber().intValue());
+    assertEquals(511, ((PrologInt) tokenizeOf("8'77_7.").readNextToken().getResult()).getNumber().intValue());
+    assertEquals(511, ((PrologInt) tokenizeOf("0o77_7.").readNextToken().getResult()).getNumber().intValue());
 
     assertEquals(12345, ((PrologInt) tokenizeOf("12_345.").readNextToken().getResult()).getNumber().intValue());
     assertEquals(12345, ((PrologInt) tokenizeOf("12_34_5.").readNextToken().getResult()).getNumber().intValue());