Skip to content

Commit f086d29

Browse files
raydacraydac
committed
fixed tokenizer operator processing
1 parent 035d499 commit f086d29

File tree

3 files changed

+80
-54
lines changed

3 files changed

+80
-54
lines changed

‎src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@
2121

2222
package com.igormaznitsa.prologparser.tokenizer;
2323

24+
import static com.igormaznitsa.prologparser.tokenizer.TokenizerState.INTEGER;
25+
import static com.igormaznitsa.prologparser.tokenizer.TokenizerState.LOOK_FOR;
26+
import static com.igormaznitsa.prologparser.tokenizer.TokenizerState.STRING;
27+
import static com.igormaznitsa.prologparser.utils.StringUtils.isCharAllowedForUnquotedAtom;
28+
2429
import com.igormaznitsa.prologparser.ParserContext;
2530
import com.igormaznitsa.prologparser.exceptions.CriticalUnexpectedError;
2631
import com.igormaznitsa.prologparser.exceptions.PrologParserException;
@@ -37,10 +42,6 @@
3742
import java.io.IOException;
3843
import java.io.Reader;
3944

40-
import static com.igormaznitsa.prologparser.tokenizer.TokenizerState.LOOK_FOR;
41-
import static com.igormaznitsa.prologparser.tokenizer.TokenizerState.STRING;
42-
import static com.igormaznitsa.prologparser.utils.StringUtils.isCharAllowedForUnquotedAtom;
43-
4445
/**
4546
* Internal tokenizer to gen next token from reader.
4647
*/
@@ -614,6 +615,8 @@ TokenizerResult readNextToken() {
614615
getLastTokenPos()
615616
);
616617
} else {
618+
calcDiffAndPushResultBack(
619+
lastFoundFullOperator.getTermText(), strBuffer);
617620
return this.tokenizerResultPool.find().setData(
618621
lastFoundFullOperator,
619622
state,

‎src/main/java/com/igormaznitsa/prologparser/tokenizer/TokenizerResult.java‎

Lines changed: 44 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,50 +29,55 @@
2929
* Internal object representing tokenizer result.
3030
*/
3131
final class TokenizerResult {
32-
private final SoftObjectPool<TokenizerResult> pool;
33-
private TokenizerState parserState;
34-
private PrologTerm resultTerm;
35-
private int pos;
36-
private int line;
32+
private final SoftObjectPool<TokenizerResult> pool;
33+
private TokenizerState parserState;
34+
private PrologTerm resultTerm;
35+
private int pos;
36+
private int line;
3737

38-
TokenizerResult(final SoftObjectPool<TokenizerResult> pool) {
39-
this.pool = pool;
40-
}
38+
TokenizerResult(final SoftObjectPool<TokenizerResult> pool) {
39+
this.pool = pool;
40+
}
4141

42-
TokenizerResult setData(
43-
final PrologTerm term,
44-
final TokenizerState parserState,
45-
final int line,
46-
final int pos
47-
) {
48-
this.resultTerm = AssertUtils.assertNotNull(term);
49-
this.parserState = AssertUtils.assertNotNull(parserState);
50-
this.pos = pos;
51-
this.line = line;
52-
return this;
53-
}
42+
TokenizerResult setData(
43+
final PrologTerm term,
44+
final TokenizerState parserState,
45+
final int line,
46+
final int pos
47+
) {
48+
this.resultTerm = AssertUtils.assertNotNull(term);
49+
this.parserState = AssertUtils.assertNotNull(parserState);
50+
this.pos = pos;
51+
this.line = line;
52+
return this;
53+
}
5454

55-
void release() {
56-
this.parserState = null;
57-
this.resultTerm = null;
58-
this.pos = -1;
59-
this.line = -1;
60-
this.pool.push(this);
61-
}
55+
void release() {
56+
this.parserState = null;
57+
this.resultTerm = null;
58+
this.pos = -1;
59+
this.line = -1;
60+
this.pool.push(this);
61+
}
6262

63-
TokenizerState getTokenizerState() {
64-
return this.parserState;
65-
}
63+
TokenizerState getTokenizerState() {
64+
return this.parserState;
65+
}
6666

67-
PrologTerm getResult() {
68-
return this.resultTerm;
69-
}
67+
PrologTerm getResult() {
68+
return this.resultTerm;
69+
}
7070

71-
int getPos() {
72-
return this.pos;
73-
}
71+
int getPos() {
72+
return this.pos;
73+
}
7474

75-
int getLine() {
76-
return this.line;
77-
}
75+
int getLine() {
76+
return this.line;
77+
}
78+
79+
@Override
80+
public String toString() {
81+
return "TokenizerResult(" + this.resultTerm + ')';
82+
}
7883
}

‎src/test/java/com/igormaznitsa/prologparser/IntegrationTest.java‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,33 @@
11
package com.igormaznitsa.prologparser;
22

3+
import static com.igormaznitsa.prologparser.DefaultParserContext.of;
4+
import static com.igormaznitsa.prologparser.ParserContext.FLAG_ALLOW_ZERO_STRUCT;
5+
import static com.igormaznitsa.prologparser.ParserContext.FLAG_BLOCK_COMMENTS;
6+
import static com.igormaznitsa.prologparser.ParserContext.FLAG_NONE;
7+
import static com.igormaznitsa.prologparser.ParserContext.FLAG_VAR_AS_FUNCTOR;
8+
import static com.igormaznitsa.prologparser.ParserContext.FLAG_ZERO_SINGLE_QUOTATION_CHAR_CODE;
9+
import static com.igormaznitsa.prologparser.terms.OpContainer.make;
10+
import static com.igormaznitsa.prologparser.terms.PrologTerm.QuotingType.BACK_QUOTED;
11+
import static com.igormaznitsa.prologparser.terms.PrologTerm.QuotingType.DOUBLE_QUOTED;
12+
import static com.igormaznitsa.prologparser.terms.PrologTerm.QuotingType.NO_QUOTED;
13+
import static com.igormaznitsa.prologparser.terms.PrologTerm.QuotingType.SINGLE_QUOTED;
14+
import static com.igormaznitsa.prologparser.terms.TermType.ATOM;
15+
import static com.igormaznitsa.prologparser.tokenizer.OpAssoc.FY;
16+
import static com.igormaznitsa.prologparser.tokenizer.OpAssoc.XFX;
17+
import static com.igormaznitsa.prologparser.tokenizer.OpAssoc.XFY;
18+
import static java.util.stream.Collectors.joining;
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
20+
import static org.junit.jupiter.api.Assertions.assertFalse;
21+
import static org.junit.jupiter.api.Assertions.assertNotNull;
22+
import static org.junit.jupiter.api.Assertions.assertNotSame;
23+
import static org.junit.jupiter.api.Assertions.assertSame;
24+
import static org.junit.jupiter.api.Assertions.assertThrows;
25+
import static org.junit.jupiter.api.Assertions.assertTrue;
26+
import static org.junit.jupiter.api.Assertions.fail;
27+
import static org.mockito.Mockito.clearInvocations;
28+
import static org.mockito.Mockito.mock;
29+
import static org.mockito.Mockito.when;
30+
331
import com.igormaznitsa.prologparser.exceptions.PrologParserException;
432
import com.igormaznitsa.prologparser.terms.OpContainer;
533
import com.igormaznitsa.prologparser.terms.PrologAtom;
@@ -32,16 +60,6 @@
3260
import java.util.Set;
3361
import java.util.concurrent.atomic.AtomicInteger;
3462

35-
import static com.igormaznitsa.prologparser.DefaultParserContext.of;
36-
import static com.igormaznitsa.prologparser.ParserContext.*;
37-
import static com.igormaznitsa.prologparser.terms.OpContainer.make;
38-
import static com.igormaznitsa.prologparser.terms.PrologTerm.QuotingType.*;
39-
import static com.igormaznitsa.prologparser.terms.TermType.ATOM;
40-
import static com.igormaznitsa.prologparser.tokenizer.OpAssoc.*;
41-
import static java.util.stream.Collectors.joining;
42-
import static org.junit.jupiter.api.Assertions.*;
43-
import static org.mockito.Mockito.*;
44-
4563
public class IntegrationTest {
4664

4765
private static PrologParser parseCpl(final String str) {
@@ -977,7 +995,7 @@ public void testOperatorAsFunctor() throws Exception {
977995

978996
@Test
979997
public void testPairOfOperatorsWithIncompatiblePrecedence() throws Exception {
980-
// assertEquals("", parseEd("1---1.").next().toString());
998+
assertEquals("1 - - -1", parseEd("1---1.").next().toString());
981999
assertEquals("1 + 1 * a * a + a - 1", parseEd("1+1*a*a+a-1.").next().toString());
9821000
assertEquals("-1 + 2 ** (- 3 ** (-4))", parseEd("-1+2**-3**-4.").next().toString());
9831001
assertEquals("X = (discontiguous)", parseEd("X=discontiguous.").next().toString());

0 commit comments

Comments
 (0)