Skip to content

Commit 3e90f48

Browse files
rlerdorfrlerdorf
committed
fix semicolon handling and test expectations
1 parent 7ef28a0 commit 3e90f48

22 files changed

+371
-180
lines changed

‎src/Parser.php‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3431,14 +3431,20 @@ private function parsePropertyDeclaration($parentNode, $modifiers, $questionToke
34313431
$propertyDeclaration->typeDeclarationList = new MissingToken(TokenKind::PropertyType, $this->token->fullStart);
34323432
}
34333433
$propertyDeclaration->propertyElements = $this->parsePropertyElementList($propertyDeclaration);
3434-
$requiresSemicolon = true;
3434+
$hasPropertyWithHooks = false;
3435+
$hasPlainProperty = false;
34353436
foreach ($propertyDeclaration->propertyElements->children ?? [] as $child) {
3436-
if ($child instanceof PropertyElement && $child->hookList !== null) {
3437-
$requiresSemicolon = false;
3438-
break;
3437+
if (!$child instanceof PropertyElement) {
3438+
continue;
3439+
}
3440+
if ($child->hookList !== null) {
3441+
$hasPropertyWithHooks = true;
3442+
continue;
34393443
}
3444+
$hasPlainProperty = true;
3445+
break;
34403446
}
3441-
if ($requiresSemicolon) {
3447+
if ($hasPlainProperty || !$hasPropertyWithHooks) {
34423448
$propertyDeclaration->semicolon = $this->eat1(TokenKind::SemicolonToken);
34433449
} else {
34443450
$propertyDeclaration->semicolon = $this->eatOptional1(TokenKind::SemicolonToken);

‎src/PhpTokenizer.php‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,50 @@ public static function getTokensArrayFromContent(
149149
$start = $fullStart = $pos;
150150
break;
151151
case \T_OPEN_TAG:
152+
if ($token[1] === '<?' && $i + 1 < $tokenCount) {
153+
$nextToken = $tokens[$keys[$i + 1]];
154+
if (\is_array($nextToken) && $nextToken[0] === \T_STRING && \strcasecmp($nextToken[1], 'php') === 0) {
155+
$pos += \strlen($nextToken[1]);
156+
$i++;
157+
$strlen += \strlen($nextToken[1]);
158+
}
159+
}
152160
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
153161
$start = $fullStart = $pos;
154162
break;
163+
case \T_YIELD:
164+
if ($treatCommentsAsTrivia) {
165+
$yieldFromEndIndex = $i;
166+
for ($lookahead = $i + 1; $lookahead < $tokenCount; $lookahead++) {
167+
$lookaheadToken = $tokens[$keys[$lookahead]];
168+
if (\is_array($lookaheadToken)) {
169+
$lookaheadKind = $lookaheadToken[0];
170+
if ($lookaheadKind === \T_WHITESPACE) {
171+
continue;
172+
}
173+
if (($lookaheadKind === \T_COMMENT || $lookaheadKind === \T_DOC_COMMENT) && $treatCommentsAsTrivia) {
174+
continue;
175+
}
176+
if ($lookaheadKind === \T_STRING && \strcasecmp($lookaheadToken[1], 'from') === 0) {
177+
$yieldFromEndIndex = $lookahead;
178+
}
179+
}
180+
break;
181+
}
182+
if ($yieldFromEndIndex !== $i) {
183+
for ($consume = $i + 1; $consume <= $yieldFromEndIndex; $consume++) {
184+
$consumed = $tokens[$keys[$consume]];
185+
$pos += \is_array($consumed) ? \strlen($consumed[1]) : \strlen($consumed);
186+
}
187+
$i = $yieldFromEndIndex;
188+
$arr[] = new Token(TokenKind::YieldFromKeyword, $fullStart, $start, $pos - $fullStart);
189+
$start = $fullStart = $pos;
190+
break;
191+
}
192+
}
193+
$arr[] = new Token(TokenKind::YieldKeyword, $fullStart, $start, $pos - $fullStart);
194+
$start = $fullStart = $pos;
195+
break;
155196
case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
156197
case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
157198
// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
@@ -167,8 +208,8 @@ public static function getTokensArrayFromContent(
167208
//
168209
// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
169210
// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
170-
foreach (\explode('\\', $token[1]) as $i => $name) {
171-
if ($i) {
211+
foreach (\explode('\\', $token[1]) as $segmentIndex => $name) {
212+
if ($segmentIndex) {
172213
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
173214
$start++;
174215
$fullStart = $start;
@@ -185,9 +226,9 @@ public static function getTokensArrayFromContent(
185226
break;
186227
case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
187228
// This is a namespace-relative name: namespace\...
188-
foreach (\explode('\\', $token[1]) as $i => $name) {
229+
foreach (\explode('\\', $token[1]) as $segmentIndex => $name) {
189230
$len = \strlen($name);
190-
if (!$i) {
231+
if (!$segmentIndex) {
191232
$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
192233
$start += $len;
193234
$fullStart = $start;

‎test‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
php -d auto_prepend_file=tests/no_git_checkout.php vendor/bin/phpunit

‎tests/cases/lexical/keyword5.php.tokens‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,8 @@
44
"textLength": 6
55
},
66
{
7-
"kind": "YieldKeyword",
8-
"textLength": 5
9-
},
10-
{
11-
"kind": "Name",
12-
"textLength": 4
7+
"kind": "YieldFromKeyword",
8+
"textLength": 22
139
},
1410
{
1511
"kind": "EndOfFileToken",

‎tests/cases/parser/dnfTypesProperty1.php.tree‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,22 @@
121121
}
122122
},
123123
"propertyElements": {
124-
"ExpressionList": {
124+
"PropertyElementList": {
125125
"children": [
126126
{
127-
"Variable": {
128-
"dollar": null,
129-
"name": {
130-
"kind": "VariableName",
131-
"textLength": 2
132-
}
127+
"PropertyElement": {
128+
"variable": {
129+
"Variable": {
130+
"dollar": null,
131+
"name": {
132+
"kind": "VariableName",
133+
"textLength": 2
134+
}
135+
}
136+
},
137+
"equalsToken": null,
138+
"initializer": null,
139+
"hookList": null
133140
}
134141
}
135142
]

‎tests/cases/parser/programStructure13.php.tree‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,20 @@
44
{
55
"InlineHtml": {
66
"scriptSectionEndTag": null,
7-
"text": {
8-
"kind": "InlineHtml",
9-
"textLength": 7
7+
"text": null,
8+
"scriptSectionStartTag": {
9+
"kind": "ScriptSectionStartTag",
10+
"textLength": 5
11+
}
12+
}
13+
},
14+
{
15+
"InlineHtml": {
16+
"scriptSectionEndTag": {
17+
"kind": "ScriptSectionEndTag",
18+
"textLength": 2
1019
},
20+
"text": null,
1121
"scriptSectionStartTag": null
1222
}
1323
}

‎tests/cases/parser/programStructure21.php.diag‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,8 @@
1-
[]
1+
[
2+
{
3+
"kind": 0,
4+
"message": "';' expected.",
5+
"start": 6,
6+
"length": 0
7+
}
8+
]

‎tests/cases/parser/programStructure21.php.tree‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,32 @@
44
{
55
"InlineHtml": {
66
"scriptSectionEndTag": null,
7-
"text": {
8-
"kind": "InlineHtml",
9-
"textLength": 6
7+
"text": null,
8+
"scriptSectionStartTag": {
9+
"kind": "ScriptSectionStartTag",
10+
"textLength": 2
11+
}
12+
}
13+
},
14+
{
15+
"ExpressionStatement": {
16+
"expression": {
17+
"QualifiedName": {
18+
"globalSpecifier": null,
19+
"relativeSpecifier": null,
20+
"nameParts": [
21+
{
22+
"kind": "Name",
23+
"textLength": 3
24+
}
25+
]
26+
}
1027
},
11-
"scriptSectionStartTag": null
28+
"semicolon": {
29+
"error": "MissingToken",
30+
"kind": "SemicolonToken",
31+
"textLength": 0
32+
}
1233
}
1334
}
1435
],

‎tests/cases/parser/propertyDeclaration1.php.tree‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,22 @@
4545
"questionToken": null,
4646
"typeDeclarationList": null,
4747
"propertyElements": {
48-
"ExpressionList": {
48+
"PropertyElementList": {
4949
"children": [
5050
{
51-
"Variable": {
52-
"dollar": null,
53-
"name": {
54-
"kind": "VariableName",
55-
"textLength": 2
56-
}
51+
"PropertyElement": {
52+
"variable": {
53+
"Variable": {
54+
"dollar": null,
55+
"name": {
56+
"kind": "VariableName",
57+
"textLength": 2
58+
}
59+
}
60+
},
61+
"equalsToken": null,
62+
"initializer": null,
63+
"hookList": null
5764
}
5865
}
5966
]

‎tests/cases/parser/propertyDeclaration2.php.tree‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@
4545
"questionToken": null,
4646
"typeDeclarationList": null,
4747
"propertyElements": {
48-
"ExpressionList": {
48+
"PropertyElementList": {
4949
"children": [
5050
{
51-
"AssignmentExpression": {
52-
"leftOperand": {
51+
"PropertyElement": {
52+
"variable": {
5353
"Variable": {
5454
"dollar": null,
5555
"name": {
@@ -58,12 +58,11 @@
5858
}
5959
}
6060
},
61-
"operator": {
61+
"equalsToken": {
6262
"kind": "EqualsToken",
6363
"textLength": 1
6464
},
65-
"byRef": null,
66-
"rightOperand": {
65+
"initializer": {
6766
"AssignmentExpression": {
6867
"leftOperand": {
6968
"Variable": {
@@ -88,7 +87,8 @@
8887
}
8988
}
9089
}
91-
}
90+
},
91+
"hookList": null
9292
}
9393
}
9494
]

0 commit comments

Comments
 (0)