Skip to content

Commit 73c8e3e

Browse files
shmaxondrejmirtes
shmax
authored and
ondrejmirtes
committed
TypeParser - support comments at EOL with // 
1 parent 19ecfcb commit 73c8e3e

13 files changed

+1627
-472
lines changed

‎src/Ast/Attribute.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,6 @@ final class Attribute
1313

1414
public const ORIGINAL_NODE = 'originalNode';
1515

16+
public const COMMENTS = 'comments';
17+
1618
}

‎src/Ast/Comment.php

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php declare(strict_types = 1);
2+
3+
namespace PHPStan\PhpDocParser\Ast;
4+
5+
use function trim;
6+
7+
class Comment
8+
{
9+
10+
public string $text;
11+
12+
public int $startLine;
13+
14+
public int $startIndex;
15+
16+
public function __construct(string $text, int $startLine = -1, int $startIndex = -1)
17+
{
18+
$this->text = $text;
19+
$this->startLine = $startLine;
20+
$this->startIndex = $startIndex;
21+
}
22+
23+
public function getReformattedText(): ?string
24+
{
25+
return trim($this->text);
26+
}
27+
28+
}

‎src/Lexer/Lexer.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ class Lexer
5151
public const TOKEN_NEGATED = 35;
5252
public const TOKEN_ARROW = 36;
5353

54+
public const TOKEN_COMMENT = 37;
55+
5456
public const TOKEN_LABELS = [
5557
self::TOKEN_REFERENCE => '\'&\'',
5658
self::TOKEN_UNION => '\'|\'',
@@ -66,6 +68,7 @@ class Lexer
6668
self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
6769
self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
6870
self::TOKEN_COMMA => '\',\'',
71+
self::TOKEN_COMMENT => '\'//\'',
6972
self::TOKEN_COLON => '\':\'',
7073
self::TOKEN_VARIADIC => '\'...\'',
7174
self::TOKEN_DOUBLE_COLON => '\'::\'',
@@ -160,6 +163,7 @@ private function generateRegexp(): string
160163
self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
161164

162165
self::TOKEN_COMMA => ',',
166+
self::TOKEN_COMMENT => '((?<![:/])\/\/[^\n]*)',
163167
self::TOKEN_VARIADIC => '\\.\\.\\.',
164168
self::TOKEN_DOUBLE_COLON => '::',
165169
self::TOKEN_DOUBLE_ARROW => '=>',

‎src/Parser/TokenIterator.php

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace PHPStan\PhpDocParser\Parser;
44

55
use LogicException;
6+
use PHPStan\PhpDocParser\Ast\Comment;
67
use PHPStan\PhpDocParser\Lexer\Lexer;
78
use function array_pop;
89
use function assert;
@@ -19,11 +20,16 @@ class TokenIterator
1920

2021
private int $index;
2122

23+
/** @var array<Comment> */
24+
private array $comments = [];
25+
2226
/** @var int[] */
2327
private array $savePoints = [];
2428

2529
/** @var list<int> */
26-
private array $skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
30+
private array $skippedTokenTypes = [
31+
Lexer::TOKEN_HORIZONTAL_WS,
32+
Lexer::TOKEN_COMMENT];
2733

2834
private ?string $newline = null;
2935

@@ -152,8 +158,7 @@ public function consumeTokenType(int $tokenType): void
152158
}
153159
}
154160

155-
$this->index++;
156-
$this->skipIrrelevantTokens();
161+
$this->next();
157162
}
158163

159164

@@ -166,8 +171,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
166171
$this->throwError($tokenType, $tokenValue);
167172
}
168173

169-
$this->index++;
170-
$this->skipIrrelevantTokens();
174+
$this->next();
171175
}
172176

173177

@@ -178,12 +182,20 @@ public function tryConsumeTokenValue(string $tokenValue): bool
178182
return false;
179183
}
180184

181-
$this->index++;
182-
$this->skipIrrelevantTokens();
185+
$this->next();
183186

184187
return true;
185188
}
186189

190+
/**
191+
* @return Comment[]
192+
*/
193+
public function flushComments(): array
194+
{
195+
$res = $this->comments;
196+
$this->comments = [];
197+
return $res;
198+
}
187199

188200
/** @phpstan-impure */
189201
public function tryConsumeTokenType(int $tokenType): bool
@@ -198,8 +210,7 @@ public function tryConsumeTokenType(int $tokenType): bool
198210
}
199211
}
200212

201-
$this->index++;
202-
$this->skipIrrelevantTokens();
213+
$this->next();
203214

204215
return true;
205216
}
@@ -267,6 +278,11 @@ private function skipIrrelevantTokens(): void
267278
if (!isset($this->tokens[$this->index + 1])) {
268279
break;
269280
}
281+
282+
if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
283+
$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
284+
}
285+
270286
$this->index++;
271287
}
272288
}
@@ -310,7 +326,6 @@ public function rollback(): void
310326
$this->index = $index;
311327
}
312328

313-
314329
/**
315330
* @throws ParserException
316331
*/

‎src/Parser/TypeParser.php

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ private function enrichTypeOnUnionOrIntersection(TokenIterator $tokens, Ast\Type
8282
* @internal
8383
* @template T of Ast\Node
8484
* @param T $type
85+
*
8586
* @return T
8687
*/
8788
public function enrichWithAttributes(TokenIterator $tokens, Ast\Node $type, int $startLine, int $startIndex): Ast\Node
@@ -91,6 +92,11 @@ public function enrichWithAttributes(TokenIterator $tokens, Ast\Node $type, int
9192
$type->setAttribute(Ast\Attribute::END_LINE, $tokens->currentTokenLine());
9293
}
9394

95+
$comments = $tokens->flushComments();
96+
if ($this->config->useCommentsAttributes) {
97+
$type->setAttribute(Ast\Attribute::COMMENTS, $comments);
98+
}
99+
94100
if ($this->config->useIndexAttributes) {
95101
$type->setAttribute(Ast\Attribute::START_INDEX, $startIndex);
96102
$type->setAttribute(Ast\Attribute::END_INDEX, $tokens->endIndexOfLastRelevantToken());
@@ -429,6 +435,7 @@ public function isHtml(TokenIterator $tokens): bool
429435
public function parseGeneric(TokenIterator $tokens, Ast\Type\IdentifierTypeNode $baseType): Ast\Type\GenericTypeNode
430436
{
431437
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET);
438+
$tokens->skipNewLineTokens();
432439

433440
$startLine = $baseType->getAttribute(Ast\Attribute::START_LINE);
434441
$startIndex = $baseType->getAttribute(Ast\Attribute::START_INDEX);
@@ -859,6 +866,8 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
859866
$sealed = true;
860867
$unsealedType = null;
861868

869+
$done = false;
870+
862871
do {
863872
$tokens->skipNewLineTokens();
864873

@@ -884,9 +893,17 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
884893
}
885894

886895
$items[] = $this->parseArrayShapeItem($tokens);
887-
888896
$tokens->skipNewLineTokens();
889-
} while ($tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA));
897+
if (!$tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
898+
$done = true;
899+
}
900+
if ($tokens->currentTokenType() !== Lexer::TOKEN_COMMENT) {
901+
continue;
902+
}
903+
904+
$tokens->next();
905+
906+
} while (!$done);
890907

891908
$tokens->skipNewLineTokens();
892909
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET);
@@ -904,12 +921,17 @@ private function parseArrayShapeItem(TokenIterator $tokens): Ast\Type\ArrayShape
904921
{
905922
$startLine = $tokens->currentTokenLine();
906923
$startIndex = $tokens->currentTokenIndex();
924+
925+
// parse any comments above the item
926+
$tokens->skipNewLineTokens();
927+
907928
try {
908929
$tokens->pushSavePoint();
909930
$key = $this->parseArrayShapeKey($tokens);
910931
$optional = $tokens->tryConsumeTokenType(Lexer::TOKEN_NULLABLE);
911932
$tokens->consumeTokenType(Lexer::TOKEN_COLON);
912933
$value = $this->parse($tokens);
934+
913935
$tokens->dropSavePoint();
914936

915937
return $this->enrichWithAttributes(
@@ -1056,12 +1078,19 @@ private function parseObjectShapeItem(TokenIterator $tokens): Ast\Type\ObjectSha
10561078
$startLine = $tokens->currentTokenLine();
10571079
$startIndex = $tokens->currentTokenIndex();
10581080

1081+
$tokens->skipNewLineTokens();
1082+
10591083
$key = $this->parseObjectShapeKey($tokens);
10601084
$optional = $tokens->tryConsumeTokenType(Lexer::TOKEN_NULLABLE);
10611085
$tokens->consumeTokenType(Lexer::TOKEN_COLON);
10621086
$value = $this->parse($tokens);
10631087

1064-
return $this->enrichWithAttributes($tokens, new Ast\Type\ObjectShapeItemNode($key, $optional, $value), $startLine, $startIndex);
1088+
return $this->enrichWithAttributes(
1089+
$tokens,
1090+
new Ast\Type\ObjectShapeItemNode($key, $optional, $value),
1091+
$startLine,
1092+
$startIndex,
1093+
);
10651094
}
10661095

10671096
/**

‎src/ParserConfig.php

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@ class ParserConfig
99

1010
public bool $useIndexAttributes;
1111

12+
public bool $useCommentsAttributes;
13+
1214
/**
13-
* @param array{lines?: bool, indexes?: bool} $usedAttributes
15+
* @param array{lines?: bool, indexes?: bool, comments?: bool} $usedAttributes
1416
*/
1517
public function __construct(array $usedAttributes)
1618
{
1719
$this->useLinesAttributes = $usedAttributes['lines'] ?? false;
1820
$this->useIndexAttributes = $usedAttributes['indexes'] ?? false;
21+
$this->useCommentsAttributes = $usedAttributes['comments'] ?? false;
1922
}
2023

2124
}

0 commit comments

Comments
 (0)