Skip to content

Commit 299f508

Browse files
mousetrapsmousetraps
authored
Merge pull request microsoft#94 from mousetraps/api
microsoft#88 Add API for `getDocCommentText`, `getFirst*Node`
2 parents 4f155b2 + 8e4ef4a commit 299f508

File tree

7 files changed

+191
-34
lines changed

7 files changed

+191
-34
lines changed

‎src/Lexer.php renamed to ‎experiments/Lexer.php

File renamed without changes.

‎src/Node.php

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,42 @@ public function getFirstAncestor(...$classNames) {
7777
return null;
7878
}
7979

80+
/**
81+
* Get's first child that is an instance of one of the provided classes.
82+
* Returns null if there is no match.
83+
*
84+
* @param array ...$classNames
85+
* @return Node|null
86+
*/
87+
public function getFirstChildNode(...$classNames) {
88+
foreach ($this->getChildNodes() as $child) {
89+
foreach ($classNames as $className) {
90+
if ($child instanceof $className) {
91+
return $child;
92+
}
93+
}
94+
}
95+
return null;
96+
}
97+
98+
/**
99+
* Get's first descendant node that is an instance of one of the provided classes.
100+
* Returns null if there is no match.
101+
*
102+
* @param array ...$classNames
103+
* @return Node|null
104+
*/
105+
public function getFirstDescendantNode(...$classNames) {
106+
foreach ($this->getDescendantNodes() as $descendant) {
107+
foreach ($classNames as $className) {
108+
if ($descendant instanceof $className) {
109+
return $descendant;
110+
}
111+
}
112+
}
113+
return null;
114+
}
115+
80116
/**
81117
* Gets root of the syntax tree (returns self if has no parents)
82118
* @return Node
@@ -349,6 +385,7 @@ public function getEndPosition() {
349385
}
350386

351387
public function & getFileContents() : string {
388+
// TODO consider renaming to getSourceText
352389
return $this->getRoot()->fileContents;
353390
}
354391

@@ -369,6 +406,27 @@ public function getDescendantNodeAtPosition(int $pos) {
369406
return null;
370407
}
371408

409+
/**
410+
* Gets leading PHP Doc Comment text corresponding to the current Node.
411+
* Returns last doc comment in leading comment / whitespace trivia,
412+
* and returns null if there is no preceding doc comment.
413+
*
414+
* @return string | null
415+
*/
416+
public function getDocCommentText() {
417+
$leadingTriviaText = $this->getLeadingCommentAndWhitespaceText();
418+
$leadingTriviaTokens = PhpTokenizer::getTokensArrayFromContent(
419+
$leadingTriviaText, ParseContext::SourceElements, $this->getFullStart(), false
420+
);
421+
for ($i = \count($leadingTriviaTokens) - 1; $i >= 0; $i--) {
422+
$token = $leadingTriviaTokens[$i];
423+
if ($token->kind === TokenKind::DocCommentToken) {
424+
return $token->getText($this->getFileContents());
425+
}
426+
}
427+
return null;
428+
}
429+
372430
public function __toString() {
373431
return $this->getText();
374432
}

‎src/ParseContext.php

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
/*---------------------------------------------------------------------------------------------
3+
* Copyright (c) Microsoft Corporation. All rights reserved.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
namespace Microsoft\PhpParser;
8+
9+
class ParseContext {
10+
const SourceElements = 0;
11+
const BlockStatements = 1;
12+
const ClassMembers = 2;
13+
const IfClause2Elements = 3;
14+
const SwitchStatementElements = 4;
15+
const CaseStatementElements = 5;
16+
const WhileStatementElements = 6;
17+
const ForStatementElements = 7;
18+
const ForeachStatementElements = 8;
19+
const DeclareStatementElements = 9;
20+
const InterfaceMembers = 10;
21+
const TraitMembers = 11;
22+
const Count = 12;
23+
}

‎src/Parser.php

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2927,20 +2927,4 @@ class Associativity {
29272927
const None = 0;
29282928
const Left = 1;
29292929
const Right = 2;
2930-
}
2931-
2932-
class ParseContext {
2933-
const SourceElements = 0;
2934-
const BlockStatements = 1;
2935-
const ClassMembers = 2;
2936-
const IfClause2Elements = 3;
2937-
const SwitchStatementElements = 4;
2938-
const CaseStatementElements = 5;
2939-
const WhileStatementElements = 6;
2940-
const ForStatementElements = 7;
2941-
const ForeachStatementElements = 8;
2942-
const DeclareStatementElements = 9;
2943-
const InterfaceMembers = 10;
2944-
const TraitMembers = 11;
2945-
const Count = 12;
2946-
}
2930+
}

‎src/PhpTokenizer.php

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,24 @@
66

77
namespace Microsoft\PhpParser;
88

9+
/**
10+
* Tokenizes content using PHP's built-in `tokens_get_all`, and converts to "lightweight" Token representation.
11+
*
12+
* Initially we tried hand-spinning the lexer (see `experiments/Lexer.php`), but we had difficulties optimizing
13+
* performance (especially when working with Unicode characters.)
14+
*
15+
* Class PhpTokenizer
16+
* @package Microsoft\PhpParser
17+
*/
918
class PhpTokenizer implements ITokenStreamProvider {
1019
public $pos;
1120
public $endOfFilePos;
12-
private $token;
13-
14-
public $inScriptSection = false;
1521

1622
private $tokensArray;
1723

1824
public function __construct($content) {
19-
$tokens = \token_get_all($content);
20-
$this->initialize($tokens);
25+
$this->tokensArray = $this->getTokensArrayFromContent($content);
26+
$this->endOfFilePos = \count($this->tokensArray) - 1;
2127
$this->pos = 0;
2228
}
2329

@@ -43,11 +49,19 @@ public function getTokensArray() : array {
4349
return $this->tokensArray;
4450
}
4551

46-
private function initialize($tokens) {
52+
public static function getTokensArrayFromContent(
53+
$content, $parseContext = null, $initialPos = 0, $treatCommentsAsTrivia = true
54+
) : array {
55+
if ($parseContext !== null) {
56+
$prefix = self::PARSE_CONTEXT_TO_PREFIX[$parseContext];
57+
$content = $prefix . $content;
58+
$passedPrefix = false;
59+
}
60+
61+
$tokens = \token_get_all($content);
62+
4763
$arr = array();
48-
$fullStart = 0;
49-
$start = 0;
50-
$pos = 0;
64+
$fullStart = $start = $pos = $initialPos;
5165

5266
foreach ($tokens as $token) {
5367
if (\is_array($token)) {
@@ -60,15 +74,21 @@ private function initialize($tokens) {
6074

6175
$pos += $strlen;
6276

77+
if ($parseContext !== null && !$passedPrefix) {
78+
$passedPrefix = \count($prefix) < $pos;
79+
if ($passedPrefix) {
80+
$fullStart = $start = $pos = $initialPos;
81+
}
82+
continue;
83+
}
84+
6385
switch ($tokenKind) {
6486
case T_OPEN_TAG:
6587
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
6688
$start = $fullStart = $pos;
6789
continue;
6890

6991
case T_WHITESPACE:
70-
case T_COMMENT:
71-
case T_DOC_COMMENT:
7292
$start += $strlen;
7393
continue;
7494

@@ -82,6 +102,11 @@ private function initialize($tokens) {
82102
}
83103

84104
default:
105+
if (($tokenKind === T_COMMENT || $tokenKind === T_DOC_COMMENT) && $treatCommentsAsTrivia) {
106+
$start += $strlen;
107+
continue;
108+
}
109+
85110
$newTokenKind = isset(self::TOKEN_MAP[$tokenKind])
86111
? self::TOKEN_MAP[$tokenKind]
87112
: $newTokenKind = TokenKind::Unknown;
@@ -92,8 +117,7 @@ private function initialize($tokens) {
92117
}
93118

94119
$arr[] = new Token(TokenKind::EndOfFileToken, $fullStart, $start, $pos - $fullStart);
95-
$this->tokensArray = $arr;
96-
$this->endOfFilePos = \count($arr) - 1;
120+
return $arr;
97121
}
98122

99123
const TOKEN_MAP = [
@@ -267,6 +291,12 @@ private function initialize($tokens) {
267291
T_UNSET_CAST => TokenKind::UnsetCastToken,
268292
T_START_HEREDOC => TokenKind::HeredocStart,
269293
T_END_HEREDOC => TokenKind::HeredocEnd,
270-
T_STRING_VARNAME => TokenKind::VariableName
294+
T_STRING_VARNAME => TokenKind::VariableName,
295+
T_COMMENT => TokenKind::CommentToken,
296+
T_DOC_COMMENT => TokenKind::DocCommentToken
297+
];
298+
299+
const PARSE_CONTEXT_TO_PREFIX = [
300+
ParseContext::SourceElements => "<?php "
271301
];
272302
}

‎src/TokenKind.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,8 @@ class TokenKind {
212212
const BoolCastToken = 414;
213213
const ArrayCastToken = 415;
214214
const IntegerLiteralToken = 416;
215+
const CommentToken = 417;
216+
const DocCommentToken = 418;
215217

216218
// TODO type annotations - PHP7
217219
}

‎tests/NodeApiTest.php

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@
44
* Licensed under the MIT License. See License.txt in the project root for license information.
55
*--------------------------------------------------------------------------------------------*/
66

7-
use Microsoft\PhpParser\Node;
87
use Microsoft\PhpParser\Node\SourceFileNode;
8+
use Microsoft\PhpParser\Node\Statement\FunctionDeclaration;
99
use Microsoft\PhpParser\Node\Statement\IfStatementNode;
1010
use Microsoft\PhpParser\Node\Statement\NamespaceDefinition;
1111
use Microsoft\PhpParser\Parser;
1212
use PHPUnit\Framework\TestCase;
13-
use Microsoft\PhpParser\TokenKind;
1413

1514
class NodeApiTest extends TestCase {
1615
const FILENAME_PATTERN = __dir__ . "/cases/{parser,}/*.php";
@@ -33,7 +32,7 @@ public static function setUpBeforeClass() {
3332

3433
public function testSourceFileNodePosition() {
3534
$node = self::$sourceFileNode;
36-
$this->assertInstanceOf(\Microsoft\PhpParser\Node\Statement\FunctionDeclaration::class, $node->getDescendantNodeAtPosition(15));
35+
$this->assertInstanceOf(FunctionDeclaration::class, $node->getDescendantNodeAtPosition(15));
3736
$this->assertInstanceOf(\Microsoft\PhpParser\Node\Expression\Variable::class, $node->getDescendantNodeAtPosition(28));
3837
}
3938

@@ -130,4 +129,65 @@ class A {
130129
"getFirstAncestor with no specified class names should return null."
131130
);
132131
}
132+
133+
public function testGetDocCommentText() {
134+
$this->AssertDocCommentTextOfNode(
135+
FunctionDeclaration::class,
136+
"<?php /** */ function b () { }",
137+
"/** */"
138+
);
139+
140+
$this->AssertDocCommentTextOfNode(
141+
FunctionDeclaration::class,
142+
"<?php /***/ function b () { }",
143+
null
144+
);
145+
146+
$this->AssertDocCommentTextOfNode(
147+
FunctionDeclaration::class,
148+
"<?php /*/** */ function b () { }",
149+
null
150+
);
151+
152+
$this->AssertDocCommentTextOfNode(
153+
FunctionDeclaration::class,
154+
"<?php /**d */ function b () { }",
155+
null
156+
);
157+
158+
$this->AssertDocCommentTextOfNode(
159+
FunctionDeclaration::class,
160+
"<?php /** hello */\n/** */ function b () { }",
161+
"/** */"
162+
);
163+
164+
$this->AssertDocCommentTextOfNode(
165+
FunctionDeclaration::class,
166+
"<?php /** hello */\n/**\n*/ function b () { }",
167+
"/**\n*/"
168+
);
169+
170+
$this->AssertDocCommentTextOfNode(
171+
FunctionDeclaration::class,
172+
"<?php function b () { }",
173+
null
174+
);
175+
176+
$this->AssertDocCommentTextOfNode(
177+
\Microsoft\PhpParser\Node\Statement\InlineHtml::class,
178+
"/** hello */ <?php function b () { }",
179+
null
180+
);
181+
}
182+
183+
private function AssertDocCommentTextOfNode($nodeKind, $contents, $expectedDocCommentText) : array {
184+
$parser = new Parser();
185+
$ast = $parser->parseSourceFile($contents);
186+
$functionDeclaration = $ast->getFirstDescendantNode($nodeKind);
187+
$this->assertEquals(
188+
$expectedDocCommentText,
189+
$functionDeclaration->getDocCommentText()
190+
);
191+
return array($contents, $parser, $ast, $functionDeclaration);
192+
}
133193
}

0 commit comments

Comments
 (0)