Skip to content

Commit 3c01426

Browse files
alekittoalekitto
committed
tokenizer: add support for php8 attributes
1 parent 285de0d commit 3c01426

File tree

6 files changed

+394
-0
lines changed

6 files changed

+394
-0
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
181181
<dir name="Tokenizer">
182182
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
183183
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
184+
<file baseinstalldir="" name="AttributesTest.inc" role="test" />
185+
<file baseinstalldir="" name="AttributesTest.php" role="test" />
184186
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
185187
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
186188
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
@@ -2085,6 +2087,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20852087
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
20862088
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
20872089
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2090+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2091+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
20882092
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
20892093
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
20902094
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
@@ -2161,6 +2165,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21612165
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
21622166
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
21632167
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2168+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2169+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
21642170
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
21652171
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
21662172
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />

‎src/Files/File.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
18131813
T_SEMICOLON,
18141814
T_OPEN_CURLY_BRACKET,
18151815
T_CLOSE_CURLY_BRACKET,
1816+
T_ATTRIBUTE_END,
18161817
],
18171818
($stackPtr - 1)
18181819
);

‎src/Tokenizers/PHP.php‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -893,6 +893,60 @@ protected function tokenize($string)
893893
continue;
894894
}//end if
895895

896+
/*
897+
PHP 8.0 Attributes
898+
*/
899+
900+
if (PHP_VERSION_ID < 80000
901+
&& $token[0] === T_COMMENT
902+
&& strpos($token[1], '#[') === 0
903+
) {
904+
$subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
905+
if ($subTokens !== null) {
906+
array_splice($tokens, $stackPtr, 1, $subTokens);
907+
$numTokens = count($tokens);
908+
909+
$tokenIsArray = true;
910+
$token = $tokens[$stackPtr];
911+
}
912+
}
913+
914+
if ($tokenIsArray === true
915+
&& $token[0] === T_ATTRIBUTE
916+
) {
917+
// Go looking for the close bracket.
918+
$bracketStack = [$stackPtr];
919+
$bracketCloser = null;
920+
for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
921+
if ($tokens[$x] === '[') {
922+
$bracketStack[] = $x;
923+
} else if ($tokens[$x] === ']') {
924+
array_pop($bracketStack);
925+
if (empty($bracketStack) === true) {
926+
$bracketCloser = $x;
927+
break;
928+
}
929+
}
930+
}
931+
932+
$newToken = [];
933+
$newToken['code'] = T_ATTRIBUTE;
934+
$newToken['type'] = 'T_ATTRIBUTE';
935+
$newToken['content'] = '#[';
936+
$finalTokens[$newStackPtr] = $newToken;
937+
938+
$tokens[$bracketCloser] = [];
939+
$tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
940+
$tokens[$bracketCloser][1] = ']';
941+
942+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
943+
echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
944+
}
945+
946+
$newStackPtr++;
947+
continue;
948+
}//end if
949+
896950
/*
897951
Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
898952
token and ensure that the colon after it is always T_COLON.
@@ -1700,6 +1754,7 @@ function return types. We want to keep the parenthesis map clean,
17001754
T_CLASS => true,
17011755
T_EXTENDS => true,
17021756
T_IMPLEMENTS => true,
1757+
T_ATTRIBUTE => true,
17031758
T_NEW => true,
17041759
T_CONST => true,
17051760
T_NS_SEPARATOR => true,
@@ -2500,6 +2555,17 @@ protected function processAdditional()
25002555
$this->tokens[$x]['code'] = T_STRING;
25012556
$this->tokens[$x]['type'] = 'T_STRING';
25022557
}
2558+
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
2559+
for ($x = ($i + 1); $x < $numTokens; $x++) {
2560+
if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) {
2561+
break;
2562+
}
2563+
}
2564+
2565+
$this->tokens[$i]['attribute_opener'] = $i;
2566+
$this->tokens[$i]['attribute_closer'] = $x;
2567+
$this->tokens[$x]['attribute_opener'] = $i;
2568+
$this->tokens[$x]['attribute_closer'] = $x;
25032569
}//end if
25042570

25052571
if (($this->tokens[$i]['code'] !== T_CASE
@@ -2837,4 +2903,58 @@ public static function resolveSimpleToken($token)
28372903
}//end resolveSimpleToken()
28382904

28392905

2906+
/**
2907+
* PHP 8 attributes parser for PHP < 8
2908+
* Handles single-line and multiline attributes.
2909+
*
2910+
* @param array $tokens The original array of tokens (as returned by token_get_all)
2911+
* @param int $stackPtr The current position in token array
2912+
*
2913+
* @return array|null The array of parsed attribute tokens
2914+
*/
2915+
private function parsePhpAttribute(array &$tokens, $stackPtr)
2916+
{
2917+
2918+
$token = $tokens[$stackPtr];
2919+
2920+
$commentBody = substr($token[1], 2);
2921+
$subTokens = @token_get_all('<?php '.$commentBody);
2922+
array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
2923+
2924+
// Go looking for the close bracket.
2925+
$findCloser = static function (array $tokens, $start=1) {
2926+
$numTokens = count($tokens);
2927+
$bracketStack = [0];
2928+
$bracketCloser = null;
2929+
for ($x = $start; $x < $numTokens; $x++) {
2930+
if ($tokens[$x] === '[') {
2931+
$bracketStack[] = $x;
2932+
} else if ($tokens[$x] === ']') {
2933+
array_pop($bracketStack);
2934+
if (empty($bracketStack) === true) {
2935+
$bracketCloser = $x;
2936+
break;
2937+
}
2938+
}
2939+
}
2940+
2941+
return $bracketCloser;
2942+
};
2943+
2944+
$bracketCloser = $findCloser($subTokens);
2945+
if ($bracketCloser === null) {
2946+
$bracketCloser = $findCloser($tokens, $stackPtr);
2947+
if ($bracketCloser === null) {
2948+
return null;
2949+
}
2950+
2951+
array_splice($subTokens, count($subTokens), 0, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
2952+
array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
2953+
}
2954+
2955+
return $subTokens;
2956+
2957+
}//end parsePhpAttribute()
2958+
2959+
28402960
}//end class

‎src/Util/Tokens.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,12 @@
143143
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
144144
}
145145

146+
if (defined('T_ATTRIBUTE') === false) {
147+
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
148+
}
149+
150+
define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');
151+
146152
// Tokens used for parsing doc blocks.
147153
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
148154
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/Tokenizer/AttributesTest.inc‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?php
2+
3+
/* testAttribute */
4+
#[Attribute]
5+
class CustomAttribute {}
6+
7+
/* testAttributeWithParams */
8+
#[Attribute(Attribute::TARGET_CLASS)]
9+
class SecondCustomAttribute {}
10+
11+
/* testAttributeWithNamedParam */
12+
#[Attribute(flags: Attribute::TARGET_ALL)]
13+
class AttributeWithParams {
14+
public function __construct($foo, array $bar) {}
15+
}
16+
17+
/* testAttributeOnFunction */
18+
#[CustomAttribute]
19+
function attribute_on_function_test() {}
20+
21+
/* testAttributeOnFunctionWithParams */
22+
#[AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
23+
function attribute_with_params_on_function_test() {}
24+
25+
/* testTwoAttributeOnTheSameLine */
26+
#[CustomAttribute] #[AttributeWithParams('foo')]
27+
function two_attribute_on_same_line_test() {}
28+
29+
/* testAttributeAndCommentOnTheSameLine */
30+
#[CustomAttribute] // This is a comment
31+
function attribute_and_line_comment_on_same_line_test() {}
32+
33+
/* testAttributeGrouping */
34+
#[CustomAttribute, AttributeWithParams('foo'), AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
35+
function attribute_grouping_test() {}
36+
37+
/* testAttributeMultiline */
38+
#[
39+
CustomAttribute,
40+
AttributeWithParams('foo'),
41+
AttributeWithParams('foo', bar: ['bar' => 'foobar'])
42+
]
43+
function attribute_multiline_test() {}
44+
45+
/* testAttributeMultilineWithComment */
46+
#[
47+
CustomAttribute, // comment
48+
AttributeWithParams(/* another comment */ 'foo'),
49+
AttributeWithParams('foo', bar: ['bar' => 'foobar'])
50+
]
51+
function attribute_multiline_with_comment_test() {}
52+
53+
/* testSingleAttributeOnParameter */
54+
function single_attribute_on_parameter_test(#[ParamAttribute] int $param) {}
55+
56+
/* testMultipleAttributesOnParameter */
57+
function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithParams(/* another comment */ 'foo')] int $param) {}
58+
59+
/* testMultilineAttributesOnParameter */
60+
function multiline_attributes_on_parameter_test(#[
61+
AttributeWithParams(
62+
'foo'
63+
)
64+
] int $param) {}
65+
66+

0 commit comments

Comments
 (0)