Skip to content

Commit 7c1f941

Browse files
jrfnljrfnl
committed
PHP 8.0: "undo" namespaced names as single token
As per the proposal in 3041. This effectively "undoes" the new PHP 8.0 tokenization of identifier names for PHPCS 3.x. Includes extensive unit tests to ensure the correct re-tokenization as well as that the rest of the tokenization is not aversely affected by this change. Includes preventing `function ...` within a group use statement from breaking the retokenization. Includes fixing the nullable tokenization when combined with any of the new PHP 8 identifier name tokens.
1 parent d33a6a9 commit 7c1f941

File tree

5 files changed

+1531
-1
lines changed

5 files changed

+1531
-1
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
109109
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
110110
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
111111
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
112+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
113+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
112114
</dir>
113115
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
114116
<file baseinstalldir="" name="AllTests.php" role="test" />
@@ -1979,6 +1981,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
19791981
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
19801982
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
19811983
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
1984+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
1985+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
19821986
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
19831987
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
19841988
</filelist>
@@ -2038,6 +2042,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20382042
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
20392043
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
20402044
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2045+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2046+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20412047
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20422048
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20432049
<ignore name="bin/phpcs.bat" />

‎src/Tokenizers/PHP.php‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -812,6 +812,81 @@ protected function tokenize($string)
812812
continue;
813813
}//end if
814814

815+
/*
816+
As of PHP 8.0 fully qualified, partially qualified and namespace relative
817+
identifier names are tokenized differently.
818+
This "undoes" the new tokenization so the tokenization will be the same in
819+
in PHP 5, 7 and 8.
820+
*/
821+
822+
if (PHP_VERSION_ID >= 80000
823+
&& $tokenIsArray === true
824+
&& ($token[0] === T_NAME_QUALIFIED
825+
|| $token[0] === T_NAME_FULLY_QUALIFIED
826+
|| $token[0] === T_NAME_RELATIVE)
827+
) {
828+
$name = $token[1];
829+
830+
if ($token[0] === T_NAME_FULLY_QUALIFIED) {
831+
$newToken = [];
832+
$newToken['code'] = T_NS_SEPARATOR;
833+
$newToken['type'] = 'T_NS_SEPARATOR';
834+
$newToken['content'] = '\\';
835+
$finalTokens[$newStackPtr] = $newToken;
836+
++$newStackPtr;
837+
838+
$name = ltrim($name, '\\');
839+
}
840+
841+
if ($token[0] === T_NAME_RELATIVE) {
842+
$newToken = [];
843+
$newToken['code'] = T_NAMESPACE;
844+
$newToken['type'] = 'T_NAMESPACE';
845+
$newToken['content'] = substr($name, 0, 9);
846+
$finalTokens[$newStackPtr] = $newToken;
847+
++$newStackPtr;
848+
849+
$newToken = [];
850+
$newToken['code'] = T_NS_SEPARATOR;
851+
$newToken['type'] = 'T_NS_SEPARATOR';
852+
$newToken['content'] = '\\';
853+
$finalTokens[$newStackPtr] = $newToken;
854+
++$newStackPtr;
855+
856+
$name = substr($name, 10);
857+
}
858+
859+
$parts = explode('\\', $name);
860+
$partCount = count($parts);
861+
$lastPart = ($partCount - 1);
862+
863+
foreach ($parts as $i => $part) {
864+
$newToken = [];
865+
$newToken['code'] = T_STRING;
866+
$newToken['type'] = 'T_STRING';
867+
$newToken['content'] = $part;
868+
$finalTokens[$newStackPtr] = $newToken;
869+
++$newStackPtr;
870+
871+
if ($i !== $lastPart) {
872+
$newToken = [];
873+
$newToken['code'] = T_NS_SEPARATOR;
874+
$newToken['type'] = 'T_NS_SEPARATOR';
875+
$newToken['content'] = '\\';
876+
$finalTokens[$newStackPtr] = $newToken;
877+
++$newStackPtr;
878+
}
879+
}
880+
881+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
882+
$type = Util\Tokens::tokenName($token[0]);
883+
$content = Util\Common::prepareForOutput($token[1]);
884+
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
885+
}
886+
887+
continue;
888+
}//end if
889+
815890
/*
816891
Before PHP 7.0, the "yield from" was tokenized as
817892
T_YIELD, T_WHITESPACE and T_STRING. So look for
@@ -1122,6 +1197,9 @@ protected function tokenize($string)
11221197
}
11231198

11241199
if ($tokenType === T_STRING
1200+
|| $tokenType === T_NAME_FULLY_QUALIFIED
1201+
|| $tokenType === T_NAME_RELATIVE
1202+
|| $tokenType === T_NAME_QUALIFIED
11251203
|| $tokenType === T_ARRAY
11261204
|| $tokenType === T_NS_SEPARATOR
11271205
) {
@@ -1278,6 +1356,10 @@ protected function tokenize($string)
12781356
tokenized as T_STRING even if it appears to be a different token,
12791357
such as when writing code like: function default(): foo
12801358
so go forward and change the token type before it is processed.
1359+
1360+
Note: this should not be done for `function Level\Name` within a
1361+
group use statement for the PHP 8 identifier name tokens as it
1362+
would interfere with the re-tokenization of those.
12811363
*/
12821364

12831365
if ($tokenIsArray === true
@@ -1295,7 +1377,10 @@ protected function tokenize($string)
12951377
}
12961378
}
12971379

1298-
if ($x < $numTokens && is_array($tokens[$x]) === true) {
1380+
if ($x < $numTokens
1381+
&& is_array($tokens[$x]) === true
1382+
&& $tokens[$x][0] !== T_NAME_QUALIFIED
1383+
) {
12991384
if (PHP_CODESNIFFER_VERBOSITY > 1) {
13001385
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
13011386
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;

‎src/Util/Tokens.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,19 @@
124124
define('T_FN', 'PHPCS_T_FN');
125125
}
126126

127+
// Some PHP 8.0 tokens, replicated for lower versions.
128+
if (defined('T_NAME_QUALIFIED') === false) {
129+
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
130+
}
131+
132+
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
133+
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
134+
}
135+
136+
if (defined('T_NAME_RELATIVE') === false) {
137+
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
138+
}
139+
127140
// Tokens used for parsing doc blocks.
128141
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
129142
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
<?php
2+
3+
/* testNamespaceDeclaration */
4+
namespace Package;
5+
6+
/* testNamespaceDeclarationWithLevels */
7+
namespace Vendor\SubLevel\Domain;
8+
9+
/* testUseStatement */
10+
use ClassName;
11+
12+
/* testUseStatementWithLevels */
13+
use Vendor\Level\Domain;
14+
15+
/* testFunctionUseStatement */
16+
use function function_name;
17+
18+
/* testFunctionUseStatementWithLevels */
19+
use function Vendor\Level\function_in_ns;
20+
21+
/* testConstantUseStatement */
22+
use const CONSTANT_NAME;
23+
24+
/* testConstantUseStatementWithLevels */
25+
use const Vendor\Level\OTHER_CONSTANT;
26+
27+
/* testMultiUseUnqualified */
28+
use UnqualifiedClassName,
29+
/* testMultiUsePartiallyQualified */
30+
Sublevel\PartiallyClassName;
31+
32+
/* testGroupUseStatement */
33+
use Vendor\Level\{
34+
AnotherDomain,
35+
function function_grouped,
36+
const CONSTANT_GROUPED,
37+
Sub\YetAnotherDomain,
38+
function SubLevelA\function_grouped_too,
39+
const SubLevelB\CONSTANT_GROUPED_TOO,
40+
};
41+
42+
/* testClassName */
43+
class MyClass
44+
/* testExtendedFQN */
45+
extends \Vendor\Level\FQN
46+
/* testImplementsRelative */
47+
implements namespace\Name,
48+
/* testImplementsFQN */
49+
\Fully\Qualified,
50+
/* testImplementsUnqualified */
51+
Unqualified,
52+
/* testImplementsPartiallyQualified */
53+
Sub\Level\Name
54+
{
55+
/* testFunctionName */
56+
public function function_name(
57+
/* testTypeDeclarationRelative */
58+
namespace\Name $paramA,
59+
60+
/* testTypeDeclarationFQN */
61+
\Fully\Qualified\Name $paramB,
62+
63+
/* testTypeDeclarationUnqualified */
64+
Unqualified $paramC,
65+
66+
/* testTypeDeclarationPartiallyQualified */
67+
Sublevel\Name $paramD,
68+
69+
/* testReturnTypeFQN */
70+
) : \Name {
71+
72+
try {
73+
/* testFunctionCallRelative */
74+
echo NameSpace\function_name();
75+
76+
/* testFunctionCallFQN */
77+
echo \Vendor\Package\function_name();
78+
79+
/* testFunctionCallUnqualified */
80+
echo function_name();
81+
82+
/* testFunctionPartiallyQualified */
83+
echo Level\function_name();
84+
85+
/* testCatchRelative */
86+
} catch (namespace\SubLevel\Exception $e) {
87+
88+
/* testCatchFQN */
89+
} catch (\Exception $e) {
90+
91+
/* testCatchUnqualified */
92+
} catch (Exception $e) {
93+
94+
/* testCatchPartiallyQualified */
95+
} catch (Level\Exception $e) {
96+
}
97+
98+
/* testNewRelative */
99+
$obj = new namespace\ClassName();
100+
101+
/* testNewFQN */
102+
$obj = new \Vendor\ClassName();
103+
104+
/* testNewUnqualified */
105+
$obj = new ClassName;
106+
107+
/* testNewPartiallyQualified */
108+
$obj = new Level\ClassName;
109+
110+
/* testDoubleColonRelative */
111+
$value = namespace\ClassName::property;
112+
113+
/* testDoubleColonFQN */
114+
$value = \ClassName::static_function();
115+
116+
/* testDoubleColonUnqualified */
117+
$value = ClassName::CONSTANT_NAME;
118+
119+
/* testDoubleColonPartiallyQualified */
120+
$value = Level\ClassName::CONSTANT_NAME['key'];
121+
122+
/* testInstanceOfRelative */
123+
$is = $obj instanceof namespace\ClassName;
124+
125+
/* testInstanceOfFQN */
126+
if ($obj instanceof \Full\ClassName) {}
127+
128+
/* testInstanceOfUnqualified */
129+
if ($a === $b && $obj instanceof ClassName && true) {}
130+
131+
/* testInstanceOfPartiallyQualified */
132+
$is = $obj instanceof Partially\ClassName;
133+
}
134+
}
135+
136+
/* testInvalidInPHP8Whitespace */
137+
namespace \ Sublevel
138+
\ function_name();
139+
140+
/* testInvalidInPHP8Comments */
141+
$value = \Fully
142+
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
143+
\Qualified
144+
/* comment */
145+
\Name
146+
// comment
147+
:: function_name();

0 commit comments

Comments
 (0)