diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 1ba26363af..4e335a1513 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -646,6 +646,29 @@ protected function tokenize($string) }//end if }//end if + /* + For Explicit Octal Notation prior to PHP 8.1 we need to combine the + T_LNUMBER and T_STRING token values into a single token value, and + then ignore the T_STRING token. + */ + + if (PHP_VERSION_ID < 80100 + && $tokenIsArray === true && $token[1] === '0' + && (isset($tokens[($stackPtr + 1)]) === true + && is_array($tokens[($stackPtr + 1)]) === true + && $tokens[($stackPtr + 1)][0] === T_STRING + && strtolower($tokens[($stackPtr + 1)][1][0]) === 'o') + ) { + $finalTokens[$newStackPtr] = [ + 'code' => T_LNUMBER, + 'type' => 'T_LNUMBER', + 'content' => $token[1] .= $tokens[($stackPtr + 1)][1], + ]; + $stackPtr++; + $newStackPtr++; + continue; + } + /* PHP 8.1 introduced two dedicated tokens for the & character. Retokenizing both of these to T_BITWISE_AND, which is the @@ -1330,6 +1353,7 @@ protected function tokenize($string) if ($newType === T_LNUMBER && ((stripos($newContent, '0x') === 0 && hexdec(str_replace('_', '', $newContent)) > PHP_INT_MAX) || (stripos($newContent, '0b') === 0 && bindec(str_replace('_', '', $newContent)) > PHP_INT_MAX) + || (stripos($newContent, '0o') === 0 && octdec(str_replace('_', '', $newContent)) > PHP_INT_MAX) || (stripos($newContent, '0x') !== 0 && stripos($newContent, 'e') !== false || strpos($newContent, '.') !== false) || (strpos($newContent, '0') === 0 && stripos($newContent, '0x') !== 0 diff --git a/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc b/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc new file mode 100644 index 0000000000..d08553db69 --- /dev/null +++ b/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc @@ -0,0 +1,7 @@ + + * @copyright 2019 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; + +class BackfillExplicitOctalNotationTest extends AbstractMethodUnitTest +{ + + + /** + * Test that explicitly-defined octal values are tokenized as a single number and not as a number and a string. + * + * @param array $testData The data required for the specific test case. + * + * @dataProvider dataExplicitOctalNotation + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testExplicitOctalNotation($testData) + { + $tokens = self::$phpcsFile->getTokens(); + + $number = $this->getTargetToken($testData['marker'], [T_LNUMBER]); + + $this->assertSame(constant($testData['type']), $tokens[$number]['code']); + $this->assertSame($testData['type'], $tokens[$number]['type']); + $this->assertSame($testData['value'], $tokens[$number]['content']); + + }//end testExplicitOctalNotation() + + + /** + * Data provider. + * + * @see testExplicitOctalNotation() + * + * @return array + */ + public function dataExplicitOctalNotation() + { + return [ + [ + [ + 'marker' => '/* testExplicitOctal */', + 'type' => 'T_LNUMBER', + 'value' => '0o137041', + ], + ], + [ + [ + 'marker' => '/* testExplicitOctal capitalised */', + 'type' => 'T_LNUMBER', + 'value' => '0O137041', + ], + ], + ]; + + }//end dataExplicitOctalNotation() + + +}//end class diff --git a/tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc b/tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc index eef53f593b..ba07f958f7 100644 --- a/tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc +++ b/tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc @@ -34,6 +34,9 @@ $foo = 0b0101_1111; /* testOctal */ $foo = 0137_041; +/* testExplicitOctal */ +$foo = 0o137_041; + /* testIntMoreThanMax */ $foo = 10_223_372_036_854_775_807; diff --git a/tests/Core/Tokenizer/BackfillNumericSeparatorTest.php b/tests/Core/Tokenizer/BackfillNumericSeparatorTest.php index ee4275a214..a177b8e01f 100644 --- a/tests/Core/Tokenizer/BackfillNumericSeparatorTest.php +++ b/tests/Core/Tokenizer/BackfillNumericSeparatorTest.php @@ -132,6 +132,13 @@ public function dataTestBackfill() 'value' => '0137_041', ], ], + [ + [ + 'marker' => '/* testExplicitOctal */', + 'type' => 'T_LNUMBER', + 'value' => '0o137_041', + ], + ], [ [ 'marker' => '/* testIntMoreThanMax */',