Support for PHP 8.1 Explicit Octal Notation in the Tokenizer

src/Tokenizers/PHP.php

@@ -646,6 +646,29 @@ protected function tokenize($string)
                 }//end if
             }//end if
 
+            /*
+                For Explicit Octal Notation prior to PHP 8.1 we need to combine the
+                T_LNUMBER and T_STRING token values into a single token value, and
+                then ignore the T_STRING token.
+            */
+
+            if (PHP_VERSION_ID < 80100
+                && $tokenIsArray === true && $token[1] === '0'
+                && (isset($tokens[($stackPtr + 1)]) === true
+                && is_array($tokens[($stackPtr + 1)]) === true
+                && $tokens[($stackPtr + 1)][0] === T_STRING
+                && strtolower($tokens[($stackPtr + 1)][1][0]) === 'o')
+            ) {
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_LNUMBER,
+                    'type'    => 'T_LNUMBER',
+                    'content' => $token[1] .= $tokens[($stackPtr + 1)][1],
+                ];
+                $stackPtr++;
+                $newStackPtr++;
+                continue;
+            }
+
             /*
                 PHP 8.1 introduced two dedicated tokens for the & character.
                 Retokenizing both of these to T_BITWISE_AND, which is the
@@ -1330,6 +1353,7 @@ protected function tokenize($string)
                 if ($newType === T_LNUMBER
                     && ((stripos($newContent, '0x') === 0 && hexdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
                     || (stripos($newContent, '0b') === 0 && bindec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
+                    || (stripos($newContent, '0o') === 0 && octdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
                     || (stripos($newContent, '0x') !== 0
                     && stripos($newContent, 'e') !== false || strpos($newContent, '.') !== false)
                     || (strpos($newContent, '0') === 0 && stripos($newContent, '0x') !== 0

tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc

@@ -0,0 +1,7 @@
+<?php
+
+/* testExplicitOctal */
+$foo = 0o137041;
+
+/* testExplicitOctal capitalised */
+$bar = 0O137041;

tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.php

@@ -0,0 +1,70 @@
+<?php
+/**
+ * Tests the tokenization of explicit octal notation to PHP < 8.1.
+ *
+ * @author    Mark Baker <[email protected]>
+ * @copyright 2019 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+
+class BackfillExplicitOctalNotationTest extends AbstractMethodUnitTest
+{
+
+
+    /**
+     * Test that explicitly-defined octal values are tokenized as a single number and not as a number and a string.
+     *
+     * @param array $testData The data required for the specific test case.
+     *
+     * @dataProvider dataExplicitOctalNotation
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testExplicitOctalNotation($testData)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $number = $this->getTargetToken($testData['marker'], [T_LNUMBER]);
+
+        $this->assertSame(constant($testData['type']), $tokens[$number]['code']);
+        $this->assertSame($testData['type'], $tokens[$number]['type']);
+        $this->assertSame($testData['value'], $tokens[$number]['content']);
+
+    }//end testExplicitOctalNotation()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testExplicitOctalNotation()
+     *
+     * @return array
+     */
+    public function dataExplicitOctalNotation()
+    {
+        return [
+            [
+                [
+                    'marker' => '/* testExplicitOctal */',
+                    'type'   => 'T_LNUMBER',
+                    'value'  => '0o137041',
+                ],
+            ],
+            [
+                [
+                    'marker' => '/* testExplicitOctal capitalised */',
+                    'type'   => 'T_LNUMBER',
+                    'value'  => '0O137041',
+                ],
+            ],
+        ];
+
+    }//end dataExplicitOctalNotation()
+
+
+}//end class

tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc

@@ -34,6 +34,9 @@ $foo = 0b0101_1111;
 /* testOctal */
 $foo = 0137_041;
 
+/* testExplicitOctal */
+$foo = 0o137_041;
+
 /* testIntMoreThanMax */
 $foo = 10_223_372_036_854_775_807;
 

tests/Core/Tokenizer/BackfillNumericSeparatorTest.php

@@ -132,6 +132,13 @@ public function dataTestBackfill()
                     'value'  => '0137_041',
                 ],
             ],
+            [
+                [
+                    'marker' => '/* testExplicitOctal */',
+                    'type'   => 'T_LNUMBER',
+                    'value'  => '0o137_041',
+                ],
+            ],
             [
                 [
                     'marker' => '/* testIntMoreThanMax */',