Debian Codegraph @

vendor/phpstan/phpdoc-parser/src/Lexer/Lexer.php

raw
1 <?php declare(strict_types = 1);
2
3 namespace PHPStan\PhpDocParser\Lexer;
4
5 use PHPStan\PhpDocParser\ParserConfig;
6 use function implode;
7 use function preg_match_all;
8 use const PREG_SET_ORDER;
9
10 /**
11 * Implementation based on Nette Tokenizer (New BSD License; https://github.com/nette/tokenizer)
12 */
13 class Lexer
14 {
15
16 public const TOKEN_REFERENCE = 0;
17 public const TOKEN_UNION = 1;
18 public const TOKEN_INTERSECTION = 2;
19 public const TOKEN_NULLABLE = 3;
20 public const TOKEN_OPEN_PARENTHESES = 4;
21 public const TOKEN_CLOSE_PARENTHESES = 5;
22 public const TOKEN_OPEN_ANGLE_BRACKET = 6;
23 public const TOKEN_CLOSE_ANGLE_BRACKET = 7;
24 public const TOKEN_OPEN_SQUARE_BRACKET = 8;
25 public const TOKEN_CLOSE_SQUARE_BRACKET = 9;
26 public const TOKEN_COMMA = 10;
27 public const TOKEN_VARIADIC = 11;
28 public const TOKEN_DOUBLE_COLON = 12;
29 public const TOKEN_DOUBLE_ARROW = 13;
30 public const TOKEN_EQUAL = 14;
31 public const TOKEN_OPEN_PHPDOC = 15;
32 public const TOKEN_CLOSE_PHPDOC = 16;
33 public const TOKEN_PHPDOC_TAG = 17;
34 public const TOKEN_DOCTRINE_TAG = 18;
35 public const TOKEN_FLOAT = 19;
36 public const TOKEN_INTEGER = 20;
37 public const TOKEN_SINGLE_QUOTED_STRING = 21;
38 public const TOKEN_DOUBLE_QUOTED_STRING = 22;
39 public const TOKEN_DOCTRINE_ANNOTATION_STRING = 23;
40 public const TOKEN_IDENTIFIER = 24;
41 public const TOKEN_THIS_VARIABLE = 25;
42 public const TOKEN_VARIABLE = 26;
43 public const TOKEN_HORIZONTAL_WS = 27;
44 public const TOKEN_PHPDOC_EOL = 28;
45 public const TOKEN_OTHER = 29;
46 public const TOKEN_END = 30;
47 public const TOKEN_COLON = 31;
48 public const TOKEN_WILDCARD = 32;
49 public const TOKEN_OPEN_CURLY_BRACKET = 33;
50 public const TOKEN_CLOSE_CURLY_BRACKET = 34;
51 public const TOKEN_NEGATED = 35;
52 public const TOKEN_ARROW = 36;
53
54 public const TOKEN_COMMENT = 37;
55
56 public const TOKEN_LABELS = [
57 self::TOKEN_REFERENCE => '\'&\'',
58 self::TOKEN_UNION => '\'|\'',
59 self::TOKEN_INTERSECTION => '\'&\'',
60 self::TOKEN_NULLABLE => '\'?\'',
61 self::TOKEN_NEGATED => '\'!\'',
62 self::TOKEN_OPEN_PARENTHESES => '\'(\'',
63 self::TOKEN_CLOSE_PARENTHESES => '\')\'',
64 self::TOKEN_OPEN_ANGLE_BRACKET => '\'<\'',
65 self::TOKEN_CLOSE_ANGLE_BRACKET => '\'>\'',
66 self::TOKEN_OPEN_SQUARE_BRACKET => '\'[\'',
67 self::TOKEN_CLOSE_SQUARE_BRACKET => '\']\'',
68 self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
69 self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
70 self::TOKEN_COMMA => '\',\'',
71 self::TOKEN_COMMENT => '\'//\'',
72 self::TOKEN_COLON => '\':\'',
73 self::TOKEN_VARIADIC => '\'...\'',
74 self::TOKEN_DOUBLE_COLON => '\'::\'',
75 self::TOKEN_DOUBLE_ARROW => '\'=>\'',
76 self::TOKEN_ARROW => '\'->\'',
77 self::TOKEN_EQUAL => '\'=\'',
78 self::TOKEN_OPEN_PHPDOC => '\'/**\'',
79 self::TOKEN_CLOSE_PHPDOC => '\'*/\'',
80 self::TOKEN_PHPDOC_TAG => 'TOKEN_PHPDOC_TAG',
81 self::TOKEN_DOCTRINE_TAG => 'TOKEN_DOCTRINE_TAG',
82 self::TOKEN_PHPDOC_EOL => 'TOKEN_PHPDOC_EOL',
83 self::TOKEN_FLOAT => 'TOKEN_FLOAT',
84 self::TOKEN_INTEGER => 'TOKEN_INTEGER',
85 self::TOKEN_SINGLE_QUOTED_STRING => 'TOKEN_SINGLE_QUOTED_STRING',
86 self::TOKEN_DOUBLE_QUOTED_STRING => 'TOKEN_DOUBLE_QUOTED_STRING',
87 self::TOKEN_DOCTRINE_ANNOTATION_STRING => 'TOKEN_DOCTRINE_ANNOTATION_STRING',
88 self::TOKEN_IDENTIFIER => 'type',
89 self::TOKEN_THIS_VARIABLE => '\'$this\'',
90 self::TOKEN_VARIABLE => 'variable',
91 self::TOKEN_HORIZONTAL_WS => 'TOKEN_HORIZONTAL_WS',
92 self::TOKEN_OTHER => 'TOKEN_OTHER',
93 self::TOKEN_END => 'TOKEN_END',
94 self::TOKEN_WILDCARD => '*',
95 ];
96
97 public const VALUE_OFFSET = 0;
98 public const TYPE_OFFSET = 1;
99 public const LINE_OFFSET = 2;
100
101 private ParserConfig $config; // @phpstan-ignore property.onlyWritten
102
103 private ?string $regexp = null;
104
105 public function __construct(ParserConfig $config)
106 {
107 $this->config = $config;
108 }
109
110 /**
111 * @return list<array{string, int, int}>
112 */
113 public function tokenize(string $s): array
114 {
115 if ($this->regexp === null) {
116 $this->regexp = $this->generateRegexp();
117 }
118
119 preg_match_all($this->regexp, $s, $matches, PREG_SET_ORDER);
120
121 $tokens = [];
122 $line = 1;
123 foreach ($matches as $match) {
124 $type = (int) $match['MARK'];
125 $tokens[] = [$match[0], $type, $line];
126 if ($type !== self::TOKEN_PHPDOC_EOL) {
127 continue;
128 }
129
130 $line++;
131 }
132
133 $tokens[] = ['', self::TOKEN_END, $line];
134
135 return $tokens;
136 }
137
138 private function generateRegexp(): string
139 {
140 $patterns = [
141 self::TOKEN_HORIZONTAL_WS => '[\\x09\\x20]++',
142
143 self::TOKEN_IDENTIFIER => '(?:[\\\\]?+[a-z_\\x80-\\xFF][0-9a-z_\\x80-\\xFF-]*+)++',
144 self::TOKEN_THIS_VARIABLE => '\\$this(?![0-9a-z_\\x80-\\xFF])',
145 self::TOKEN_VARIABLE => '\\$[a-z_\\x80-\\xFF][0-9a-z_\\x80-\\xFF]*+',
146
147 // '&' followed by TOKEN_VARIADIC, TOKEN_VARIABLE, TOKEN_EQUAL, TOKEN_EQUAL or TOKEN_CLOSE_PARENTHESES
148 self::TOKEN_REFERENCE => '&(?=\\s*+(?:[.,=)]|(?:\\$(?!this(?![0-9a-z_\\x80-\\xFF])))))',
149 self::TOKEN_UNION => '\\|',
150 self::TOKEN_INTERSECTION => '&',
151 self::TOKEN_NULLABLE => '\\?',
152 self::TOKEN_NEGATED => '!',
153
154 self::TOKEN_OPEN_PARENTHESES => '\\(',
155 self::TOKEN_CLOSE_PARENTHESES => '\\)',
156 self::TOKEN_OPEN_ANGLE_BRACKET => '<',
157 self::TOKEN_CLOSE_ANGLE_BRACKET => '>',
158 self::TOKEN_OPEN_SQUARE_BRACKET => '\\[',
159 self::TOKEN_CLOSE_SQUARE_BRACKET => '\\]',
160 self::TOKEN_OPEN_CURLY_BRACKET => '\\{',
161 self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
162
163 self::TOKEN_COMMA => ',',
164 self::TOKEN_COMMENT => '\/\/[^\\r\\n]*(?=\n|\r|\*/)',
165 self::TOKEN_VARIADIC => '\\.\\.\\.',
166 self::TOKEN_DOUBLE_COLON => '::',
167 self::TOKEN_DOUBLE_ARROW => '=>',
168 self::TOKEN_ARROW => '->',
169 self::TOKEN_EQUAL => '=',
170 self::TOKEN_COLON => ':',
171
172 self::TOKEN_OPEN_PHPDOC => '/\\*\\*(?=\\s)\\x20?+',
173 self::TOKEN_CLOSE_PHPDOC => '\\*/',
174 self::TOKEN_PHPDOC_TAG => '@(?:[a-z][a-z0-9-\\\\]+:)?[a-z][a-z0-9-\\\\]*+',
175 self::TOKEN_DOCTRINE_TAG => '@[a-z_\\\\][a-z0-9_\:\\\\]*[a-z_][a-z0-9_]*',
176 self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/)\\x20?+)?',
177
178 self::TOKEN_FLOAT => '[+\-]?(?:(?:[0-9]++(_[0-9]++)*\\.[0-9]*+(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]*+(_[0-9]++)*\\.[0-9]++(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]++(_[0-9]++)*e[+\-]?[0-9]++(_[0-9]++)*))',
179 self::TOKEN_INTEGER => '[+\-]?(?:(?:0b[0-1]++(_[0-1]++)*)|(?:0o[0-7]++(_[0-7]++)*)|(?:0x[0-9a-f]++(_[0-9a-f]++)*)|(?:[0-9]++(_[0-9]++)*))',
180 self::TOKEN_SINGLE_QUOTED_STRING => '\'(?:\\\\[^\\r\\n]|[^\'\\r\\n\\\\])*+\'',
181 self::TOKEN_DOUBLE_QUOTED_STRING => '"(?:\\\\[^\\r\\n]|[^"\\r\\n\\\\])*+"',
182 self::TOKEN_DOCTRINE_ANNOTATION_STRING => '"(?:""|[^"])*+"',
183
184 self::TOKEN_WILDCARD => '\\*',
185
186 // anything but TOKEN_CLOSE_PHPDOC or TOKEN_HORIZONTAL_WS or TOKEN_EOL
187 self::TOKEN_OTHER => '(?:(?!\\*/)[^\\s])++',
188 ];
189
190 foreach ($patterns as $type => &$pattern) {
191 $pattern = '(?:' . $pattern . ')(*MARK:' . $type . ')';
192 }
193
194 return '~' . implode('|', $patterns) . '~Asi';
195 }
196
197 }