Debian Codegraph @

src/tokenize.c

raw
1
2 /*
3 * This file is part of Complexity.
4 * Complexity Copyright (c) 2011-2020 by Bruce Korb - all rights reserved
5 *
6 * Complexity is free software: you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * Complexity is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14 * See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #define DEFINE_CPLX_CHAR_TYPE
21
22 #include "opts.h"
23
24 static bool
25 skip_comment(fstate_t * fs)
26 {
27 char const * p = fs->fs_scan + 1; // skip the '*' from the "/*"
28
29 for (;;) {
30 p = BRK_STAR_OR_NL_CHARS(p);
31
32 switch (*p) {
33 case NUL:
34 fs->fs_scan = p;
35 return false;
36
37 case NL:
38 fs->cur_line++;
39 /* FALLTHROUGH */
40
41 case CR:
42 p++;
43 break;
44
45 case '*':
46 if (*++p == FSLASH) {
47 fs->fs_scan = p + 1;
48 return true;
49 }
50 }
51 }
52 }
53
54 static bool
55 skip_to_eol(fstate_t * fs)
56 {
57 fs->fs_scan = BRK_END_OF_LINE_CHARS(fs->fs_scan);
58 switch (fs->fs_scan[0]) {
59 case CR:
60 if (fs->fs_scan[1] == NL)
61 fs->fs_scan++;
62 /* FALLTHROUGH */
63 case NL:
64 return true;
65 default:
66 return false;
67 }
68 }
69
70 static token_t
71 bang_check(fstate_t * fs)
72 {
73 if (fs->fs_scan[0] != '=')
74 return TKN_ARITH_OP; // logical not
75 fs->fs_scan++;
76 return TKN_REL_OP; // not equal
77 }
78
79 static token_t
80 check_quote(fstate_t * fs, char q)
81 {
82 token_t res = TKN_NAME;
83 char const * s = fs->fs_scan;
84
85 while (*s != q) {
86 switch (*s) {
87 case BSLASH:
88 if (*++s != NUL)
89 break;
90 /* FALLTHROUGH */
91
92 case NUL:
93 res = TKN_EOF;
94 goto leave;
95
96 default: ;
97 }
98 s++;
99 }
100
101 leave:
102 fs->fs_scan = s + 1;
103 return res; // string, actually
104 }
105
106 static token_t
107 dblquot_check(fstate_t * fs)
108 {
109 return check_quote(fs, DQUOT);
110 }
111
112 static token_t
113 sglquot_check(fstate_t * fs)
114 {
115 return check_quote(fs, SQUOT);
116 }
117
118 /**
119 * handle comments
120 */
121 static inline bool
122 comment_check(fstate_t * fs, char const * s, token_t * res)
123 {
124 fs->fs_scan = s;
125 switch (*s) {
126 case '*': // "C" comment
127 if (! skip_comment(fs)) {
128 *res = TKN_EOF;
129 return true;
130 }
131 return false;
132
133 case FSLASH: // "C++" comment to end of line
134 if (! skip_to_eol(fs))
135 *res = TKN_EOF;
136 return true;
137
138 case CR:
139 if (s[1] == NL)
140 fs->fs_scan++;
141 /* FALLTHROUGH */
142 case NL:
143 return true;
144
145 case NUL: // Wooops
146 *res = TKN_EOF;
147 return true;
148
149 default:
150 return false;
151 }
152 }
153
154 /**
155 * handle a preprocessing directive. If the line ends with a backslash,
156 * then the next line is considered part of that directive.
157 */
158 static token_t
159 hash_check(fstate_t * fs)
160 {
161 if (! fs->fs_bol)
162 return TKN_ARITH_OP;
163
164 token_t res = TKN_EMPTY;
165 char const * s = fs->fs_scan;
166 char ch;
167
168 for (;;) {
169 s = BRK_END_OF_LINE_CHARS(s);
170 if (*s == NUL) {
171 res = TKN_EOF;
172 break;
173 }
174
175 if ((s > fs->fs_scan) && (s[-1] == BSLASH)) {
176 if (*(s++) == NL)
177 fs->cur_line++;
178 continue;
179 }
180
181 s += ((s[0] == CR) && (s[1] == NL)) ? 2 : 1;
182 break;
183 }
184
185 fs->fs_scan = s;
186 fs->fs_bol = true;
187 return res;
188 }
189
190 static token_t
191 assign_check(fstate_t * fs)
192 {
193 if (fs->fs_scan[0] != '=')
194 return TKN_ARITH_OP;
195 fs->fs_scan++;
196 return TKN_ASSIGN;
197 }
198
199 static token_t
200 amph_check(fstate_t * fs)
201 {
202 if (fs->fs_scan[0] != '&')
203 return assign_check(fs);
204 fs->fs_scan++;
205 return TKN_LOGIC_AND;
206 }
207
208 static token_t
209 dot_check(fstate_t * fs)
210 {
211 token_t res = TKN_ARITH_OP;
212 if ((fs->fs_scan[0] == '.') && (fs->fs_scan[1] == '.')) {
213 fs->fs_scan += 2;
214 res = TKN_ELLIPSIS;
215 }
216 return res;
217 }
218
219 static token_t
220 plus_check(fstate_t * fs)
221 {
222 token_t res = TKN_ARITH_OP;
223
224 switch (fs->fs_scan[0]) {
225 case '+': res = TKN_ARITH_OP; fs->fs_scan++; break;
226 case '=': res = TKN_ASSIGN; fs->fs_scan++; break;
227 case NUL: res = TKN_EOF; break;
228 }
229
230 return res;
231 }
232
233 static token_t
234 hyphen_check(fstate_t * fs)
235 {
236 token_t res = TKN_ARITH_OP;
237
238 switch (fs->fs_scan[0]) {
239 case '>':
240 case '-': res = TKN_ARITH_OP; fs->fs_scan++; break;
241 case '=': res = TKN_ASSIGN; fs->fs_scan++; break;
242 case NUL: res = TKN_EOF; break;
243 }
244
245 return res;
246 }
247
248 static token_t
249 slash_check(fstate_t * fs)
250 {
251 token_t res = TKN_ARITH_OP;
252
253 switch (fs->fs_scan[0]) {
254 case FSLASH:
255 skip_to_eol(fs);
256 res = TKN_EMPTY;
257 break;
258
259 case '=':
260 fs->fs_scan++;
261 res = TKN_ASSIGN;
262 break;
263
264 case '*':
265 res = skip_comment(fs) ? TKN_EMPTY : TKN_EOF;
266 break;
267 }
268
269 return res;
270 }
271
272 static token_t
273 less_check(fstate_t * fs)
274 {
275 token_t res = TKN_REL_OP;
276
277 switch (fs->fs_scan[0]) {
278 case '<':
279 fs->fs_scan++;
280 if (fs->fs_scan[0] == '=') {
281 fs->fs_scan++;
282 res = TKN_ASSIGN;
283 }
284 else
285 res = TKN_ARITH_OP;
286 break;
287
288 case '=':
289 fs->fs_scan++;
290 break;
291 }
292
293 return res;
294 }
295
296 static token_t
297 equal_check(fstate_t * fs)
298 {
299 token_t res = TKN_ASSIGN;
300 if (fs->fs_scan[0] == '=') {
301 fs->fs_scan++;
302 res = TKN_REL_OP;
303 }
304 return res;
305 }
306
307 static token_t
308 greater_check(fstate_t * fs)
309 {
310 token_t res = TKN_REL_OP;
311
312 switch (fs->fs_scan[0]) {
313 case '>':
314 fs->fs_scan++;
315 if (fs->fs_scan[0] == '=') {
316 fs->fs_scan++;
317 res = TKN_ASSIGN;
318 }
319 else
320 res = TKN_ARITH_OP;
321 break;
322
323 case '=':
324 fs->fs_scan++;
325 break;
326 }
327
328 return res;
329 }
330
331 static token_t
332 carat_check(fstate_t * fs)
333 {
334 token_t res = TKN_ARITH_OP;
335
336 if (fs->fs_scan[0] == '=') {
337 fs->fs_scan++;
338 res = TKN_ASSIGN;
339 }
340
341 return res;
342 }
343
344 static token_t
345 vertbar_check(fstate_t * fs)
346 {
347 if (fs->fs_scan[0] != '|')
348 return assign_check(fs);
349 fs->fs_scan++;
350 return TKN_LOGIC_OR;
351 }
352
353 static token_t
354 unknown_check(fstate_t * fs)
355 {
356 unsigned char ch = fs->fs_scan[-1];
357
358 fprintf(stderr, "invalid character in %s on line %d: 0x%02X (%c)\n",
359 fs->fs_fname, fs->cur_line, ch,
360 (isprint(ch) ? ch : '?'));
361
362 return TKN_EOF;
363 }
364
365 static token_t
366 keyword_check(fstate_t * fs)
367 {
368 static struct {
369 char const * name;
370 token_t tval;
371 int nlen;
372 } const kw_table[] = {
373 #define _Ktbl_(_n, _e) { #_n, _e, sizeof(#_n) - 1 },
374 RES_WORD_TABLE
375 #undef _Ktbl_
376 };
377 static int const kw_size = sizeof(kw_table) / sizeof(kw_table[0]);
378
379 int lo = 0;
380 int hi = kw_size - 1;
381 fs->fs_scan--;
382
383 do {
384 int ix = (lo + hi) / 2;
385 int cmp = strncmp(kw_table[ix].name, fs->fs_scan, kw_table[ix].nlen);
386
387 if (cmp == 0) {
388 if (! IS_NAME_CHAR(fs->fs_scan[kw_table[ix].nlen])) {
389 fs->fs_scan += kw_table[ix].nlen;
390 return kw_table[ix].tval;
391 }
392 break;
393 }
394
395 if (cmp < 0)
396 lo = ix + 1;
397 else
398 hi = ix - 1;
399 } while (lo <= hi);
400
401 for (;;) {
402 fs->fs_scan = SPN_NAME_CHARS(fs->fs_scan + 1);
403 if ((fs->fs_scan[0] != ':') || (fs->fs_scan[1] != ':'))
404 return TKN_NAME;
405 fs->fs_scan += 2;
406 }
407 }
408
409 void
410 unget_token(fstate_t * fs)
411 {
412 fs->fs_scan = fs->tkn_text;
413 fs->cur_line = fs->tkn_line;
414 }
415
416 token_t
417 extern_c_check(fstate_t * fs)
418 {
419 int nl_ct = 0;
420 char const * s = fs->fs_scan;
421 while (IS_SPACE_CHAR(*s)) {
422 if (*s == NL)
423 nl_ct++;
424 s++;
425 }
426
427 if (strncmp(s, "\"C\"", 3) != 0)
428 return TKN_NAME;
429 s += 3;
430 while (IS_SPACE_CHAR(*s)) {
431 if (*s == NL)
432 nl_ct++;
433 s++;
434 }
435 if (*s != '{') return TKN_NAME;
436 fs->fs_scan = s + 1;
437 fs->cur_line += nl_ct;
438 return TKN_EMPTY;
439 }
440
441 static inline bool
442 next_nonblank(fstate_t * fs)
443 {
444 for (;;) {
445 switch (*(fs->fs_scan)) {
446 case NL:
447 fs->cur_line++;
448 fs->fs_bol = true;
449 /* FALLTHROUGH */
450
451 case ' ': case HT: case FF: case VT: case CR:
452 fs->fs_scan++;
453 break;
454
455 case NUL:
456 return false;
457
458 default:
459 fs->tkn_text = fs->fs_scan;
460 fs->tkn_line = fs->cur_line;
461 return true;
462 }
463 }
464 }
465
466 token_t
467 next_token(fstate_t * fs)
468 {
469 token_t res = TKN_EOF;
470
471 do {
472 if (! next_nonblank(fs))
473 return TKN_EOF;
474
475 switch (*(fs->fs_scan++)) {
476 case NUL:
477 return TKN_EOF;
478
479 case 'A' ... 'Z':
480 case '_': case '$':
481 fs->fs_scan = SPN_NAME_CHARS(fs->fs_scan);
482 res = TKN_NAME;
483 break;
484
485 case 'a' ... 'z':
486 res = keyword_check(fs);
487 if (res == TKN_KW_EXTERN)
488 res = extern_c_check(fs);
489 break;
490
491 case '0' ... '9':
492 fs->fs_scan = SPN_NAME_CHARS(fs->fs_scan);
493 res = TKN_NUMBER;
494 break;
495
496 case '!': res = bang_check( fs); break;
497 case DQUOT: res = dblquot_check(fs); break;
498 case '#': res = hash_check( fs); break;
499 case '%': res = assign_check( fs); break;
500 case '&': res = amph_check( fs); break;
501 case '*': res = assign_check( fs); break;
502 case '+': res = plus_check( fs); break;
503 case '-': res = hyphen_check( fs); break;
504 case '.': res = dot_check( fs); break;
505 case FSLASH: res = slash_check( fs); break;
506 case '<': res = less_check( fs); break;
507 case '=': res = equal_check( fs); break;
508 case '>': res = greater_check(fs); break;
509 case SQUOT: res = sglquot_check(fs); break;
510 case '^': res = carat_check( fs); break;
511 case '|': res = vertbar_check(fs); break;
512
513 case '(': res = TKN_LIT_OPNPAREN; break;
514 case ')': res = TKN_LIT_CLSPAREN; break;
515 case ',': res = TKN_LIT_COMMA; break;
516 case ';': res = TKN_LIT_SEMI; break;
517 case '?': res = TKN_LIT_QUESTION; break;
518 case '[': res = TKN_LIT_OPNBRACK; break;
519 case ']': res = TKN_LIT_CLSBRACK; break;
520 case '{': res = TKN_LIT_OBRACE; break;
521 case '}': res = TKN_LIT_CBRACE; break;
522
523 case BSLASH: res = TKN_EMPTY; break;
524 case '~': res = TKN_ARITH_OP; break;
525
526 case ':':
527 if (fs->fs_scan[0] == ':') {
528 /*
529 * double colon name space operator -- basically
530 * part of the name.
531 */
532 fs->fs_scan++;
533 (void)keyword_check(fs);
534 res = TKN_NAME;
535 } else {
536 res = TKN_LIT_COLON;
537 }
538 break;
539
540 case '@':
541 case '`':
542 default: res = unknown_check(fs); break;
543 }
544 } while (res == TKN_EMPTY);
545
546 fs->tkn_len = fs->fs_scan - fs->tkn_text;
547 if (fs->fs_bol) {
548 fs->fs_bol = 0;
549 fs->nc_line++;
550 }
551
552 return fs->last_tkn = res;
553 }
554
555 static token_t
556 skip_params(fstate_t * fs)
557 {
558 int depth = 1;
559 for (;;) {
560 token_t tk = next_token(fs);
561 switch (tk) {
562 case TKN_LIT_OPNPAREN:
563 depth++;
564 break;
565
566 case TKN_LIT_CLSPAREN:
567 if (--depth > 0)
568 continue;
569 /* FALLTHROUGH */
570 case TKN_EOF:
571 return tk;
572
573 default:
574 continue;
575 }
576 }
577 }
578
579 /*
580 * quit when you find either a semi-colon or a closing brace in col 1.
581 */
582 static token_t
583 skip_to_semi(fstate_t * fs)
584 {
585 int depth = (fs->last_tkn == TKN_LIT_OBRACE) ? 1 : 0;
586
587 for (;;) {
588 token_t tkn = next_token(fs);
589 switch (tkn) {
590 case TKN_LIT_OBRACE:
591 depth++;
592 break;
593
594 case TKN_LIT_CBRACE:
595 if ((--depth == 0) && (fs->tkn_text[-1] == NL))
596 return tkn;
597 break;
598
599 case TKN_LIT_SEMI:
600 if (depth)
601 continue;
602 /* FALLTHROUGH */
603 case TKN_EOF:
604 return tkn;
605
606 default:
607 continue;
608 }
609 }
610 }
611
612 bool
613 find_proc_start(fstate_t * fs)
614 {
615 bool res = false;
616
617 for (;;) {
618 token_t tkn = next_token(fs);
619 char const * proc_name;
620 size_t pname_len;
621
622 switch (tkn) {
623 case TKN_NAME: break;
624 case TKN_EOF: return res;
625 case TKN_LIT_SEMI: continue;
626 default:
627 tkn = skip_to_semi(fs);
628 if (tkn == TKN_EOF)
629 return res;
630 continue;
631 }
632
633 do {
634 proc_name = fs->tkn_text;
635 pname_len = fs->tkn_len;
636 do
637 tkn = next_token(fs);
638 while (tkn == TKN_ARITH_OP);
639 } while (tkn == TKN_NAME);
640
641 switch (tkn) {
642 case TKN_LIT_OPNPAREN: break;
643 case TKN_EOF: return res;
644 case TKN_LIT_SEMI: continue;
645 default:
646 tkn = skip_to_semi(fs);
647 if (tkn == TKN_EOF)
648 return res;
649 continue;
650 }
651
652 tkn = skip_params(fs);
653
654 switch (tkn) {
655 case TKN_LIT_CLSPAREN: break;
656 case TKN_EOF: return res;
657 case TKN_LIT_SEMI: continue;
658 default:
659 tkn = skip_to_semi(fs);
660 if (tkn == TKN_EOF)
661 return res;
662 continue;
663 }
664
665 tkn = next_token(fs);
666 if (tkn == TKN_LIT_OBRACE) {
667 fs->tkn_text = proc_name;
668 fs->tkn_len = pname_len;
669 return true;
670 }
671 }
672 }
673 /*
674 * Local Variables:
675 * mode: C
676 * c-file-style: "stroustrup"
677 * indent-tabs-mode: nil
678 * End:
679 * end of tokenize.c */