Debian Codegraph @

basic/source/comp/token.cxx

raw
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3 * This file is part of the LibreOffice project.
4 *
5 * This Source Code Form is subject to the terms of the Mozilla Public
6 * License, v. 2.0. If a copy of the MPL was not distributed with this
7 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 *
9 * This file incorporates work covered by the following license notice:
10 *
11 * Licensed to the Apache Software Foundation (ASF) under one or more
12 * contributor license agreements. See the NOTICE file distributed
13 * with this work for additional information regarding copyright
14 * ownership. The ASF licenses this file to you under the Apache
15 * License, Version 2.0 (the "License"); you may not use this file
16 * except in compliance with the License. You may obtain a copy of
17 * the License at http://www.apache.org/licenses/LICENSE-2.0 .
18 */
19
20
21 #include <array>
22
23 #include <basic/sberrors.hxx>
24 #include <o3tl/string_view.hxx>
25 #include <basiccharclass.hxx>
26 #include <token.hxx>
27
28 namespace {
29
30 struct TokenTable { SbiToken t; const char *s; };
31
32 }
33
34 const TokenTable aTokTable_Basic [] = {
35 { CAT, "&" },
36 { MUL, "*" },
37 { PLUS, "+" },
38 { MINUS, "-" },
39 { DIV, "/" },
40 { EOS, ":" },
41 { ASSIGN, ":=" },
42 { LT, "<" },
43 { LE, "<=" },
44 { NE, "<>" },
45 { EQ, "=" },
46 { GT, ">" },
47 { GE, ">=" },
48 { ACCESS, "Access" },
49 { ALIAS, "Alias" },
50 { AND, "And" },
51 { ANY, "Any" },
52 { APPEND, "Append" },
53 { AS, "As" },
54 { ATTRIBUTE,"Attribute" },
55 { BASE, "Base" },
56 { BINARY, "Binary" },
57 { TBOOLEAN, "Boolean" },
58 { BYREF, "ByRef", },
59 { TBYTE, "Byte", },
60 { BYVAL, "ByVal", },
61 { CALL, "Call" },
62 { CASE, "Case" },
63 { CDECL_, "Cdecl" },
64 { CLASSMODULE, "ClassModule" },
65 { CLOSE, "Close" },
66 { COMPARE, "Compare" },
67 { COMPATIBLE,"Compatible" },
68 { CONST_, "Const" },
69 { TCURRENCY,"Currency" },
70 { TDATE, "Date" },
71 { DECLARE, "Declare" },
72 { DEFBOOL, "DefBool" },
73 { DEFCUR, "DefCur" },
74 { DEFDATE, "DefDate" },
75 { DEFDBL, "DefDbl" },
76 { DEFERR, "DefErr" },
77 { DEFINT, "DefInt" },
78 { DEFLNG, "DefLng" },
79 { DEFOBJ, "DefObj" },
80 { DEFSNG, "DefSng" },
81 { DEFSTR, "DefStr" },
82 { DEFVAR, "DefVar" },
83 { DIM, "Dim" },
84 { DO, "Do" },
85 { TDOUBLE, "Double" },
86 { EACH, "Each" },
87 { ELSE, "Else" },
88 { ELSEIF, "ElseIf" },
89 { END, "End" },
90 { ENDENUM, "End Enum" },
91 { ENDFUNC, "End Function" },
92 { ENDIF, "End If" },
93 { ENDPROPERTY, "End Property" },
94 { ENDSELECT,"End Select" },
95 { ENDSUB, "End Sub" },
96 { ENDTYPE, "End Type" },
97 { ENDIF, "EndIf" },
98 { ENUM, "Enum" },
99 { EQV, "Eqv" },
100 { ERASE, "Erase" },
101 { ERROR_, "Error" },
102 { EXIT, "Exit" },
103 { BASIC_EXPLICIT, "Explicit" },
104 { FOR, "For" },
105 { FUNCTION, "Function" },
106 { GET, "Get" },
107 { GLOBAL, "Global" },
108 { GOSUB, "GoSub" },
109 { GOTO, "GoTo" },
110 { IF, "If" },
111 { IMP, "Imp" },
112 { IMPLEMENTS, "Implements" },
113 { IN_, "In" },
114 { INPUT, "Input" }, // also INPUT #
115 { TINTEGER, "Integer" },
116 { IS, "Is" },
117 { LET, "Let" },
118 { LIB, "Lib" },
119 { LIKE, "Like" },
120 { LINE, "Line" },
121 { LINEINPUT,"Line Input" },
122 { LOCAL, "Local" },
123 { LOCK, "Lock" },
124 { TLONG, "Long" },
125 { LOOP, "Loop" },
126 { LPRINT, "LPrint" },
127 { LSET, "LSet" }, // JSM
128 { MOD, "Mod" },
129 { NAME, "Name" },
130 { NEW, "New" },
131 { NEXT, "Next" },
132 { NOT, "Not" },
133 { TOBJECT, "Object" },
134 { ON, "On" },
135 { OPEN, "Open" },
136 { OPTION, "Option" },
137 { OPTIONAL_, "Optional" },
138 { OR, "Or" },
139 { OUTPUT, "Output" },
140 { PARAMARRAY, "ParamArray" },
141 { PRESERVE, "Preserve" },
142 { PRINT, "Print" },
143 { PRIVATE, "Private" },
144 { PROPERTY, "Property" },
145 { PTRSAFE, "PtrSafe" },
146 { PUBLIC, "Public" },
147 { RANDOM, "Random" },
148 { READ, "Read" },
149 { REDIM, "ReDim" },
150 { REM, "Rem" },
151 { RESUME, "Resume" },
152 { RETURN, "Return" },
153 { RSET, "RSet" }, // JSM
154 { SELECT, "Select" },
155 { SET, "Set" },
156 { SHARED, "Shared" },
157 { TSINGLE, "Single" },
158 { STATIC, "Static" },
159 { STEP, "Step" },
160 { STOP, "Stop" },
161 { TSTRING, "String" },
162 { SUB, "Sub" },
163 { STOP, "System" },
164 { TEXT, "Text" },
165 { THEN, "Then" },
166 { TO, "To", },
167 { TYPE, "Type" },
168 { TYPEOF, "TypeOf" },
169 { UNTIL, "Until" },
170 { TVARIANT, "Variant" },
171 { VBASUPPORT, "VbaSupport" },
172 { WEND, "Wend" },
173 { WHILE, "While" },
174 { WITH, "With" },
175 { WITHEVENTS, "WithEvents" },
176 { WRITE, "Write" }, // also WRITE #
177 { XOR, "Xor" },
178 };
179
180 namespace {
181
182 // #i109076
183 class TokenLabelInfo
184 {
185 std::array<bool,VBASUPPORT+1> m_pTokenCanBeLabelTab;
186
187 public:
188 TokenLabelInfo();
189
190 bool canTokenBeLabel( SbiToken eTok )
191 { return m_pTokenCanBeLabelTab[eTok]; }
192 };
193
194 }
195
196 // #i109076
197 TokenLabelInfo::TokenLabelInfo()
198 {
199 m_pTokenCanBeLabelTab.fill(false);
200
201 // Token accepted as label by VBA
202 static const SbiToken eLabelToken[] = { ACCESS, ALIAS, APPEND, BASE, BINARY, CLASSMODULE,
203 COMPARE, COMPATIBLE, DEFERR, ERROR_, BASIC_EXPLICIT, LIB, LINE, LPRINT, NAME,
204 TOBJECT, OUTPUT, PROPERTY, RANDOM, READ, STEP, STOP, TEXT, VBASUPPORT };
205 for( SbiToken eTok : eLabelToken )
206 {
207 m_pTokenCanBeLabelTab[eTok] = true;
208 }
209 }
210
211
212 SbiTokenizer::SbiTokenizer( const OUString& rSrc, StarBASIC* pb )
213 : SbiScanner(rSrc, pb)
214 , eCurTok(NIL)
215 , ePush(NIL)
216 , nPLine(0)
217 , nPCol1(0)
218 , nPCol2(0)
219 , bEof(false)
220 , bEos(true)
221 , bAs(false)
222 , bErrorIsSymbol(true)
223 {
224 }
225
226 void SbiTokenizer::Push( SbiToken t )
227 {
228 if( ePush != NIL )
229 Error( ERRCODE_BASIC_INTERNAL_ERROR, u"PUSH"_ustr );
230 else ePush = t;
231 }
232
233 void SbiTokenizer::Error( ErrCode code, const OUString &aMsg )
234 {
235 aError = aMsg;
236 Error( code );
237 }
238
239 void SbiTokenizer::Error( ErrCode code, SbiToken tok )
240 {
241 aError = Symbol( tok );
242 Error( code );
243 }
244
245 // reading in the next token without absorbing it
246
247 SbiToken SbiTokenizer::Peek()
248 {
249 if( ePush == NIL )
250 {
251 sal_Int32 nOldLine = nLine;
252 sal_Int32 nOldCol1 = nCol1;
253 sal_Int32 nOldCol2 = nCol2;
254 ePush = Next();
255 nPLine = nLine; nLine = nOldLine;
256 nPCol1 = nCol1; nCol1 = nOldCol1;
257 nPCol2 = nCol2; nCol2 = nOldCol2;
258 }
259 eCurTok = ePush;
260 return eCurTok;
261 }
262
263 // For decompilation. Numbers and symbols return an empty string.
264
265 const OUString& SbiTokenizer::Symbol( SbiToken t )
266 {
267 // character token?
268 if( t < FIRSTKWD )
269 {
270 aSym = OUString(sal::static_int_cast<sal_Unicode>(t));
271 return aSym;
272 }
273 switch( t )
274 {
275 case NEG :
276 aSym = u"-"_ustr;
277 return aSym;
278 case EOS :
279 aSym = u":/CRLF"_ustr;
280 return aSym;
281 case EOLN :
282 aSym = u"CRLF"_ustr;
283 return aSym;
284 default:
285 break;
286 }
287 for( auto& rTok : aTokTable_Basic )
288 {
289 if( rTok.t == t )
290 {
291 aSym = OStringToOUString(rTok.s, RTL_TEXTENCODING_ASCII_US);
292 return aSym;
293 }
294 }
295 const sal_Unicode *p = aSym.getStr();
296 if (*p <= ' ')
297 {
298 aSym = u"???"_ustr;
299 }
300 return aSym;
301 }
302
303 // Reading in the next token and put it down.
304 // Tokens that don't appear in the token table
305 // are directly returned as a character.
306 // Some words are treated in a special way.
307
308 SbiToken SbiTokenizer::Next()
309 {
310 if (bEof)
311 {
312 return EOLN;
313 }
314 // have read in one already?
315 if( ePush != NIL )
316 {
317 eCurTok = ePush;
318 ePush = NIL;
319 nLine = nPLine;
320 nCol1 = nPCol1;
321 nCol2 = nPCol2;
322 bEos = IsEoln( eCurTok );
323 return eCurTok;
324 }
325 const TokenTable *tp;
326
327 if( !NextSym() )
328 {
329 bEof = bEos = true;
330 eCurTok = EOLN;
331 return eCurTok;
332 }
333
334 if( aSym.startsWith("\n") )
335 {
336 bEos = true;
337 eCurTok = EOLN;
338 return eCurTok;
339 }
340 bEos = false;
341
342 if( bNumber )
343 {
344 eCurTok = NUMBER;
345 return eCurTok;
346 }
347 else if( ( eScanType == SbxDATE || eScanType == SbxSTRING ) && !bSymbol )
348 {
349 eCurTok = FIXSTRING;
350 return eCurTok;
351 }
352 else if( aSym.isEmpty() )
353 {
354 //something went wrong
355 bEof = bEos = true;
356 eCurTok = EOLN;
357 return eCurTok;
358 }
359 // Special cases of characters that are between "Z" and "a". ICompare()
360 // evaluates the position of these characters in different ways.
361 else if( aSym[0] == '^' )
362 {
363 eCurTok = EXPON;
364 return eCurTok;
365 }
366 else if( aSym[0] == '\\' )
367 {
368 eCurTok = IDIV;
369 return eCurTok;
370 }
371 else
372 {
373 if( eScanType != SbxVARIANT )
374 {
375 eCurTok = SYMBOL;
376 return eCurTok;
377 }
378 // valid token?
379 short lb = 0;
380 short ub = std::size(aTokTable_Basic)-1;
381 short delta;
382 do
383 {
384 delta = (ub - lb) >> 1;
385 tp = &aTokTable_Basic[ lb + delta ];
386 sal_Int32 res = aSym.compareToIgnoreAsciiCaseAscii( tp->s );
387
388 if( res == 0 )
389 {
390 goto special;
391 }
392 if( res < 0 )
393 {
394 if ((ub - lb) == 2)
395 {
396 ub = lb;
397 }
398 else
399 {
400 ub = ub - delta;
401 }
402 }
403 else
404 {
405 if ((ub -lb) == 2)
406 {
407 lb = ub;
408 }
409 else
410 {
411 lb = lb + delta;
412 }
413 }
414 }
415 while( delta );
416 // Symbol? if not >= token
417 sal_Unicode ch = aSym[0];
418 if( !BasicCharClass::isAlpha( ch, bCompatible ) && !bSymbol )
419 {
420 eCurTok = static_cast<SbiToken>(ch & 0x00FF);
421 return eCurTok;
422 }
423 eCurTok = SYMBOL;
424 return eCurTok;
425 }
426 special:
427 // #i92642
428 bool bStartOfLine = (eCurTok == NIL || eCurTok == REM || eCurTok == EOLN ||
429 eCurTok == THEN || eCurTok == ELSE); // single line If
430 if( !bStartOfLine && (tp->t == NAME || tp->t == LINE) )
431 {
432 eCurTok = SYMBOL;
433 return eCurTok;
434 }
435 else if( tp->t == TEXT )
436 {
437 eCurTok = SYMBOL;
438 return eCurTok;
439 }
440 // maybe we can expand this for other statements that have parameters
441 // that are keywords ( and those keywords are only used within such
442 // statements )
443 // what's happening here is that if we come across 'append' ( and we are
444 // not in the middle of parsing a special statement ( like 'Open')
445 // we just treat keyword 'append' as a normal 'SYMBOL'.
446 // Also we accept Dim APPEND
447 else if ( ( !bInStatement || eCurTok == DIM ) && tp->t == APPEND )
448 {
449 eCurTok = SYMBOL;
450 return eCurTok;
451 }
452 // #i92642: Special LINE token handling -> SbiParser::Line()
453
454 // END IF, CASE, SUB, DEF, FUNCTION, TYPE, CLASS, WITH
455 if( tp->t == END )
456 {
457 // from 15.3.96, special treatment for END, at Peek() the current
458 // time is lost, so memorize everything and restore after
459 sal_Int32 nOldLine = nLine;
460 sal_Int32 nOldCol = nCol;
461 sal_Int32 nOldCol1 = nCol1;
462 sal_Int32 nOldCol2 = nCol2;
463 OUString aOldSym = aSym;
464 SaveLine(); // save pLine in the scanner
465
466 eCurTok = Peek();
467 switch( eCurTok )
468 {
469 case IF: Next(); eCurTok = ENDIF; break;
470 case SELECT: Next(); eCurTok = ENDSELECT; break;
471 case SUB: Next(); eCurTok = ENDSUB; break;
472 case FUNCTION: Next(); eCurTok = ENDFUNC; break;
473 case PROPERTY: Next(); eCurTok = ENDPROPERTY; break;
474 case TYPE: Next(); eCurTok = ENDTYPE; break;
475 case ENUM: Next(); eCurTok = ENDENUM; break;
476 case WITH: Next(); eCurTok = ENDWITH; break;
477 default : eCurTok = END; break;
478 }
479 nCol1 = nOldCol1;
480 if( eCurTok == END )
481 {
482 // reset everything so that token is read completely newly after END
483 ePush = NIL;
484 nLine = nOldLine;
485 nCol = nOldCol;
486 nCol2 = nOldCol2;
487 aSym = aOldSym;
488 RestoreLine();
489 }
490 return eCurTok;
491 }
492 // are data types keywords?
493 // there is ERROR(), DATA(), STRING() etc.
494 eCurTok = tp->t;
495 // AS: data types are keywords
496 if( tp->t == AS )
497 {
498 bAs = true;
499 }
500 else
501 {
502 if( bAs )
503 {
504 bAs = false;
505 }
506 else if( eCurTok >= DATATYPE1 && eCurTok <= DATATYPE2 && (bErrorIsSymbol || eCurTok != ERROR_) )
507 {
508 eCurTok = SYMBOL;
509 }
510 }
511
512 // CLASSMODULE, PROPERTY, GET, ENUM token only visible in compatible mode
513 SbiToken eTok = tp->t;
514 if( bCompatible )
515 {
516 // #129904 Suppress system
517 if( eTok == STOP && aSym.equalsIgnoreAsciiCase("system") )
518 {
519 eCurTok = SYMBOL;
520 }
521 if( eTok == GET && bStartOfLine )
522 {
523 eCurTok = SYMBOL;
524 }
525 }
526 else
527 {
528 if( eTok == CLASSMODULE ||
529 eTok == IMPLEMENTS ||
530 eTok == PARAMARRAY ||
531 eTok == ENUM ||
532 eTok == PROPERTY ||
533 eTok == GET ||
534 eTok == TYPEOF )
535 {
536 eCurTok = SYMBOL;
537 }
538 }
539
540 bEos = IsEoln( eCurTok );
541 return eCurTok;
542 }
543
544 bool SbiTokenizer::MayBeLabel( bool bNeedsColon )
545 {
546 static TokenLabelInfo gaStaticTokenLabelInfo;
547
548 if( eCurTok == SYMBOL || gaStaticTokenLabelInfo.canTokenBeLabel( eCurTok ) )
549 {
550 return !bNeedsColon || DoesColonFollow();
551 }
552 else
553 {
554 return ( eCurTok == NUMBER
555 && eScanType == SbxINTEGER
556 && nVal >= 0 );
557 }
558 }
559
560
561 OUString SbiTokenizer::GetKeywordCase( std::u16string_view sKeyword )
562 {
563 for( auto& rTok : aTokTable_Basic )
564 {
565 if( o3tl::equalsIgnoreAsciiCase(sKeyword, rTok.s) )
566 return OStringToOUString(rTok.s, RTL_TEXTENCODING_ASCII_US);
567 }
568 return OUString();
569 }
570
571 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */