Debian Codegraph @

visa/iga/IGALibrary/Frontend/BufferedLexer.hpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #ifndef _BUFFEREDLEXER_H
10 #define _BUFFEREDLEXER_H
11
12 #include "../IR/Loc.hpp"
13 #include "../asserts.hpp"
14 #include "Lexemes.hpp"
15
16 #include <iostream>
17 #include <ostream>
18 #include <sstream>
19 #include <string>
20 #include <vector>
21
22 // #define DUMP_LEXEMES
23
24 #define YY_DECL iga::Lexeme yylex(yyscan_t yyscanner, unsigned int &inp_off)
25 #ifndef YY_NO_UNISTD_H
26 #define YY_NO_UNISTD_H
27 #endif
28 #include "lex.yy.hpp"
29
30 YY_DECL;
31
32 namespace iga {
33
34 struct Token {
35 Lexeme lexeme;
36 Loc loc;
37
38 Token() : lexeme(Lexeme::LEXICAL_ERROR) {}
39 Token(const Lexeme &lxm, uint32_t ln, uint32_t cl, uint32_t off, uint32_t len)
40 : lexeme(lxm), loc(ln, cl, off, len) {}
41 };
42
43 static void WriteTokenContext(const std::string &inp, const struct Loc &loc,
44 std::ostream &os) {
45 if (loc.offset >= (PC)inp.size()) {
46 os << "<<EOF>>" << std::endl;
47 } else if (loc.line > 0) {
48 size_t k = static_cast<size_t>(loc.offset) - loc.col + 1;
49 while (k < inp.size() && inp[k] != '\n' && inp[k] != '\r')
50 os << inp[k++];
51 os << std::endl;
52 if (loc.col > 0) {
53 for (size_t i = 1; i < loc.col; i++)
54 os << ' ';
55 if (loc.line == 0) {
56 os << '^';
57 } else {
58 for (size_t i = 0; i < loc.extent; i++) {
59 os << '~';
60 }
61 }
62 }
63 os << std::endl;
64 }
65 }
66
67 static std::string GetTokenString(const Token &token, const std::string &inp) {
68 std::stringstream ss;
69 ss << token.loc.line << "." << token.loc.col << ": (" << token.loc.offset
70 << "/" << token.loc.extent << "): " << LexemeString(token.lexeme)
71 << std::endl;
72 WriteTokenContext(inp, token.loc, ss);
73 return ss.str();
74 }
75
76 class BufferedLexer {
77 std::vector<Token> m_tokens;
78 size_t m_offset, m_mark; // token index of the scanner
79
80 const std::string m_input;
81
82 Token m_eof;
83
84 public:
85 BufferedLexer(const std::string &inp)
86 : m_offset(0), m_mark(0), m_input(inp),
87 m_eof(Lexeme::END_OF_FILE, 0, 0, 0, 0) {
88 yyscan_t yy;
89
90 yylex_init(&yy);
91 yy_scan_string(inp.c_str(), yy);
92 yyset_lineno(1, yy);
93 yyset_column(1, yy);
94
95 unsigned int inpOff = 0, bolOff = 0;
96
97 Lexeme lxm;
98 while (true) {
99 lxm = yylex(yy, inpOff);
100
101 uint32_t lno = (uint32_t)yyget_lineno(yy);
102 uint32_t len = (uint32_t)yyget_leng(yy);
103 uint32_t col = (uint32_t)yyget_column(yy) - len;
104 uint32_t off = (uint32_t)inpOff;
105 if (lxm == Lexeme::NEWLINE) {
106 // flex increments yylineno and clear's column before this
107 // we fix this by backing up the newline for that case
108 // and inferring the final column from the beginning of
109 // the last line
110 lno--;
111 col = inpOff - bolOff + 1;
112 bolOff = inpOff;
113 }
114 // const char *str = yyget_text(yy);
115 // printf("AT %u.%u(%u:%u:\"%s\"): %s\n",
116 // lno,col,off,len,str,LexemeString(lxm));
117 // struct Loc loc(lno,col,off,len);
118 // ShowToken(inp,loc,std::cout);
119
120 if (lxm == Lexeme::END_OF_FILE) {
121 m_eof = Token(lxm, lno, col, off, len); // update EOF w/ loc
122 m_tokens.push_back(m_eof);
123 break;
124 }
125 m_tokens.emplace_back(lxm, lno, col, off, len);
126 inpOff += len;
127 }
128
129 yylex_destroy(yy);
130 }
131 const std::string &GetSource() const { return m_input; }
132
133 size_t GetTokenOffset() const { return m_offset; }
134 void SetTokenOffset(size_t off) { m_offset = off; }
135 void Mark() { m_mark = m_offset; }
136 void Reset() { SetTokenOffset(m_mark); }
137
138 void DumpTokens(std::ostream &out, const std::string &inp) const {
139 for (const auto &t : m_tokens) {
140 out << "AT" << t.loc.line << "." << t.loc.col << "(" << t.loc.offset
141 << ":" << t.loc.extent << ": " << LexemeString(t.lexeme) << std::endl;
142 WriteTokenContext(inp, t.loc, out);
143 }
144 }
145
146 void DumpLookaheads(std::ostream &os, int n = 1) const {
147 os << "LEXER: Next " << n << " lookaheads are:\n";
148 for (int i = 0; i < n; i++) {
149 const Token &tk = Next(i);
150 os << " " << GetTokenString(tk, m_input).c_str() << "\n";
151 if (tk.lexeme == Lexeme::END_OF_FILE) {
152 break;
153 }
154 }
155 }
156
157 bool EndOfFile() const {
158 return m_tokens[m_offset].lexeme == Lexeme::END_OF_FILE;
159 }
160
161 bool Skip(int i) {
162 int k = (int)m_offset + i;
163 if (k < 0 || k >= (int)m_tokens.size()) {
164 return false;
165 }
166 m_offset = k;
167 #ifdef DUMP_LEXEMES
168 DumpLookahead(1);
169 #endif
170 return true;
171 }
172 bool Consume(Lexeme lxm) {
173 if (LookingAt(lxm)) {
174 (void)Skip(1);
175 return true;
176 }
177 return false;
178 }
179
180 bool LookingAt(Lexeme lxm) const { return LookingAtFrom(0, lxm); }
181 bool LookingAtFrom(int i, Lexeme lx) const { return Next(i).lexeme == lx; }
182
183 const Token &Next(int i) const {
184 int k = (int)m_offset + i;
185 if (k < 0 || k >= (int)m_tokens.size()) {
186 return m_eof;
187 } else {
188 return m_tokens[k];
189 }
190 }
191 }; // class BufferedLexer
192
193 } // namespace iga
194
195 #endif // _BUFFEREDLEXER_H