| 1 |
/*========================== begin_copyright_notice ============================ |
| 2 |
|
| 3 |
Copyright (C) 2017-2021 Intel Corporation |
| 4 |
|
| 5 |
SPDX-License-Identifier: MIT |
| 6 |
|
| 7 |
============================= end_copyright_notice ===========================*/ |
| 8 |
|
| 9 |
#ifndef _BUFFEREDLEXER_H |
| 10 |
#define _BUFFEREDLEXER_H |
| 11 |
|
| 12 |
#include "../IR/Loc.hpp" |
| 13 |
#include "../asserts.hpp" |
| 14 |
#include "Lexemes.hpp" |
| 15 |
|
| 16 |
#include <iostream> |
| 17 |
#include <ostream> |
| 18 |
#include <sstream> |
| 19 |
#include <string> |
| 20 |
#include <vector> |
| 21 |
|
| 22 |
// #define DUMP_LEXEMES |
| 23 |
|
| 24 |
#define YY_DECL iga::Lexeme yylex(yyscan_t yyscanner, unsigned int &inp_off) |
| 25 |
#ifndef YY_NO_UNISTD_H |
| 26 |
#define YY_NO_UNISTD_H |
| 27 |
#endif |
| 28 |
#include "lex.yy.hpp" |
| 29 |
|
| 30 |
YY_DECL; |
| 31 |
|
| 32 |
namespace iga { |
| 33 |
|
| 34 |
struct Token { |
| 35 |
Lexeme lexeme; |
| 36 |
Loc loc; |
| 37 |
|
| 38 |
Token() : lexeme(Lexeme::LEXICAL_ERROR) {} |
| 39 |
Token(const Lexeme &lxm, uint32_t ln, uint32_t cl, uint32_t off, uint32_t len) |
| 40 |
: lexeme(lxm), loc(ln, cl, off, len) {} |
| 41 |
}; |
| 42 |
|
| 43 |
static void WriteTokenContext(const std::string &inp, const struct Loc &loc, |
| 44 |
std::ostream &os) { |
| 45 |
if (loc.offset >= (PC)inp.size()) { |
| 46 |
os << "<<EOF>>" << std::endl; |
| 47 |
} else if (loc.line > 0) { |
| 48 |
size_t k = static_cast<size_t>(loc.offset) - loc.col + 1; |
| 49 |
while (k < inp.size() && inp[k] != '\n' && inp[k] != '\r') |
| 50 |
os << inp[k++]; |
| 51 |
os << std::endl; |
| 52 |
if (loc.col > 0) { |
| 53 |
for (size_t i = 1; i < loc.col; i++) |
| 54 |
os << ' '; |
| 55 |
if (loc.line == 0) { |
| 56 |
os << '^'; |
| 57 |
} else { |
| 58 |
for (size_t i = 0; i < loc.extent; i++) { |
| 59 |
os << '~'; |
| 60 |
} |
| 61 |
} |
| 62 |
} |
| 63 |
os << std::endl; |
| 64 |
} |
| 65 |
} |
| 66 |
|
| 67 |
static std::string GetTokenString(const Token &token, const std::string &inp) { |
| 68 |
std::stringstream ss; |
| 69 |
ss << token.loc.line << "." << token.loc.col << ": (" << token.loc.offset |
| 70 |
<< "/" << token.loc.extent << "): " << LexemeString(token.lexeme) |
| 71 |
<< std::endl; |
| 72 |
WriteTokenContext(inp, token.loc, ss); |
| 73 |
return ss.str(); |
| 74 |
} |
| 75 |
|
| 76 |
class BufferedLexer { |
| 77 |
std::vector<Token> m_tokens; |
| 78 |
size_t m_offset, m_mark; // token index of the scanner |
| 79 |
|
| 80 |
const std::string m_input; |
| 81 |
|
| 82 |
Token m_eof; |
| 83 |
|
| 84 |
public: |
| 85 |
BufferedLexer(const std::string &inp) |
| 86 |
: m_offset(0), m_mark(0), m_input(inp), |
| 87 |
m_eof(Lexeme::END_OF_FILE, 0, 0, 0, 0) { |
| 88 |
yyscan_t yy; |
| 89 |
|
| 90 |
yylex_init(&yy); |
| 91 |
yy_scan_string(inp.c_str(), yy); |
| 92 |
yyset_lineno(1, yy); |
| 93 |
yyset_column(1, yy); |
| 94 |
|
| 95 |
unsigned int inpOff = 0, bolOff = 0; |
| 96 |
|
| 97 |
Lexeme lxm; |
| 98 |
while (true) { |
| 99 |
lxm = yylex(yy, inpOff); |
| 100 |
|
| 101 |
uint32_t lno = (uint32_t)yyget_lineno(yy); |
| 102 |
uint32_t len = (uint32_t)yyget_leng(yy); |
| 103 |
uint32_t col = (uint32_t)yyget_column(yy) - len; |
| 104 |
uint32_t off = (uint32_t)inpOff; |
| 105 |
if (lxm == Lexeme::NEWLINE) { |
| 106 |
// flex increments yylineno and clear's column before this |
| 107 |
// we fix this by backing up the newline for that case |
| 108 |
// and inferring the final column from the beginning of |
| 109 |
// the last line |
| 110 |
lno--; |
| 111 |
col = inpOff - bolOff + 1; |
| 112 |
bolOff = inpOff; |
| 113 |
} |
| 114 |
// const char *str = yyget_text(yy); |
| 115 |
// printf("AT %u.%u(%u:%u:\"%s\"): %s\n", |
| 116 |
// lno,col,off,len,str,LexemeString(lxm)); |
| 117 |
// struct Loc loc(lno,col,off,len); |
| 118 |
// ShowToken(inp,loc,std::cout); |
| 119 |
|
| 120 |
if (lxm == Lexeme::END_OF_FILE) { |
| 121 |
m_eof = Token(lxm, lno, col, off, len); // update EOF w/ loc |
| 122 |
m_tokens.push_back(m_eof); |
| 123 |
break; |
| 124 |
} |
| 125 |
m_tokens.emplace_back(lxm, lno, col, off, len); |
| 126 |
inpOff += len; |
| 127 |
} |
| 128 |
|
| 129 |
yylex_destroy(yy); |
| 130 |
} |
| 131 |
const std::string &GetSource() const { return m_input; } |
| 132 |
|
| 133 |
size_t GetTokenOffset() const { return m_offset; } |
| 134 |
void SetTokenOffset(size_t off) { m_offset = off; } |
| 135 |
void Mark() { m_mark = m_offset; } |
| 136 |
void Reset() { SetTokenOffset(m_mark); } |
| 137 |
|
| 138 |
void DumpTokens(std::ostream &out, const std::string &inp) const { |
| 139 |
for (const auto &t : m_tokens) { |
| 140 |
out << "AT" << t.loc.line << "." << t.loc.col << "(" << t.loc.offset |
| 141 |
<< ":" << t.loc.extent << ": " << LexemeString(t.lexeme) << std::endl; |
| 142 |
WriteTokenContext(inp, t.loc, out); |
| 143 |
} |
| 144 |
} |
| 145 |
|
| 146 |
void DumpLookaheads(std::ostream &os, int n = 1) const { |
| 147 |
os << "LEXER: Next " << n << " lookaheads are:\n"; |
| 148 |
for (int i = 0; i < n; i++) { |
| 149 |
const Token &tk = Next(i); |
| 150 |
os << " " << GetTokenString(tk, m_input).c_str() << "\n"; |
| 151 |
if (tk.lexeme == Lexeme::END_OF_FILE) { |
| 152 |
break; |
| 153 |
} |
| 154 |
} |
| 155 |
} |
| 156 |
|
| 157 |
bool EndOfFile() const { |
| 158 |
return m_tokens[m_offset].lexeme == Lexeme::END_OF_FILE; |
| 159 |
} |
| 160 |
|
| 161 |
bool Skip(int i) { |
| 162 |
int k = (int)m_offset + i; |
| 163 |
if (k < 0 || k >= (int)m_tokens.size()) { |
| 164 |
return false; |
| 165 |
} |
| 166 |
m_offset = k; |
| 167 |
#ifdef DUMP_LEXEMES |
| 168 |
DumpLookahead(1); |
| 169 |
#endif |
| 170 |
return true; |
| 171 |
} |
| 172 |
bool Consume(Lexeme lxm) { |
| 173 |
if (LookingAt(lxm)) { |
| 174 |
(void)Skip(1); |
| 175 |
return true; |
| 176 |
} |
| 177 |
return false; |
| 178 |
} |
| 179 |
|
| 180 |
bool LookingAt(Lexeme lxm) const { return LookingAtFrom(0, lxm); } |
| 181 |
bool LookingAtFrom(int i, Lexeme lx) const { return Next(i).lexeme == lx; } |
| 182 |
|
| 183 |
const Token &Next(int i) const { |
| 184 |
int k = (int)m_offset + i; |
| 185 |
if (k < 0 || k >= (int)m_tokens.size()) { |
| 186 |
return m_eof; |
| 187 |
} else { |
| 188 |
return m_tokens[k]; |
| 189 |
} |
| 190 |
} |
| 191 |
}; // class BufferedLexer |
| 192 |
|
| 193 |
} // namespace iga |
| 194 |
|
| 195 |
#endif // _BUFFEREDLEXER_H |