Debian Codegraph @

lualib/lua_lexer.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[ Lua LPEG grammar based on https://github.com/xolox/lua-lxsh/ ]]
18
19
20 local lpeg = require "lpeg"
21
22 local P = lpeg.P
23 local R = lpeg.R
24 local S = lpeg.S
25 local D = R '09' -- Digits
26 local I = R('AZ', 'az', '\127\255') + '_' -- Identifiers
27 local B = -(I + D) -- Word boundary
28 local EOS = -lpeg.P(1) -- end of string
29
30 -- Pattern for long strings and long comments.
31 local longstring = #(P '[[' + (P '[' * P '=' ^ 0 * '[')) * P(function(input, index)
32 local level = input:match('^%[(=*)%[', index)
33 if level then
34 local _, last = input:find(']' .. level .. ']', index, true)
35 if last then
36 return last + 1
37 end
38 end
39 end)
40
41 -- String literals.
42 local singlequoted = P "'" * ((1 - S "'\r\n\f\\") + (P '\\' * 1)) ^ 0 * "'"
43 local doublequoted = P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"'
44
45 -- Comments.
46 local eol = P '\r\n' + '\n'
47 local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
48 local singleline = P '--' * line
49 local multiline = P '--' * longstring
50
51 -- Numbers.
52 local sign = S '+-' ^ -1
53 local decimal = D ^ 1
54 local hexadecimal = P '0' * S 'xX' * R('09', 'AF', 'af') ^ 1
55 local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
56 local maybeexp = (float + decimal) * (S 'eE' * sign * D ^ 1) ^ -1
57
58 local function compile_keywords(keywords)
59 local list = {}
60 for word in keywords:gmatch('%S+') do
61 list[#list + 1] = word
62 end
63 -- Sort by length
64 table.sort(list, function(a, b)
65 return #a > #b
66 end)
67
68 local pattern
69 for _, word in ipairs(list) do
70 local p = lpeg.P(word)
71 pattern = pattern and (pattern + p) or p
72 end
73
74 local AB = B + EOS -- ending boundary
75 return pattern * AB
76 end
77
78 -- Identifiers
79 local ident = I * (I + D) ^ 0
80 local expr = ('.' * ident) ^ 0
81
82 local patterns = {
83 { 'whitespace', S '\r\n\f\t\v ' ^ 1 },
84 { 'constant', (P 'true' + 'false' + 'nil') * B },
85 { 'string', singlequoted + doublequoted + longstring },
86 { 'comment', multiline + singleline },
87 { 'number', hexadecimal + maybeexp },
88 { 'operator', P 'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<='
89 + 'or' + S ']{=>^[<;)*(%}+-:,/.#' },
90 { 'keyword', compile_keywords([[
91 break do else elseif end for function if in local repeat return then until while
92 ]]) },
93 { 'identifier', lpeg.Cmt(ident,
94 function(input, index)
95 return expr:match(input, index)
96 end)
97 },
98 { 'error', 1 },
99 }
100
101 local compiled
102
103 local function compile_patterns()
104 if not compiled then
105 local function process(elt)
106 local n, grammar = elt[1], elt[2]
107 return lpeg.Cc(n) * lpeg.P(grammar) * lpeg.Cp()
108 end
109 local any = process(patterns[1])
110 for i = 2, #patterns do
111 any = any + process(patterns[i])
112 end
113 compiled = any
114 end
115
116 return compiled
117 end
118
119 local function sync(token, lnum, cnum)
120 local lastidx
121 lnum, cnum = lnum or 1, cnum or 1
122 if token:find '\n' then
123 for i in token:gmatch '()\n' do
124 lnum = lnum + 1
125 lastidx = i
126 end
127 cnum = #token - lastidx + 1
128 else
129 cnum = cnum + #token
130 end
131 return lnum, cnum
132 end
133
134 local exports = {}
135
136 exports.gmatch = function(input)
137 local parser = compile_patterns()
138 local index, lnum, cnum = 1, 1, 1
139
140 return function()
141 local kind, after = parser:match(input, index)
142 if kind and after then
143 local text = input:sub(index, after - 1)
144 local oldlnum, oldcnum = lnum, cnum
145 index = after
146 lnum, cnum = sync(text, lnum, cnum)
147 return kind, text, oldlnum, oldcnum
148 end
149 end
150 end
151
152 exports.lex_to_table = function(input)
153 local out = {}
154
155 for kind, text, lnum, cnum in exports.gmatch(input) do
156 out[#out + 1] = { kind, text, lnum, cnum }
157 end
158
159 return out
160 end
161
162 return exports
163