Debian Codegraph @

src/core/Lexer.nml

raw
1 /*
2 * Copyright (C)2005-2022 Haxe Foundation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 type pos {
24 psource : string;
25 pline : int;
26 pmin : int;
27 pmax : int;
28 }
29
30 type 'a t {
31 mutable data : 'a;
32 mutable current : string;
33 mutable buffer : string;
34 mutable bsize : int;
35 mutable bin : int;
36 mutable bpos : int;
37 mutable cin : int;
38 mutable cpos : int;
39 mutable input : IO.input;
40 mutable source : string;
41 mutable line : int;
42 mutable pos : int;
43 mutable carriage : bool;
44 }
45
46 type ('a,'b) tables {
47 engine : LexEngine.tables;
48 cases : ('b t -> 'a) array;
49 def : 'b t -> 'a;
50 }
51
52 exception Invalid_rule : string
53
54 var null_pos = { pmin = 0; pmax = 0; pline = 0; psource = "<null>" };
55
56 function source(p) { p.psource }
57 function line(p) { p.pline }
58
59 function punion(p,p2) {
60 {
61 psource = p.psource;
62 pline = p.pline;
63 pmin = min p.pmin p2.pmin;
64 pmax = max p.pmax p2.pmax;
65 }
66 }
67
68 function create(data) {
69 var bufsize = 4096;
70 {
71 data = data;
72 carriage = false;
73 current = "";
74 buffer = String.create bufsize;
75 bsize = bufsize;
76 bin = 0;
77 cin = 0;
78 bpos = bufsize;
79 cpos = bufsize;
80 input = IO.read_string "";
81 source = "<null>";
82 line = 0;
83 pos = 0;
84 }
85 }
86
87 function data(l) {
88 l.data;
89 }
90
91 function set(l,data) {
92 l.data := data;
93 }
94
95 function input(l,source,input,line,pos) {
96 l.bin := 0;
97 l.cin := 0;
98 l.bpos := l.bsize;
99 l.cpos := l.bsize;
100 l.input := input;
101 l.source := source;
102 l.line := line;
103 l.pos := pos;
104 }
105
106
107 function curpos(l) {
108 {
109 psource = l.source;
110 pline = l.line;
111 pmin = l.pos - String.length l.current;
112 pmax = l.pos;
113 }
114 }
115
116 function current(l) : string {
117 l.current
118 }
119
120 function read(l) : char {
121 if l.bin == 0 then {
122 if l.bpos == l.bsize then {
123 var buf = String.create (l.bsize * 2);
124 String.blit buf l.bsize l.buffer 0 l.bsize;
125 l.cpos := l.cpos + l.bsize;
126 l.bpos := l.bpos + l.bsize;
127 l.buffer := buf;
128 l.bsize := l.bsize * 2;
129 }
130 var delta = l.bpos - l.cpos;
131 String.blit l.buffer 0 l.buffer l.cpos delta;
132 l.bpos := delta;
133 l.cpos := 0;
134 var k = IO.input l.input l.buffer delta (l.bsize - delta);
135 l.bin := l.bin + k;
136 l.cin := l.cin + k;
137 };
138 var c = String.get l.buffer l.bpos;
139 l.bpos := l.bpos + 1;
140 l.bin := l.bin - 1;
141 c
142 }
143
144 function inc_line(l,c) {
145 if c == '\r' then
146 l.carriage := true
147 else if c == '\n' || l.carriage then {
148 l.carriage := false;
149 l.line := l.line + 1;
150 };
151 }
152
153 function char(l) : char option {
154 try {
155 var c = read l;
156 l.bpos := l.bpos - 1;
157 l.bin := l.bin + 1;
158 inc_line l c;
159 Some c
160 } catch {
161 IO.Eof -> None
162 }
163 }
164
165 type cur {
166 mutable cur_n : int;
167 mutable cur_res : int;
168 }
169
170 function token(l,t) : 'a {
171 var tbl = fst t.engine;
172 var exits = snd t.engine;
173 var cur = { cur_n = 0; cur_res = -1 };
174 var last = { cur_n = 0; cur_res = -1 };
175 function rec loop(n,k) {
176 var res = neko("tbl[0][k][0][read(l)]");
177 var e = exits.[k];
178 cur.cur_n := n;
179 cur.cur_res := res;
180 if e != -1 then {
181 last.cur_n := n;
182 last.cur_res := e;
183 }
184 if res == -1 then throw Exit;
185 neko("loop[0](n+1,res)");
186 };
187 function process(eof) {
188 var n = last.cur_n;
189 var k = last.cur_res;
190 if k == -1 then {
191 l.current := "";
192 if !eof then {
193 l.bpos := l.bpos - (cur.cur_n + 1);
194 l.bin := l.bin + (cur.cur_n + 1);
195 }
196 -1;
197 } else {
198 l.cin := l.cin - n;
199 l.bin := l.cin;
200 l.current := String.sub l.buffer l.cpos n;
201 l.cpos := l.cpos + n;
202 l.bpos := l.cpos;
203 l.pos := l.pos + n;
204 var i = &0;
205 while *i < n {
206 inc_line l (String.get l.current (*i));
207 i := *i + 1;
208 }
209 k
210 }
211 };
212 var k = try {
213 loop 0 0;
214 assert();
215 } catch {
216 | IO.Eof ->
217 if cur.cur_res != -1 then {
218 last.cur_n := cur.cur_n + 1;
219 last.cur_res := exits.[cur.cur_res];
220 }
221 process true;
222 | Exit ->
223 process false
224 }
225 if k == -1 then
226 t.def(l)
227 else
228 t.cases.[k](l)
229 }
230
231 function build(rules,def) {
232 var nfa = List.map (function((r,_)) { try LexEngine.parse r catch { _ -> throw Invalid_rule(r) } }) rules;
233 var cases = List.array (List.map snd rules);
234 {
235 engine = LexEngine.make_tables (LexEngine.determinize (List.array nfa));
236 cases = cases;
237 def = def;
238 }
239 }
240
241 function empty() {
242 function empty_table(_) { invalid_arg() };
243 build [] empty_table;
244 }