Debian Codegraph @

pattern/lexer.go

raw
1 package pattern
2
3 import (
4 "fmt"
5 "go/token"
6 "iter"
7 "unicode"
8 "unicode/utf8"
9 )
10
11 // lex returns the sequence of tokens in the input.
12 func lex(f *token.File, input string) iter.Seq[item] {
13 return func(yield func(item) bool) {
14 lex := &lexer{
15 f: f,
16 input: input,
17 yield: yield,
18 }
19 lex.run()
20 }
21 }
22
23 // lexer holds the state of a single [lex] iteration.
24 type lexer struct {
25 f *token.File
26
27 input string
28 start int
29 pos int
30 width int
31
32 yield func(item) bool
33 }
34
35 type itemType int
36
37 const eof = -1
38
39 const (
40 itemError itemType = iota
41 itemLeftParen
42 itemRightParen
43 itemLeftBracket
44 itemRightBracket
45 itemTypeName
46 itemVariable
47 itemAt
48 itemColon
49 itemBlank
50 itemString
51 itemEOF
52 )
53
54 func (typ itemType) String() string {
55 switch typ {
56 case itemError:
57 return "ERROR"
58 case itemLeftParen:
59 return "("
60 case itemRightParen:
61 return ")"
62 case itemLeftBracket:
63 return "["
64 case itemRightBracket:
65 return "]"
66 case itemTypeName:
67 return "TYPE"
68 case itemVariable:
69 return "VAR"
70 case itemAt:
71 return "@"
72 case itemColon:
73 return ":"
74 case itemBlank:
75 return "_"
76 case itemString:
77 return "STRING"
78 case itemEOF:
79 return "EOF"
80 default:
81 return fmt.Sprintf("itemType(%d)", typ)
82 }
83 }
84
85 type item struct {
86 typ itemType
87 val string
88 pos int
89 }
90
91 type stateFn func(*lexer) stateFn
92
93 func (l *lexer) run() {
94 for state := lexStart; state != nil; {
95 state = state(l)
96 }
97 }
98
99 func (l *lexer) emitValue(t itemType, value string) bool {
100 ok := l.yield(item{t, value, l.start})
101 l.start = l.pos
102 return ok
103 }
104
105 func (l *lexer) emit(t itemType) bool {
106 ok := l.yield(item{t, l.input[l.start:l.pos], l.start})
107 l.start = l.pos
108 return ok
109 }
110
111 func lexStart(l *lexer) stateFn {
112 switch r := l.next(); {
113 case r == eof:
114 _ = l.emit(itemEOF)
115 return nil
116 case unicode.IsSpace(r):
117 l.ignore()
118 case r == '(':
119 if !l.emit(itemLeftParen) {
120 return nil
121 }
122 case r == ')':
123 if !l.emit(itemRightParen) {
124 return nil
125 }
126 case r == '[':
127 if !l.emit(itemLeftBracket) {
128 return nil
129 }
130 case r == ']':
131 if !l.emit(itemRightBracket) {
132 return nil
133 }
134 case r == '@':
135 if !l.emit(itemAt) {
136 return nil
137 }
138 case r == ':':
139 if !l.emit(itemColon) {
140 return nil
141 }
142 case r == '_':
143 if !l.emit(itemBlank) {
144 return nil
145 }
146 case r == '"':
147 l.backup()
148 return lexString
149 case unicode.IsUpper(r):
150 l.backup()
151 return lexType
152 case unicode.IsLower(r):
153 l.backup()
154 return lexVariable
155 default:
156 return l.errorf("unexpected character %c", r)
157 }
158 return lexStart
159 }
160
161 func (l *lexer) next() (r rune) {
162 if l.pos >= len(l.input) {
163 l.width = 0
164 return eof
165 }
166 r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
167
168 if r == '\n' {
169 l.f.AddLine(l.pos)
170 }
171
172 l.pos += l.width
173
174 return r
175 }
176
177 func (l *lexer) ignore() {
178 l.start = l.pos
179 }
180
181 func (l *lexer) backup() {
182 l.pos -= l.width
183 }
184
185 func (l *lexer) errorf(format string, args ...any) stateFn {
186 // TODO(dh): emit position information in errors
187 _ = l.yield(item{
188 itemError,
189 fmt.Sprintf(format, args...),
190 l.start,
191 })
192 return nil
193 }
194
195 func isAlphaNumeric(r rune) bool {
196 return r >= '0' && r <= '9' ||
197 r >= 'a' && r <= 'z' ||
198 r >= 'A' && r <= 'Z'
199 }
200
201 func lexString(l *lexer) stateFn {
202 l.next() // skip quote
203 escape := false
204
205 var runes []rune
206 for {
207 switch r := l.next(); r {
208 case eof:
209 return l.errorf("unterminated string")
210 case '"':
211 if !escape {
212 if !l.emitValue(itemString, string(runes)) {
213 return nil
214 }
215 return lexStart
216 } else {
217 runes = append(runes, '"')
218 escape = false
219 }
220 case '\\':
221 if escape {
222 runes = append(runes, '\\')
223 escape = false
224 } else {
225 escape = true
226 }
227 default:
228 runes = append(runes, r)
229 }
230 }
231 }
232
233 func lexType(l *lexer) stateFn {
234 l.next()
235 for {
236 if !isAlphaNumeric(l.next()) {
237 l.backup()
238 if !l.emit(itemTypeName) {
239 return nil
240 }
241 return lexStart
242 }
243 }
244 }
245
246 func lexVariable(l *lexer) stateFn {
247 l.next()
248 for {
249 if !isAlphaNumeric(l.next()) {
250 l.backup()
251 if !l.emit(itemVariable) {
252 return nil
253 }
254 return lexStart
255 }
256 }
257 }