Debian Codegraph @

golang-x-tools/expect/extract.go

raw ◀ ▶
1 Debian patch touch this file
•1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 package expect
6
7 import (
8 "fmt"
9 "go/ast"
10 "go/parser"
11 goscanner "go/scanner"
12 "go/token"
13 "os"
14 "path/filepath"
15 "regexp"
16 "strconv"
17 "strings"
18 "text/scanner"
19
20 "golang.org/x/mod/modfile"
21 )
22
23 const commentStart = "@"
24 const commentStartLen = len(commentStart)
25
26 // Identifier is the type for an identifier in a Note argument list.
27 type Identifier string
28
29 // Parse collects all the notes present in a file.
30 // If content is nil, the filename specified is read and parsed, otherwise the
31 // content is used and the filename is used for positions and error messages.
32 // Each comment whose text starts with @ is parsed as a comma-separated
33 // sequence of notes.
34 // See the package documentation for details about the syntax of those
35 // notes.
36 func Parse(fset *token.FileSet, filename string, content []byte) ([]*Note, error) {
37 if content == nil {
38 data, err := os.ReadFile(filename)
39 if err != nil {
40 return nil, err
41 }
42 content = data
43 }
44
45 switch filepath.Ext(filename) {
46 case ".s":
47 // The assembler uses a custom scanner,
48 // but the go/scanner package is close
49 // enough: we only want the comments.
50 file := fset.AddFile(filename, -1, len(content))
51 var scan goscanner.Scanner
52 scan.Init(file, content, nil, goscanner.ScanComments)
53
54 var notes []*Note
55 for {
56 pos, tok, lit := scan.Scan()
57 if tok == token.EOF {
58 break
59 }
60 if tok == token.COMMENT {
61 text, adjust := getAdjustedNote(lit)
62 if text == "" {
63 continue
64 }
65 parsed, err := parse(fset, pos+token.Pos(adjust), text)
66 if err != nil {
67 return nil, err
68 }
69 notes = append(notes, parsed...)
70 }
71 }
72 return notes, nil
73
74 case ".go":
75 // TODO: We should write this in terms of the scanner, like the .s case above.
76 // there are ways you can break the parser such that it will not add all the
77 // comments to the ast, which may result in files where the tests are silently
78 // not run.
79 file, err := parser.ParseFile(fset, filename, content, parser.ParseComments|parser.AllErrors|parser.SkipObjectResolution)
80 if file == nil {
81 return nil, err
82 }
83 return ExtractGo(fset, file)
84
85 case ".mod":
86 file, err := modfile.Parse(filename, content, nil)
87 if err != nil {
88 return nil, err
89 }
90 f := fset.AddFile(filename, -1, len(content))
91 f.SetLinesForContent(content)
92 notes, err := extractModWork(fset, file.Syntax.Stmt)
93 if err != nil {
94 return nil, err
95 }
96 // Since modfile.Parse does not return an *ast, we need to add the offset
97 // within the file's contents to the file's base relative to the fileset.
98 for _, note := range notes {
99 note.Pos += token.Pos(f.Base())
100 }
101 return notes, nil
102
103 case ".work":
104 file, err := modfile.ParseWork(filename, content, nil)
105 if err != nil {
106 return nil, err
107 }
108 f := fset.AddFile(filename, -1, len(content))
109 f.SetLinesForContent(content)
110 notes, err := extractModWork(fset, file.Syntax.Stmt)
111 if err != nil {
112 return nil, err
113 }
114 // As with go.mod files, we need to compute a synthetic token.Pos.
115 for _, note := range notes {
116 note.Pos += token.Pos(f.Base())
117 }
118 return notes, nil
119 }
120 return nil, nil
121 }
122
123 // extractModWork collects all the notes present in a go.mod file or go.work
124 // file, by way of the shared modfile.Expr statement node.
125 //
126 // Each comment whose text starts with @ is parsed as a comma-separated
127 // sequence of notes.
128 // See the package documentation for details about the syntax of those
129 // notes.
130 // Only allow notes to appear with the following format: "//@mark()" or // @mark()
131 func extractModWork(fset *token.FileSet, exprs []modfile.Expr) ([]*Note, error) {
132 var notes []*Note
133 for _, stmt := range exprs {
134 comment := stmt.Comment()
135 if comment == nil {
136 continue
137 }
138 var allComments []modfile.Comment
139 allComments = append(allComments, comment.Before...)
140 allComments = append(allComments, comment.Suffix...)
141 for _, cmt := range allComments {
142 text, adjust := getAdjustedNote(cmt.Token)
143 if text == "" {
144 continue
145 }
146 parsed, err := parse(fset, token.Pos(int(cmt.Start.Byte)+adjust), text)
147 if err != nil {
148 return nil, err
149 }
150 notes = append(notes, parsed...)
151 }
152 }
153 return notes, nil
154 }
155
156 // ExtractGo collects all the notes present in an AST.
157 // Each comment whose text starts with @ is parsed as a comma-separated
158 // sequence of notes.
159 // See the package documentation for details about the syntax of those
160 // notes.
161 func ExtractGo(fset *token.FileSet, file *ast.File) ([]*Note, error) {
162 var notes []*Note
163 for _, g := range file.Comments {
164 for _, c := range g.List {
165 text, adjust := getAdjustedNote(c.Text)
166 if text == "" {
167 continue
168 }
169 parsed, err := parse(fset, token.Pos(int(c.Pos())+adjust), text)
170 if err != nil {
171 return nil, err
172 }
173 notes = append(notes, parsed...)
174 }
175 }
176 return notes, nil
177 }
178
179 func getAdjustedNote(text string) (string, int) {
180 if strings.HasPrefix(text, "/*") {
181 text = strings.TrimSuffix(text, "*/")
182 }
183 text = text[2:] // remove "//" or "/*" prefix
184
185 // Allow notes to appear within comments.
186 // For example:
187 // "// //@mark()" is valid.
188 // "// @mark()" is not valid.
189 // "// /*@mark()*/" is not valid.
190 var adjust int
191 if i := strings.Index(text, commentStart); i > 2 {
192 // Get the text before the commentStart.
193 pre := text[i-2 : i]
194 if pre != "//" {
195 return "", 0
196 }
197 text = text[i:]
198 adjust = i
199 }
200 if !strings.HasPrefix(text, commentStart) {
201 return "", 0
202 }
203 text = text[commentStartLen:]
204 return text, commentStartLen + adjust + 1
205 }
206
207 const invalidToken rune = 0
208
209 type tokens struct {
210 scanner scanner.Scanner
211 current rune
212 err error
213 base token.Pos
214 }
215
216 func (t *tokens) Init(base token.Pos, text string) *tokens {
217 t.base = base
218 t.scanner.Init(strings.NewReader(text))
219 t.scanner.Mode = scanner.GoTokens
220 t.scanner.Whitespace ^= 1 << '\n' // don't skip new lines
221 t.scanner.Error = func(s *scanner.Scanner, msg string) {
222 t.Errorf("%v", msg)
223 }
224 return t
225 }
226
227 func (t *tokens) Consume() string {
228 t.current = invalidToken
229 return t.scanner.TokenText()
230 }
231
232 func (t *tokens) Token() rune {
233 if t.err != nil {
234 return scanner.EOF
235 }
236 if t.current == invalidToken {
237 t.current = t.scanner.Scan()
238 }
239 return t.current
240 }
241
242 func (t *tokens) Skip(r rune) int {
243 i := 0
244 for t.Token() == '\n' {
245 t.Consume()
246 i++
247 }
248 return i
249 }
250
251 func (t *tokens) TokenString() string {
252 return scanner.TokenString(t.Token())
253 }
254
255 func (t *tokens) Pos() token.Pos {
256 return t.base + token.Pos(t.scanner.Position.Offset)
257 }
258
259 func (t *tokens) Errorf(msg string, args ...any) {
260 if t.err != nil {
261 return
262 }
263 t.err = fmt.Errorf(msg, args...)
264 }
265
266 func parse(fset *token.FileSet, base token.Pos, text string) ([]*Note, error) {
267 t := new(tokens).Init(base, text)
268 notes := parseComment(t)
269 if t.err != nil {
270 return nil, fmt.Errorf("%v: %s", fset.Position(t.Pos()), t.err)
271 }
272 return notes, nil
273 }
274
275 func parseComment(t *tokens) []*Note {
276 var notes []*Note
277 for {
278 t.Skip('\n')
279 switch t.Token() {
280 case scanner.EOF:
281 return notes
282 case scanner.Ident:
283 notes = append(notes, parseNote(t))
284 default:
285 t.Errorf("unexpected %s parsing comment, expect identifier", t.TokenString())
286 return nil
287 }
288 switch t.Token() {
289 case scanner.EOF:
290 return notes
291 case ',', '\n':
292 t.Consume()
293 default:
294 t.Errorf("unexpected %s parsing comment, expect separator", t.TokenString())
295 return nil
296 }
297 }
298 }
299
300 func parseNote(t *tokens) *Note {
301 n := &Note{
302 Pos: t.Pos(),
303 Name: t.Consume(),
304 }
305
306 switch t.Token() {
307 case ',', '\n', scanner.EOF:
308 // no argument list present
309 return n
310 case '(':
311 n.Args, n.NamedArgs = parseArgumentList(t)
312 return n
313 default:
314 t.Errorf("unexpected %s parsing note", t.TokenString())
315 return nil
316 }
317 }
318
319 func parseArgumentList(t *tokens) (args []any, named map[string]any) {
320 args = []any{} // @name() is represented by a non-nil empty slice.
321 t.Consume() // '('
322 t.Skip('\n')
323 for t.Token() != ')' {
324 name, arg := parseArgument(t)
325 if name != "" {
326 // f(k=v)
327 if named == nil {
328 named = make(map[string]any)
329 }
330 if _, dup := named[name]; dup {
331 t.Errorf("duplicate named argument %q", name)
332 return nil, nil
333 }
334 named[name] = arg
335 } else {
336 // f(v)
337 if named != nil {
338 t.Errorf("positional argument follows named argument")
339 return nil, nil
340 }
341 args = append(args, arg)
342 }
343 if t.Token() != ',' {
344 break
345 }
346 t.Consume()
347 t.Skip('\n')
348 }
349 if t.Token() != ')' {
350 t.Errorf("unexpected %s parsing argument list", t.TokenString())
351 return nil, nil
352 }
353 t.Consume() // ')'
354 return args, named
355 }
356
357 // parseArgument returns the value of the argument ("f(value)"),
358 // and its name if named "f(name=value)".
359 func parseArgument(t *tokens) (name string, value any) {
360 again:
361 switch t.Token() {
362 case scanner.Ident:
363 v := t.Consume()
364 switch v {
365 case "true":
366 value = true
367 case "false":
368 value = false
369 case "nil":
370 value = nil
371 case "re":
372 if t.Token() != scanner.String && t.Token() != scanner.RawString {
373 t.Errorf("re must be followed by string, got %s", t.TokenString())
374 return
375 }
376 pattern, _ := strconv.Unquote(t.Consume()) // can't fail
377 re, err := regexp.Compile(pattern)
378 if err != nil {
379 t.Errorf("invalid regular expression %s: %v", pattern, err)
380 return
381 }
382 value = re
383 default:
384 // f(name=value)?
385 if name == "" && t.Token() == '=' {
386 t.Consume() // '='
387 name = v
388 goto again
389 }
390 value = Identifier(v)
391 }
392
393 case scanner.String, scanner.RawString:
394 value, _ = strconv.Unquote(t.Consume()) // can't fail
395
396 case scanner.Int:
397 s := t.Consume()
398 v, err := strconv.ParseInt(s, 0, 0)
399 if err != nil {
400 t.Errorf("cannot convert %v to int: %v", s, err)
401 }
402 value = v
403
404 case scanner.Float:
405 s := t.Consume()
406 v, err := strconv.ParseFloat(s, 64)
407 if err != nil {
408 t.Errorf("cannot convert %v to float: %v", s, err)
409 }
410 value = v
411
412 case '-':
413 t.Consume()
414 switch t.Token() {
415 case scanner.Int:
416 s := "-" + t.Consume()
417 v, err := strconv.ParseInt(s, 0, 0)
418 if err != nil {
419 t.Errorf("cannot convert %v to int: %v", s, err)
420 }
421 value = v
422
423 case scanner.Float:
424 s := "-" + t.Consume()
425 v, err := strconv.ParseFloat(s, 64)
426 if err != nil {
427 t.Errorf("cannot convert %v to float: %v", s, err)
428 }
429 value = v
430
431 default:
432 t.Errorf("unexpected '-' not followed by number, got %s", t.TokenString())
433 }
434
435 case scanner.Char:
436 t.Errorf("unexpected char literal %s", t.Consume())
437
438 default:
439 t.Errorf("unexpected %s parsing argument", t.TokenString())
440 }
441 return
442 }