| 1 |
package loader |
| 2 |
|
| 3 |
import ( |
| 4 |
"errors" |
| 5 |
"fmt" |
| 6 |
"go/ast" |
| 7 |
"go/build" |
| 8 |
"go/parser" |
| 9 |
"go/scanner" |
| 10 |
"go/token" |
| 11 |
"go/types" |
| 12 |
"os" |
| 13 |
"time" |
| 14 |
|
| 15 |
"honnef.co/go/tools/config" |
| 16 |
"honnef.co/go/tools/lintcmd/cache" |
| 17 |
|
| 18 |
"golang.org/x/tools/go/gcexportdata" |
| 19 |
"golang.org/x/tools/go/packages" |
| 20 |
) |
| 21 |
|
| 22 |
const MaxFileSize = 50 * 1024 * 1024 // 50 MB |
| 23 |
|
| 24 |
var errMaxFileSize = errors.New("file exceeds max file size") |
| 25 |
|
| 26 |
type PackageSpec struct { |
| 27 |
ID string |
| 28 |
Name string |
| 29 |
PkgPath string |
| 30 |
// Errors that occurred while building the import graph. These will |
| 31 |
// primarily be parse errors or failure to resolve imports, but |
| 32 |
// may also be other errors. |
| 33 |
Errors []packages.Error |
| 34 |
GoFiles []string |
| 35 |
CompiledGoFiles []string |
| 36 |
OtherFiles []string |
| 37 |
ExportFile string |
| 38 |
Imports map[string]*PackageSpec |
| 39 |
TypesSizes types.Sizes |
| 40 |
Hash cache.ActionID |
| 41 |
Module *packages.Module |
| 42 |
|
| 43 |
Config config.Config |
| 44 |
} |
| 45 |
|
| 46 |
func (spec *PackageSpec) String() string { |
| 47 |
return spec.ID |
| 48 |
} |
| 49 |
|
| 50 |
type Package struct { |
| 51 |
*PackageSpec |
| 52 |
|
| 53 |
// Errors that occurred while loading the package. These will |
| 54 |
// primarily be parse or type errors, but may also be lower-level |
| 55 |
// failures such as file-system ones. |
| 56 |
Errors []packages.Error |
| 57 |
Types *types.Package |
| 58 |
Fset *token.FileSet |
| 59 |
Syntax []*ast.File |
| 60 |
TypesInfo *types.Info |
| 61 |
} |
| 62 |
|
| 63 |
// Graph resolves patterns and returns packages with all the |
| 64 |
// information required to later load type information, and optionally |
| 65 |
// syntax trees. |
| 66 |
// |
| 67 |
// The provided config can set any setting with the exception of Mode. |
| 68 |
func Graph(c *cache.Cache, cfg *packages.Config, patterns ...string) ([]*PackageSpec, error) { |
| 69 |
var dcfg packages.Config |
| 70 |
if cfg != nil { |
| 71 |
dcfg = *cfg |
| 72 |
} |
| 73 |
dcfg.Mode = packages.NeedName | |
| 74 |
packages.NeedImports | |
| 75 |
packages.NeedDeps | |
| 76 |
packages.NeedExportFile | |
| 77 |
packages.NeedFiles | |
| 78 |
packages.NeedCompiledGoFiles | |
| 79 |
packages.NeedTypesSizes | |
| 80 |
packages.NeedModule |
| 81 |
pkgs, err := packages.Load(&dcfg, patterns...) |
| 82 |
if err != nil { |
| 83 |
return nil, err |
| 84 |
} |
| 85 |
|
| 86 |
m := map[*packages.Package]*PackageSpec{} |
| 87 |
packages.Visit(pkgs, nil, func(pkg *packages.Package) { |
| 88 |
spec := &PackageSpec{ |
| 89 |
ID: pkg.ID, |
| 90 |
Name: pkg.Name, |
| 91 |
PkgPath: pkg.PkgPath, |
| 92 |
Errors: pkg.Errors, |
| 93 |
GoFiles: pkg.GoFiles, |
| 94 |
CompiledGoFiles: pkg.CompiledGoFiles, |
| 95 |
OtherFiles: pkg.OtherFiles, |
| 96 |
ExportFile: pkg.ExportFile, |
| 97 |
Imports: map[string]*PackageSpec{}, |
| 98 |
TypesSizes: pkg.TypesSizes, |
| 99 |
Module: pkg.Module, |
| 100 |
} |
| 101 |
for path, imp := range pkg.Imports { |
| 102 |
spec.Imports[path] = m[imp] |
| 103 |
} |
| 104 |
if cdir := config.Dir(pkg.GoFiles); cdir != "" { |
| 105 |
cfg, err := config.Load(cdir) |
| 106 |
if err != nil { |
| 107 |
spec.Errors = append(spec.Errors, convertError(err)...) |
| 108 |
} |
| 109 |
spec.Config = cfg |
| 110 |
} else { |
| 111 |
spec.Config = config.DefaultConfig |
| 112 |
} |
| 113 |
spec.Hash, err = computeHash(c, spec) |
| 114 |
if err != nil { |
| 115 |
spec.Errors = append(spec.Errors, convertError(err)...) |
| 116 |
} |
| 117 |
m[pkg] = spec |
| 118 |
}) |
| 119 |
out := make([]*PackageSpec, 0, len(pkgs)) |
| 120 |
for _, pkg := range pkgs { |
| 121 |
if len(pkg.CompiledGoFiles) == 0 && len(pkg.Errors) == 0 && pkg.PkgPath != "unsafe" { |
| 122 |
// If a package consists only of test files, then |
| 123 |
// go/packages incorrectly(?) returns an empty package for |
| 124 |
// the non-test variant. Get rid of those packages. See |
| 125 |
// #646. |
| 126 |
// |
| 127 |
// Do not, however, skip packages that have errors. Those, |
| 128 |
// too, may have no files, but we want to print the |
| 129 |
// errors. |
| 130 |
continue |
| 131 |
} |
| 132 |
out = append(out, m[pkg]) |
| 133 |
} |
| 134 |
|
| 135 |
return out, nil |
| 136 |
} |
| 137 |
|
| 138 |
type program struct { |
| 139 |
fset *token.FileSet |
| 140 |
packages map[string]*types.Package |
| 141 |
options *Options |
| 142 |
} |
| 143 |
|
| 144 |
type Stats struct { |
| 145 |
Source time.Duration |
| 146 |
Export map[*PackageSpec]time.Duration |
| 147 |
} |
| 148 |
|
| 149 |
type Options struct { |
| 150 |
// The Go language version to use for the type checker. If unset, or if set |
| 151 |
// to "module", it will default to the Go version specified in the module; |
| 152 |
// if there is no module, it will default to the version of Go the |
| 153 |
// executable was built with. |
| 154 |
GoVersion string |
| 155 |
} |
| 156 |
|
| 157 |
// Load loads the package described in spec. Imports will be loaded |
| 158 |
// from export data, while the package itself will be loaded from |
| 159 |
// source. |
| 160 |
// |
| 161 |
// An error will only be returned for system failures, such as failure |
| 162 |
// to read export data from disk. Syntax and type errors, among |
| 163 |
// others, will only populate the returned package's Errors field. |
| 164 |
func Load(spec *PackageSpec, opts *Options) (*Package, Stats, error) { |
| 165 |
if opts == nil { |
| 166 |
opts = &Options{} |
| 167 |
} |
| 168 |
if opts.GoVersion == "" { |
| 169 |
opts.GoVersion = "module" |
| 170 |
} |
| 171 |
prog := &program{ |
| 172 |
fset: token.NewFileSet(), |
| 173 |
packages: map[string]*types.Package{}, |
| 174 |
options: opts, |
| 175 |
} |
| 176 |
|
| 177 |
stats := Stats{ |
| 178 |
Export: map[*PackageSpec]time.Duration{}, |
| 179 |
} |
| 180 |
for _, imp := range spec.Imports { |
| 181 |
if imp.PkgPath == "unsafe" { |
| 182 |
continue |
| 183 |
} |
| 184 |
t := time.Now() |
| 185 |
_, err := prog.loadFromExport(imp) |
| 186 |
stats.Export[imp] = time.Since(t) |
| 187 |
if err != nil { |
| 188 |
return nil, stats, err |
| 189 |
} |
| 190 |
} |
| 191 |
t := time.Now() |
| 192 |
pkg, err := prog.loadFromSource(spec) |
| 193 |
if err == errMaxFileSize { |
| 194 |
pkg, err = prog.loadFromExport(spec) |
| 195 |
} |
| 196 |
stats.Source = time.Since(t) |
| 197 |
return pkg, stats, err |
| 198 |
} |
| 199 |
|
| 200 |
// loadFromExport loads a package from export data. |
| 201 |
func (prog *program) loadFromExport(spec *PackageSpec) (*Package, error) { |
| 202 |
// log.Printf("Loading package %s from export", spec) |
| 203 |
if spec.ExportFile == "" { |
| 204 |
return nil, fmt.Errorf("no export data for %q", spec.ID) |
| 205 |
} |
| 206 |
f, err := os.Open(spec.ExportFile) |
| 207 |
if err != nil { |
| 208 |
return nil, err |
| 209 |
} |
| 210 |
defer f.Close() |
| 211 |
|
| 212 |
r, err := gcexportdata.NewReader(f) |
| 213 |
if err != nil { |
| 214 |
return nil, err |
| 215 |
} |
| 216 |
tpkg, err := gcexportdata.Read(r, prog.fset, prog.packages, spec.PkgPath) |
| 217 |
if err != nil { |
| 218 |
return nil, err |
| 219 |
} |
| 220 |
pkg := &Package{ |
| 221 |
PackageSpec: spec, |
| 222 |
Types: tpkg, |
| 223 |
Fset: prog.fset, |
| 224 |
} |
| 225 |
// runtime.SetFinalizer(pkg, func(pkg *Package) { |
| 226 |
// log.Println("Unloading package", pkg.PkgPath) |
| 227 |
// }) |
| 228 |
return pkg, nil |
| 229 |
} |
| 230 |
|
| 231 |
// loadFromSource loads a package from source. All of its dependencies |
| 232 |
// must have been loaded already. |
| 233 |
func (prog *program) loadFromSource(spec *PackageSpec) (*Package, error) { |
| 234 |
if len(spec.Errors) > 0 { |
| 235 |
panic("LoadFromSource called on package with errors") |
| 236 |
} |
| 237 |
|
| 238 |
pkg := &Package{ |
| 239 |
PackageSpec: spec, |
| 240 |
Types: types.NewPackage(spec.PkgPath, spec.Name), |
| 241 |
Syntax: make([]*ast.File, len(spec.CompiledGoFiles)), |
| 242 |
Fset: prog.fset, |
| 243 |
TypesInfo: &types.Info{ |
| 244 |
Types: make(map[ast.Expr]types.TypeAndValue), |
| 245 |
Defs: make(map[*ast.Ident]types.Object), |
| 246 |
Uses: make(map[*ast.Ident]types.Object), |
| 247 |
Implicits: make(map[ast.Node]types.Object), |
| 248 |
Scopes: make(map[ast.Node]*types.Scope), |
| 249 |
Selections: make(map[*ast.SelectorExpr]*types.Selection), |
| 250 |
Instances: make(map[*ast.Ident]types.Instance), |
| 251 |
FileVersions: make(map[*ast.File]string), |
| 252 |
}, |
| 253 |
} |
| 254 |
// runtime.SetFinalizer(pkg, func(pkg *Package) { |
| 255 |
// log.Println("Unloading package", pkg.PkgPath) |
| 256 |
// }) |
| 257 |
|
| 258 |
// OPT(dh): many packages have few files, much fewer than there |
| 259 |
// are CPU cores. Additionally, parsing each individual file is |
| 260 |
// very fast. A naive parallel implementation of this loop won't |
| 261 |
// be faster, and tends to be slower due to extra scheduling, |
| 262 |
// bookkeeping and potentially false sharing of cache lines. |
| 263 |
for i, file := range spec.CompiledGoFiles { |
| 264 |
f, err := os.Open(file) |
| 265 |
if err != nil { |
| 266 |
return nil, err |
| 267 |
} |
| 268 |
fi, err := f.Stat() |
| 269 |
if err != nil { |
| 270 |
return nil, err |
| 271 |
} |
| 272 |
if fi.Size() >= MaxFileSize { |
| 273 |
return nil, errMaxFileSize |
| 274 |
} |
| 275 |
af, err := parser.ParseFile(prog.fset, file, f, parser.ParseComments|parser.SkipObjectResolution) |
| 276 |
f.Close() |
| 277 |
if err != nil { |
| 278 |
pkg.Errors = append(pkg.Errors, convertError(err)...) |
| 279 |
return pkg, nil |
| 280 |
} |
| 281 |
pkg.Syntax[i] = af |
| 282 |
} |
| 283 |
importer := func(path string) (*types.Package, error) { |
| 284 |
if path == "unsafe" { |
| 285 |
return types.Unsafe, nil |
| 286 |
} |
| 287 |
if path == "C" { |
| 288 |
// go/packages doesn't tell us that cgo preprocessing |
| 289 |
// failed. When we subsequently try to parse the package, |
| 290 |
// we'll encounter the raw C import. |
| 291 |
return nil, errors.New("cgo preprocessing failed") |
| 292 |
} |
| 293 |
ispecpkg := spec.Imports[path] |
| 294 |
if ispecpkg == nil { |
| 295 |
return nil, fmt.Errorf("trying to import %q in the context of %q returned nil PackageSpec", path, spec) |
| 296 |
} |
| 297 |
ipkg := prog.packages[ispecpkg.PkgPath] |
| 298 |
if ipkg == nil { |
| 299 |
return nil, fmt.Errorf("trying to import %q (%q) in the context of %q returned nil PackageSpec", ispecpkg.PkgPath, path, spec) |
| 300 |
} |
| 301 |
return ipkg, nil |
| 302 |
} |
| 303 |
tc := &types.Config{ |
| 304 |
Importer: importerFunc(importer), |
| 305 |
Error: func(err error) { |
| 306 |
pkg.Errors = append(pkg.Errors, convertError(err)...) |
| 307 |
}, |
| 308 |
} |
| 309 |
if prog.options.GoVersion == "module" { |
| 310 |
if spec.Module != nil && spec.Module.GoVersion != "" { |
| 311 |
tc.GoVersion = "go" + spec.Module.GoVersion |
| 312 |
} else { |
| 313 |
tags := build.Default.ReleaseTags |
| 314 |
tc.GoVersion = tags[len(tags)-1] |
| 315 |
} |
| 316 |
} else { |
| 317 |
tc.GoVersion = prog.options.GoVersion |
| 318 |
} |
| 319 |
// Note that the type-checker can return a non-nil error even though the Go |
| 320 |
// compiler has already successfully built this package (which is an |
| 321 |
// invariant of getting to this point), for example because of the Go |
| 322 |
// version passed to the type checker. |
| 323 |
err := types.NewChecker(tc, pkg.Fset, pkg.Types, pkg.TypesInfo).Files(pkg.Syntax) |
| 324 |
return pkg, err |
| 325 |
} |
| 326 |
|
| 327 |
func convertError(err error) []packages.Error { |
| 328 |
var errs []packages.Error |
| 329 |
// taken from go/packages |
| 330 |
switch err := err.(type) { |
| 331 |
case packages.Error: |
| 332 |
// from driver |
| 333 |
errs = append(errs, err) |
| 334 |
|
| 335 |
case *os.PathError: |
| 336 |
// from parser |
| 337 |
errs = append(errs, packages.Error{ |
| 338 |
Pos: err.Path + ":1", |
| 339 |
Msg: err.Err.Error(), |
| 340 |
Kind: packages.ParseError, |
| 341 |
}) |
| 342 |
|
| 343 |
case scanner.ErrorList: |
| 344 |
// from parser |
| 345 |
for _, err := range err { |
| 346 |
errs = append(errs, packages.Error{ |
| 347 |
Pos: err.Pos.String(), |
| 348 |
Msg: err.Msg, |
| 349 |
Kind: packages.ParseError, |
| 350 |
}) |
| 351 |
} |
| 352 |
|
| 353 |
case types.Error: |
| 354 |
// from type checker |
| 355 |
errs = append(errs, packages.Error{ |
| 356 |
Pos: err.Fset.Position(err.Pos).String(), |
| 357 |
Msg: err.Msg, |
| 358 |
Kind: packages.TypeError, |
| 359 |
}) |
| 360 |
|
| 361 |
case config.ParseError: |
| 362 |
errs = append(errs, packages.Error{ |
| 363 |
Pos: fmt.Sprintf("%s:%d:%d", err.Filename, err.Position.Line, err.Position.Col), |
| 364 |
Msg: fmt.Sprintf("%s (last key parsed: %q)", err.Message, err.LastKey), |
| 365 |
Kind: packages.ParseError, |
| 366 |
}) |
| 367 |
default: |
| 368 |
errs = append(errs, packages.Error{ |
| 369 |
Pos: "-", |
| 370 |
Msg: err.Error(), |
| 371 |
Kind: packages.UnknownError, |
| 372 |
}) |
| 373 |
} |
| 374 |
return errs |
| 375 |
} |
| 376 |
|
| 377 |
type importerFunc func(path string) (*types.Package, error) |
| 378 |
|
| 379 |
func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } |