diff --git a/go.mod b/go.mod index cfc184e5fb..2216421d51 100644 --- a/go.mod +++ b/go.mod @@ -8,3 +8,5 @@ require ( golang.org/x/net v0.1.0 golang.org/x/sys v0.1.0 ) + +require golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index da2cda7d75..50d82dec89 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/gopls/go.mod b/gopls/go.mod index efb9be1189..2e4fb3261a 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -8,7 +8,7 @@ require ( github.com/jba/templatecheck v0.6.0 github.com/sergi/go-diff v1.1.0 golang.org/x/mod v0.6.0 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + golang.org/x/sync v0.1.0 golang.org/x/sys v0.1.0 golang.org/x/text v0.4.0 golang.org/x/tools v0.1.13-0.20220928184430-f80e98464e27 diff --git a/gopls/go.sum b/gopls/go.sum index 78ff483dc7..dc2d738b31 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -55,8 +55,9 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/gcimporter/bexport.go b/internal/gcimporter/bexport.go index 196cb3f9b4..30582ed6d3 100644 --- a/internal/gcimporter/bexport.go +++ b/internal/gcimporter/bexport.go @@ -12,7 +12,6 @@ import ( "bytes" "encoding/binary" "fmt" - "go/ast" "go/constant" "go/token" "go/types" @@ -145,7 +144,7 @@ func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) objcount := 0 scope := pkg.Scope() for _, name := range scope.Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } if trace { @@ -482,7 +481,7 @@ func (p *exporter) method(m *types.Func) { p.pos(m) p.string(m.Name()) - if m.Name() != "_" && !ast.IsExported(m.Name()) { + if m.Name() != "_" && !token.IsExported(m.Name()) { p.pkg(m.Pkg(), false) } @@ -501,7 +500,7 @@ func (p *exporter) fieldName(f *types.Var) { // 3) field name doesn't match base type name (alias name) bname := basetypeName(f.Type()) if name == bname { - if ast.IsExported(name) { + if token.IsExported(name) { name = "" // 1) we don't need to know the field name or package } else { name = "?" // 2) use unexported name "?" to force package export @@ -514,7 +513,7 @@ func (p *exporter) fieldName(f *types.Var) { } p.string(name) - if name != "" && !ast.IsExported(name) { + if name != "" && !token.IsExported(name) { p.pkg(f.Pkg(), false) } } diff --git a/internal/gcimporter/bexport_test.go b/internal/gcimporter/bexport_test.go index 93ee3cd21a..b5e9ce1004 100644 --- a/internal/gcimporter/bexport_test.go +++ b/internal/gcimporter/bexport_test.go @@ -109,7 +109,7 @@ type UnknownType undefined // Compare the packages' corresponding members. for _, name := range pkg.Scope().Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } obj1 := pkg.Scope().Lookup(name) diff --git a/internal/gcimporter/iexport.go b/internal/gcimporter/iexport.go index db2753217a..7d90f00f32 100644 --- a/internal/gcimporter/iexport.go +++ b/internal/gcimporter/iexport.go @@ -12,7 +12,6 @@ import ( "bytes" "encoding/binary" "fmt" - "go/ast" "go/constant" "go/token" "go/types" @@ -26,6 +25,41 @@ import ( "golang.org/x/tools/internal/typeparams" ) +// IExportShallow encodes "shallow" export data for the specified package. +// +// No promises are made about the encoding other than that it can be +// decoded by the same version of IIExportShallow. If you plan to save +// export data in the file system, be sure to include a cryptographic +// digest of the executable in the key to avoid version skew. +func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) { + // In principle this operation can only fail if out.Write fails, + // but that's impossible for bytes.Buffer---and as a matter of + // fact iexportCommon doesn't even check for I/O errors. + // TODO(adonovan): handle I/O errors properly. + // TODO(adonovan): use byte slices throughout, avoiding copying. + const bundle, shallow = false, true + var out bytes.Buffer + err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) + return out.Bytes(), err +} + +// IImportShallow decodes "shallow" types.Package data encoded by IExportShallow +// in the same executable. This function cannot import data from +// cmd/compile or gcexportdata.Write. +func IImportShallow(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string, insert InsertType) (*types.Package, error) { + const bundle = false + pkgs, err := iimportCommon(fset, imports, data, bundle, path, insert) + if err != nil { + return nil, err + } + return pkgs[0], nil +} + +// InsertType is the type of a function that creates a types.TypeName +// object for a named type and inserts it into the scope of the +// specified Package. +type InsertType = func(pkg *types.Package, name string) + // Current bundled export format version. Increase with each format change. // 0: initial implementation const bundleVersion = 0 @@ -36,15 +70,17 @@ const bundleVersion = 0 // The package path of the top-level package will not be recorded, // so that calls to IImportData can override with a provided package path. func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error { - return iexportCommon(out, fset, false, iexportVersion, []*types.Package{pkg}) + const bundle, shallow = false, false + return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) } // IExportBundle writes an indexed export bundle for pkgs to out. func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error { - return iexportCommon(out, fset, true, iexportVersion, pkgs) + const bundle, shallow = true, false + return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs) } -func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, pkgs []*types.Package) (err error) { +func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package) (err error) { if !debug { defer func() { if e := recover(); e != nil { @@ -61,6 +97,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, p := iexporter{ fset: fset, version: version, + shallow: shallow, allPkgs: map[*types.Package]bool{}, stringIndex: map[string]uint64{}, declIndex: map[types.Object]uint64{}, @@ -82,7 +119,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, for _, pkg := range pkgs { scope := pkg.Scope() for _, name := range scope.Names() { - if ast.IsExported(name) { + if token.IsExported(name) { p.pushDecl(scope.Lookup(name)) } } @@ -205,7 +242,8 @@ type iexporter struct { out *bytes.Buffer version int - localpkg *types.Package + shallow bool // don't put types from other packages in the index + localpkg *types.Package // (nil in bundle mode) // allPkgs tracks all packages that have been referenced by // the export data, so we can ensure to include them in the @@ -256,6 +294,11 @@ func (p *iexporter) pushDecl(obj types.Object) { panic("cannot export package unsafe") } + // Shallow export data: don't index decls from other packages. + if p.shallow && obj.Pkg() != p.localpkg { + return + } + if _, ok := p.declIndex[obj]; ok { return } @@ -497,7 +540,7 @@ func (w *exportWriter) pkg(pkg *types.Package) { w.string(w.exportPath(pkg)) } -func (w *exportWriter) qualifiedIdent(obj types.Object) { +func (w *exportWriter) qualifiedType(obj *types.TypeName) { name := w.p.exportName(obj) // Ensure any referenced declarations are written out too. @@ -556,11 +599,11 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { return } w.startType(definedType) - w.qualifiedIdent(t.Obj()) + w.qualifiedType(t.Obj()) case *typeparams.TypeParam: w.startType(typeParamType) - w.qualifiedIdent(t.Obj()) + w.qualifiedType(t.Obj()) case *types.Pointer: w.startType(pointerType) diff --git a/internal/gcimporter/iexport_test.go b/internal/gcimporter/iexport_test.go index 702528aef3..93183f9dc6 100644 --- a/internal/gcimporter/iexport_test.go +++ b/internal/gcimporter/iexport_test.go @@ -59,7 +59,8 @@ func readExportFile(filename string) ([]byte, error) { func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) { var buf bytes.Buffer - if err := gcimporter.IExportCommon(&buf, fset, false, version, []*types.Package{pkg}); err != nil { + const bundle, shallow = false, false + if err := gcimporter.IExportCommon(&buf, fset, bundle, shallow, version, []*types.Package{pkg}); err != nil { return nil, err } return buf.Bytes(), nil @@ -197,7 +198,7 @@ func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, // Compare the packages' corresponding members. for _, name := range pkg.Scope().Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } obj1 := pkg.Scope().Lookup(name) diff --git a/internal/gcimporter/iimport.go b/internal/gcimporter/iimport.go index 6e4c066b69..a1c4696535 100644 --- a/internal/gcimporter/iimport.go +++ b/internal/gcimporter/iimport.go @@ -85,7 +85,7 @@ const ( // If the export data version is not recognized or the format is otherwise // compromised, an error is returned. func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { - pkgs, err := iimportCommon(fset, imports, data, false, path) + pkgs, err := iimportCommon(fset, imports, data, false, path, nil) if err != nil { return 0, nil, err } @@ -94,10 +94,10 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // IImportBundle imports a set of packages from the serialized package bundle. func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) { - return iimportCommon(fset, imports, data, true, "") + return iimportCommon(fset, imports, data, true, "", nil) } -func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) { +func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) { const currentVersion = iexportVersionCurrent version := int64(-1) if !debug { @@ -147,6 +147,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data p := iimporter{ version: int(version), ipath: path, + insert: insert, stringData: stringData, stringCache: make(map[uint64]string), @@ -187,11 +188,18 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data } else if pkg.Name() != pkgName { errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path) } + if i == 0 && !bundle { + p.localpkg = pkg + } p.pkgCache[pkgPathOff] = pkg + // Read index for package. nameIndex := make(map[string]uint64) - for nSyms := r.uint64(); nSyms > 0; nSyms-- { + nSyms := r.uint64() + // In shallow mode we don't expect an index for other packages. + assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil) + for ; nSyms > 0; nSyms-- { name := p.stringAt(r.uint64()) nameIndex[name] = r.uint64() } @@ -267,6 +275,9 @@ type iimporter struct { version int ipath string + localpkg *types.Package + insert func(pkg *types.Package, name string) // "shallow" mode only + stringData []byte stringCache map[uint64]string pkgCache map[uint64]*types.Package @@ -310,6 +321,13 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) { off, ok := p.pkgIndex[pkg][name] if !ok { + // In "shallow" mode, call back to the application to + // find the object and insert it into the package scope. + if p.insert != nil { + assert(pkg != p.localpkg) + p.insert(pkg, name) // "can't fail" + return + } errorf("%v.%v not in index", pkg, name) } diff --git a/internal/gcimporter/shallow_test.go b/internal/gcimporter/shallow_test.go new file mode 100644 index 0000000000..084604c7e0 --- /dev/null +++ b/internal/gcimporter/shallow_test.go @@ -0,0 +1,153 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gcimporter_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "golang.org/x/sync/errgroup" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/gcimporter" + "golang.org/x/tools/internal/testenv" +) + +// TestStd type-checks the standard library using shallow export data. +func TestShallowStd(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode; too slow (https://golang.org/issue/14113)") + } + testenv.NeedsTool(t, "go") + + // Load import graph of the standard library. + // (No parsing or type-checking.) + cfg := &packages.Config{ + Mode: packages.LoadImports, + Tests: false, + } + pkgs, err := packages.Load(cfg, "std") + if err != nil { + t.Fatalf("load: %v", err) + } + if len(pkgs) < 200 { + t.Fatalf("too few packages: %d", len(pkgs)) + } + + // Type check the packages in parallel postorder. + done := make(map[*packages.Package]chan struct{}) + packages.Visit(pkgs, nil, func(p *packages.Package) { + done[p] = make(chan struct{}) + }) + packages.Visit(pkgs, nil, + func(pkg *packages.Package) { + go func() { + // Wait for all deps to be done. + for _, imp := range pkg.Imports { + <-done[imp] + } + typecheck(t, pkg) + close(done[pkg]) + }() + }) + for _, root := range pkgs { + <-done[root] + } +} + +// typecheck reads, parses, and type-checks a package. +// It squirrels the export data in the the ppkg.ExportFile field. +func typecheck(t *testing.T, ppkg *packages.Package) { + if ppkg.PkgPath == "unsafe" { + return // unsafe is special + } + + // Create a local FileSet just for this package. + fset := token.NewFileSet() + + // Parse files in parallel. + syntax := make([]*ast.File, len(ppkg.CompiledGoFiles)) + var group errgroup.Group + for i, filename := range ppkg.CompiledGoFiles { + i, filename := i, filename + group.Go(func() error { + f, err := parser.ParseFile(fset, filename, nil, parser.SkipObjectResolution) + if err != nil { + return err // e.g. missing file + } + syntax[i] = f + return nil + }) + } + if err := group.Wait(); err != nil { + t.Fatal(err) + } + // Inv: all files were successfully parsed. + + // importer state + var ( + insert func(p *types.Package, name string) + importMap = make(map[string]*types.Package) // keys are PackagePaths + ) + + loadFromExportData := func(imp *packages.Package) (*types.Package, error) { + data := []byte(imp.ExportFile) + return gcimporter.IImportShallow(fset, importMap, data, imp.PkgPath, insert) + } + insert = func(p *types.Package, name string) { + // Hunt for p among the transitive dependencies (inefficient). + var imp *packages.Package + packages.Visit([]*packages.Package{ppkg}, func(q *packages.Package) bool { + if q.PkgPath == p.Path() { + imp = q + return false + } + return true + }, nil) + if imp == nil { + t.Fatalf("can't find dependency: %q", p.Path()) + } + imported, err := loadFromExportData(imp) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + obj := imported.Scope().Lookup(name) + if obj == nil { + t.Fatalf("lookup %q.%s failed", imported.Path(), name) + } + if imported != p { + t.Fatalf("internal error: inconsistent packages") + } + } + + cfg := &types.Config{ + Error: func(e error) { + t.Error(e) + }, + Importer: importerFunc(func(importPath string) (*types.Package, error) { + if importPath == "unsafe" { + return types.Unsafe, nil // unsafe has no exportdata + } + imp, ok := ppkg.Imports[importPath] + if !ok { + return nil, fmt.Errorf("missing import %q", importPath) + } + return loadFromExportData(imp) + }), + } + + // Type-check the syntax trees. + tpkg, _ := cfg.Check(ppkg.PkgPath, fset, syntax, nil) + + // Save the export data. + data, err := gcimporter.IExportShallow(fset, tpkg) + if err != nil { + t.Fatalf("internal error marshalling export data: %v", err) + } + ppkg.ExportFile = string(data) +}