mirror
/
go
mirror of https://github.com/golang/go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

internal/gcimporter: API for shallow export data 

This change adds an internal API for marshalling and
unmarshalling a types.Package to "shallow" export data,
which does not index packages other than the main one.
The import function accepts a function that loads symbols
on demand (e.g. by recursively reading export data for
indirect dependencies).

The CL includes a test that the entire standard
library can be type-checked using shallow data.

Also:
- break dependency on go/ast.
- narrow the name and type of qualifiedObject.
- add (test) dependency on errgroup, and tidy go.mod.

Change-Id: I92d31efd343cf5dd6fca6d7b918a23749e2d1e83
Reviewed-on: https://go-review.googlesource.com/c/tools/+/447737
Run-TryBot: Alan Donovan <adonovan@google.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
gopls-CI: kokoro <noreply+kokoro@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
This commit is contained in:
Alan Donovan 2022-11-03 14:55:29 -04:00
parent affa603132
commit 2b29c66d7e
10 changed files with 242 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -8,3 +8,5 @@ require (
golang.org/x/net v0.1.0
golang.org/x/sys v0.1.0
)
require golang.org/x/sync v0.1.0

2
go.sum
View File

@ -13,6 +13,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

2
gopls/go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/jba/templatecheck v0.6.0
github.com/sergi/go-diff v1.1.0
golang.org/x/mod v0.6.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/sync v0.1.0
golang.org/x/sys v0.1.0
golang.org/x/text v0.4.0
golang.org/x/tools v0.1.13-0.20220928184430-f80e98464e27

3
gopls/go.sum
View File

@ -55,8 +55,9 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

9
internal/gcimporter/bexport.go
View File

@ -12,7 +12,6 @@ import (
"bytes"
"encoding/binary"
"fmt"
"go/ast"
"go/constant"
"go/token"
"go/types"
@ -145,7 +144,7 @@ func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
objcount := 0
scope := pkg.Scope()
for _, name := range scope.Names() {
if !ast.IsExported(name) {
if !token.IsExported(name) {
continue
}
if trace {
@ -482,7 +481,7 @@ func (p *exporter) method(m *types.Func) {
p.pos(m)
p.string(m.Name())
if m.Name() != "_" && !ast.IsExported(m.Name()) {
if m.Name() != "_" && !token.IsExported(m.Name()) {
p.pkg(m.Pkg(), false)
}
@ -501,7 +500,7 @@ func (p *exporter) fieldName(f *types.Var) {
// 3) field name doesn't match base type name (alias name)
bname := basetypeName(f.Type())
if name == bname {
if ast.IsExported(name) {
if token.IsExported(name) {
name = "" // 1) we don't need to know the field name or package
} else {
name = "?" // 2) use unexported name "?" to force package export
@ -514,7 +513,7 @@ func (p *exporter) fieldName(f *types.Var) {
}
p.string(name)
if name != "" && !ast.IsExported(name) {
if name != "" && !token.IsExported(name) {
p.pkg(f.Pkg(), false)
}
}

2
internal/gcimporter/bexport_test.go
View File

@ -109,7 +109,7 @@ type UnknownType undefined
// Compare the packages' corresponding members.
for _, name := range pkg.Scope().Names() {
if !ast.IsExported(name) {
if !token.IsExported(name) {
continue
}
obj1 := pkg.Scope().Lookup(name)

61
internal/gcimporter/iexport.go
View File

@ -12,7 +12,6 @@ import (
"bytes"
"encoding/binary"
"fmt"
"go/ast"
"go/constant"
"go/token"
"go/types"
@ -26,6 +25,41 @@ import (
"golang.org/x/tools/internal/typeparams"
)
// IExportShallow encodes "shallow" export data for the specified package.
//
// No promises are made about the encoding other than that it can be
// decoded by the same version of IIExportShallow. If you plan to save
// export data in the file system, be sure to include a cryptographic
// digest of the executable in the key to avoid version skew.
func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) {
// In principle this operation can only fail if out.Write fails,
// but that's impossible for bytes.Buffer---and as a matter of
// fact iexportCommon doesn't even check for I/O errors.
// TODO(adonovan): handle I/O errors properly.
// TODO(adonovan): use byte slices throughout, avoiding copying.
const bundle, shallow = false, true
var out bytes.Buffer
err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg})
return out.Bytes(), err
}
// IImportShallow decodes "shallow" types.Package data encoded by IExportShallow
// in the same executable. This function cannot import data from
// cmd/compile or gcexportdata.Write.
func IImportShallow(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string, insert InsertType) (*types.Package, error) {
const bundle = false
pkgs, err := iimportCommon(fset, imports, data, bundle, path, insert)
if err != nil {
return nil, err
}
return pkgs[0], nil
}
// InsertType is the type of a function that creates a types.TypeName
// object for a named type and inserts it into the scope of the
// specified Package.
type InsertType = func(pkg *types.Package, name string)
// Current bundled export format version. Increase with each format change.
// 0: initial implementation
const bundleVersion = 0
@ -36,15 +70,17 @@ const bundleVersion = 0
// The package path of the top-level package will not be recorded,
// so that calls to IImportData can override with a provided package path.
func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
return iexportCommon(out, fset, false, iexportVersion, []*types.Package{pkg})
const bundle, shallow = false, false
return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg})
}
// IExportBundle writes an indexed export bundle for pkgs to out.
func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
return iexportCommon(out, fset, true, iexportVersion, pkgs)
const bundle, shallow = true, false
return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs)
}
func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, pkgs []*types.Package) (err error) {
func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package) (err error) {
if !debug {
defer func() {
if e := recover(); e != nil {
@ -61,6 +97,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int,
p := iexporter{
fset: fset,
version: version,
shallow: shallow,
allPkgs: map[*types.Package]bool{},
stringIndex: map[string]uint64{},
declIndex: map[types.Object]uint64{},
@ -82,7 +119,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int,
for _, pkg := range pkgs {
scope := pkg.Scope()
for _, name := range scope.Names() {
if ast.IsExported(name) {
if token.IsExported(name) {
p.pushDecl(scope.Lookup(name))
}
}
@ -205,7 +242,8 @@ type iexporter struct {
out *bytes.Buffer
version int
localpkg *types.Package
shallow bool // don't put types from other packages in the index
localpkg *types.Package // (nil in bundle mode)
// allPkgs tracks all packages that have been referenced by
// the export data, so we can ensure to include them in the
@ -256,6 +294,11 @@ func (p *iexporter) pushDecl(obj types.Object) {
panic("cannot export package unsafe")
}
// Shallow export data: don't index decls from other packages.
if p.shallow && obj.Pkg() != p.localpkg {
return
}
if _, ok := p.declIndex[obj]; ok {
return
}
@ -497,7 +540,7 @@ func (w *exportWriter) pkg(pkg *types.Package) {
w.string(w.exportPath(pkg))
}
func (w *exportWriter) qualifiedIdent(obj types.Object) {
func (w *exportWriter) qualifiedType(obj *types.TypeName) {
name := w.p.exportName(obj)
// Ensure any referenced declarations are written out too.
@ -556,11 +599,11 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) {
return
}
w.startType(definedType)
w.qualifiedIdent(t.Obj())
w.qualifiedType(t.Obj())
case *typeparams.TypeParam:
w.startType(typeParamType)
w.qualifiedIdent(t.Obj())
w.qualifiedType(t.Obj())
case *types.Pointer:
w.startType(pointerType)

5
internal/gcimporter/iexport_test.go
View File

@ -59,7 +59,8 @@ func readExportFile(filename string) ([]byte, error) {
func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) {
var buf bytes.Buffer
if err := gcimporter.IExportCommon(&buf, fset, false, version, []*types.Package{pkg}); err != nil {
const bundle, shallow = false, false
if err := gcimporter.IExportCommon(&buf, fset, bundle, shallow, version, []*types.Package{pkg}); err != nil {
return nil, err
}
return buf.Bytes(), nil
@ -197,7 +198,7 @@ func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package,
// Compare the packages' corresponding members.
for _, name := range pkg.Scope().Names() {
if !ast.IsExported(name) {
if !token.IsExported(name) {
continue
}
obj1 := pkg.Scope().Lookup(name)

26
internal/gcimporter/iimport.go
View File

@ -85,7 +85,7 @@ const (
// If the export data version is not recognized or the format is otherwise
// compromised, an error is returned.
func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
pkgs, err := iimportCommon(fset, imports, data, false, path)
pkgs, err := iimportCommon(fset, imports, data, false, path, nil)
if err != nil {
return 0, nil, err
}
@ -94,10 +94,10 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
// IImportBundle imports a set of packages from the serialized package bundle.
func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) {
return iimportCommon(fset, imports, data, true, "")
return iimportCommon(fset, imports, data, true, "", nil)
}
func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) {
func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) {
const currentVersion = iexportVersionCurrent
version := int64(-1)
if !debug {
@ -147,6 +147,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
p := iimporter{
version: int(version),
ipath: path,
insert: insert,
stringData: stringData,
stringCache: make(map[uint64]string),
@ -187,11 +188,18 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
} else if pkg.Name() != pkgName {
errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path)
}
if i == 0 && !bundle {
p.localpkg = pkg
}
p.pkgCache[pkgPathOff] = pkg
// Read index for package.
nameIndex := make(map[string]uint64)
for nSyms := r.uint64(); nSyms > 0; nSyms-- {
nSyms := r.uint64()
// In shallow mode we don't expect an index for other packages.
assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil)
for ; nSyms > 0; nSyms-- {
name := p.stringAt(r.uint64())
nameIndex[name] = r.uint64()
}
@ -267,6 +275,9 @@ type iimporter struct {
version int
ipath string
localpkg *types.Package
insert func(pkg *types.Package, name string) // "shallow" mode only
stringData []byte
stringCache map[uint64]string
pkgCache map[uint64]*types.Package
@ -310,6 +321,13 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) {
off, ok := p.pkgIndex[pkg][name]
if !ok {
// In "shallow" mode, call back to the application to
// find the object and insert it into the package scope.
if p.insert != nil {
assert(pkg != p.localpkg)
p.insert(pkg, name) // "can't fail"
return
}
errorf("%v.%v not in index", pkg, name)
}

153
internal/gcimporter/shallow_test.go Normal file
View File

@ -0,0 +1,153 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gcimporter_test
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"go/types"
"testing"
"golang.org/x/sync/errgroup"
"golang.org/x/tools/go/packages"
"golang.org/x/tools/internal/gcimporter"
"golang.org/x/tools/internal/testenv"
)
// TestStd type-checks the standard library using shallow export data.
func TestShallowStd(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode; too slow (https://golang.org/issue/14113)")
}
testenv.NeedsTool(t, "go")
// Load import graph of the standard library.
// (No parsing or type-checking.)
cfg := &packages.Config{
Mode: packages.LoadImports,
Tests: false,
}
pkgs, err := packages.Load(cfg, "std")
if err != nil {
t.Fatalf("load: %v", err)
}
if len(pkgs) < 200 {
t.Fatalf("too few packages: %d", len(pkgs))
}
// Type check the packages in parallel postorder.
done := make(map[*packages.Package]chan struct{})
packages.Visit(pkgs, nil, func(p *packages.Package) {
done[p] = make(chan struct{})
})
packages.Visit(pkgs, nil,
func(pkg *packages.Package) {
go func() {
// Wait for all deps to be done.
for _, imp := range pkg.Imports {
<-done[imp]
}
typecheck(t, pkg)
close(done[pkg])
}()
})
for _, root := range pkgs {
<-done[root]
}
}
// typecheck reads, parses, and type-checks a package.
// It squirrels the export data in the the ppkg.ExportFile field.
func typecheck(t *testing.T, ppkg *packages.Package) {
if ppkg.PkgPath == "unsafe" {
return // unsafe is special
}
// Create a local FileSet just for this package.
fset := token.NewFileSet()
// Parse files in parallel.
syntax := make([]*ast.File, len(ppkg.CompiledGoFiles))
var group errgroup.Group
for i, filename := range ppkg.CompiledGoFiles {
i, filename := i, filename
group.Go(func() error {
f, err := parser.ParseFile(fset, filename, nil, parser.SkipObjectResolution)
if err != nil {
return err // e.g. missing file
}
syntax[i] = f
return nil
})
}
if err := group.Wait(); err != nil {
t.Fatal(err)
}
// Inv: all files were successfully parsed.
// importer state
var (
insert func(p *types.Package, name string)
importMap = make(map[string]*types.Package) // keys are PackagePaths
)
loadFromExportData := func(imp *packages.Package) (*types.Package, error) {
data := []byte(imp.ExportFile)
return gcimporter.IImportShallow(fset, importMap, data, imp.PkgPath, insert)
}
insert = func(p *types.Package, name string) {
// Hunt for p among the transitive dependencies (inefficient).
var imp *packages.Package
packages.Visit([]*packages.Package{ppkg}, func(q *packages.Package) bool {
if q.PkgPath == p.Path() {
imp = q
return false
}
return true
}, nil)
if imp == nil {
t.Fatalf("can't find dependency: %q", p.Path())
}
imported, err := loadFromExportData(imp)
if err != nil {
t.Fatalf("unmarshal: %v", err)
}
obj := imported.Scope().Lookup(name)
if obj == nil {
t.Fatalf("lookup %q.%s failed", imported.Path(), name)
}
if imported != p {
t.Fatalf("internal error: inconsistent packages")
}
}
cfg := &types.Config{
Error: func(e error) {
t.Error(e)
},
Importer: importerFunc(func(importPath string) (*types.Package, error) {
if importPath == "unsafe" {
return types.Unsafe, nil // unsafe has no exportdata
}
imp, ok := ppkg.Imports[importPath]
if !ok {
return nil, fmt.Errorf("missing import %q", importPath)
}
return loadFromExportData(imp)
}),
}
// Type-check the syntax trees.
tpkg, _ := cfg.Check(ppkg.PkgPath, fset, syntax, nil)
// Save the export data.
data, err := gcimporter.IExportShallow(fset, tpkg)
if err != nil {
t.Fatalf("internal error marshalling export data: %v", err)
}
ppkg.ExportFile = string(data)
}