go/build: deps check all std packages

Instead of only checking packages that are already listed in pkgDeps,
apply deps checks to all standard library packages.

To avoid slowing testing down too much, instead of running "go list
std" in a subprocess like cmd/api or cmd/dist, this test manually
walks the GOROOT src directory to enumerate packages.

Timings on an HP Z620 using linux/amd64:

    	short	full
before	0.092s	4.880s
after	0.137s	5.104s

Additionally, a handful of packages that were previously unchecked are
now listed, along with their current dependencies.  These should
probably eventually be moved elsewhere and assigned appropriate
allowable-dependency sets.  For now, they've been grandfathered in by
simply assigning them their current dependencies, so that followup CLs
can review them individually as appropriate.

Fixes #10475.

Change-Id: I83ffd8ff329092f664bf3e3f2c9e3dad8e77ac02
Reviewed-on: https://go-review.googlesource.com/9001
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Russ Cox <rsc@golang.org>

src/go/build/deps_test.go

@@ -8,8 +8,11 @@
 package build
 
 import (
+	"os"
+	"path/filepath"
 	"runtime"
 	"sort"
+	"strings"
 	"testing"
 )
 
@@ -330,6 +333,29 @@ var pkgDeps = map[string][]string{
 	"net/http/pprof":    {"L4", "OS", "html/template", "net/http", "runtime/pprof"},
 	"net/rpc":           {"L4", "NET", "encoding/gob", "html/template", "net/http"},
 	"net/rpc/jsonrpc":   {"L4", "NET", "encoding/json", "net/rpc"},
+
+	// Packages below are grandfathered because of issue 10475.
+	// When updating these entries, move them to an appropriate
+	// location above and assign them a justified set of
+	// dependencies.  Do not simply update them in situ.
+	"container/heap":           {"sort"},
+	"debug/plan9obj":           {"encoding/binary", "errors", "fmt", "io", "os"},
+	"go/exact":                 {"fmt", "go/token", "math/big", "strconv"},
+	"go/format":                {"bytes", "fmt", "go/ast", "go/parser", "go/printer", "go/token", "internal/format", "io"},
+	"go/importer":              {"go/internal/gcimporter", "go/types", "io", "runtime"},
+	"go/internal/gcimporter":   {"bufio", "errors", "fmt", "go/build", "go/exact", "go/token", "go/types", "io", "os", "path/filepath", "strconv", "strings", "text/scanner"},
+	"go/types":                 {"bytes", "container/heap", "fmt", "go/ast", "go/exact", "go/parser", "go/token", "io", "math", "path", "sort", "strconv", "strings", "sync", "unicode"},
+	"image/internal/imageutil": {"image"},
+	"internal/format":          {"bytes", "go/ast", "go/parser", "go/printer", "go/token", "strings"},
+	"internal/mime":            {"bytes", "encoding/base64", "errors", "fmt", "io", "io/ioutil", "strconv", "strings", "unicode"},
+	"internal/singleflight":    {"sync"},
+	"internal/syscall/unix":    {"runtime", "sync/atomic", "syscall", "unsafe"},
+	"internal/syscall/windows": {"syscall", "unsafe"},
+	"internal/trace":           {"bufio", "bytes", "fmt", "io", "os", "os/exec", "sort", "strconv", "strings"},
+	"mime/quotedprintable":     {"bufio", "bytes", "fmt", "io"},
+	"net/http/cookiejar":       {"errors", "fmt", "net", "net/http", "net/url", "sort", "strings", "sync", "time", "unicode/utf8"},
+	"net/http/internal":        {"bufio", "bytes", "errors", "fmt", "io"},
+	"net/internal/socktest":    {"fmt", "sync", "syscall"},
 }
 
 // isMacro reports whether p is a package dependency macro
@@ -375,6 +401,36 @@ var allowedErrors = map[osPkg]bool{
 	osPkg{"plan9", "log/syslog"}:   true,
 }
 
+// listStdPkgs returns the same list of packages as "go list std".
+func listStdPkgs(goroot string) ([]string, error) {
+	// Based on cmd/go's matchPackages function.
+	var pkgs []string
+
+	src := filepath.Join(goroot, "src") + string(filepath.Separator)
+	walkFn := func(path string, fi os.FileInfo, err error) error {
+		if err != nil || !fi.IsDir() || path == src {
+			return nil
+		}
+
+		base := filepath.Base(path)
+		if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") || base == "testdata" {
+			return filepath.SkipDir
+		}
+
+		name := filepath.ToSlash(path[len(src):])
+		if name == "builtin" || name == "cmd" || strings.Contains(name, ".") {
+			return filepath.SkipDir
+		}
+
+		pkgs = append(pkgs, name)
+		return nil
+	}
+	if err := filepath.Walk(src, walkFn); err != nil {
+		return nil, err
+	}
+	return pkgs, nil
+}
+
 func TestDependencies(t *testing.T) {
 	iOS := runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64")
 	if runtime.GOOS == "nacl" || iOS {
@@ -382,24 +438,24 @@ func TestDependencies(t *testing.T) {
 		// provide access to every source file.
 		t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
 	}
-	var all []string
 
-	for k := range pkgDeps {
-		all = append(all, k)
+	ctxt := Default
+	all, err := listStdPkgs(ctxt.GOROOT)
+	if err != nil {
+		t.Fatal(err)
 	}
 	sort.Strings(all)
 
-	ctxt := Default
 	test := func(mustImport bool) {
 		for _, pkg := range all {
-			if isMacro(pkg) {
-				continue
-			}
 			if pkg == "runtime/cgo" && !ctxt.CgoEnabled {
 				continue
 			}
 			p, err := ctxt.Import(pkg, "", 0)
 			if err != nil {
+				if _, ok := err.(*NoGoError); ok {
+					continue
+				}
 				if allowedErrors[osPkg{ctxt.GOOS, pkg}] {
 					continue
 				}