mirror
/
go
mirror of https://github.com/golang/go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cmd/dist: support GOROOT vendoring 

In the second step of make.bash, cmd/dist builds cmd/go by invoking
the compiler, linker, and other tools directly on transitive
dependencies of cmd/go. Essentially, cmd/dist acts as a minimal
version of 'go install' when building go_toolchain.

Until now, cmd/go has had no transitive dependencies in vendor
directories. This changes in CL 202698, where several packages are
deleted and equivalent versions in golang.org/x/mod are used
instead. So this CL adds support to cmd/dist for vendor directories.

Updates #31761

Change-Id: Iab4cdc7e505069a8df296287d16fbaa871944955
Reviewed-on: https://go-review.googlesource.com/c/go/+/203537
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
This commit is contained in:
Jay Conrod 2019-10-23 17:14:09 -04:00
parent 3e457030d9
commit 60f34f739d
2 changed files with 101 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
src/cmd/dist/build.go vendored
View File

@ -605,26 +605,26 @@ func startInstall(dir string) chan struct{} {
// runInstall installs the library, package, or binary associated with dir,
// which is relative to $GOROOT/src.
func runInstall(dir string, ch chan struct{}) {
if dir == "net" || dir == "os/user" || dir == "crypto/x509" {
fatalf("go_bootstrap cannot depend on cgo package %s", dir)
func runInstall(pkg string, ch chan struct{}) {
if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" {
fatalf("go_bootstrap cannot depend on cgo package %s", pkg)
}
defer close(ch)
if dir == "unsafe" {
if pkg == "unsafe" {
return
}
if vflag > 0 {
if goos != gohostos || goarch != gohostarch {
errprintf("%s (%s/%s)\n", dir, goos, goarch)
errprintf("%s (%s/%s)\n", pkg, goos, goarch)
} else {
errprintf("%s\n", dir)
errprintf("%s\n", pkg)
}
}
workdir := pathf("%s/%s", workdir, dir)
workdir := pathf("%s/%s", workdir, pkg)
xmkdirall(workdir)
var clean []string
@ -634,11 +634,14 @@ func runInstall(dir string, ch chan struct{}) {
}
}()
// path = full path to dir.
path := pathf("%s/src/%s", goroot, dir)
// dir = full path to pkg.
dir := pathf("%s/src/%s", goroot, pkg)
name := filepath.Base(dir)
ispkg := !strings.HasPrefix(dir, "cmd/") || strings.Contains(dir, "/internal/")
// ispkg predicts whether the package should be linked as a binary, based
// on the name. There should be no "main" packages in vendor, since
// 'go mod vendor' will only copy imported packages there.
ispkg := !strings.HasPrefix(pkg, "cmd/") || strings.Contains(pkg, "/internal/") || strings.Contains(pkg, "/vendor/")
// Start final link command line.
// Note: code below knows that link.p[targ] is the target.
@ -650,7 +653,7 @@ func runInstall(dir string, ch chan struct{}) {
if ispkg {
// Go library (package).
ispackcmd = true
link = []string{"pack", pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, dir)}
link = []string{"pack", packagefile(pkg)}
targ = len(link) - 1
xmkdirall(filepath.Dir(link[targ]))
} else {
@ -675,7 +678,7 @@ func runInstall(dir string, ch chan struct{}) {
// Gather files that are sources for this target.
// Everything in that directory, and any target-specific
// additions.
files := xreaddir(path)
files := xreaddir(dir)
// Remove files beginning with . or _,
// which are likely to be editor temporary files.
@ -687,7 +690,7 @@ func runInstall(dir string, ch chan struct{}) {
})
for _, dt := range deptab {
if dir == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(dir, dt.prefix) {
if pkg == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(pkg, dt.prefix) {
for _, p := range dt.dep {
p = os.ExpandEnv(p)
files = append(files, p)
@ -699,7 +702,7 @@ func runInstall(dir string, ch chan struct{}) {
// Convert to absolute paths.
for i, p := range files {
if !filepath.IsAbs(p) {
files[i] = pathf("%s/%s", path, p)
files[i] = pathf("%s/%s", dir, p)
}
}
@ -715,7 +718,7 @@ func runInstall(dir string, ch chan struct{}) {
return false
ok:
t := mtime(p)
if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, dir) {
if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) {
return false
}
if strings.HasSuffix(p, ".go") {
@ -742,7 +745,7 @@ func runInstall(dir string, ch chan struct{}) {
}
// For package runtime, copy some files into the work space.
if dir == "runtime" {
if pkg == "runtime" {
xmkdirall(pathf("%s/pkg/include", goroot))
// For use by assembly and C files.
copyfile(pathf("%s/pkg/include/textflag.h", goroot),
@ -764,7 +767,7 @@ func runInstall(dir string, ch chan struct{}) {
if vflag > 1 {
errprintf("generate %s\n", p)
}
gt.gen(path, p)
gt.gen(dir, p)
// Do not add generated file to clean list.
// In runtime, we want to be able to
// build the package with the go tool,
@ -782,22 +785,31 @@ func runInstall(dir string, ch chan struct{}) {
built:
}
// Make sure dependencies are installed.
var deps []string
// Resolve imported packages to actual package paths.
// Make sure they're installed.
importMap := make(map[string]string)
for _, p := range gofiles {
deps = append(deps, readimports(p)...)
for _, imp := range readimports(p) {
importMap[imp] = resolveVendor(imp, dir)
}
}
for _, dir1 := range deps {
startInstall(dir1)
sortedImports := make([]string, 0, len(importMap))
for imp := range importMap {
sortedImports = append(sortedImports, imp)
}
for _, dir1 := range deps {
install(dir1)
sort.Strings(sortedImports)
for _, dep := range importMap {
startInstall(dep)
}
for _, dep := range importMap {
install(dep)
}
if goos != gohostos || goarch != gohostarch {
// We've generated the right files; the go command can do the build.
if vflag > 1 {
errprintf("skip build for cross-compile %s\n", dir)
errprintf("skip build for cross-compile %s\n", pkg)
}
return
}
@ -830,18 +842,35 @@ func runInstall(dir string, ch chan struct{}) {
if err := ioutil.WriteFile(goasmh, nil, 0666); err != nil {
fatalf("cannot write empty go_asm.h: %s", err)
}
bgrun(&wg, path, asmabis...)
bgrun(&wg, dir, asmabis...)
bgwait(&wg)
}
// Build an importcfg file for the compiler.
buf := &bytes.Buffer{}
for _, imp := range sortedImports {
if imp == "unsafe" {
continue
}
dep := importMap[imp]
if imp != dep {
fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep)
}
fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep))
}
importcfg := pathf("%s/importcfg", workdir)
if err := ioutil.WriteFile(importcfg, buf.Bytes(), 0666); err != nil {
fatalf("cannot write importcfg file: %v", err)
}
var archive string
// The next loop will compile individual non-Go files.
// Hand the Go files to the compiler en masse.
// For packages containing assembly, this writes go_asm.h, which
// the assembly files will need.
pkg := dir
if strings.HasPrefix(dir, "cmd/") && strings.Count(dir, "/") == 1 {
pkg = "main"
pkgName := pkg
if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 {
pkgName = "main"
}
b := pathf("%s/_go_.a", workdir)
clean = append(clean, b)
@ -852,11 +881,11 @@ func runInstall(dir string, ch chan struct{}) {
}
// Compile Go code.
compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkg}
compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg}
if gogcflags != "" {
compile = append(compile, strings.Fields(gogcflags)...)
}
if dir == "runtime" {
if pkg == "runtime" {
compile = append(compile, "-+")
}
if len(sfiles) > 0 {
@ -874,7 +903,7 @@ func runInstall(dir string, ch chan struct{}) {
// We use bgrun and immediately wait for it instead of calling run() synchronously.
// This executes all jobs through the bgwork channel and allows the process
// to exit cleanly in case an error occurs.
bgrun(&wg, path, compile...)
bgrun(&wg, dir, compile...)
bgwait(&wg)
// Compile the files.
@ -888,7 +917,7 @@ func runInstall(dir string, ch chan struct{}) {
// Change the last character of the output file (which was c or s).
b = b[:len(b)-1] + "o"
compile = append(compile, "-o", b, p)
bgrun(&wg, path, compile...)
bgrun(&wg, dir, compile...)
link = append(link, b)
if doclean {
@ -909,6 +938,12 @@ func runInstall(dir string, ch chan struct{}) {
bgwait(&wg)
}
// packagefile returns the path to a compiled .a file for the given package
// path. Paths may need to be resolved with resolveVendor first.
func packagefile(pkg string) string {
return pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, pkg)
}
// matchfield reports whether the field (x,y,z) matches this build.
// all the elements in the field must be satisfied.
func matchfield(f string) bool {
@ -940,7 +975,7 @@ func matchtag(tag string) bool {
// of GOOS and GOARCH.
// We also allow the special tag cmd_go_bootstrap.
// See ../go/bootstrap.go and package go/build.
func shouldbuild(file, dir string) bool {
func shouldbuild(file, pkg string) bool {
// Check file name for GOOS or GOARCH.
name := filepath.Base(file)
excluded := func(list []string, ok string) bool {
@ -982,7 +1017,7 @@ func shouldbuild(file, dir string) bool {
if code == "package documentation" {
return false
}
if code == "package main" && dir != "cmd/go" && dir != "cmd/cgo" {
if code == "package main" && pkg != "cmd/go" && pkg != "cmd/cgo" {
return false
}
if !strings.HasPrefix(p, "//") {

31
src/cmd/dist/imports.go vendored
View File

@ -11,7 +11,10 @@ package main
import (
"bufio"
"errors"
"fmt"
"io"
"path"
"path/filepath"
"strconv"
"strings"
"unicode/utf8"
@ -243,3 +246,31 @@ func readimports(file string) []string {
return imports
}
// resolveVendor returns a unique package path imported with the given import
// path from srcDir.
//
// resolveVendor assumes that a package is vendored if and only if its first
// path component contains a dot. If a package is vendored, its import path
// is returned with a "vendor" or "cmd/vendor" prefix, depending on srcDir.
// Otherwise, the import path is returned verbatim.
func resolveVendor(imp, srcDir string) string {
var first string
if i := strings.Index(imp, "/"); i < 0 {
first = imp
} else {
first = imp[:i]
}
isStandard := !strings.Contains(first, ".")
if isStandard {
return imp
}
if strings.HasPrefix(srcDir, filepath.Join(goroot, "src", "cmd")) {
return path.Join("cmd", "vendor", imp)
} else if strings.HasPrefix(srcDir, filepath.Join(goroot, "src")) {
return path.Join("vendor", imp)
} else {
panic(fmt.Sprintf("srcDir %q not in GOOROT/src", srcDir))
}
}