go/src/cmd/doc/dirs.go

// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"golang.org/x/mod/semver"
)

// A Dir describes a directory holding code by specifying
// the expected import path and the file system directory.
type Dir struct {
	importPath string // import path for that dir
	dir        string // file system directory
	inModule   bool
}

// Dirs is a structure for scanning the directory tree.
// Its Next method returns the next Go source directory it finds.
// Although it can be used to scan the tree multiple times, it
// only walks the tree once, caching the data it finds.
type Dirs struct {
	scan   chan Dir // Directories generated by walk.
	hist   []Dir    // History of reported Dirs.
	offset int      // Counter for Next.
}

var dirs Dirs

// dirsInit starts the scanning of package directories in GOROOT and GOPATH. Any
// extra paths passed to it are included in the channel.
func dirsInit(extra ...Dir) {
	if buildCtx.GOROOT == "" {
		stdout, err := exec.Command("go", "env", "GOROOT").Output()
		if err != nil {
			if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
				log.Fatalf("failed to determine GOROOT: $GOROOT is not set and 'go env GOROOT' failed:\n%s", ee.Stderr)
			}
			log.Fatalf("failed to determine GOROOT: $GOROOT is not set and could not run 'go env GOROOT':\n\t%s", err)
		}
		buildCtx.GOROOT = string(bytes.TrimSpace(stdout))
	}

	dirs.hist = make([]Dir, 0, 1000)
	dirs.hist = append(dirs.hist, extra...)
	dirs.scan = make(chan Dir)
	go dirs.walk(codeRoots())
}

// goCmd returns the "go" command path corresponding to buildCtx.GOROOT.
func goCmd() string {
	if buildCtx.GOROOT == "" {
		return "go"
	}
	return filepath.Join(buildCtx.GOROOT, "bin", "go")
}

// Reset puts the scan back at the beginning.
func (d *Dirs) Reset() {
	d.offset = 0
}

// Next returns the next directory in the scan. The boolean
// is false when the scan is done.
func (d *Dirs) Next() (Dir, bool) {
	if d.offset < len(d.hist) {
		dir := d.hist[d.offset]
		d.offset++
		return dir, true
	}
	dir, ok := <-d.scan
	if !ok {
		return Dir{}, false
	}
	d.hist = append(d.hist, dir)
	d.offset++
	return dir, ok
}

// walk walks the trees in GOROOT and GOPATH.
func (d *Dirs) walk(roots []Dir) {
	for _, root := range roots {
		d.bfsWalkRoot(root)
	}
	close(d.scan)
}

// bfsWalkRoot walks a single directory hierarchy in breadth-first lexical order.
// Each Go source directory it finds is delivered on d.scan.
func (d *Dirs) bfsWalkRoot(root Dir) {
	root.dir = filepath.Clean(root.dir) // because filepath.Join will do it anyway

	// this is the queue of directories to examine in this pass.
	this := []string{}
	// next is the queue of directories to examine in the next pass.
	next := []string{root.dir}

	for len(next) > 0 {
		this, next = next, this[0:0]
		for _, dir := range this {
			fd, err := os.Open(dir)
			if err != nil {
				log.Print(err)
				continue
			}
			entries, err := fd.Readdir(0)
			fd.Close()
			if err != nil {
				log.Print(err)
				continue
			}
			hasGoFiles := false
			for _, entry := range entries {
				name := entry.Name()
				// For plain files, remember if this directory contains any .go
				// source files, but ignore them otherwise.
				if !entry.IsDir() {
					if !hasGoFiles && strings.HasSuffix(name, ".go") {
						hasGoFiles = true
					}
					continue
				}
				// Entry is a directory.

				// The go tool ignores directories starting with ., _, or named "testdata".
				if name[0] == '.' || name[0] == '_' || name == "testdata" {
					continue
				}
				// When in a module, ignore vendor directories and stop at module boundaries.
				if root.inModule {
					if name == "vendor" {
						continue
					}
					if fi, err := os.Stat(filepath.Join(dir, name, "go.mod")); err == nil && !fi.IsDir() {
						continue
					}
				}
				// Remember this (fully qualified) directory for the next pass.
				next = append(next, filepath.Join(dir, name))
			}
			if hasGoFiles {
				// It's a candidate.
				importPath := root.importPath
				if len(dir) > len(root.dir) {
					if importPath != "" {
						importPath += "/"
					}
					importPath += filepath.ToSlash(dir[len(root.dir)+1:])
				}
				d.scan <- Dir{importPath, dir, root.inModule}
			}
		}

	}
}

var testGOPATH = false // force GOPATH use for testing

// codeRoots returns the code roots to search for packages.
// In GOPATH mode this is GOROOT/src and GOPATH/src, with empty import paths.
// In module mode, this is each module root, with an import path set to its module path.
func codeRoots() []Dir {
	codeRootsCache.once.Do(func() {
		codeRootsCache.roots = findCodeRoots()
	})
	return codeRootsCache.roots
}

var codeRootsCache struct {
	once  sync.Once
	roots []Dir
}

var usingModules bool

func findCodeRoots() []Dir {
	var list []Dir
	if !testGOPATH {
		// Check for use of modules by 'go env GOMOD',
		// which reports a go.mod file path if modules are enabled.
		stdout, _ := exec.Command(goCmd(), "env", "GOMOD").Output()
		gomod := string(bytes.TrimSpace(stdout))

		usingModules = len(gomod) > 0
		if usingModules && buildCtx.GOROOT != "" {
			list = append(list,
				Dir{dir: filepath.Join(buildCtx.GOROOT, "src"), inModule: true},
				Dir{importPath: "cmd", dir: filepath.Join(buildCtx.GOROOT, "src", "cmd"), inModule: true})
		}

		if gomod == os.DevNull {
			// Modules are enabled, but the working directory is outside any module.
			// We can still access std, cmd, and packages specified as source files
			// on the command line, but there are no module roots.
			// Avoid 'go list -m all' below, since it will not work.
			return list
		}
	}

	if !usingModules {
		if buildCtx.GOROOT != "" {
			list = append(list, Dir{dir: filepath.Join(buildCtx.GOROOT, "src")})
		}
		for _, root := range splitGopath() {
			list = append(list, Dir{dir: filepath.Join(root, "src")})
		}
		return list
	}

	// Find module root directories from go list.
	// Eventually we want golang.org/x/tools/go/packages
	// to handle the entire file system search and become go/packages,
	// but for now enumerating the module roots lets us fit modules
	// into the current code with as few changes as possible.
	mainMod, vendorEnabled, err := vendorEnabled()
	if err != nil {
		return list
	}
	if vendorEnabled {
		// Add the vendor directory to the search path ahead of "std".
		// That way, if the main module *is* "std", we will identify the path
		// without the "vendor/" prefix before the one with that prefix.
		list = append([]Dir{{dir: filepath.Join(mainMod.Dir, "vendor"), inModule: false}}, list...)
		if mainMod.Path != "std" {
			list = append(list, Dir{importPath: mainMod.Path, dir: mainMod.Dir, inModule: true})
		}
		return list
	}

	cmd := exec.Command(goCmd(), "list", "-m", "-f={{.Path}}\t{{.Dir}}", "all")
	cmd.Stderr = os.Stderr
	out, _ := cmd.Output()
	for _, line := range strings.Split(string(out), "\n") {
		path, dir, _ := strings.Cut(line, "\t")
		if dir != "" {
			list = append(list, Dir{importPath: path, dir: dir, inModule: true})
		}
	}

	return list
}

// The functions below are derived from x/tools/internal/imports at CL 203017.

type moduleJSON struct {
	Path, Dir, GoVersion string
}

var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)

// vendorEnabled indicates if vendoring is enabled.
// Inspired by setDefaultBuildMod in modload/init.go
func vendorEnabled() (*moduleJSON, bool, error) {
	mainMod, go114, err := getMainModuleAnd114()
	if err != nil {
		return nil, false, err
	}

	stdout, _ := exec.Command(goCmd(), "env", "GOFLAGS").Output()
	goflags := string(bytes.TrimSpace(stdout))
	matches := modFlagRegexp.FindStringSubmatch(goflags)
	var modFlag string
	if len(matches) != 0 {
		modFlag = matches[1]
	}
	if modFlag != "" {
		// Don't override an explicit '-mod=' argument.
		return mainMod, modFlag == "vendor", nil
	}
	if mainMod == nil || !go114 {
		return mainMod, false, nil
	}
	// Check 1.14's automatic vendor mode.
	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
			// The Go version is at least 1.14, and a vendor directory exists.
			// Set -mod=vendor by default.
			return mainMod, true, nil
		}
	}
	return mainMod, false, nil
}

// getMainModuleAnd114 gets the main module's information and whether the
// go command in use is 1.14+. This is the information needed to figure out
// if vendoring should be enabled.
func getMainModuleAnd114() (*moduleJSON, bool, error) {
	const format = `{{.Path}}
{{.Dir}}
{{.GoVersion}}
{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
`
	cmd := exec.Command(goCmd(), "list", "-m", "-f", format)
	cmd.Stderr = os.Stderr
	stdout, err := cmd.Output()
	if err != nil {
		return nil, false, nil
	}
	lines := strings.Split(string(stdout), "\n")
	if len(lines) < 5 {
		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
	}
	mod := &moduleJSON{
		Path:      lines[0],
		Dir:       lines[1],
		GoVersion: lines[2],
	}
	return mod, lines[3] == "go1.14", nil
}