cmd/go: implement "go build -json"

This adds support for a "-json" flag in all build-related go
subcommands. This causes build output and build failures to be
reported to stdout in a machine-readable way.

For #62067.
Fixes #23037.

Change-Id: Id045c5bd5dde9d16cc09dde6248a4b9637896a30
Reviewed-on: https://go-review.googlesource.com/c/go/+/536397
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Russ Cox <rsc@golang.org>

src/cmd/go/alldocs.go

@@ -38,6 +38,7 @@
 // Additional help topics:
 //
 //	buildconstraint build constraints
+//	buildjson       build -json encoding
 //	buildmode       build modes
 //	c               calling between Go and C
 //	cache           build and test caching
@@ -184,6 +185,9 @@
 //		or, if set explicitly, has _race appended to it. Likewise for the -msan
 //		and -asan flags. Using a -buildmode option that requires non-default compile
 //		flags has a similar effect.
+//	-json
+//		Emit build output in JSON suitable for automated processing.
+//		See 'go help buildjson' for the encoding details.
 //	-ldflags '[pattern=]arg list'
 //		arguments to pass on each go tool link invocation.
 //	-linkshared
@@ -2139,6 +2143,39 @@
 // has a term for a Go major release, the language version used when compiling
 // the file will be the minimum version implied by the build constraint.
 //
+// # Build -json encoding
+//
+// The 'go build' and 'go install' commands take a -json flag that reports
+// build output and failures as structured JSON output on standard output.
+//
+// The JSON stream is a newline-separated sequence of BuildEvent objects
+// corresponding to the Go struct:
+//
+//	type BuildEvent struct {
+//		ImportPath string
+//		Action     string
+//		Output     string
+//	}
+//
+// The ImportPath field gives the package ID of the package being built.
+// This matches the Package.ImportPath field of go list -json.
+//
+// The Action field is one of the following:
+//
+//	build-output - The toolchain printed output
+//	build-fail - The build failed
+//
+// The Output field is set for Action == "build-output" and is a portion of
+// the build's output. The concatenation of the Output fields of all output
+// events is the exact output of the build. A single event may contain one
+// or more lines of output and there may be more than one output event for
+// a given ImportPath. This matches the definition of the TestEvent.Output
+// field produced by go test -json.
+//
+// Note that there may also be non-JSON error text on stdnard error, even
+// with the -json flag. Typically, this indicates an early, serious error.
+// Consumers should be robust to this.
+//
 // # Build modes
 //
 // The 'go build' and 'go install' commands take a -buildmode argument which

src/cmd/go/internal/cfg/cfg.go

@@ -65,31 +65,34 @@ func ToolExeSuffix() string {
 
 // These are general "build flags" used by build and other commands.
 var (
-	BuildA             bool     // -a flag
-	BuildBuildmode     string   // -buildmode flag
-	BuildBuildvcs      = "auto" // -buildvcs flag: "true", "false", or "auto"
-	BuildContext       = defaultContext()
-	BuildMod           string                  // -mod flag
-	BuildModExplicit   bool                    // whether -mod was set explicitly
-	BuildModReason     string                  // reason -mod was set, if set by default
-	BuildLinkshared    bool                    // -linkshared flag
-	BuildMSan          bool                    // -msan flag
-	BuildASan          bool                    // -asan flag
-	BuildCover         bool                    // -cover flag
-	BuildCoverMode     string                  // -covermode flag
-	BuildCoverPkg      []string                // -coverpkg flag
-	BuildN             bool                    // -n flag
-	BuildO             string                  // -o flag
-	BuildP             = runtime.GOMAXPROCS(0) // -p flag
-	BuildPGO           string                  // -pgo flag
-	BuildPkgdir        string                  // -pkgdir flag
-	BuildRace          bool                    // -race flag
-	BuildToolexec      []string                // -toolexec flag
-	BuildToolchainName string
-	BuildTrimpath      bool // -trimpath flag
-	BuildV             bool // -v flag
-	BuildWork          bool // -work flag
-	BuildX             bool // -x flag
+	BuildA                 bool     // -a flag
+	BuildBuildmode         string   // -buildmode flag
+	BuildBuildvcs          = "auto" // -buildvcs flag: "true", "false", or "auto"
+	BuildContext           = defaultContext()
+	BuildMod               string                  // -mod flag
+	BuildModExplicit       bool                    // whether -mod was set explicitly
+	BuildModReason         string                  // reason -mod was set, if set by default
+	BuildLinkshared        bool                    // -linkshared flag
+	BuildMSan              bool                    // -msan flag
+	BuildASan              bool                    // -asan flag
+	BuildCover             bool                    // -cover flag
+	BuildCoverMode         string                  // -covermode flag
+	BuildCoverPkg          []string                // -coverpkg flag
+	BuildJSON              bool                    // -json flag
+	BuildN                 bool                    // -n flag
+	BuildO                 string                  // -o flag
+	BuildP                 = runtime.GOMAXPROCS(0) // -p flag
+	BuildPGO               string                  // -pgo flag
+	BuildPkgdir            string                  // -pkgdir flag
+	BuildRace              bool                    // -race flag
+	BuildToolexec          []string                // -toolexec flag
+	BuildToolchainName     string
+	BuildToolchainCompiler func() string
+	BuildToolchainLinker   func() string
+	BuildTrimpath          bool // -trimpath flag
+	BuildV                 bool // -v flag
+	BuildWork              bool // -work flag
+	BuildX                 bool // -x flag
 
 	ModCacheRW bool   // -modcacherw flag
 	ModFile    string // -modfile flag

src/cmd/go/internal/clean/clean.go

@@ -114,7 +114,7 @@ func init() {
 	// mentioned explicitly in the docs but they
 	// are part of the build flags.
 
-	work.AddBuildFlags(CmdClean, work.DefaultBuildFlags)
+	work.AddBuildFlags(CmdClean, work.OmitBuildOnlyFlags)
 }
 
 func runClean(ctx context.Context, cmd *base.Command, args []string) {

src/cmd/go/internal/fix/fix.go

@@ -40,7 +40,7 @@ See also: go fmt, go vet.
 var fixes = CmdFix.Flag.String("fix", "", "comma-separated list of fixes to apply")
 
 func init() {
-	work.AddBuildFlags(CmdFix, work.DefaultBuildFlags)
+	work.AddBuildFlags(CmdFix, work.OmitBuildOnlyFlags)
 	CmdFix.Run = runFix // fix cycle
 }
 

src/cmd/go/internal/generate/generate.go

@@ -176,7 +176,7 @@ var (
 )
 
 func init() {
-	work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
+	work.AddBuildFlags(CmdGenerate, work.OmitBuildOnlyFlags)
 	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
 	CmdGenerate.Flag.StringVar(&generateSkipFlag, "skip", "", "")
 }

src/cmd/go/internal/help/helpdoc.go

@@ -1044,3 +1044,40 @@ If the server responds with an error again, the fetch fails: a URL-specific
 GOAUTH will only be attempted once per fetch.
 `,
 }
+
+var HelpBuildJSON = &base.Command{
+	UsageLine: "buildjson",
+	Short:     "build -json encoding",
+	Long: `
+The 'go build' and 'go install' commands take a -json flag that reports
+build output and failures as structured JSON output on standard output.
+
+The JSON stream is a newline-separated sequence of BuildEvent objects
+corresponding to the Go struct:
+
+	type BuildEvent struct {
+		ImportPath string
+		Action     string
+		Output     string
+	}
+
+The ImportPath field gives the package ID of the package being built.
+This matches the Package.ImportPath field of go list -json.
+
+The Action field is one of the following:
+
+	build-output - The toolchain printed output
+	build-fail - The build failed
+
+The Output field is set for Action == "build-output" and is a portion of
+the build's output. The concatenation of the Output fields of all output
+events is the exact output of the build. A single event may contain one
+or more lines of output and there may be more than one output event for
+a given ImportPath. This matches the definition of the TestEvent.Output
+field produced by go test -json.
+
+Note that there may also be non-JSON error text on stdnard error, even
+with the -json flag. Typically, this indicates an early, serious error.
+Consumers should be robust to this.
+	`,
+}

src/cmd/go/internal/list/list.go

@@ -345,7 +345,8 @@ For more about modules, see https://golang.org/ref/mod.
 
 func init() {
 	CmdList.Run = runList // break init cycle
-	work.AddBuildFlags(CmdList, work.DefaultBuildFlags)
+	// Omit build -json because list has its own -json
+	work.AddBuildFlags(CmdList, work.OmitJSONFlag)
 	if cfg.Experiment != nil && cfg.Experiment.CoverageRedesign {
 		work.AddCoverFlags(CmdList, nil)
 	}

src/cmd/go/internal/load/printer.go

@@ -6,6 +6,8 @@ package load
 
 import (
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -45,7 +47,9 @@ func DefaultPrinter() Printer {
 }
 
 var defaultPrinter = sync.OnceValue(func() Printer {
-	// TODO: This will return a JSON printer once that's an option.
+	if cfg.BuildJSON {
+		return NewJSONPrinter(os.Stdout)
+	}
 	return &TextPrinter{os.Stderr}
 })
 
@@ -72,3 +76,51 @@ func (p *TextPrinter) Errorf(_ *Package, format string, args ...any) {
 	fmt.Fprint(p.Writer, ensureNewline(fmt.Sprintf(format, args...)))
 	base.SetExitStatus(1)
 }
+
+// A JSONPrinter emits output about a build in JSON format.
+type JSONPrinter struct {
+	enc *json.Encoder
+}
+
+func NewJSONPrinter(w io.Writer) *JSONPrinter {
+	return &JSONPrinter{json.NewEncoder(w)}
+}
+
+type jsonBuildEvent struct {
+	ImportPath string
+	Action     string
+	Output     string `json:",omitempty"` // Non-empty if Action == “build-output”
+}
+
+func (p *JSONPrinter) Output(pkg *Package, args ...any) {
+	ev := &jsonBuildEvent{
+		Action: "build-output",
+		Output: fmt.Sprint(args...),
+	}
+	if ev.Output == "" {
+		// There's no point in emitting a completely empty output event.
+		return
+	}
+	if pkg != nil {
+		ev.ImportPath = pkg.Desc()
+	}
+	p.enc.Encode(ev)
+}
+
+func (p *JSONPrinter) Errorf(pkg *Package, format string, args ...any) {
+	s := ensureNewline(fmt.Sprintf(format, args...))
+	// For clarity, emit each line as a separate output event.
+	for len(s) > 0 {
+		i := strings.IndexByte(s, '\n')
+		p.Output(pkg, s[:i+1])
+		s = s[i+1:]
+	}
+	ev := &jsonBuildEvent{
+		Action: "build-fail",
+	}
+	if pkg != nil {
+		ev.ImportPath = pkg.Desc()
+	}
+	p.enc.Encode(ev)
+	base.SetExitStatus(1)
+}

src/cmd/go/internal/test/testflag.go

@@ -26,13 +26,14 @@ import (
 // some are for both.
 
 func init() {
-	work.AddBuildFlags(CmdTest, work.OmitVFlag)
+	work.AddBuildFlags(CmdTest, work.OmitVFlag|work.OmitJSONFlag)
 
 	cf := CmdTest.Flag
 	cf.BoolVar(&testC, "c", false, "")
 	cf.StringVar(&testO, "o", "", "")
 	work.AddCoverFlags(CmdTest, &testCoverProfile)
 	cf.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "")
+	// TODO(austin): Make test -json imply build -json.
 	cf.BoolVar(&testJSON, "json", false, "")
 	cf.Var(&testVet, "vet", "")
 

src/cmd/go/internal/vet/vetflag.go

@@ -38,7 +38,10 @@ import (
 var vetTool string // -vettool
 
 func init() {
-	work.AddBuildFlags(CmdVet, work.DefaultBuildFlags)
+	// For now, we omit the -json flag for vet because we could plausibly
+	// support -json specific to the vet command in the future (perhaps using
+	// the same format as build -json).
+	work.AddBuildFlags(CmdVet, work.OmitJSONFlag)
 	CmdVet.Flag.StringVar(&vetTool, "vettool", "", "")
 }
 

src/cmd/go/internal/work/action.go

@@ -269,6 +269,7 @@ func NewBuilder(workDir string) *Builder {
 	b.toolIDCache = make(map[string]string)
 	b.buildIDCache = make(map[string]string)
 
+	printWorkDir := false
 	if workDir != "" {
 		b.WorkDir = workDir
 	} else if cfg.BuildN {
@@ -291,13 +292,15 @@ func NewBuilder(workDir string) *Builder {
 		}
 		b.WorkDir = tmp
 		builderWorkDirs.Store(b, b.WorkDir)
-		if cfg.BuildX || cfg.BuildWork {
-			fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir)
-		}
+		printWorkDir = cfg.BuildX || cfg.BuildWork
 	}
 
 	b.backgroundSh = NewShell(b.WorkDir, nil)
 
+	if printWorkDir {
+		b.BackgroundShell().Print("WORK=", b.WorkDir, "\n")
+	}
+
 	if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil {
 		fmt.Fprintf(os.Stderr, "go: %v\n", err)
 		base.SetExitStatus(2)

src/cmd/go/internal/work/build.go

@@ -141,6 +141,9 @@ and test commands:
 		or, if set explicitly, has _race appended to it. Likewise for the -msan
 		and -asan flags. Using a -buildmode option that requires non-default compile
 		flags has a similar effect.
+	-json
+		Emit build output in JSON suitable for automated processing.
+		See 'go help buildjson' for the encoding details.
 	-ldflags '[pattern=]arg list'
 		arguments to pass on each go tool link invocation.
 	-linkshared
@@ -300,6 +303,8 @@ const (
 	OmitModFlag       BuildFlagMask = 1 << iota
 	OmitModCommonFlags
 	OmitVFlag
+	OmitBuildOnlyFlags // Omit flags that only affect building packages
+	OmitJSONFlag
 )
 
 // AddBuildFlags adds the flags common to the build, clean, get,
@@ -332,6 +337,13 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) {
 		cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "")
 	}
 	cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "")
+	if mask&(OmitBuildOnlyFlags|OmitJSONFlag) == 0 {
+		// TODO(#62250): OmitBuildOnlyFlags should apply to many more flags
+		// here, but we let a bunch of flags slip in before we realized that
+		// many of them don't make sense for most subcommands. We might even
+		// want to separate "AddBuildFlags" and "AddSelectionFlags".
+		cmd.Flag.BoolVar(&cfg.BuildJSON, "json", false, "")
+	}
 	cmd.Flag.Var(&load.BuildLdflags, "ldflags", "")
 	cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "")
 	cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "")

src/cmd/go/main.go

@@ -71,6 +71,7 @@ func init() {
 		vet.CmdVet,
 
 		help.HelpBuildConstraint,
+		help.HelpBuildJSON,
 		help.HelpBuildmode,
 		help.HelpC,
 		help.HelpCache,

src/cmd/go/testdata/script/build_json.txt

@@ -0,0 +1,48 @@
+[short] skip
+
+# Basic build error. This test also checks that the output is fully-formed JSON.
+! go build -json -o=$devnull ./compileerror
+stdout '^\{"ImportPath":"m/compileerror","Action":"build-output","Output":"# m/compileerror\\n"\}$'
+stdout '^\{"ImportPath":"m/compileerror","Action":"build-output","Output":"compileerror/main.go:3:11: undefined: y\\n"}$'
+stdout '^\{"ImportPath":"m/compileerror","Action":"build-fail"\}$'
+! stderr '.'
+
+# Check that a build failure in an imported package is attributed correctly.
+! go build -json -o=$devnull ./importerror
+stdout '"ImportPath":"m/compileerror","Action":"build-fail"'
+! stderr '.'
+
+# TODO(#65335): Attributing this to "x" doesn't make much sense,
+# especially since the reported line is the import statement.
+! go build -json -o=$devnull ./loaderror
+stdout '"ImportPath":"x","Action":"build-output","Output":".*package x is not in std.*\\n"'
+stdout '"ImportPath":"x","Action":"build-fail"'
+! stderr '.'
+
+# Check that a load error in an imported package is attributed correctly.
+! go build -json -o=$devnull ./loadimporterror
+stdout '"ImportPath":"x","Action":"build-output","Output":".*package x is not in std.*\\n"'
+stdout '"ImportPath":"x","Action":"build-fail"'
+! stderr '.'
+
+-- go.mod --
+module m
+go 1.21
+-- compileerror/main.go --
+package compileerror
+
+const x = y
+-- importerror/main.go --
+package main
+
+import _ "m/compileerror"
+-- loaderror/main.go --
+// A bad import causes a failure directly in cmd/go during import processing.
+
+package loaderror
+
+import _ "x"
+-- loadimporterror/main.go --
+package loadimporterror
+
+import _ "m/loaderror"