cmd/go: add support for build IDs with gccgo

This just adds support on ELF systems, which is OK for now since that
is all that gccgo works on.

For the archive file generated by the compiler we add a new file
_buildid.o that has a section .go.buildid containing the build ID.
Using a new file lets us set the SHF_EXCLUDE bit in the section header,
so the linker will discard the section. It would be nicer to use
`objcopy --add-section`, but objcopy doesn't support setting the
SHF_EXCLUDE bit.

For an executable we just use an ordinary GNU build ID. Doing this
required modifying cmd/internal/buildid to look for a GNU build ID,
and use it if there is no other Go-specific note.

This CL fixes a minor bug in gccgoTOolchain.link: it was using .Target
instead of .built, so it failed for a cached file.

This CL fixes a bug reading note segments: the notes are aligned as
reported by the PT_NOTE's alignment field.

Updates #22472

Change-Id: I4d9e9978ef060bafc5b9574d9af16d97c13f3102
Reviewed-on: https://go-review.googlesource.com/85555
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

src/cmd/go/internal/work/buildid.go

@@ -7,6 +7,7 @@ package work
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -203,6 +204,132 @@ func (b *Builder) toolID(name string) string {
 	return id
 }
 
+// gccToolID returns the unique ID to use for a tool that is invoked
+// by the GCC driver. This is in particular gccgo, but this can also
+// be used for gcc, g++, gfortran, etc.; those tools all use the GCC
+// driver under different names. The approach used here should also
+// work for sufficiently new versions of clang. Unlike toolID, the
+// name argument is the program to run. The language argument is the
+// type of input file as passed to the GCC driver's -x option.
+//
+// For these tools we have no -V=full option to dump the build ID,
+// but we can run the tool with -v -### to reliably get the compiler proper
+// and hash that. That will work in the presence of -toolexec.
+//
+// In order to get reproducible builds for released compilers, we
+// detect a released compiler by the absence of "experimental" in the
+// --version output, and in that case we just use the version string.
+func (b *Builder) gccgoToolID(name, language string) (string, error) {
+	key := name + "." + language
+	b.id.Lock()
+	id := b.toolIDCache[key]
+	b.id.Unlock()
+
+	if id != "" {
+		return id, nil
+	}
+
+	// Invoke the driver with -### to see the subcommands and the
+	// version strings. Use -x to set the language. Pretend to
+	// compile an empty file on standard input.
+	cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-")
+	cmd := exec.Command(cmdline[0], cmdline[1:]...)
+	cmd.Env = base.EnvForDir(cmd.Dir, os.Environ())
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("%s: %v; output: %q", name, err, out)
+	}
+
+	version := ""
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" {
+			version = line
+			break
+		}
+	}
+	if version == "" {
+		return "", fmt.Errorf("%s: can not find version number in %q", name, out)
+	}
+
+	if !strings.Contains(version, "experimental") {
+		// This is a release. Use this line as the tool ID.
+		id = version
+	} else {
+		// This is a development version. The first line with
+		// a leading space is the compiler proper.
+		compiler := ""
+		for _, line := range lines {
+			if len(line) > 1 && line[0] == ' ' {
+				compiler = line
+				break
+			}
+		}
+		if compiler == "" {
+			return "", fmt.Errorf("%s: can not find compilation command in %q", name, out)
+		}
+
+		fields := strings.Fields(compiler)
+		if len(fields) == 0 {
+			return "", fmt.Errorf("%s: compilation command confusion %q", name, out)
+		}
+		exe := fields[0]
+		if !strings.ContainsAny(exe, `/\`) {
+			if lp, err := exec.LookPath(exe); err == nil {
+				exe = lp
+			}
+		}
+		if _, err := os.Stat(exe); err != nil {
+			return "", fmt.Errorf("%s: can not find compiler %q: %v; output %q", name, exe, err, out)
+		}
+		id = b.fileHash(exe)
+	}
+
+	b.id.Lock()
+	b.toolIDCache[name] = id
+	b.id.Unlock()
+
+	return id, nil
+}
+
+// gccgoBuildIDELFFile creates an assembler file that records the
+// action's build ID in an SHF_EXCLUDE section.
+func (b *Builder) gccgoBuildIDELFFile(a *Action) (string, error) {
+	sfile := a.Objdir + "_buildid.s"
+
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n")
+	fmt.Fprintf(&buf, "\t.byte ")
+	for i := 0; i < len(a.buildID); i++ {
+		if i > 0 {
+			if i%8 == 0 {
+				fmt.Fprintf(&buf, "\n\t.byte ")
+			} else {
+				fmt.Fprintf(&buf, ",")
+			}
+		}
+		fmt.Fprintf(&buf, "%#02x", a.buildID[i])
+	}
+	fmt.Fprintf(&buf, "\n")
+	fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",@progbits`+"\n")
+	fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",@progbits`+"\n")
+
+	if cfg.BuildN || cfg.BuildX {
+		for _, line := range bytes.Split(buf.Bytes(), []byte("\n")) {
+			b.Showcmd("", "echo '%s' >> %s", line, sfile)
+		}
+		if cfg.BuildN {
+			return sfile, nil
+		}
+	}
+
+	if err := ioutil.WriteFile(sfile, buf.Bytes(), 0666); err != nil {
+		return "", err
+	}
+
+	return sfile, nil
+}
+
 // buildID returns the build ID found in the given file.
 // If no build ID is found, buildID returns the content hash of the file.
 func (b *Builder) buildID(file string) string {

src/cmd/go/internal/work/exec.go

@@ -252,6 +252,20 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
 			// essentially unfindable.
 			fmt.Fprintf(h, "nocache %d\n", time.Now().UnixNano())
 		}
+
+	case "gccgo":
+		id, err := b.gccgoToolID(BuildToolchain.compiler(), "go")
+		if err != nil {
+			base.Fatalf("%v", err)
+		}
+		fmt.Fprintf(h, "compile %s %q %q\n", id, forcedGccgoflags, p.Internal.Gccgoflags)
+		fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p))
+		if len(p.SFiles) > 0 {
+			id, err = b.gccgoToolID(BuildToolchain.compiler(), "assembler-with-cpp")
+			// Ignore error; different assembler versions
+			// are unlikely to make any difference anyhow.
+			fmt.Fprintf(h, "asm %q\n", id)
+		}
 	}
 
 	// Input files.
@@ -608,6 +622,24 @@ func (b *Builder) build(a *Action) (err error) {
 		objects = append(objects, ofiles...)
 	}
 
+	// For gccgo on ELF systems, we write the build ID as an assembler file.
+	// This lets us set the the SHF_EXCLUDE flag.
+	// This is read by readGccgoArchive in cmd/internal/buildid/buildid.go.
+	if a.buildID != "" && cfg.BuildToolchainName == "gccgo" {
+		switch cfg.Goos {
+		case "android", "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris":
+			asmfile, err := b.gccgoBuildIDELFFile(a)
+			if err != nil {
+				return err
+			}
+			ofiles, err := BuildToolchain.asm(b, a, []string{asmfile})
+			if err != nil {
+				return err
+			}
+			objects = append(objects, ofiles...)
+		}
+	}
+
 	// NOTE(rsc): On Windows, it is critically important that the
 	// gcc-compiled objects (cgoObjects) be listed after the ordinary
 	// objects in the archive. I do not know why this is.
@@ -692,12 +724,17 @@ func (b *Builder) vet(a *Action) error {
 		return err
 	}
 
+	var env []string
+	if cfg.BuildToolchainName == "gccgo" {
+		env = append(env, "GCCGO="+BuildToolchain.compiler())
+	}
+
 	p := a.Package
 	tool := VetTool
 	if tool == "" {
 		tool = base.Tool("vet")
 	}
-	return b.run(a, p.Dir, p.ImportPath, nil, cfg.BuildToolexec, tool, VetFlags, a.Objdir+"vet.cfg")
+	return b.run(a, p.Dir, p.ImportPath, env, cfg.BuildToolexec, tool, VetFlags, a.Objdir+"vet.cfg")
 }
 
 // linkActionID computes the action ID for a link action.
@@ -776,6 +813,14 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) {
 
 		// TODO(rsc): Do cgo settings and flags need to be included?
 		// Or external linker settings and flags?
+
+	case "gccgo":
+		id, err := b.gccgoToolID(BuildToolchain.linker(), "go")
+		if err != nil {
+			base.Fatalf("%v", err)
+		}
+		fmt.Fprintf(h, "link %s %s\n", id, ldBuildmode)
+		// TODO(iant): Should probably include cgo flags here.
 	}
 }
 

src/cmd/go/internal/work/gccgo.go

@@ -154,7 +154,8 @@ func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]strin
 	p := a.Package
 	var ofiles []string
 	for _, sfile := range sfiles {
-		ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o"
+		base := filepath.Base(sfile)
+		ofile := a.Objdir + base[:len(base)-len(".s")] + ".o"
 		ofiles = append(ofiles, ofile)
 		sfile = mkAbs(p.Dir, sfile)
 		defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch}
@@ -285,7 +286,7 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string
 			// doesn't work.
 			if !apackagePathsSeen[a.Package.ImportPath] {
 				apackagePathsSeen[a.Package.ImportPath] = true
-				target := a.Target
+				target := a.built
 				if len(a.Package.CgoFiles) > 0 || a.Package.UsesSwig() {
 					target, err = readAndRemoveCgoFlags(target)
 					if err != nil {
@@ -353,6 +354,15 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string
 
 	ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)")
 
+	if root.buildID != "" {
+		// On systems that normally use gold or the GNU linker,
+		// use the --build-id option to write a GNU build ID note.
+		switch cfg.Goos {
+		case "android", "dragonfly", "linux", "netbsd":
+			ldflags = append(ldflags, fmt.Sprintf("-Wl,--build-id=0x%x", root.buildID))
+		}
+	}
+
 	for _, shlib := range shlibs {
 		ldflags = append(
 			ldflags,
@@ -392,7 +402,9 @@ func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string
 		}
 
 		// We are creating an object file, so we don't want a build ID.
-		ldflags = b.disableBuildID(ldflags)
+		if root.buildID == "" {
+			ldflags = b.disableBuildID(ldflags)
+		}
 
 		realOut = out
 		out = out + ".o"

src/cmd/internal/buildid/buildid.go

@@ -6,10 +6,12 @@ package buildid
 
 import (
 	"bytes"
+	"debug/elf"
 	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"strings"
 )
 
 var (
@@ -26,8 +28,6 @@ var (
 )
 
 // ReadFile reads the build ID from an archive or executable file.
-// It only supports archives from the gc toolchain.
-// TODO(rsc): Figure out what gccgo and llvm are going to do for archives.
 func ReadFile(name string) (id string, err error) {
 	f, err := os.Open(name)
 	if err != nil {
@@ -59,30 +59,30 @@ func ReadFile(name string) (id string, err error) {
 		return "", err
 	}
 
-	bad := func() (string, error) {
-		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+	tryGccgo := func() (string, error) {
+		return readGccgoArchive(name, f)
 	}
 
 	// Archive header.
 	for i := 0; ; i++ { // returns during i==3
 		j := bytes.IndexByte(data, '\n')
 		if j < 0 {
-			return bad()
+			return tryGccgo()
 		}
 		line := data[:j]
 		data = data[j+1:]
 		switch i {
 		case 0:
 			if !bytes.Equal(line, bangArch) {
-				return bad()
+				return tryGccgo()
 			}
 		case 1:
 			if !bytes.HasPrefix(line, pkgdef) {
-				return bad()
+				return tryGccgo()
 			}
 		case 2:
 			if !bytes.HasPrefix(line, goobject) {
-				return bad()
+				return tryGccgo()
 			}
 		case 3:
 			if !bytes.HasPrefix(line, buildid) {
@@ -92,13 +92,71 @@ func ReadFile(name string) (id string, err error) {
 			}
 			id, err := strconv.Unquote(string(line[len(buildid):]))
 			if err != nil {
-				return bad()
+				return tryGccgo()
 			}
 			return id, nil
 		}
 	}
 }
 
+// readGccgoArchive tries to parse the archive as a standard Unix
+// archive file, and fetch the build ID from the _buildid.o entry.
+// The _buildid.o entry is written by (*Builder).gccgoBuildIDELFFile
+// in cmd/go/internal/work/exec.go.
+func readGccgoArchive(name string, f *os.File) (string, error) {
+	bad := func() (string, error) {
+		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+	}
+
+	off := int64(8)
+	for {
+		if _, err := f.Seek(off, io.SeekStart); err != nil {
+			return "", err
+		}
+
+		// TODO(iant): Make a debug/ar package, and use it
+		// here and in cmd/link.
+		var hdr [60]byte
+		if _, err := io.ReadFull(f, hdr[:]); err != nil {
+			if err == io.EOF {
+				// No more entries, no build ID.
+				return "", nil
+			}
+			return "", err
+		}
+		off += 60
+
+		sizeStr := strings.TrimSpace(string(hdr[48:58]))
+		size, err := strconv.ParseInt(sizeStr, 0, 64)
+		if err != nil {
+			return bad()
+		}
+
+		name := strings.TrimSpace(string(hdr[:16]))
+		if name == "_buildid.o/" {
+			sr := io.NewSectionReader(f, off, size)
+			e, err := elf.NewFile(sr)
+			if err != nil {
+				return bad()
+			}
+			s := e.Section(".go.buildid")
+			if s == nil {
+				return bad()
+			}
+			data, err := s.Data()
+			if err != nil {
+				return bad()
+			}
+			return string(data), nil
+		}
+
+		off += size
+		if off&1 != 0 {
+			off++
+		}
+	}
+}
+
 var (
 	goBuildPrefix = []byte("\xff Go build ID: \"")
 	goBuildEnd    = []byte("\"\n \xff")

src/cmd/internal/buildid/note.go

@@ -69,6 +69,7 @@ func ReadELFNote(filename, name string, typ int32) ([]byte, error) {
 }
 
 var elfGoNote = []byte("Go\x00\x00")
+var elfGNUNote = []byte("GNU\x00")
 
 // The Go build ID is stored in a note described by an ELF PT_NOTE prog
 // header. The caller has already opened filename, to get f, and read
@@ -90,11 +91,13 @@ func readELF(name string, f *os.File, data []byte) (buildid string, err error) {
 	}
 
 	const elfGoBuildIDTag = 4
+	const gnuBuildIDTag = 3
 
 	ef, err := elf.NewFile(bytes.NewReader(data))
 	if err != nil {
 		return "", &os.PathError{Path: name, Op: "parse", Err: err}
 	}
+	var gnu string
 	for _, p := range ef.Progs {
 		if p.Type != elf.PT_NOTE || p.Filesz < 16 {
 			continue
@@ -123,26 +126,42 @@ func readELF(name string, f *os.File, data []byte) (buildid string, err error) {
 		}
 
 		filesz := p.Filesz
+		off := p.Off
 		for filesz >= 16 {
 			nameSize := ef.ByteOrder.Uint32(note)
 			valSize := ef.ByteOrder.Uint32(note[4:])
 			tag := ef.ByteOrder.Uint32(note[8:])
-			name := note[12:16]
-			if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(name, elfGoNote) {
+			nname := note[12:16]
+			if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(nname, elfGoNote) {
 				return string(note[16 : 16+valSize]), nil
 			}
 
+			if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == gnuBuildIDTag && bytes.Equal(nname, elfGNUNote) {
+				gnu = string(note[16 : 16+valSize])
+			}
+
 			nameSize = (nameSize + 3) &^ 3
 			valSize = (valSize + 3) &^ 3
 			notesz := uint64(12 + nameSize + valSize)
 			if filesz <= notesz {
 				break
 			}
+			off += notesz
+			align := uint64(p.Align)
+			alignedOff := (off + align - 1) &^ (align - 1)
+			notesz += alignedOff - off
+			off = alignedOff
 			filesz -= notesz
 			note = note[notesz:]
 		}
 	}
 
+	// If we didn't find a Go note, use a GNU note if available.
+	// This is what gccgo uses.
+	if gnu != "" {
+		return gnu, nil
+	}
+
 	// No note. Treat as successful but build ID empty.
 	return "", nil
 }