From dc8453964a519032bfd338a2cc027d038c9b2e79 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 11 Sep 2019 16:03:59 -0400 Subject: [PATCH 01/79] [dev.link] cmd/compile: finish all data generation before writing object file Currently, at the end of compilation, the compiler writes out the export data, the linker object file header, then does more code/data generation, then writes the main content of the linker object file. This CL refactors it to finish all the code/data generation before writing any output file. A later CL will inject some code that operates on all defined symbols before writing the output. This ensures all the symbols are available at that point. Change-Id: I97d946553fd0ffd298234c520219540d29783576 Reviewed-on: https://go-review.googlesource.com/c/go/+/196027 Reviewed-by: Than McIntosh Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/main.go | 1 + src/cmd/compile/internal/gc/obj.go | 32 +++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index f75e35c3be..eec5ece0db 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -723,6 +723,7 @@ func Main(archInit func(*Arch)) { // Write object data to disk. timings.Start("be", "dumpobj") + dumpdata() dumpobj() if asmhdr != "" { dumpasmhdr() diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index be13b27892..e703e8a302 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -111,21 +111,7 @@ func dumpCompilerObj(bout *bio.Writer) { dumpexport(bout) } -func dumpLinkerObj(bout *bio.Writer) { - printObjHeader(bout) - - if len(pragcgobuf) != 0 { - // write empty export section; must be before cgo section - fmt.Fprintf(bout, "\n$$\n\n$$\n\n") - fmt.Fprintf(bout, "\n$$ // cgo\n") - if err := json.NewEncoder(bout).Encode(pragcgobuf); err != nil { - Fatalf("serializing pragcgobuf: %v", err) - } - fmt.Fprintf(bout, "\n$$\n\n") - } - - fmt.Fprintf(bout, "\n!\n") - +func dumpdata() { externs := len(externdcl) dumpglobls() @@ -163,6 +149,22 @@ func dumpLinkerObj(bout *bio.Writer) { } addGCLocals() +} + +func dumpLinkerObj(bout *bio.Writer) { + printObjHeader(bout) + + if len(pragcgobuf) != 0 { + // write empty export section; must be before cgo section + fmt.Fprintf(bout, "\n$$\n\n$$\n\n") + fmt.Fprintf(bout, "\n$$ // cgo\n") + if err := json.NewEncoder(bout).Encode(pragcgobuf); err != nil { + Fatalf("serializing pragcgobuf: %v", err) + } + fmt.Fprintf(bout, "\n$$\n\n") + } + + fmt.Fprintf(bout, "\n!\n") obj.WriteObjFile(Ctxt, bout.Writer, myimportpath) } From cd75cf4bc02a741f7b59e30dd1170364000fd134 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 11 Sep 2019 16:11:31 -0400 Subject: [PATCH 02/79] [dev.link] cmd/compile, cmd/asm: use bio.Writer for object file writing It is convenient to have a seekable writer. A later CL will make use of Seek. Change-Id: Iba0107ce2975d9a451d97f16aa91a318dd4c90e2 Reviewed-on: https://go-review.googlesource.com/c/go/+/196028 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/asm/main.go | 8 +++----- src/cmd/compile/internal/gc/obj.go | 2 +- src/cmd/internal/obj/objfile.go | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cmd/asm/main.go b/src/cmd/asm/main.go index fc6acc74c0..91b48975d2 100644 --- a/src/cmd/asm/main.go +++ b/src/cmd/asm/main.go @@ -46,12 +46,11 @@ func main() { architecture.Init(ctxt) // Create object file, write header. - out, err := os.Create(*flags.OutputFile) + buf, err := bio.Create(*flags.OutputFile) if err != nil { log.Fatal(err) } - defer bio.MustClose(out) - buf := bufio.NewWriter(bio.MustWriter(out)) + defer buf.Close() if !*flags.SymABIs { fmt.Fprintf(buf, "go object %s %s %s\n", objabi.GOOS, objabi.GOARCH, objabi.Version) @@ -91,9 +90,8 @@ func main() { } else { log.Print("assembly failed") } - out.Close() + buf.Close() os.Remove(*flags.OutputFile) os.Exit(1) } - buf.Flush() } diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index e703e8a302..ae0fc1dbc1 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -166,7 +166,7 @@ func dumpLinkerObj(bout *bio.Writer) { fmt.Fprintf(bout, "\n!\n") - obj.WriteObjFile(Ctxt, bout.Writer, myimportpath) + obj.WriteObjFile(Ctxt, bout, myimportpath) } func addptabs() { diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index ab5627c0dd..c51a11c51f 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -8,6 +8,7 @@ package obj import ( "bufio" + "cmd/internal/bio" "cmd/internal/dwarf" "cmd/internal/objabi" "cmd/internal/sys" @@ -80,7 +81,8 @@ func newObjWriter(ctxt *Link, b *bufio.Writer, pkgpath string) *objWriter { } } -func WriteObjFile(ctxt *Link, b *bufio.Writer, pkgpath string) { +func WriteObjFile(ctxt *Link, bout *bio.Writer, pkgpath string) { + b := bout.Writer w := newObjWriter(ctxt, b, pkgpath) // Magic header From 53b7c18284a404de6ca814bc3313d980b8e5ecc3 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 11 Sep 2019 16:17:01 -0400 Subject: [PATCH 03/79] [dev.link] cmd/compile, cmd/asm: assign index to symbols We are planning to use indices for symbol references, instead of symbol names. Here we assign indices to symbols defined in the package being compiled, and propagate the indices to the dependent packages in the export data. A symbol is referenced by a tuple, (package index, symbol index). Normally, for a given symbol, this index is unique, and the symbol index is globally consistent (but with exceptions, see below). The package index is local to a compilation. For example, when compiling the fmt package, fmt.Println gets assigned index 25, then all packages that reference fmt.Println will refer it as (X, 25) with some X. X is the index for the fmt package, which may differ in different compilations. There are some symbols that do not have clear package affiliation, such as dupOK symbols and linknamed symbols. We cannot give them globally consistent indices. We categorize them as non-package symbols, assign them with package index 1 and a symbol index that is only meaningful locally. Currently nothing will consume the indices. All this is behind a flag, -newobj. The flag needs to be set for all builds (-gcflags=all=-newobj -asmflags=all=-newobj), or none. Change-Id: I18e489c531e9a9fbc668519af92c6116b7308cab Reviewed-on: https://go-review.googlesource.com/c/go/+/196029 Reviewed-by: Than McIntosh --- src/cmd/asm/internal/flags/flags.go | 1 + src/cmd/asm/main.go | 2 + src/cmd/compile/internal/gc/iexport.go | 14 +++ src/cmd/compile/internal/gc/iimport.go | 17 ++++ src/cmd/compile/internal/gc/main.go | 5 +- src/cmd/compile/internal/types/sym.go | 15 +++- src/cmd/internal/obj/link.go | 30 ++++++- src/cmd/internal/obj/sizeof_test.go | 2 +- src/cmd/internal/obj/sym.go | 114 +++++++++++++++++++++++++ 9 files changed, 194 insertions(+), 6 deletions(-) diff --git a/src/cmd/asm/internal/flags/flags.go b/src/cmd/asm/internal/flags/flags.go index 5fe3fd9d53..fad87b221a 100644 --- a/src/cmd/asm/internal/flags/flags.go +++ b/src/cmd/asm/internal/flags/flags.go @@ -23,6 +23,7 @@ var ( Dynlink = flag.Bool("dynlink", false, "support references to Go symbols defined in other shared libraries") AllErrors = flag.Bool("e", false, "no limit on number of errors reported") SymABIs = flag.Bool("gensymabis", false, "write symbol ABI information to output file, don't assemble") + Newobj = flag.Bool("newobj", false, "use new object file format") ) var ( diff --git a/src/cmd/asm/main.go b/src/cmd/asm/main.go index 91b48975d2..6b0a609071 100644 --- a/src/cmd/asm/main.go +++ b/src/cmd/asm/main.go @@ -40,6 +40,7 @@ func main() { } ctxt.Flag_dynlink = *flags.Dynlink ctxt.Flag_shared = *flags.Shared || *flags.Dynlink + ctxt.Flag_newobj = *flags.Newobj ctxt.Bso = bufio.NewWriter(os.Stdout) defer ctxt.Bso.Flush() @@ -82,6 +83,7 @@ func main() { } } if ok && !*flags.SymABIs { + ctxt.NumberSyms(true) obj.WriteObjFile(ctxt, buf, "") } if !ok || diag { diff --git a/src/cmd/compile/internal/gc/iexport.go b/src/cmd/compile/internal/gc/iexport.go index 873de46fa4..da81331b82 100644 --- a/src/cmd/compile/internal/gc/iexport.go +++ b/src/cmd/compile/internal/gc/iexport.go @@ -202,6 +202,7 @@ import ( "bufio" "bytes" "cmd/compile/internal/types" + "cmd/internal/obj" "cmd/internal/src" "encoding/binary" "fmt" @@ -932,10 +933,12 @@ func (w *exportWriter) string(s string) { w.uint64(w.p.stringOff(s)) } func (w *exportWriter) varExt(n *Node) { w.linkname(n.Sym) + w.symIdx(n.Sym) } func (w *exportWriter) funcExt(n *Node) { w.linkname(n.Sym) + w.symIdx(n.Sym) // Escape analysis. for _, fs := range types.RecvsParams { @@ -974,6 +977,17 @@ func (w *exportWriter) linkname(s *types.Sym) { w.string(s.Linkname) } +func (w *exportWriter) symIdx(s *types.Sym) { + if Ctxt.Flag_newobj { + lsym := s.Linksym() + if lsym.PkgIdx > obj.PkgIdxSelf || lsym.PkgIdx == obj.PkgIdxInvalid || s.Linkname != "" { + w.int64(-1) + } else { + w.int64(int64(lsym.SymIdx)) + } + } +} + // Inline bodies. func (w *exportWriter) stmtList(list Nodes) { diff --git a/src/cmd/compile/internal/gc/iimport.go b/src/cmd/compile/internal/gc/iimport.go index 28808c51c5..96d7e0257a 100644 --- a/src/cmd/compile/internal/gc/iimport.go +++ b/src/cmd/compile/internal/gc/iimport.go @@ -10,6 +10,7 @@ package gc import ( "cmd/compile/internal/types" "cmd/internal/bio" + "cmd/internal/obj" "cmd/internal/src" "encoding/binary" "fmt" @@ -650,10 +651,12 @@ func (r *importReader) byte() byte { func (r *importReader) varExt(n *Node) { r.linkname(n.Sym) + r.symIdx(n.Sym) } func (r *importReader) funcExt(n *Node) { r.linkname(n.Sym) + r.symIdx(n.Sym) // Escape analysis. for _, fs := range types.RecvsParams { @@ -682,6 +685,20 @@ func (r *importReader) linkname(s *types.Sym) { s.Linkname = r.string() } +func (r *importReader) symIdx(s *types.Sym) { + if Ctxt.Flag_newobj { + lsym := s.Linksym() + idx := int32(r.int64()) + if idx != -1 { + if s.Linkname != "" { + Fatalf("bad index for linknamed symbol: %v %d\n", lsym, idx) + } + lsym.SymIdx = idx + lsym.Set(obj.AttrIndexed, true) + } + } +} + func (r *importReader) doInline(n *Node) { if len(n.Func.Inl.Body) != 0 { Fatalf("%v already has inline body", n) diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index eec5ece0db..78d702d868 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -264,12 +264,14 @@ func Main(archInit func(*Arch)) { flag.StringVar(&benchfile, "bench", "", "append benchmark times to `file`") flag.BoolVar(&smallFrames, "smallframes", false, "reduce the size limit for stack allocated objects") flag.BoolVar(&Ctxt.UseBASEntries, "dwarfbasentries", Ctxt.UseBASEntries, "use base address selection entries in DWARF") + flag.BoolVar(&Ctxt.Flag_newobj, "newobj", false, "use new object file format") + objabi.Flagparse(usage) // Record flags that affect the build result. (And don't // record flags that don't, since that would cause spurious // changes in the binary.) - recordFlags("B", "N", "l", "msan", "race", "shared", "dynlink", "dwarflocationlists", "dwarfbasentries", "smallframes") + recordFlags("B", "N", "l", "msan", "race", "shared", "dynlink", "dwarflocationlists", "dwarfbasentries", "smallframes", "newobj") if smallFrames { maxStackVarSize = 128 * 1024 @@ -724,6 +726,7 @@ func Main(archInit func(*Arch)) { // Write object data to disk. timings.Start("be", "dumpobj") dumpdata() + Ctxt.NumberSyms(false) dumpobj() if asmhdr != "" { dumpasmhdr() diff --git a/src/cmd/compile/internal/types/sym.go b/src/cmd/compile/internal/types/sym.go index c9dd9f399e..d43efd3bd0 100644 --- a/src/cmd/compile/internal/types/sym.go +++ b/src/cmd/compile/internal/types/sym.go @@ -76,15 +76,24 @@ func (sym *Sym) LinksymName() string { return sym.Pkg.Prefix + "." + sym.Name } -func (sym *Sym) Linksym() *obj.LSym { +func (sym *Sym) Linksym() (r *obj.LSym) { if sym == nil { return nil } if sym.Func() { // This is a function symbol. Mark it as "internal ABI". - return Ctxt.LookupABI(sym.LinksymName(), obj.ABIInternal) + r = Ctxt.LookupABI(sym.LinksymName(), obj.ABIInternal) + } else { + r = Ctxt.Lookup(sym.LinksymName()) } - return Ctxt.Lookup(sym.LinksymName()) + if r.Pkg == "" { + if sym.Linkname != "" { + r.Pkg = "_" + } else { + r.Pkg = sym.Pkg.Prefix + } + } + return } // Less reports whether symbol a is ordered before symbol b. diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index 1c101bfc27..f1cf342d3d 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -388,6 +388,10 @@ type LSym struct { R []Reloc Func *FuncInfo + + Pkg string + PkgIdx int32 + SymIdx int32 // TODO: replace RefIdx } // A FuncInfo contains extra fields for STEXT symbols. @@ -460,7 +464,7 @@ const ( ) // Attribute is a set of symbol attributes. -type Attribute uint16 +type Attribute uint32 const ( AttrDuplicateOK Attribute = 1 << iota @@ -501,6 +505,10 @@ const ( // keep unwinding beyond this frame. AttrTopFrame + // Indexed indicates this symbol has been assigned with an index (when using the + // new object file format). + AttrIndexed + // attrABIBase is the value at which the ABI is encoded in // Attribute. This must be last; all bits after this are // assumed to be an ABI value. @@ -524,6 +532,7 @@ func (a Attribute) NoFrame() bool { return a&AttrNoFrame != 0 } func (a Attribute) Static() bool { return a&AttrStatic != 0 } func (a Attribute) WasInlined() bool { return a&AttrWasInlined != 0 } func (a Attribute) TopFrame() bool { return a&AttrTopFrame != 0 } +func (a Attribute) Indexed() bool { return a&AttrIndexed != 0 } func (a *Attribute) Set(flag Attribute, value bool) { if value { @@ -558,6 +567,7 @@ var textAttrStrings = [...]struct { {bit: AttrStatic, s: "STATIC"}, {bit: AttrWasInlined, s: ""}, {bit: AttrTopFrame, s: "TOPFRAME"}, + {bit: AttrIndexed, s: ""}, } // TextAttrString formats a for printing in as part of a TEXT prog. @@ -626,6 +636,15 @@ type Pcdata struct { P []byte } +// Package Index. +const ( + PkgIdxNone = (1<<31 - 1) - iota // Non-package symbols + PkgIdxBuiltin // Predefined symbols // TODO: not used for now, we could use it for compiler-generated symbols like runtime.newobject + PkgIdxSelf // Symbols defined in the current package + PkgIdxInvalid = 0 + // The index of other referenced packages starts from 1. +) + // Link holds the context for writing object code from a compiler // to be linker input or for reading that input into the linker. type Link struct { @@ -638,6 +657,7 @@ type Link struct { Flag_dynlink bool Flag_optimize bool Flag_locationlists bool + Flag_newobj bool // use new object file format Bso *bufio.Writer Pathname string hashmu sync.Mutex // protects hash, funchash @@ -671,6 +691,14 @@ type Link struct { // TODO(austin): Replace this with ABI wrappers once the ABIs // actually diverge. ABIAliases []*LSym + + // pkgIdx maps package path to index. The index is used for + // symbol reference in the object file. + pkgIdx map[string]int32 + + defs []*LSym // list of defined symbols in the current package + nonpkgdefs []*LSym // list of defined non-package symbols + nonpkgrefs []*LSym // list of referenced non-package symbols } func (ctxt *Link) Diag(format string, args ...interface{}) { diff --git a/src/cmd/internal/obj/sizeof_test.go b/src/cmd/internal/obj/sizeof_test.go index e70d174637..306bf10ee6 100644 --- a/src/cmd/internal/obj/sizeof_test.go +++ b/src/cmd/internal/obj/sizeof_test.go @@ -23,7 +23,7 @@ func TestSizeof(t *testing.T) { _64bit uintptr // size on 64bit platforms }{ {Addr{}, 32, 48}, - {LSym{}, 56, 104}, + //{LSym{}, 56, 104}, // TODO: re-enable {Prog{}, 132, 200}, } diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index 15a501c3aa..c4eabe7806 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -147,3 +147,117 @@ func (ctxt *Link) Int64Sym(i int64) *LSym { s.Set(AttrLocal, true) }) } + +// Assign index to symbols. +// asm is set to true if this is called by the assembler (i.e. not the compiler), +// in which case all the symbols are non-package (for now). +func (ctxt *Link) NumberSyms(asm bool) { + if !ctxt.Flag_newobj { + return + } + + ctxt.pkgIdx = make(map[string]int32) + ctxt.defs = []*LSym{} + ctxt.nonpkgdefs = []*LSym{} + + var idx, nonpkgidx int32 = 0, 0 + ctxt.traverseSyms(traverseDefs, func(s *LSym) { + if asm || s.Pkg == "_" || s.DuplicateOK() { + s.PkgIdx = PkgIdxNone + s.SymIdx = nonpkgidx + if nonpkgidx != int32(len(ctxt.nonpkgdefs)) { + panic("bad index") + } + ctxt.nonpkgdefs = append(ctxt.nonpkgdefs, s) + nonpkgidx++ + } else { + s.PkgIdx = PkgIdxSelf + s.SymIdx = idx + if idx != int32(len(ctxt.defs)) { + panic("bad index") + } + ctxt.defs = append(ctxt.defs, s) + idx++ + } + s.Set(AttrIndexed, true) + }) + + ipkg := int32(1) // 0 is invalid index + nonpkgdef := nonpkgidx + ctxt.traverseSyms(traverseRefs|traverseAux, func(rs *LSym) { + if rs.PkgIdx != PkgIdxInvalid { + return + } + pkg := rs.Pkg + if pkg == "" || pkg == "\"\"" || pkg == "_" || !rs.Indexed() { + rs.PkgIdx = PkgIdxNone + rs.SymIdx = nonpkgidx + rs.Set(AttrIndexed, true) + if nonpkgidx != nonpkgdef+int32(len(ctxt.nonpkgrefs)) { + panic("bad index") + } + ctxt.nonpkgrefs = append(ctxt.nonpkgrefs, rs) + nonpkgidx++ + return + } + if k, ok := ctxt.pkgIdx[pkg]; ok { + rs.PkgIdx = k + return + } + rs.PkgIdx = ipkg + ctxt.pkgIdx[pkg] = ipkg + ipkg++ + }) +} + +type traverseFlag uint32 + +const ( + traverseDefs traverseFlag = 1 << iota + traverseRefs + traverseAux + + traverseAll = traverseDefs | traverseRefs | traverseAux +) + +// Traverse symbols based on flag, call fn for each symbol. +func (ctxt *Link) traverseSyms(flag traverseFlag, fn func(*LSym)) { + lists := [][]*LSym{ctxt.Text, ctxt.Data, ctxt.ABIAliases} + for _, list := range lists { + for _, s := range list { + if flag&traverseDefs != 0 { + fn(s) + } + if flag&traverseRefs != 0 { + for _, r := range s.R { + if r.Sym != nil { + fn(r.Sym) + } + } + } + if flag&traverseAux != 0 { + if s.Gotype != nil { + fn(s.Gotype) + } + if s.Type == objabi.STEXT { + pc := &s.Func.Pcln + for _, d := range pc.Funcdata { + if d != nil { + fn(d) + } + } + for _, f := range pc.File { + if fsym := ctxt.Lookup(f); fsym != nil { + fn(fsym) + } + } + for _, call := range pc.InlTree.nodes { + if call.Func != nil { + fn(call.Func) + } + } + } + } + } + } +} From a09cd8ccb372d760bd21d7e56f42b9dbea6ecc2b Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 2 Oct 2019 17:45:05 -0400 Subject: [PATCH 04/79] [dev.link] cmd/compile: fix data race on LSym.Pkg LSym may be created concurrently. Reading/writing LSym.Pkg may cause data race (see https://build.golang.org/log/f0351c5cc7bf4c92e3aa5e78e294c2d009ebf118). Fix this by setting LSym.Pkg only when holding the lock. Change-Id: Ib3160ecf47c4ca530b09369e0e8284db6597cfd0 Reviewed-on: https://go-review.googlesource.com/c/go/+/198492 Reviewed-by: Than McIntosh --- src/cmd/compile/internal/types/sym.go | 16 +++++++--------- src/cmd/internal/obj/sym.go | 10 ++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/cmd/compile/internal/types/sym.go b/src/cmd/compile/internal/types/sym.go index d43efd3bd0..07bce4d5cd 100644 --- a/src/cmd/compile/internal/types/sym.go +++ b/src/cmd/compile/internal/types/sym.go @@ -76,24 +76,22 @@ func (sym *Sym) LinksymName() string { return sym.Pkg.Prefix + "." + sym.Name } -func (sym *Sym) Linksym() (r *obj.LSym) { +func (sym *Sym) Linksym() *obj.LSym { if sym == nil { return nil } - if sym.Func() { - // This is a function symbol. Mark it as "internal ABI". - r = Ctxt.LookupABI(sym.LinksymName(), obj.ABIInternal) - } else { - r = Ctxt.Lookup(sym.LinksymName()) - } - if r.Pkg == "" { + initPkg := func(r *obj.LSym) { if sym.Linkname != "" { r.Pkg = "_" } else { r.Pkg = sym.Pkg.Prefix } } - return + if sym.Func() { + // This is a function symbol. Mark it as "internal ABI". + return Ctxt.LookupABIInit(sym.LinksymName(), obj.ABIInternal, initPkg) + } + return Ctxt.LookupInit(sym.LinksymName(), initPkg) } // Less reports whether symbol a is ordered before symbol b. diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index c4eabe7806..e47c511ddc 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -78,6 +78,13 @@ func (ctxt *Link) LookupStatic(name string) *LSym { // LookupABI looks up a symbol with the given ABI. // If it does not exist, it creates it. func (ctxt *Link) LookupABI(name string, abi ABI) *LSym { + return ctxt.LookupABIInit(name, abi, nil) +} + +// LookupABI looks up a symbol with the given ABI. +// If it does not exist, it creates it and +// passes it to init for one-time initialization. +func (ctxt *Link) LookupABIInit(name string, abi ABI, init func(s *LSym)) *LSym { var hash map[string]*LSym switch abi { case ABI0: @@ -94,6 +101,9 @@ func (ctxt *Link) LookupABI(name string, abi ABI) *LSym { s = &LSym{Name: name} s.SetABI(abi) hash[name] = s + if init != nil { + init(s) + } } ctxt.hashmu.Unlock() return s From 2c484c0356c5e13a480b3842de0d345224a7bbf8 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 22 Aug 2019 17:26:41 -0400 Subject: [PATCH 05/79] [dev.link] cmd/internal/obj: write object file in new format If -newobj is set, write object file in new format, which uses indices for symbol references instead of symbol names. The file format is described at the beginning of cmd/internal/goobj2/objfile.go. A new package, cmd/internal/goobj2, is introduced for reading and writing new object files. (The package name is temporary.) It is written in a way that trys to make the encoding as regular as possible, and the reader and writer as symmetric as possible. This is incomplete, and currently nothing will consume the new object file. Change-Id: Ifefedbf6456d760d15a9f40a28af6486c93100fe Reviewed-on: https://go-review.googlesource.com/c/go/+/196030 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/compile/internal/gc/iexport.go | 4 +- src/cmd/dist/buildtool.go | 1 + src/cmd/internal/goobj2/funcinfo.go | 114 ++++++ src/cmd/internal/goobj2/objfile.go | 513 +++++++++++++++++++++++++ src/cmd/internal/obj/link.go | 11 +- src/cmd/internal/obj/objfile.go | 5 + src/cmd/internal/obj/objfile2.go | 350 +++++++++++++++++ src/cmd/internal/obj/sym.go | 9 +- 8 files changed, 992 insertions(+), 15 deletions(-) create mode 100644 src/cmd/internal/goobj2/funcinfo.go create mode 100644 src/cmd/internal/goobj2/objfile.go create mode 100644 src/cmd/internal/obj/objfile2.go diff --git a/src/cmd/compile/internal/gc/iexport.go b/src/cmd/compile/internal/gc/iexport.go index da81331b82..39f7770136 100644 --- a/src/cmd/compile/internal/gc/iexport.go +++ b/src/cmd/compile/internal/gc/iexport.go @@ -202,7 +202,7 @@ import ( "bufio" "bytes" "cmd/compile/internal/types" - "cmd/internal/obj" + "cmd/internal/goobj2" "cmd/internal/src" "encoding/binary" "fmt" @@ -980,7 +980,7 @@ func (w *exportWriter) linkname(s *types.Sym) { func (w *exportWriter) symIdx(s *types.Sym) { if Ctxt.Flag_newobj { lsym := s.Linksym() - if lsym.PkgIdx > obj.PkgIdxSelf || lsym.PkgIdx == obj.PkgIdxInvalid || s.Linkname != "" { + if lsym.PkgIdx > goobj2.PkgIdxSelf || lsym.PkgIdx == goobj2.PkgIdxInvalid || s.Linkname != "" { w.int64(-1) } else { w.int64(int64(lsym.SymIdx)) diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index f27ea17230..e85dd9a660 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -54,6 +54,7 @@ var bootstrapDirs = []string{ "cmd/internal/gcprog", "cmd/internal/dwarf", "cmd/internal/edit", + "cmd/internal/goobj2", "cmd/internal/objabi", "cmd/internal/obj", "cmd/internal/obj/arm", diff --git a/src/cmd/internal/goobj2/funcinfo.go b/src/cmd/internal/goobj2/funcinfo.go new file mode 100644 index 0000000000..5938b5f920 --- /dev/null +++ b/src/cmd/internal/goobj2/funcinfo.go @@ -0,0 +1,114 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package goobj2 + +import ( + "bytes" + "encoding/binary" +) + +// FuncInfo is serialized as a symbol (aux symbol). The symbol data is +// the binary encoding of the struct below. +// +// TODO: make each pcdata a separate symbol? +type FuncInfo struct { + NoSplit uint8 + Flags uint8 + + Args uint32 + Locals uint32 + + Pcsp uint32 + Pcfile uint32 + Pcline uint32 + Pcinline uint32 + Pcdata []uint32 + PcdataEnd uint32 + Funcdataoff []uint32 + File []SymRef // TODO: just use string? + + // TODO: InlTree +} + +const ( + FuncFlagLeaf = 1 << iota + FuncFlagCFunc + FuncFlagReflectMethod + FuncFlagShared // This is really silly + FuncFlagTopFrame +) + +func (a *FuncInfo) Write(w *bytes.Buffer) { + w.WriteByte(a.NoSplit) + w.WriteByte(a.Flags) + + var b [4]byte + writeUint32 := func(x uint32) { + binary.LittleEndian.PutUint32(b[:], x) + w.Write(b[:]) + } + + writeUint32(a.Args) + writeUint32(a.Locals) + + writeUint32(a.Pcsp) + writeUint32(a.Pcfile) + writeUint32(a.Pcline) + writeUint32(a.Pcinline) + writeUint32(uint32(len(a.Pcdata))) + for _, x := range a.Pcdata { + writeUint32(x) + } + writeUint32(a.PcdataEnd) + writeUint32(uint32(len(a.Funcdataoff))) + for _, x := range a.Funcdataoff { + writeUint32(x) + } + writeUint32(uint32(len(a.File))) + for _, f := range a.File { + writeUint32(f.PkgIdx) + writeUint32(f.SymIdx) + } + + // TODO: InlTree +} + +func (a *FuncInfo) Read(b []byte) { + a.NoSplit = b[0] + a.Flags = b[1] + b = b[2:] + + readUint32 := func() uint32 { + x := binary.LittleEndian.Uint32(b) + b = b[4:] + return x + } + + a.Args = readUint32() + a.Locals = readUint32() + + a.Pcsp = readUint32() + a.Pcfile = readUint32() + a.Pcline = readUint32() + a.Pcinline = readUint32() + pcdatalen := readUint32() + a.Pcdata = make([]uint32, pcdatalen) + for i := range a.Pcdata { + a.Pcdata[i] = readUint32() + } + a.PcdataEnd = readUint32() + funcdataofflen := readUint32() + a.Funcdataoff = make([]uint32, funcdataofflen) + for i := range a.Funcdataoff { + a.Funcdataoff[i] = readUint32() + } + filelen := readUint32() + a.File = make([]SymRef, filelen) + for i := range a.File { + a.File[i] = SymRef{readUint32(), readUint32()} + } + + // TODO: InlTree +} diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go new file mode 100644 index 0000000000..eb9290a699 --- /dev/null +++ b/src/cmd/internal/goobj2/objfile.go @@ -0,0 +1,513 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Go new object file format, reading and writing. + +package goobj2 // TODO: replace the goobj package? + +import ( + "cmd/internal/bio" + "encoding/binary" + "errors" + "fmt" + "io" +) + +// New object file format. +// +// Header struct { +// Magic [...]byte // "\x00go114LD" +// // TODO: Fingerprint +// Offsets [...]uint32 // byte offset of each block below +// } +// +// Strings [...]struct { +// Len uint32 +// Data [...]byte +// } +// +// PkgIndex [...]stringOff // TODO: add fingerprints +// +// SymbolDefs [...]struct { +// Name stringOff +// ABI uint16 +// Type uint8 +// Flag uint8 +// Size uint32 +// } +// NonPkgDefs [...]struct { // non-pkg symbol definitions +// ... // same as SymbolDefs +// } +// NonPkgRefs [...]struct { // non-pkg symbol references +// ... // same as SymbolDefs +// } +// +// RelocIndex [...]uint32 // index to Relocs +// AuxIndex [...]uint32 // index to Aux +// DataIndex [...]uint32 // offset to Data +// +// Relocs [...]struct { +// Off int32 +// Size uint8 +// Type uint8 +// Add int64 +// Sym symRef +// } +// +// Aux [...]struct { +// Type uint8 +// Sym symRef +// } +// +// Data [...]byte +// Pcdata [...]byte +// +// stringOff is a uint32 (?) offset that points to the corresponding +// string, which is a uint32 length followed by that number of bytes. +// +// symRef is struct { PkgIdx, SymIdx uint32 }. +// +// Slice type (e.g. []symRef) is encoded as a length prefix (uint32) +// followed by that number of elements. +// +// The types below correspond to the encoded data structure in the +// object file. + +// Symbol indexing. +// +// Each symbol is referenced with a pair of indices, { PkgIdx, SymIdx }, +// as the symRef struct above. +// +// PkgIdx is either a predeclared index (see PkgIdxNone below) or +// an index of an imported package. For the latter case, PkgIdx is the +// index of the package in the PkgIndex array. 0 is an invalid index. +// +// SymIdx is the index of the symbol in the given package. +// - If PkgIdx is PkgIdxSelf, SymIdx is the index of the symbol in the +// SymbolDefs array. +// - If PkgIdx is PkgIdxNone, SymIdx is the index of the symbol in the +// NonPkgDefs array (could natually overflow to NonPkgRefs array). +// - Otherwise, SymIdx is the index of the symbol in some other package's +// SymbolDefs array. +// +// {0, 0} represents a nil symbol. Otherwise PkgIdx should not be 0. +// +// RelocIndex, AuxIndex, and DataIndex contains indices/offsets to +// Relocs/Aux/Data blocks, one element per symbol, first for all the +// defined symbols, then all the defined non-package symbols, in the +// same order of SymbolDefs/NonPkgDefs arrays. For N total defined +// symbols, the array is of length N+1. The last element is the total +// number of relocations (aux symbols, data blocks, etc.). +// +// They can be accessed by index. For the i-th symbol, its relocations +// are the RelocIndex[i]-th (inclusive) to RelocIndex[i+1]-th (exclusive) +// elements in the Relocs array. Aux/Data are likewise. (The index is +// 0-based.) + +// Auxiliary symbols. +// +// Each symbol may (or may not) be associated with a number of auxiliary +// symbols. They are described in the Aux block. See Aux struct below. +// Currently a symbol's Gotype and FuncInfo are auxiliary symbols. We +// may make use of aux symbols in more cases, e.g. DWARF symbols. + +// Package Index. +const ( + PkgIdxNone = (1<<31 - 1) - iota // Non-package symbols + PkgIdxBuiltin // Predefined symbols // TODO: not used for now, we could use it for compiler-generated symbols like runtime.newobject + PkgIdxSelf // Symbols defined in the current package + PkgIdxInvalid = 0 + // The index of other referenced packages starts from 1. +) + +// Blocks +const ( + BlkPkgIdx = iota + BlkSymdef + BlkNonpkgdef + BlkNonpkgref + BlkRelocIdx + BlkAuxIdx + BlkDataIdx + BlkReloc + BlkAux + BlkData + BlkPcdata + NBlk +) + +// File header. +// TODO: probably no need to export this. +type Header struct { + Magic string + Offsets [NBlk]uint32 +} + +const Magic = "\x00go114LD" + +func (h *Header) Write(w *Writer) { + w.RawString(h.Magic) + for _, x := range h.Offsets { + w.Uint32(x) + } +} + +func (h *Header) Read(r *Reader) error { + b := r.BytesAt(0, len(Magic)) + h.Magic = string(b) + if h.Magic != Magic { + return errors.New("wrong magic, not a Go object file") + } + off := uint32(len(h.Magic)) + for i := range h.Offsets { + h.Offsets[i] = r.uint32At(off) + off += 4 + } + return nil +} + +func (h *Header) Size() int { + return len(h.Magic) + 4*len(h.Offsets) +} + +// Symbol definition. +type Sym struct { + Name string + ABI uint16 + Type uint8 + Flag uint8 + Siz uint32 +} + +const SymABIstatic = ^uint16(0) + +const ( + SymFlagDupok = 1 << iota + SymFlagLocal + SymFlagTypelink +) + +func (s *Sym) Write(w *Writer) { + w.StringRef(s.Name) + w.Uint16(s.ABI) + w.Uint8(s.Type) + w.Uint8(s.Flag) + w.Uint32(s.Siz) +} + +func (s *Sym) Read(r *Reader, off uint32) { + s.Name = r.StringRef(off) + s.ABI = r.uint16At(off + 4) + s.Type = r.uint8At(off + 6) + s.Flag = r.uint8At(off + 7) + s.Siz = r.uint32At(off + 8) +} + +func (s *Sym) Size() int { + return 4 + 2 + 1 + 1 + 4 +} + +// Symbol reference. +type SymRef struct { + PkgIdx uint32 + SymIdx uint32 +} + +func (s *SymRef) Write(w *Writer) { + w.Uint32(s.PkgIdx) + w.Uint32(s.SymIdx) +} + +func (s *SymRef) Read(r *Reader, off uint32) { + s.PkgIdx = r.uint32At(off) + s.SymIdx = r.uint32At(off + 4) +} + +func (s *SymRef) Size() int { + return 4 + 4 +} + +// Relocation. +type Reloc struct { + Off int32 + Siz uint8 + Type uint8 + Add int64 + Sym SymRef +} + +func (r *Reloc) Write(w *Writer) { + w.Uint32(uint32(r.Off)) + w.Uint8(r.Siz) + w.Uint8(r.Type) + w.Uint64(uint64(r.Add)) + r.Sym.Write(w) +} + +func (o *Reloc) Read(r *Reader, off uint32) { + o.Off = r.int32At(off) + o.Siz = r.uint8At(off + 4) + o.Type = r.uint8At(off + 5) + o.Add = r.int64At(off + 6) + o.Sym.Read(r, off+14) +} + +func (r *Reloc) Size() int { + return 4 + 1 + 1 + 8 + r.Sym.Size() +} + +// Aux symbol info. +type Aux struct { + Type uint8 + Sym SymRef +} + +// Aux Type +const ( + AuxGotype = iota + AuxFuncInfo + AuxFuncdata + + // TODO: more. DWARF? Pcdata? +) + +func (a *Aux) Write(w *Writer) { + w.Uint8(a.Type) + a.Sym.Write(w) +} + +func (a *Aux) Read(r *Reader, off uint32) { + a.Type = r.uint8At(off) + a.Sym.Read(r, off+1) +} + +func (a *Aux) Size() int { + return 1 + a.Sym.Size() +} + +type Writer struct { + wr *bio.Writer + stringMap map[string]uint32 + off uint32 // running offset +} + +func NewWriter(wr *bio.Writer) *Writer { + return &Writer{wr: wr, stringMap: make(map[string]uint32)} +} + +func (w *Writer) AddString(s string) { + if _, ok := w.stringMap[s]; ok { + return + } + w.stringMap[s] = w.off + w.Uint32(uint32(len(s))) + w.RawString(s) +} + +func (w *Writer) StringRef(s string) { + off, ok := w.stringMap[s] + if !ok { + panic(fmt.Sprintf("writeStringRef: string not added: %q", s)) + } + w.Uint32(off) +} + +func (w *Writer) RawString(s string) { + w.wr.WriteString(s) + w.off += uint32(len(s)) +} + +func (w *Writer) Bytes(s []byte) { + w.wr.Write(s) + w.off += uint32(len(s)) +} + +func (w *Writer) Uint64(x uint64) { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], x) + w.wr.Write(b[:]) + w.off += 8 +} + +func (w *Writer) Uint32(x uint32) { + var b [4]byte + binary.LittleEndian.PutUint32(b[:], x) + w.wr.Write(b[:]) + w.off += 4 +} + +func (w *Writer) Uint16(x uint16) { + var b [2]byte + binary.LittleEndian.PutUint16(b[:], x) + w.wr.Write(b[:]) + w.off += 2 +} + +func (w *Writer) Uint8(x uint8) { + w.wr.WriteByte(x) + w.off++ +} + +func (w *Writer) Offset() uint32 { + return w.off +} + +type Reader struct { + rd io.ReaderAt + start uint32 + h Header // keep block offsets +} + +func NewReader(rd io.ReaderAt, off uint32) *Reader { + r := &Reader{rd: rd, start: off} + err := r.h.Read(r) + if err != nil { + return nil + } + return r +} + +func (r *Reader) BytesAt(off uint32, len int) []byte { + // TODO: read from mapped memory + b := make([]byte, len) + _, err := r.rd.ReadAt(b[:], int64(r.start+off)) + if err != nil { + panic("corrupted input") + } + return b +} + +func (r *Reader) uint64At(off uint32) uint64 { + var b [8]byte + n, err := r.rd.ReadAt(b[:], int64(r.start+off)) + if n != 8 || err != nil { + panic("corrupted input") + } + return binary.LittleEndian.Uint64(b[:]) +} + +func (r *Reader) int64At(off uint32) int64 { + return int64(r.uint64At(off)) +} + +func (r *Reader) uint32At(off uint32) uint32 { + var b [4]byte + n, err := r.rd.ReadAt(b[:], int64(r.start+off)) + if n != 4 || err != nil { + panic("corrupted input") + } + return binary.LittleEndian.Uint32(b[:]) +} + +func (r *Reader) int32At(off uint32) int32 { + return int32(r.uint32At(off)) +} + +func (r *Reader) uint16At(off uint32) uint16 { + var b [2]byte + n, err := r.rd.ReadAt(b[:], int64(r.start+off)) + if n != 2 || err != nil { + panic("corrupted input") + } + return binary.LittleEndian.Uint16(b[:]) +} + +func (r *Reader) uint8At(off uint32) uint8 { + var b [1]byte + n, err := r.rd.ReadAt(b[:], int64(r.start+off)) + if n != 1 || err != nil { + panic("corrupted input") + } + return b[0] +} + +func (r *Reader) StringAt(off uint32) string { + // TODO: have some way to construct a string without copy + l := r.uint32At(off) + b := make([]byte, l) + n, err := r.rd.ReadAt(b, int64(r.start+off+4)) + if n != int(l) || err != nil { + panic("corrupted input") + } + return string(b) +} + +func (r *Reader) StringRef(off uint32) string { + return r.StringAt(r.uint32At(off)) +} + +func (r *Reader) Pkglist() []string { + n := (r.h.Offsets[BlkPkgIdx+1] - r.h.Offsets[BlkPkgIdx]) / 4 + s := make([]string, n) + for i := range s { + off := r.h.Offsets[BlkPkgIdx] + uint32(i)*4 + s[i] = r.StringRef(off) + } + return s +} + +func (r *Reader) NSym() int { + symsiz := (&Sym{}).Size() + return int(r.h.Offsets[BlkSymdef+1]-r.h.Offsets[BlkSymdef]) / symsiz +} + +func (r *Reader) NNonpkgdef() int { + symsiz := (&Sym{}).Size() + return int(r.h.Offsets[BlkNonpkgdef+1]-r.h.Offsets[BlkNonpkgdef]) / symsiz +} + +func (r *Reader) NNonpkgref() int { + symsiz := (&Sym{}).Size() + return int(r.h.Offsets[BlkNonpkgref+1]-r.h.Offsets[BlkNonpkgref]) / symsiz +} + +// SymOff returns the offset of the i-th symbol. +func (r *Reader) SymOff(i int) uint32 { + symsiz := (&Sym{}).Size() + return r.h.Offsets[BlkSymdef] + uint32(i*symsiz) +} + +// NReloc returns the number of relocations of the i-th symbol. +func (r *Reader) NReloc(i int) int { + relocIdxOff := r.h.Offsets[BlkRelocIdx] + uint32(i*4) + return int(r.uint32At(relocIdxOff+4) - r.uint32At(relocIdxOff)) +} + +// RelocOff returns the offset of the j-th relocation of the i-th symbol. +func (r *Reader) RelocOff(i int, j int) uint32 { + relocIdxOff := r.h.Offsets[BlkRelocIdx] + uint32(i*4) + relocIdx := r.uint32At(relocIdxOff) + relocsiz := (&Reloc{}).Size() + return r.h.Offsets[BlkReloc] + (relocIdx+uint32(j))*uint32(relocsiz) +} + +// NAux returns the number of aux symbols of the i-th symbol. +func (r *Reader) NAux(i int) int { + auxIdxOff := r.h.Offsets[BlkAuxIdx] + uint32(i*4) + return int(r.uint32At(auxIdxOff+4) - r.uint32At(auxIdxOff)) +} + +// AuxOff returns the offset of the j-th aux symbol of the i-th symbol. +func (r *Reader) AuxOff(i int, j int) uint32 { + auxIdxOff := r.h.Offsets[BlkAuxIdx] + uint32(i*4) + auxIdx := r.uint32At(auxIdxOff) + auxsiz := (&Aux{}).Size() + return r.h.Offsets[BlkAux] + (auxIdx+uint32(j))*uint32(auxsiz) +} + +// DataOff returns the offset of the i-th symbol's data. +func (r *Reader) DataOff(i int) uint32 { + dataIdxOff := r.h.Offsets[BlkDataIdx] + uint32(i*4) + return r.h.Offsets[BlkData] + r.uint32At(dataIdxOff) +} + +// DataSize returns the size of the i-th symbol's data. +func (r *Reader) DataSize(i int) int { + return int(r.DataOff(i+1) - r.DataOff(i)) +} + +// AuxDataBase returns the base offset of the aux data block. +func (r *Reader) PcdataBase() uint32 { + return r.h.Offsets[BlkPcdata] +} diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index f1cf342d3d..2c106bab30 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -413,6 +413,8 @@ type FuncInfo struct { GCLocals *LSym GCRegs *LSym StackObjects *LSym + + FuncInfoSym *LSym } type InlMark struct { @@ -636,15 +638,6 @@ type Pcdata struct { P []byte } -// Package Index. -const ( - PkgIdxNone = (1<<31 - 1) - iota // Non-package symbols - PkgIdxBuiltin // Predefined symbols // TODO: not used for now, we could use it for compiler-generated symbols like runtime.newobject - PkgIdxSelf // Symbols defined in the current package - PkgIdxInvalid = 0 - // The index of other referenced packages starts from 1. -) - // Link holds the context for writing object code from a compiler // to be linker input or for reading that input into the linker. type Link struct { diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index c51a11c51f..a27004a389 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -82,6 +82,11 @@ func newObjWriter(ctxt *Link, b *bufio.Writer, pkgpath string) *objWriter { } func WriteObjFile(ctxt *Link, bout *bio.Writer, pkgpath string) { + if ctxt.Flag_newobj { + WriteObjFile2(ctxt, bout, pkgpath) + return + } + b := bout.Writer w := newObjWriter(ctxt, b, pkgpath) diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go new file mode 100644 index 0000000000..42f050a940 --- /dev/null +++ b/src/cmd/internal/obj/objfile2.go @@ -0,0 +1,350 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Writing Go object files. + +package obj + +import ( + "bytes" + "cmd/internal/bio" + "cmd/internal/goobj2" + "cmd/internal/objabi" + "fmt" + "strings" +) + +// Entry point of writing new object file. +func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { + genFuncInfoSyms(ctxt) + + w := writer{ + Writer: goobj2.NewWriter(b), + ctxt: ctxt, + pkgpath: objabi.PathToPrefix(pkgpath), + } + + start := b.Offset() + w.init() + + // Header + // We just reserve the space. We'll fill in the offsets later. + h := goobj2.Header{Magic: goobj2.Magic} + h.Write(w.Writer) + + // String table + w.StringTable() + + // Package references + h.Offsets[goobj2.BlkPkgIdx] = w.Offset() + for _, pkg := range w.pkglist { + w.StringRef(pkg) + } + + // Symbol definitions + h.Offsets[goobj2.BlkSymdef] = w.Offset() + for _, s := range ctxt.defs { + w.Sym(s) + } + + // Non-pkg symbol definitions + h.Offsets[goobj2.BlkNonpkgdef] = w.Offset() + for _, s := range ctxt.nonpkgdefs { + w.Sym(s) + } + + // Non-pkg symbol references + h.Offsets[goobj2.BlkNonpkgref] = w.Offset() + for _, s := range ctxt.nonpkgrefs { + w.Sym(s) + } + + // Reloc indexes + h.Offsets[goobj2.BlkRelocIdx] = w.Offset() + nreloc := uint32(0) + lists := [][]*LSym{ctxt.defs, ctxt.nonpkgdefs} + for _, list := range lists { + for _, s := range list { + w.Uint32(nreloc) + nreloc += uint32(len(s.R)) + } + } + w.Uint32(nreloc) + + // Symbol Info indexes + h.Offsets[goobj2.BlkAuxIdx] = w.Offset() + naux := uint32(0) + for _, list := range lists { + for _, s := range list { + w.Uint32(naux) + if s.Gotype != nil { + naux++ + } + if s.Func != nil { + // FuncInfo is an aux symbol, each Funcdata is an aux symbol + naux += 1 + uint32(len(s.Func.Pcln.Funcdata)) + } + } + } + w.Uint32(naux) + + // Data indexes + h.Offsets[goobj2.BlkDataIdx] = w.Offset() + dataOff := uint32(0) + for _, list := range lists { + for _, s := range list { + w.Uint32(dataOff) + dataOff += uint32(len(s.P)) + } + } + w.Uint32(dataOff) + + // Relocs + h.Offsets[goobj2.BlkReloc] = w.Offset() + for _, list := range lists { + for _, s := range list { + for i := range s.R { + w.Reloc(&s.R[i]) + } + } + } + + // Aux symbol info + h.Offsets[goobj2.BlkAux] = w.Offset() + for _, list := range lists { + for _, s := range list { + w.Aux(s) + } + } + + // Data + h.Offsets[goobj2.BlkData] = w.Offset() + for _, list := range lists { + for _, s := range list { + w.Bytes(s.P) + } + } + + // Pcdata + h.Offsets[goobj2.BlkPcdata] = w.Offset() + for _, s := range ctxt.Text { // iteration order must match genFuncInfoSyms + if s.Func != nil { + pc := &s.Func.Pcln + w.Bytes(pc.Pcsp.P) + w.Bytes(pc.Pcfile.P) + w.Bytes(pc.Pcline.P) + w.Bytes(pc.Pcinline.P) + for i := range pc.Pcdata { + w.Bytes(pc.Pcdata[i].P) + } + } + } + + // Fix up block offsets in the header + end := start + int64(w.Offset()) + b.MustSeek(start, 0) + h.Write(w.Writer) + b.MustSeek(end, 0) +} + +type writer struct { + *goobj2.Writer + ctxt *Link + pkgpath string // the package import path (escaped), "" if unknown + pkglist []string // list of packages referenced, indexed by ctxt.pkgIdx +} + +// prepare package index list +func (w *writer) init() { + w.pkglist = make([]string, len(w.ctxt.pkgIdx)+1) + w.pkglist[0] = "" // dummy invalid package for index 0 + for pkg, i := range w.ctxt.pkgIdx { + w.pkglist[i] = pkg + } + + // Also make sure imported packages appear in the list (even if no symbol is referenced). + for _, pkg := range w.ctxt.Imports { + if _, ok := w.ctxt.pkgIdx[pkg]; !ok { + w.pkglist = append(w.pkglist, pkg) + } + } +} + +func (w *writer) StringTable() { + w.AddString("") + for _, pkg := range w.ctxt.Imports { + w.AddString(pkg) + } + for _, pkg := range w.pkglist { + w.AddString(pkg) + } + w.ctxt.traverseSyms(traverseAll, func(s *LSym) { + if w.pkgpath != "" { + s.Name = strings.Replace(s.Name, "\"\".", w.pkgpath+".", -1) + } + w.AddString(s.Name) + }) + w.ctxt.traverseSyms(traverseDefs, func(s *LSym) { + if s.Type != objabi.STEXT { + return + } + pc := &s.Func.Pcln + for _, f := range pc.File { + w.AddString(f) + } + for _, call := range pc.InlTree.nodes { + f, _ := linkgetlineFromPos(w.ctxt, call.Pos) + w.AddString(f) + } + }) +} + +func (w *writer) Sym(s *LSym) { + abi := uint16(s.ABI()) + if s.Static() { + abi = goobj2.SymABIstatic + } + flag := uint8(0) + if s.DuplicateOK() { + flag |= goobj2.SymFlagDupok + } + if s.Local() { + flag |= goobj2.SymFlagLocal + } + if s.MakeTypelink() { + flag |= goobj2.SymFlagTypelink + } + o := goobj2.Sym{ + Name: s.Name, + ABI: abi, + Type: uint8(s.Type), + Flag: flag, + Siz: uint32(s.Size), + } + o.Write(w.Writer) +} + +func makeSymRef(s *LSym) goobj2.SymRef { + if s == nil { + return goobj2.SymRef{} + } + if s.PkgIdx == 0 || !s.Indexed() { + fmt.Printf("unindexed symbol reference: %v\n", s) + panic("unindexed symbol reference") + } + return goobj2.SymRef{PkgIdx: uint32(s.PkgIdx), SymIdx: uint32(s.SymIdx)} +} + +func (w *writer) Reloc(r *Reloc) { + o := goobj2.Reloc{ + Off: r.Off, + Siz: r.Siz, + Type: uint8(r.Type), + Add: r.Add, + Sym: makeSymRef(r.Sym), + } + o.Write(w.Writer) +} + +func (w *writer) Aux(s *LSym) { + if s.Gotype != nil { + o := goobj2.Aux{ + Type: goobj2.AuxGotype, + Sym: makeSymRef(s.Gotype), + } + o.Write(w.Writer) + } + if s.Func != nil { + o := goobj2.Aux{ + Type: goobj2.AuxFuncInfo, + Sym: makeSymRef(s.Func.FuncInfoSym), + } + o.Write(w.Writer) + + for _, d := range s.Func.Pcln.Funcdata { + o := goobj2.Aux{ + Type: goobj2.AuxFuncdata, + Sym: makeSymRef(d), + } + o.Write(w.Writer) + } + } +} + +// generate symbols for FuncInfo. +func genFuncInfoSyms(ctxt *Link) { + infosyms := make([]*LSym, 0, len(ctxt.Text)) + var pcdataoff uint32 + var b bytes.Buffer + symidx := int32(len(ctxt.defs)) + for _, s := range ctxt.Text { + if s.Func == nil { + continue + } + nosplit := uint8(0) + if s.NoSplit() { + nosplit = 1 + } + flags := uint8(0) + if s.Leaf() { + flags |= goobj2.FuncFlagLeaf + } + if s.CFunc() { + flags |= goobj2.FuncFlagCFunc + } + if s.ReflectMethod() { + flags |= goobj2.FuncFlagReflectMethod + } + if ctxt.Flag_shared { // This is really silly + flags |= goobj2.FuncFlagShared + } + if s.TopFrame() { + flags |= goobj2.FuncFlagTopFrame + } + o := goobj2.FuncInfo{ + NoSplit: nosplit, + Flags: flags, + Args: uint32(s.Func.Args), + Locals: uint32(s.Func.Locals), + } + pc := &s.Func.Pcln + o.Pcsp = pcdataoff + pcdataoff += uint32(len(pc.Pcsp.P)) + o.Pcfile = pcdataoff + pcdataoff += uint32(len(pc.Pcfile.P)) + o.Pcline = pcdataoff + pcdataoff += uint32(len(pc.Pcline.P)) + o.Pcinline = pcdataoff + pcdataoff += uint32(len(pc.Pcinline.P)) + o.Pcdata = make([]uint32, len(pc.Pcdata)) + for i, pcd := range pc.Pcdata { + o.Pcdata[i] = pcdataoff + pcdataoff += uint32(len(pcd.P)) + } + o.PcdataEnd = pcdataoff + o.Funcdataoff = make([]uint32, len(pc.Funcdataoff)) + for i, x := range pc.Funcdataoff { + o.Funcdataoff[i] = uint32(x) + } + o.File = make([]goobj2.SymRef, len(pc.File)) + for i, f := range pc.File { + fsym := ctxt.Lookup(f) + o.File[i] = makeSymRef(fsym) + } + + o.Write(&b) + isym := &LSym{ + Type: objabi.SDATA, // for now, I don't think it matters + PkgIdx: goobj2.PkgIdxSelf, + SymIdx: symidx, + P: append([]byte(nil), b.Bytes()...), + } + isym.Set(AttrIndexed, true) + symidx++ + infosyms = append(infosyms, isym) + s.Func.FuncInfoSym = isym + b.Reset() + } + ctxt.defs = append(ctxt.defs, infosyms...) +} diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index e47c511ddc..e72ec3e701 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -32,6 +32,7 @@ package obj import ( + "cmd/internal/goobj2" "cmd/internal/objabi" "fmt" "log" @@ -173,7 +174,7 @@ func (ctxt *Link) NumberSyms(asm bool) { var idx, nonpkgidx int32 = 0, 0 ctxt.traverseSyms(traverseDefs, func(s *LSym) { if asm || s.Pkg == "_" || s.DuplicateOK() { - s.PkgIdx = PkgIdxNone + s.PkgIdx = goobj2.PkgIdxNone s.SymIdx = nonpkgidx if nonpkgidx != int32(len(ctxt.nonpkgdefs)) { panic("bad index") @@ -181,7 +182,7 @@ func (ctxt *Link) NumberSyms(asm bool) { ctxt.nonpkgdefs = append(ctxt.nonpkgdefs, s) nonpkgidx++ } else { - s.PkgIdx = PkgIdxSelf + s.PkgIdx = goobj2.PkgIdxSelf s.SymIdx = idx if idx != int32(len(ctxt.defs)) { panic("bad index") @@ -195,12 +196,12 @@ func (ctxt *Link) NumberSyms(asm bool) { ipkg := int32(1) // 0 is invalid index nonpkgdef := nonpkgidx ctxt.traverseSyms(traverseRefs|traverseAux, func(rs *LSym) { - if rs.PkgIdx != PkgIdxInvalid { + if rs.PkgIdx != goobj2.PkgIdxInvalid { return } pkg := rs.Pkg if pkg == "" || pkg == "\"\"" || pkg == "_" || !rs.Indexed() { - rs.PkgIdx = PkgIdxNone + rs.PkgIdx = goobj2.PkgIdxNone rs.SymIdx = nonpkgidx rs.Set(AttrIndexed, true) if nonpkgidx != nonpkgdef+int32(len(ctxt.nonpkgrefs)) { From d79380026cd9f3a65e4896e77a7f3fbe1a954934 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 11 Sep 2019 15:57:39 -0400 Subject: [PATCH 06/79] [dev.link] cmd/internal/goobj: support parsing new object file Add support of parsing new object file format. We use the new parser if the magic string matches the new one, otherwise use the old one. The parsed data are still filled into the current goobj API. In the future we may consider to change the goobj API to a close match of the object file data. Now objdump and nm commands support new object file format. For a reference to a symbol defined in another package, with the new object file format we don't know its name. Write it as pkg.<#nn> for now, where nn is its symbol index. Change-Id: I06d05b2ca834ba36980da3c5d76aee16c3b0a483 Reviewed-on: https://go-review.googlesource.com/c/go/+/196031 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/read.go | 5 + src/cmd/internal/goobj/readnew.go | 163 ++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 src/cmd/internal/goobj/readnew.go diff --git a/src/cmd/internal/goobj/read.go b/src/cmd/internal/goobj/read.go index 0c70b8cd9f..2a3afffeb0 100644 --- a/src/cmd/internal/goobj/read.go +++ b/src/cmd/internal/goobj/read.go @@ -503,6 +503,11 @@ func (r *objReader) parseObject(prefix []byte) error { // TODO: extract OS + build ID if/when we need it r.readFull(r.tmp[:8]) + if bytes.Equal(r.tmp[:8], []byte("\x00go114LD")) { + r.offset -= 8 + r.readNew() + return nil + } if !bytes.Equal(r.tmp[:8], []byte("\x00go114ld")) { return r.error(errCorruptObject) } diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go new file mode 100644 index 0000000000..442784de3a --- /dev/null +++ b/src/cmd/internal/goobj/readnew.go @@ -0,0 +1,163 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package goobj + +import ( + "cmd/internal/goobj2" + "cmd/internal/objabi" + "fmt" +) + +// Read object file in new format. For now we still fill +// the data to the current goobj API. +func (r *objReader) readNew() { + start := uint32(r.offset) + rr := goobj2.NewReader(r.f, start) + if rr == nil { + panic("cannot read object file") + } + + // Imports + pkglist := rr.Pkglist() + r.p.Imports = pkglist[1:] // index 0 is a dummy invalid package + + abiToVer := func(abi uint16) int64 { + var vers int64 + if abi == goobj2.SymABIstatic { + // Static symbol + vers = r.p.MaxVersion + } + return vers + } + + resolveSymRef := func(s goobj2.SymRef) SymID { + var i int + switch p := s.PkgIdx; p { + case goobj2.PkgIdxInvalid: + if s.SymIdx != 0 { + panic("bad sym ref") + } + return SymID{} + case goobj2.PkgIdxNone: + i = int(s.SymIdx) + rr.NSym() + case goobj2.PkgIdxBuiltin: + panic("PkgIdxBuiltin is unused") + case goobj2.PkgIdxSelf: + i = int(s.SymIdx) + default: + pkg := pkglist[p] + return SymID{fmt.Sprintf("%s.<#%d>", pkg, s.SymIdx), 0} + } + sym := goobj2.Sym{} + sym.Read(rr, rr.SymOff(i)) + return SymID{sym.Name, abiToVer(sym.ABI)} + } + + // Read things for the current goobj API for now. + + // Symbols + pcdataBase := start + rr.PcdataBase() + n := rr.NSym() + rr.NNonpkgdef() + rr.NNonpkgref() + ndef := rr.NSym() + rr.NNonpkgdef() + for i := 0; i < n; i++ { + osym := goobj2.Sym{} + osym.Read(rr, rr.SymOff(i)) + if osym.Name == "" { + continue // not a real symbol + } + symID := SymID{Name: osym.Name, Version: abiToVer(osym.ABI)} + r.p.SymRefs = append(r.p.SymRefs, symID) + + if i >= ndef { + continue // not a defined symbol from here + } + + // Symbol data + dataOff := rr.DataOff(i) + siz := int64(rr.DataSize(i)) + + sym := Sym{ + SymID: symID, + Kind: objabi.SymKind(osym.Type), + DupOK: osym.Flag&goobj2.SymFlagDupok != 0, + Size: int64(osym.Siz), + Data: Data{int64(start + dataOff), siz}, + } + r.p.Syms = append(r.p.Syms, &sym) + + // Reloc + nreloc := rr.NReloc(i) + sym.Reloc = make([]Reloc, nreloc) + for j := 0; j < nreloc; j++ { + rel := goobj2.Reloc{} + rel.Read(rr, rr.RelocOff(i, j)) + sym.Reloc[j] = Reloc{ + Offset: int64(rel.Off), + Size: int64(rel.Siz), + Type: objabi.RelocType(rel.Type), + Add: rel.Add, + Sym: resolveSymRef(rel.Sym), + } + } + + // Aux symbol info + isym := -1 + funcdata := make([]goobj2.SymRef, 0, 4) + naux := rr.NAux(i) + for j := 0; j < naux; j++ { + a := goobj2.Aux{} + a.Read(rr, rr.AuxOff(i, j)) + switch a.Type { + case goobj2.AuxGotype: + sym.Type = resolveSymRef(a.Sym) + case goobj2.AuxFuncInfo: + if a.Sym.PkgIdx != goobj2.PkgIdxSelf { + panic("funcinfo symbol not defined in current package") + } + isym = int(a.Sym.SymIdx) + case goobj2.AuxFuncdata: + funcdata = append(funcdata, a.Sym) + default: + panic("unknown aux type") + } + } + + // Symbol Info + if isym == -1 { + continue + } + b := rr.BytesAt(rr.DataOff(isym), rr.DataSize(isym)) + info := goobj2.FuncInfo{} + info.Read(b) + + info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends + f := &Func{ + Args: int64(info.Args), + Frame: int64(info.Locals), + NoSplit: info.NoSplit != 0, + Leaf: info.Flags&goobj2.FuncFlagLeaf != 0, + TopFrame: info.Flags&goobj2.FuncFlagTopFrame != 0, + PCSP: Data{int64(pcdataBase + info.Pcsp), int64(info.Pcfile - info.Pcsp)}, + PCFile: Data{int64(pcdataBase + info.Pcfile), int64(info.Pcline - info.Pcfile)}, + PCLine: Data{int64(pcdataBase + info.Pcline), int64(info.Pcinline - info.Pcline)}, + PCInline: Data{int64(pcdataBase + info.Pcinline), int64(info.Pcdata[0] - info.Pcinline)}, + PCData: make([]Data, len(info.Pcdata)-1), // -1 as we appended one above + FuncData: make([]FuncData, len(info.Funcdataoff)), + File: make([]string, len(info.File)), + } + sym.Func = f + for k := range f.PCData { + f.PCData[k] = Data{int64(pcdataBase + info.Pcdata[k]), int64(info.Pcdata[k+1] - info.Pcdata[k])} + } + for k := range f.FuncData { + symID := resolveSymRef(funcdata[k]) + f.FuncData[k] = FuncData{symID, int64(info.Funcdataoff[k])} + } + for k := range f.File { + symID := resolveSymRef(info.File[k]) + f.File[k] = symID.Name + } + } +} From 6051fd0ad9f3e9e2ee405da709ee9f087747e4c1 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 17 Sep 2019 16:14:37 -0400 Subject: [PATCH 07/79] [dev.link] cmd/link: support new object file format Parse new object file format in the linker. At least we can link a hello-world program. Add a basic "loader", which handles symbol references in the object file. - mapping between local and global indices - resolve by-name references (TODO: the overwrite logic isn't implemented yet) Currently we still create sym.Symbol rather early, and, after all the object files are loaded and indexed references are resolved, add all symbols to sym.Symbols. The code here is probably not going in the final version. This is basically only for debugging purposes -- to make sure the writer and the reader work as expected. Change-Id: I895aeea68326fabdb7e5aa1371b8cac7211a09dd Reviewed-on: https://go-review.googlesource.com/c/go/+/196032 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/go.go | 7 +- src/cmd/link/internal/ld/lib.go | 75 +++- src/cmd/link/internal/ld/link.go | 4 + src/cmd/link/internal/ld/main.go | 2 + src/cmd/link/internal/objfile/objfile2.go | 419 ++++++++++++++++++++++ src/cmd/link/internal/sym/library.go | 7 + src/cmd/link/internal/sym/symbols.go | 11 + 7 files changed, 513 insertions(+), 12 deletions(-) create mode 100644 src/cmd/link/internal/objfile/objfile2.go diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index 80d7ac32f5..d1d68b0704 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -111,7 +111,12 @@ func ldpkg(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, filename s } p1 += p0 - loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]) + if *flagNewobj { + // loadcgo creates sym.Symbol. Delay this until all the symbols are added. + ctxt.cgodata = append(ctxt.cgodata, [3]string{filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]}) + } else { + loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]) + } } } diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 182e5b0769..e2efb95ff4 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -375,15 +375,8 @@ func (ctxt *Link) findLibPath(libname string) string { } func (ctxt *Link) loadlib() { - switch ctxt.BuildMode { - case BuildModeCShared, BuildModePlugin: - s := ctxt.Syms.Lookup("runtime.islibrary", 0) - s.Attr |= sym.AttrDuplicateOK - s.AddUint8(1) - case BuildModeCArchive: - s := ctxt.Syms.Lookup("runtime.isarchive", 0) - s.Attr |= sym.AttrDuplicateOK - s.AddUint8(1) + if *flagNewobj { + ctxt.loader = objfile.NewLoader() } loadinternal(ctxt, "runtime") @@ -408,6 +401,11 @@ func (ctxt *Link) loadlib() { } } + // XXX do it here for now + if *flagNewobj { + ctxt.loadlibfull() + } + for _, lib := range ctxt.Library { if lib.Shlib != "" { if ctxt.Debugvlog > 1 { @@ -417,6 +415,19 @@ func (ctxt *Link) loadlib() { } } + switch ctxt.BuildMode { + case BuildModeCShared, BuildModePlugin: + s := ctxt.Syms.Lookup("runtime.islibrary", 0) + s.Type = sym.SNOPTRDATA + s.Attr |= sym.AttrDuplicateOK + s.AddUint8(1) + case BuildModeCArchive: + s := ctxt.Syms.Lookup("runtime.isarchive", 0) + s.Type = sym.SNOPTRDATA + s.Attr |= sym.AttrDuplicateOK + s.AddUint8(1) + } + iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil // We now have enough information to determine the link mode. @@ -843,7 +854,7 @@ func loadobjfile(ctxt *Link, lib *sym.Library) { if err != nil { Exitf("cannot open file %s: %v", lib.File, err) } - defer f.Close() + //defer f.Close() defer func() { if pkg == "main" && !lib.Main { Exitf("%s: not package main", lib.File) @@ -1773,7 +1784,12 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, default: log.Fatalf("invalid -strictdups flag value %d", *FlagStrictDups) } - c := objfile.Load(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) + var c int + if *flagNewobj { + objfile.LoadNew(ctxt.loader, ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) + } else { + c = objfile.Load(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) + } strictDupMsgCount += c addImports(ctxt, lib, pn) return nil @@ -2545,3 +2561,40 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library mark[lib] = visited *order = append(*order, lib) } + +func (ctxt *Link) loadlibfull() { + // Add references of externally defined symbols. + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) + } + } + + // Load full symbol contents, resolve indexed references. + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadFull(ctxt.loader, r.Reader, lib, ctxt.Syms, r.Version, ctxt.LibraryByPkg) + } + } + + // For now, add all symbols to ctxt.Syms. + for _, s := range ctxt.loader.Syms { + if s != nil && s.Name != "" { + ctxt.Syms.Add(s) + } + } + + // Now load cgo directives. + for _, p := range ctxt.cgodata { + loadcgo(ctxt, p[0], p[1], p[2]) + } +} + +func (ctxt *Link) dumpsyms() { + for _, s := range ctxt.Syms.Allsym { + fmt.Printf("%s %s %p\n", s, s.Type, s) + for i := range s.R { + fmt.Println("\t", s.R[i].Type, s.R[i].Sym) + } + } +} diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index 53092d2e8f..bbdb0e50ed 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -35,6 +35,7 @@ import ( "cmd/internal/obj" "cmd/internal/objabi" "cmd/internal/sys" + "cmd/link/internal/objfile" "cmd/link/internal/sym" "debug/elf" "fmt" @@ -96,6 +97,9 @@ type Link struct { runtimeCU *sym.CompilationUnit // One of the runtime CUs, the last one seen. relocbuf []byte // temporary buffer for applying relocations + + loader *objfile.Loader + cgodata [][3]string // cgo directives to load, three strings are args for loadcgo } type unresolvedSymKey struct { diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 67e5ef9392..54a5fa199c 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -86,6 +86,7 @@ var ( flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker") FlagDebugTramp = flag.Int("debugtramp", 0, "debug trampolines") FlagStrictDups = flag.Int("strictdups", 0, "sanity check duplicate symbol contents during object file reading (1=warn 2=err).") + flagNewobj = flag.Bool("newobj", false, "use new object file format") FlagRound = flag.Int("R", -1, "set address rounding `quantum`") FlagTextAddr = flag.Int64("T", -1, "set text segment `address`") @@ -209,6 +210,7 @@ func Main(arch *sys.Arch, theArch Arch) { ctxt.dostrdata() deadcode(ctxt) + //ctxt.dumpsyms() // XXX dwarfGenerateDebugInfo(ctxt) if objabi.Fieldtrack_enabled != 0 { fieldtrack(ctxt) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go new file mode 100644 index 0000000000..5a92fef4ec --- /dev/null +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -0,0 +1,419 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objfile + +import ( + "cmd/internal/bio" + "cmd/internal/goobj2" + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/sys" + "cmd/link/internal/sym" + "fmt" + "log" + "os" + "sort" + "strconv" + "strings" +) + +var _ = fmt.Print + +type objIdx struct { + r *goobj2.Reader + i int // start index +} + +type nameVer struct { + name string + v int +} + +// A Loader loads new object files and resolves indexed symbol references. +// +// TODO: describe local-global index mapping. +type Loader struct { + start map[*goobj2.Reader]int // map from object file to its start index + objs []objIdx // sorted by start index (i.e. objIdx.i) + max int // current max index + + symsByName map[nameVer]int // map symbol name to index + + Syms []*sym.Symbol // indexed symbols. XXX we still make sym.Symbol for now. +} + +func NewLoader() *Loader { + return &Loader{ + start: make(map[*goobj2.Reader]int), + objs: []objIdx{{nil, 0}}, + symsByName: make(map[nameVer]int), + Syms: []*sym.Symbol{nil}, + } +} + +// Return the start index in the global index space for a given object file. +func (l *Loader) StartIndex(r *goobj2.Reader) int { + return l.start[r] +} + +// Add object file r, return the start index. +func (l *Loader) AddObj(r *goobj2.Reader) int { + if _, ok := l.start[r]; ok { + panic("already added") + } + n := r.NSym() + r.NNonpkgdef() + i := l.max + 1 + l.start[r] = i + l.objs = append(l.objs, objIdx{r, i}) + l.max += n + return i +} + +// Add a symbol with a given index, return if it is added. +func (l *Loader) AddSym(name string, ver int, i int, dupok bool) bool { + nv := nameVer{name, ver} + if _, ok := l.symsByName[nv]; ok { + if dupok || true { // TODO: "true" isn't quite right. need to implement "overwrite" logic. + return false + } + panic("duplicated definition of symbol " + name) + } + l.symsByName[nv] = i + return true +} + +// Add an external symbol (without index). Return the index of newly added +// symbol, or 0 if not added. +func (l *Loader) AddExtSym(name string, ver int) int { + nv := nameVer{name, ver} + if _, ok := l.symsByName[nv]; ok { + return 0 + } + i := l.max + 1 + l.symsByName[nv] = i + l.max++ + return i +} + +// Convert a local index to a global index. +func (l *Loader) ToGlobal(r *goobj2.Reader, i int) int { + return l.StartIndex(r) + i +} + +// Convert a global index to a global index. Is it useful? +func (l *Loader) ToLocal(i int) (*goobj2.Reader, int) { + k := sort.Search(i, func(k int) bool { + return l.objs[k].i >= i + }) + if k == len(l.objs) { + return nil, 0 + } + return l.objs[k].r, i - l.objs[k].i +} + +// Look up a symbol by name, return global index, or 0 if not found. +// This is more like Syms.ROLookup than Lookup -- it doesn't create +// new symbol. +func (l *Loader) Lookup(name string, ver int) int { + nv := nameVer{name, ver} + return l.symsByName[nv] +} + +// Preload a package: add autolibs, add symbols to the symbol table. +// Does not read symbol data yet. +func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { + start := f.Offset() + r := goobj2.NewReader(f.File(), uint32(start)) + if r == nil { + panic("cannot read object file") + } + localSymVersion := syms.IncVersion() + lib.Readers = append(lib.Readers, struct { + Reader *goobj2.Reader + Version int + }{r, localSymVersion}) + + pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + + // Autolib + lib.ImportStrings = append(lib.ImportStrings, r.Pkglist()[1:]...) + + istart := l.AddObj(r) + + ndef := r.NSym() + nnonpkgdef := r.NNonpkgdef() + + // XXX add all symbols for now + l.Syms = append(l.Syms, make([]*sym.Symbol, ndef+nnonpkgdef)...) + for i, n := 0, ndef+nnonpkgdef; i < n; i++ { + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + if name == "" { + continue // don't add unnamed aux symbol + } + v := abiToVer(osym.ABI, localSymVersion) + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + if l.AddSym(name, v, istart+i, dupok) { + s := syms.Newsym(name, v) + preprocess(arch, s) // TODO: put this at a better place + l.Syms[istart+i] = s + } + } + + // The caller expects us consuming all the data + f.MustSeek(length, os.SEEK_CUR) +} + +// Make sure referenced symbols are added. Most of them should already be added. +// This should only be needed for referenced external symbols. +func LoadRefs(l *Loader, r *goobj2.Reader, lib *sym.Library, arch *sys.Arch, syms *sym.Symbols, localSymVersion int) { + pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + ndef := r.NSym() + r.NNonpkgdef() + for i, n := 0, r.NNonpkgref(); i < n; i++ { + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(ndef+i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + v := abiToVer(osym.ABI, localSymVersion) + if ii := l.AddExtSym(name, v); ii != 0 { + s := syms.Newsym(name, v) + preprocess(arch, s) // TODO: put this at a better place + if ii != len(l.Syms) { + panic("AddExtSym returned bad index") + } + l.Syms = append(l.Syms, s) + } + } +} + +func abiToVer(abi uint16, localSymVersion int) int { + var v int + if abi == goobj2.SymABIstatic { + // Static + v = localSymVersion + } else if abiver := sym.ABIToVersion(obj.ABI(abi)); abiver != -1 { + // Note that data symbols are "ABI0", which maps to version 0. + v = abiver + } else { + log.Fatalf("invalid symbol ABI: %d", abi) + } + return v +} + +func preprocess(arch *sys.Arch, s *sym.Symbol) { + if s.Name != "" && s.Name[0] == '$' && len(s.Name) > 5 && s.Type == 0 && len(s.P) == 0 { + x, err := strconv.ParseUint(s.Name[5:], 16, 64) + if err != nil { + log.Panicf("failed to parse $-symbol %s: %v", s.Name, err) + } + s.Type = sym.SRODATA + s.Attr |= sym.AttrLocal + switch s.Name[:5] { + case "$f32.": + if uint64(uint32(x)) != x { + log.Panicf("$-symbol %s too large: %d", s.Name, x) + } + s.AddUint32(arch, uint32(x)) + case "$f64.", "$i64.": + s.AddUint64(arch, x) + default: + log.Panicf("unrecognized $-symbol: %s", s.Name) + } + s.Attr.Set(sym.AttrReachable, false) + } + if strings.HasPrefix(s.Name, "runtime.gcbits.") { + s.Attr |= sym.AttrLocal + } +} + +func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, localSymVersion int, libByPkg map[string]*sym.Library) { + // PkgIdx + pkglist := r.Pkglist() + + pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + istart := l.StartIndex(r) + + resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { + var rr *goobj2.Reader + switch p := s.PkgIdx; p { + case goobj2.PkgIdxInvalid: + if s.SymIdx != 0 { + panic("bad sym ref") + } + return nil + case goobj2.PkgIdxNone: + // Resolve by name + i := int(s.SymIdx) + r.NSym() + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + v := abiToVer(osym.ABI, localSymVersion) + nv := nameVer{name, v} + i = l.symsByName[nv] + return l.Syms[i] + case goobj2.PkgIdxBuiltin: + panic("PkgIdxBuiltin is not used") + case goobj2.PkgIdxSelf: + rr = r + default: + pkg := pkglist[p] + rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm) + } + i := l.ToGlobal(rr, int(s.SymIdx)) + return l.Syms[i] + } + + pcdataBase := r.PcdataBase() + for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { + s := l.Syms[istart+i] + if s == nil || s.Name == "" { + continue + } + + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + if s.Name != name { + fmt.Println("name mismatch:", lib, i, s.Name, name) + panic("name mismatch") + } + + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + local := osym.Flag&goobj2.SymFlagLocal != 0 + makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 + nreloc := r.NReloc(i) + datasize := r.DataSize(i) + size := osym.Siz + + t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] + if s.Type != 0 && s.Type != sym.SXREF { + fmt.Println("symbol already processed:", lib, i, s) + panic("symbol already processed") + } + + // Symbol data + s.P = r.BytesAt(r.DataOff(i), datasize) + + // Reloc + s.R = make([]sym.Reloc, nreloc) + for j := range s.R { + rel := goobj2.Reloc{} + rel.Read(r, r.RelocOff(i, j)) + s.R[j] = sym.Reloc{ + Off: rel.Off, + Siz: rel.Siz, + Type: objabi.RelocType(rel.Type), + Add: rel.Add, + Sym: resolveSymRef(rel.Sym), + } + } + + // Aux symbol info + isym := -1 + funcdata := make([]goobj2.SymRef, 0, 4) + naux := r.NAux(i) + for j := 0; j < naux; j++ { + a := goobj2.Aux{} + a.Read(r, r.AuxOff(i, j)) + switch a.Type { + case goobj2.AuxGotype: + typ := resolveSymRef(a.Sym) + if typ != nil { + s.Gotype = typ + } + case goobj2.AuxFuncInfo: + if a.Sym.PkgIdx != goobj2.PkgIdxSelf { + panic("funcinfo symbol not defined in current package") + } + isym = int(a.Sym.SymIdx) + case goobj2.AuxFuncdata: + funcdata = append(funcdata, a.Sym) + default: + panic("unknown aux type") + } + } + + s.File = pkgprefix[:len(pkgprefix)-1] + if dupok { + s.Attr |= sym.AttrDuplicateOK + } + if t == sym.SXREF { + log.Fatalf("bad sxref") + } + if t == 0 { + log.Fatalf("missing type for %s in %s", s.Name, lib) + } + if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { + t = s.Type + } + s.Type = t + if s.Size < int64(size) { + s.Size = int64(size) + } + s.Attr.Set(sym.AttrLocal, local) + s.Attr.Set(sym.AttrMakeTypelink, makeTypelink) + + if s.Type != sym.STEXT { + continue + } + if !dupok { + if s.Attr.OnList() { + log.Fatalf("symbol %s listed multiple times", s.Name) + } + s.Attr |= sym.AttrOnList + lib.Textp = append(lib.Textp, s) + } else { + // there may ba a dup in another package + // put into a temp list and add to text later + lib.DupTextSyms = append(lib.DupTextSyms, s) + } + + // FuncInfo + if isym == -1 { + continue + } + b := r.BytesAt(r.DataOff(isym), r.DataSize(isym)) + info := goobj2.FuncInfo{} + info.Read(b) + + if info.NoSplit != 0 { + s.Attr |= sym.AttrNoSplit + } + if info.Flags&goobj2.FuncFlagReflectMethod != 0 { + s.Attr |= sym.AttrReflectMethod + } + if info.Flags&goobj2.FuncFlagShared != 0 { + s.Attr |= sym.AttrShared + } + if info.Flags&goobj2.FuncFlagTopFrame != 0 { + s.Attr |= sym.AttrTopFrame + } + + info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends + pc := &sym.FuncInfo{ + Args: int32(info.Args), + Locals: int32(info.Locals), + Pcdata: make([]sym.Pcdata, len(info.Pcdata)-1), // -1 as we appended one above + Funcdata: make([]*sym.Symbol, len(info.Funcdataoff)), + Funcdataoff: make([]int64, len(info.Funcdataoff)), + File: make([]*sym.Symbol, len(info.File)), + } + s.FuncInfo = pc + pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp)) + pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile)) + pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline)) + pc.Pcinline.P = r.BytesAt(pcdataBase+info.Pcinline, int(info.Pcdata[0]-info.Pcinline)) + for k := range pc.Pcdata { + pc.Pcdata[k].P = r.BytesAt(pcdataBase+info.Pcdata[k], int(info.Pcdata[k+1]-info.Pcdata[k])) + } + for k := range pc.Funcdata { + pc.Funcdata[k] = resolveSymRef(funcdata[k]) + pc.Funcdataoff[k] = int64(info.Funcdataoff[k]) + } + for k := range pc.File { + pc.File[k] = resolveSymRef(info.File[k]) + } + } +} diff --git a/src/cmd/link/internal/sym/library.go b/src/cmd/link/internal/sym/library.go index 4f2023b8f7..b319c6b54e 100644 --- a/src/cmd/link/internal/sym/library.go +++ b/src/cmd/link/internal/sym/library.go @@ -4,6 +4,8 @@ package sym +import "cmd/internal/goobj2" + type Library struct { Objref string Srcref string @@ -18,6 +20,11 @@ type Library struct { Main bool Safe bool Units []*CompilationUnit + + Readers []struct { // TODO: probably move this to Loader + Reader *goobj2.Reader + Version int + } } func (l Library) String() string { diff --git a/src/cmd/link/internal/sym/symbols.go b/src/cmd/link/internal/sym/symbols.go index f0fcf2361b..e772496534 100644 --- a/src/cmd/link/internal/sym/symbols.go +++ b/src/cmd/link/internal/sym/symbols.go @@ -86,6 +86,17 @@ func (syms *Symbols) ROLookup(name string, v int) *Symbol { return syms.hash[v][name] } +// Add an existing symbol to the symbol table. +func (syms *Symbols) Add(s *Symbol) { + name := s.Name + v := int(s.Version) + m := syms.hash[v] + if _, ok := m[name]; ok { + panic(name + " already added") + } + m[name] = s +} + // Allocate a new version (i.e. symbol namespace). func (syms *Symbols) IncVersion() int { syms.hash = append(syms.hash, make(map[string]*Symbol)) From e63c1df34856fbf61f72fef84f810cf3306ec204 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 23 Sep 2019 22:44:02 -0400 Subject: [PATCH 08/79] [dev.link] cmd/link: move some work after deadcode This is in preparation of delaying loading symbol content to after dead code elimination. Change-Id: Ia4ce82d9a42cc1a76c82f6d0eb9a54b0325ee3ba Reviewed-on: https://go-review.googlesource.com/c/go/+/197061 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/link/internal/ld/lib.go | 351 ++++++++++++++++--------------- src/cmd/link/internal/ld/main.go | 5 +- 2 files changed, 179 insertions(+), 177 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index e2efb95ff4..5ab43cca70 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -415,37 +415,110 @@ func (ctxt *Link) loadlib() { } } - switch ctxt.BuildMode { - case BuildModeCShared, BuildModePlugin: - s := ctxt.Syms.Lookup("runtime.islibrary", 0) - s.Type = sym.SNOPTRDATA - s.Attr |= sym.AttrDuplicateOK - s.AddUint8(1) - case BuildModeCArchive: - s := ctxt.Syms.Lookup("runtime.isarchive", 0) - s.Type = sym.SNOPTRDATA - s.Attr |= sym.AttrDuplicateOK - s.AddUint8(1) - } - iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil // We now have enough information to determine the link mode. determineLinkMode(ctxt) - // Recalculate pe parameters now that we have ctxt.LinkMode set. - if ctxt.HeadType == objabi.Hwindows { - Peinit(ctxt) + // Now that we know the link mode, trim the dynexp list. + x := sym.AttrCgoExportDynamic + + if ctxt.LinkMode == LinkExternal { + x = sym.AttrCgoExportStatic + } + w := 0 + for i := range dynexp { + if dynexp[i].Attr&x != 0 { + dynexp[w] = dynexp[i] + w++ + } + } + dynexp = dynexp[:w] + + // Resolve ABI aliases in the list of cgo-exported functions. + // This is necessary because we load the ABI0 symbol for all + // cgo exports. + for i, s := range dynexp { + if s.Type != sym.SABIALIAS { + continue + } + t := resolveABIAlias(s) + t.Attr |= s.Attr + t.SetExtname(s.Extname()) + dynexp[i] = t } - if ctxt.HeadType == objabi.Hdarwin && ctxt.LinkMode == LinkExternal { - *FlagTextAddr = 0 + // In internal link mode, read the host object files. + if ctxt.LinkMode == LinkInternal { + // Drop all the cgo_import_static declarations. + // Turns out we won't be needing them. + for _, s := range ctxt.Syms.Allsym { + if s.Type == sym.SHOSTOBJ { + // If a symbol was marked both + // cgo_import_static and cgo_import_dynamic, + // then we want to make it cgo_import_dynamic + // now. + if s.Extname() != "" && s.Dynimplib() != "" && !s.Attr.CgoExport() { + s.Type = sym.SDYNIMPORT + } else { + s.Type = 0 + } + } + } + + hostobjs(ctxt) + + // If we have any undefined symbols in external + // objects, try to read them from the libgcc file. + any := false + for _, s := range ctxt.Syms.Allsym { + for i := range s.R { + r := &s.R[i] // Copying sym.Reloc has measurable impact on performance + if r.Sym != nil && r.Sym.Type == sym.SXREF && r.Sym.Name != ".got" { + any = true + break + } + } + } + if any { + if *flagLibGCC == "" { + *flagLibGCC = ctxt.findLibPathCmd("--print-libgcc-file-name", "libgcc") + } + if runtime.GOOS == "openbsd" && *flagLibGCC == "libgcc.a" { + // On OpenBSD `clang --print-libgcc-file-name` returns "libgcc.a". + // In this case we fail to load libgcc.a and can encounter link + // errors - see if we can find libcompiler_rt.a instead. + *flagLibGCC = ctxt.findLibPathCmd("--print-file-name=libcompiler_rt.a", "libcompiler_rt") + } + if *flagLibGCC != "none" { + hostArchive(ctxt, *flagLibGCC) + } + if ctxt.HeadType == objabi.Hwindows { + if p := ctxt.findLibPath("libmingwex.a"); p != "none" { + hostArchive(ctxt, p) + } + if p := ctxt.findLibPath("libmingw32.a"); p != "none" { + hostArchive(ctxt, p) + } + // TODO: maybe do something similar to peimporteddlls to collect all lib names + // and try link them all to final exe just like libmingwex.a and libmingw32.a: + /* + for: + #cgo windows LDFLAGS: -lmsvcrt -lm + import: + libmsvcrt.a libm.a + */ + } + } + } else { + hostlinksetup(ctxt) } - if ctxt.LinkMode == LinkExternal && ctxt.Arch.Family == sys.PPC64 && objabi.GOOS != "aix" { - toc := ctxt.Syms.Lookup(".TOC.", 0) - toc.Type = sym.SDYNIMPORT - } + // We've loaded all the code now. + ctxt.Loaded = true + + // Record whether we can use plugins. + ctxt.canUsePlugins = (ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil) if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { // This indicates a user requested -linkmode=external. @@ -464,24 +537,92 @@ func (ctxt *Link) loadlib() { } } - if ctxt.LinkMode == LinkInternal { - // Drop all the cgo_import_static declarations. - // Turns out we won't be needing them. - for _, s := range ctxt.Syms.Allsym { - if s.Type == sym.SHOSTOBJ { - // If a symbol was marked both - // cgo_import_static and cgo_import_dynamic, - // then we want to make it cgo_import_dynamic - // now. - if s.Extname() != "" && s.Dynimplib() != "" && !s.Attr.CgoExport() { - s.Type = sym.SDYNIMPORT - } else { - s.Type = 0 + importcycles() + + // put symbols into Textp + // do it in postorder so that packages are laid down in dependency order + // internal first, then everything else + ctxt.Library = postorder(ctxt.Library) + for _, doInternal := range [2]bool{true, false} { + for _, lib := range ctxt.Library { + if isRuntimeDepPkg(lib.Pkg) != doInternal { + continue + } + ctxt.Textp = append(ctxt.Textp, lib.Textp...) + for _, s := range lib.DupTextSyms { + if !s.Attr.OnList() { + ctxt.Textp = append(ctxt.Textp, s) + s.Attr |= sym.AttrOnList + // dupok symbols may be defined in multiple packages. its + // associated package is chosen sort of arbitrarily (the + // first containing package that the linker loads). canonicalize + // it here to the package with which it will be laid down + // in text. + s.File = objabi.PathToPrefix(lib.Pkg) } } } } + if len(ctxt.Shlibs) > 0 { + // We might have overwritten some functions above (this tends to happen for the + // autogenerated type equality/hashing functions) and we don't want to generated + // pcln table entries for these any more so remove them from Textp. + textp := make([]*sym.Symbol, 0, len(ctxt.Textp)) + for _, s := range ctxt.Textp { + if s.Type != sym.SDYNIMPORT { + textp = append(textp, s) + } + } + ctxt.Textp = textp + } +} + +// Set up flags and special symbols depending on the platform build mode. +func (ctxt *Link) linksetup() { + switch ctxt.BuildMode { + case BuildModeCShared, BuildModePlugin: + s := ctxt.Syms.Lookup("runtime.islibrary", 0) + s.Type = sym.SNOPTRDATA + s.Attr |= sym.AttrDuplicateOK + s.AddUint8(1) + case BuildModeCArchive: + s := ctxt.Syms.Lookup("runtime.isarchive", 0) + s.Type = sym.SNOPTRDATA + s.Attr |= sym.AttrDuplicateOK + s.AddUint8(1) + } + + // Recalculate pe parameters now that we have ctxt.LinkMode set. + if ctxt.HeadType == objabi.Hwindows { + Peinit(ctxt) + } + + if ctxt.HeadType == objabi.Hdarwin && ctxt.LinkMode == LinkExternal { + *FlagTextAddr = 0 + } + + // If there are no dynamic libraries needed, gcc disables dynamic linking. + // Because of this, glibc's dynamic ELF loader occasionally (like in version 2.13) + // assumes that a dynamic binary always refers to at least one dynamic library. + // Rather than be a source of test cases for glibc, disable dynamic linking + // the same way that gcc would. + // + // Exception: on OS X, programs such as Shark only work with dynamic + // binaries, so leave it enabled on OS X (Mach-O) binaries. + // Also leave it enabled on Solaris which doesn't support + // statically linked binaries. + if ctxt.BuildMode == BuildModeExe { + if havedynamic == 0 && ctxt.HeadType != objabi.Hdarwin && ctxt.HeadType != objabi.Hsolaris { + *FlagD = true + } + } + + if ctxt.LinkMode == LinkExternal && ctxt.Arch.Family == sys.PPC64 && objabi.GOOS != "aix" { + toc := ctxt.Syms.Lookup(".TOC.", 0) + toc.Type = sym.SDYNIMPORT + } + // The Android Q linker started to complain about underalignment of the our TLS // section. We don't actually use the section on android, so dont't // generate it. @@ -542,93 +683,6 @@ func (ctxt *Link) loadlib() { moduledata.Attr |= sym.AttrReachable ctxt.Moduledata = moduledata - // Now that we know the link mode, trim the dynexp list. - x := sym.AttrCgoExportDynamic - - if ctxt.LinkMode == LinkExternal { - x = sym.AttrCgoExportStatic - } - w := 0 - for i := range dynexp { - if dynexp[i].Attr&x != 0 { - dynexp[w] = dynexp[i] - w++ - } - } - dynexp = dynexp[:w] - - // In internal link mode, read the host object files. - if ctxt.LinkMode == LinkInternal { - hostobjs(ctxt) - - // If we have any undefined symbols in external - // objects, try to read them from the libgcc file. - any := false - for _, s := range ctxt.Syms.Allsym { - for i := range s.R { - r := &s.R[i] // Copying sym.Reloc has measurable impact on performance - if r.Sym != nil && r.Sym.Type == sym.SXREF && r.Sym.Name != ".got" { - any = true - break - } - } - } - if any { - if *flagLibGCC == "" { - *flagLibGCC = ctxt.findLibPathCmd("--print-libgcc-file-name", "libgcc") - } - if runtime.GOOS == "openbsd" && *flagLibGCC == "libgcc.a" { - // On OpenBSD `clang --print-libgcc-file-name` returns "libgcc.a". - // In this case we fail to load libgcc.a and can encounter link - // errors - see if we can find libcompiler_rt.a instead. - *flagLibGCC = ctxt.findLibPathCmd("--print-file-name=libcompiler_rt.a", "libcompiler_rt") - } - if *flagLibGCC != "none" { - hostArchive(ctxt, *flagLibGCC) - } - if ctxt.HeadType == objabi.Hwindows { - if p := ctxt.findLibPath("libmingwex.a"); p != "none" { - hostArchive(ctxt, p) - } - if p := ctxt.findLibPath("libmingw32.a"); p != "none" { - hostArchive(ctxt, p) - } - // TODO: maybe do something similar to peimporteddlls to collect all lib names - // and try link them all to final exe just like libmingwex.a and libmingw32.a: - /* - for: - #cgo windows LDFLAGS: -lmsvcrt -lm - import: - libmsvcrt.a libm.a - */ - } - } - } else { - hostlinksetup(ctxt) - } - - // We've loaded all the code now. - ctxt.Loaded = true - - // Record whether we can use plugins. - ctxt.canUsePlugins = (ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil) - - // If there are no dynamic libraries needed, gcc disables dynamic linking. - // Because of this, glibc's dynamic ELF loader occasionally (like in version 2.13) - // assumes that a dynamic binary always refers to at least one dynamic library. - // Rather than be a source of test cases for glibc, disable dynamic linking - // the same way that gcc would. - // - // Exception: on OS X, programs such as Shark only work with dynamic - // binaries, so leave it enabled on OS X (Mach-O) binaries. - // Also leave it enabled on Solaris which doesn't support - // statically linked binaries. - if ctxt.BuildMode == BuildModeExe { - if havedynamic == 0 && ctxt.HeadType != objabi.Hdarwin && ctxt.HeadType != objabi.Hsolaris { - *FlagD = true - } - } - // If package versioning is required, generate a hash of the // packages used in the link. if ctxt.BuildMode == BuildModeShared || ctxt.BuildMode == BuildModePlugin || ctxt.CanUsePlugins() { @@ -646,59 +700,6 @@ func (ctxt *Link) loadlib() { got.Attr |= sym.AttrReachable } } - - importcycles() - - // put symbols into Textp - // do it in postorder so that packages are laid down in dependency order - // internal first, then everything else - ctxt.Library = postorder(ctxt.Library) - for _, doInternal := range [2]bool{true, false} { - for _, lib := range ctxt.Library { - if isRuntimeDepPkg(lib.Pkg) != doInternal { - continue - } - ctxt.Textp = append(ctxt.Textp, lib.Textp...) - for _, s := range lib.DupTextSyms { - if !s.Attr.OnList() { - ctxt.Textp = append(ctxt.Textp, s) - s.Attr |= sym.AttrOnList - // dupok symbols may be defined in multiple packages. its - // associated package is chosen sort of arbitrarily (the - // first containing package that the linker loads). canonicalize - // it here to the package with which it will be laid down - // in text. - s.File = objabi.PathToPrefix(lib.Pkg) - } - } - } - } - - if len(ctxt.Shlibs) > 0 { - // We might have overwritten some functions above (this tends to happen for the - // autogenerated type equality/hashing functions) and we don't want to generated - // pcln table entries for these any more so remove them from Textp. - textp := make([]*sym.Symbol, 0, len(ctxt.Textp)) - for _, s := range ctxt.Textp { - if s.Type != sym.SDYNIMPORT { - textp = append(textp, s) - } - } - ctxt.Textp = textp - } - - // Resolve ABI aliases in the list of cgo-exported functions. - // This is necessary because we load the ABI0 symbol for all - // cgo exports. - for i, s := range dynexp { - if s.Type != sym.SABIALIAS { - continue - } - t := resolveABIAlias(s) - t.Attr |= s.Attr - t.SetExtname(s.Extname()) - dynexp[i] = t - } } // mangleTypeSym shortens the names of symbols that represent Go types diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 54a5fa199c..759e7fb744 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -208,9 +208,10 @@ func Main(arch *sys.Arch, theArch Arch) { } ctxt.loadlib() - ctxt.dostrdata() deadcode(ctxt) - //ctxt.dumpsyms() // XXX + ctxt.linksetup() + ctxt.dostrdata() + dwarfGenerateDebugInfo(ctxt) if objabi.Fieldtrack_enabled != 0 { fieldtrack(ctxt) From fcfbe25dca8525257a2c265e6bd825b5c4062c6c Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 7 Oct 2019 12:44:49 -0400 Subject: [PATCH 09/79] [dev.link] cmd/link: remove runtime.gcbits special case Follow-up of Than's comment on CL 196032. For runtime.gcbits symbols, the compiler already sets the "local" attribute (cmd/compile/internal/gc/reflect.go:dgcptrmask). No need of the special handling in the linker. Change-Id: I7c53a818ed0375083f04655f3565c1c9c45f2401 Reviewed-on: https://go-review.googlesource.com/c/go/+/199642 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/objfile/objfile2.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 5a92fef4ec..1908d21c93 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -223,9 +223,6 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { } s.Attr.Set(sym.AttrReachable, false) } - if strings.HasPrefix(s.Name, "runtime.gcbits.") { - s.Attr |= sym.AttrLocal - } } func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, localSymVersion int, libByPkg map[string]*sym.Library) { From 48151b3f0ffda8a38ca71c8f1b8a72d5f90d7061 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 24 Sep 2019 18:12:55 -0400 Subject: [PATCH 10/79] [dev.link] cmd/link: add symbols to Textp after deadcode pass Currently we add all text symbols to ctxt.Textp at load time, then the deadcode pass filters out unreachable ones. This CL delays adding symbols to ctxt.Textp to the end of the deadcode pass, where we only add reachable ones. Change-Id: Ie83b2958f915c5aaa004b8c5ed1f1bc275f4d1db Reviewed-on: https://go-review.googlesource.com/c/go/+/197257 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode.go | 62 ++++++++++++++++++++++++---- src/cmd/link/internal/ld/lib.go | 38 ----------------- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index c880c0da01..cadb92b43c 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -118,22 +118,66 @@ func deadcode(ctxt *Link) { } } - for _, lib := range ctxt.Library { - lib.Textp = lib.Textp[:0] - } - // Remove dead text but keep file information (z symbols). - textp := make([]*sym.Symbol, 0, len(ctxt.Textp)) + textp := []*sym.Symbol{} for _, s := range ctxt.Textp { if s.Attr.Reachable() { - if s.Unit != nil { - s.Unit.Lib.Textp = append(s.Unit.Lib.Textp, s) - s.Unit.Textp = append(s.Unit.Textp, s) - } textp = append(textp, s) } } + + // Put reachable text symbols into Textp. + // do it in postorder so that packages are laid down in dependency order + // internal first, then everything else + ctxt.Library = postorder(ctxt.Library) + for _, doInternal := range [2]bool{true, false} { + for _, lib := range ctxt.Library { + if isRuntimeDepPkg(lib.Pkg) != doInternal { + continue + } + libtextp := lib.Textp[:0] + for _, s := range lib.Textp { + if s.Attr.Reachable() { + textp = append(textp, s) + libtextp = append(libtextp, s) + if s.Unit != nil { + s.Unit.Textp = append(s.Unit.Textp, s) + } + } + } + for _, s := range lib.DupTextSyms { + if s.Attr.Reachable() && !s.Attr.OnList() { + textp = append(textp, s) + libtextp = append(libtextp, s) + if s.Unit != nil { + s.Unit.Textp = append(s.Unit.Textp, s) + } + s.Attr |= sym.AttrOnList + // dupok symbols may be defined in multiple packages. its + // associated package is chosen sort of arbitrarily (the + // first containing package that the linker loads). canonicalize + // it here to the package with which it will be laid down + // in text. + s.File = objabi.PathToPrefix(lib.Pkg) + } + } + lib.Textp = libtextp + } + } ctxt.Textp = textp + + if len(ctxt.Shlibs) > 0 { + // We might have overwritten some functions above (this tends to happen for the + // autogenerated type equality/hashing functions) and we don't want to generated + // pcln table entries for these any more so remove them from Textp. + textp := make([]*sym.Symbol, 0, len(ctxt.Textp)) + for _, s := range ctxt.Textp { + if s.Type != sym.SDYNIMPORT { + textp = append(textp, s) + } + } + ctxt.Textp = textp + } } // methodref holds the relocations from a receiver type symbol to its diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 5ab43cca70..f11adbcfb6 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -538,44 +538,6 @@ func (ctxt *Link) loadlib() { } importcycles() - - // put symbols into Textp - // do it in postorder so that packages are laid down in dependency order - // internal first, then everything else - ctxt.Library = postorder(ctxt.Library) - for _, doInternal := range [2]bool{true, false} { - for _, lib := range ctxt.Library { - if isRuntimeDepPkg(lib.Pkg) != doInternal { - continue - } - ctxt.Textp = append(ctxt.Textp, lib.Textp...) - for _, s := range lib.DupTextSyms { - if !s.Attr.OnList() { - ctxt.Textp = append(ctxt.Textp, s) - s.Attr |= sym.AttrOnList - // dupok symbols may be defined in multiple packages. its - // associated package is chosen sort of arbitrarily (the - // first containing package that the linker loads). canonicalize - // it here to the package with which it will be laid down - // in text. - s.File = objabi.PathToPrefix(lib.Pkg) - } - } - } - } - - if len(ctxt.Shlibs) > 0 { - // We might have overwritten some functions above (this tends to happen for the - // autogenerated type equality/hashing functions) and we don't want to generated - // pcln table entries for these any more so remove them from Textp. - textp := make([]*sym.Symbol, 0, len(ctxt.Textp)) - for _, s := range ctxt.Textp { - if s.Type != sym.SDYNIMPORT { - textp = append(textp, s) - } - } - ctxt.Textp = textp - } } // Set up flags and special symbols depending on the platform build mode. From 24950952759cb26144333b0a47eae7da44808eec Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 24 Sep 2019 17:31:12 -0400 Subject: [PATCH 11/79] [dev.link] cmd/link: load full symbol contents after deadcode pass If the new object file format is used, now we load full symbol contents after the deadcode pass, for reachable symbols only. We still load some informations early, like relocations and the contents of type symbols, which are used in the deadcode pass. If we rewrite deadcode to use index directly, we could delay more of the loading (to sym.Symbol), and perhaps delay the creation of sym.Symbol. TODO: internal linking with host objects doesn't work yet. Change-Id: I7d4880e8f150e8709ffac277e62191623440e4cf Reviewed-on: https://go-review.googlesource.com/c/go/+/197258 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode.go | 10 +- src/cmd/link/internal/ld/go.go | 6 +- src/cmd/link/internal/ld/lib.go | 92 ++++++----- src/cmd/link/internal/ld/link.go | 29 ++++ src/cmd/link/internal/ld/main.go | 3 + src/cmd/link/internal/objfile/objfile2.go | 187 ++++++++++++++++------ 6 files changed, 227 insertions(+), 100 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index cadb92b43c..575fabc259 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -60,8 +60,8 @@ func deadcode(ctxt *Link) { d.init() d.flood() - callSym := ctxt.Syms.ROLookup("reflect.Value.Call", sym.SymVerABIInternal) - methSym := ctxt.Syms.ROLookup("reflect.Value.Method", sym.SymVerABIInternal) + callSym := ctxt.Lookup("reflect.Value.Call", sym.SymVerABIInternal) + methSym := ctxt.Lookup("reflect.Value.Method", sym.SymVerABIInternal) reflectSeen := false if ctxt.DynlinkingGo() { @@ -283,7 +283,7 @@ func (d *deadcodepass) init() { // We don't keep the go.plugin.exports symbol, // but we do keep the symbols it refers to. - exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0) + exports := d.ctxt.Lookup("go.plugin.exports", 0) if exports != nil { for i := range exports.R { d.mark(exports.R[i].Sym, nil) @@ -298,9 +298,9 @@ func (d *deadcodepass) init() { for _, name := range names { // Mark symbol as an data/ABI0 symbol. - d.mark(d.ctxt.Syms.ROLookup(name, 0), nil) + d.mark(d.ctxt.Lookup(name, 0), nil) // Also mark any Go functions (internal ABI). - d.mark(d.ctxt.Syms.ROLookup(name, sym.SymVerABIInternal), nil) + d.mark(d.ctxt.Lookup(name, sym.SymVerABIInternal), nil) } } diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index d1d68b0704..13fbbed10f 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -169,7 +169,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { if i := strings.Index(remote, "#"); i >= 0 { remote, q = remote[:i], remote[i+1:] } - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ { s.SetDynimplib(lib) s.SetExtname(remote) @@ -188,7 +188,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { } local := f[1] - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) s.Type = sym.SHOSTOBJ s.Size = 0 continue @@ -209,7 +209,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { // functions. Link.loadlib will resolve any // ABI aliases we find here (since we may not // yet know it's an alias). - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) switch ctxt.BuildMode { case BuildModeCShared, BuildModeCArchive, BuildModePlugin: diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index f11adbcfb6..2ebd5d333c 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -401,21 +401,24 @@ func (ctxt *Link) loadlib() { } } - // XXX do it here for now if *flagNewobj { - ctxt.loadlibfull() - } - - for _, lib := range ctxt.Library { - if lib.Shlib != "" { - if ctxt.Debugvlog > 1 { - ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + // Add references of externally defined symbols. + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) } - ldshlibsyms(ctxt, lib.Shlib) + } + + // Load cgo directives. + for _, p := range ctxt.cgodata { + loadcgo(ctxt, p[0], p[1], p[2]) } } - iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil + iscgo = ctxt.Lookup("x_cgo_init", 0) != nil + + // Record whether we can use plugins. + ctxt.canUsePlugins = (ctxt.Lookup("plugin.Open", sym.SymVerABIInternal) != nil) // We now have enough information to determine the link mode. determineLinkMode(ctxt) @@ -448,8 +451,34 @@ func (ctxt *Link) loadlib() { dynexp[i] = t } + for _, lib := range ctxt.Library { + if lib.Shlib != "" { + if ctxt.Debugvlog > 1 { + ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + } + ldshlibsyms(ctxt, lib.Shlib) + } + } + + if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { + // This indicates a user requested -linkmode=external. + // The startup code uses an import of runtime/cgo to decide + // whether to initialize the TLS. So give it one. This could + // be handled differently but it's an unusual case. + if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil { + if lib.Shlib != "" { + ldshlibsyms(ctxt, lib.Shlib) + } else { + if ctxt.BuildMode == BuildModeShared || ctxt.linkShared { + Exitf("cannot implicitly include runtime/cgo in a shared library") + } + loadobjfile(ctxt, lib) + } + } + } + // In internal link mode, read the host object files. - if ctxt.LinkMode == LinkInternal { + if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. // Turns out we won't be needing them. for _, s := range ctxt.Syms.Allsym { @@ -510,34 +539,23 @@ func (ctxt *Link) loadlib() { */ } } - } else { + } else if ctxt.LinkMode == LinkExternal { hostlinksetup(ctxt) } // We've loaded all the code now. ctxt.Loaded = true - // Record whether we can use plugins. - ctxt.canUsePlugins = (ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil) + importcycles() - if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { - // This indicates a user requested -linkmode=external. - // The startup code uses an import of runtime/cgo to decide - // whether to initialize the TLS. So give it one. This could - // be handled differently but it's an unusual case. - if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil { - if lib.Shlib != "" { - ldshlibsyms(ctxt, lib.Shlib) - } else { - if ctxt.BuildMode == BuildModeShared || ctxt.linkShared { - Exitf("cannot implicitly include runtime/cgo in a shared library") - } - loadobjfile(ctxt, lib) + // For now, load relocations for dead-code elimination. + if *flagNewobj { + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadReloc(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) } } } - - importcycles() } // Set up flags and special symbols depending on the platform build mode. @@ -1907,7 +1925,7 @@ func ldshlibsyms(ctxt *Link, shlib string) { ver = sym.SymVerABIInternal } - lsym := ctxt.Syms.Lookup(elfsym.Name, ver) + lsym := ctxt.LookupOrCreate(elfsym.Name, ver) // Because loadlib above loads all .a files before loading any shared // libraries, any non-dynimport symbols we find that duplicate symbols // already loaded should be ignored (the symbols from the .a files @@ -2526,17 +2544,10 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library } func (ctxt *Link) loadlibfull() { - // Add references of externally defined symbols. - for _, lib := range ctxt.Library { - for _, r := range lib.Readers { - objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) - } - } - // Load full symbol contents, resolve indexed references. for _, lib := range ctxt.Library { for _, r := range lib.Readers { - objfile.LoadFull(ctxt.loader, r.Reader, lib, ctxt.Syms, r.Version, ctxt.LibraryByPkg) + objfile.LoadFull(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) } } @@ -2546,11 +2557,6 @@ func (ctxt *Link) loadlibfull() { ctxt.Syms.Add(s) } } - - // Now load cgo directives. - for _, p := range ctxt.cgodata { - loadcgo(ctxt, p[0], p[1], p[2]) - } } func (ctxt *Link) dumpsyms() { diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index bbdb0e50ed..dfb686f038 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -176,3 +176,32 @@ func addImports(ctxt *Link, l *sym.Library, pn string) { } l.ImportStrings = nil } + +// convenient helper during the transition period. +func (ctxt *Link) Lookup(name string, ver int) *sym.Symbol { + if *flagNewobj { + i := ctxt.loader.Lookup(name, ver) + if i == 0 { + return nil + } + return ctxt.loader.Syms[i] + } else { + return ctxt.Syms.ROLookup(name, ver) + } +} + +// convenient helper during the transition period. +func (ctxt *Link) LookupOrCreate(name string, ver int) *sym.Symbol { + if *flagNewobj { + i := ctxt.loader.Lookup(name, ver) + if i != 0 { + return ctxt.loader.Syms[i] + } + ctxt.loader.AddExtSym(name, ver) + s := ctxt.Syms.Newsym(name, ver) + ctxt.loader.Syms = append(ctxt.loader.Syms, s) + return s + } else { + return ctxt.Syms.Lookup(name, ver) + } +} diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 759e7fb744..e667afecc1 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -209,6 +209,9 @@ func Main(arch *sys.Arch, theArch Arch) { ctxt.loadlib() deadcode(ctxt) + if *flagNewobj { + ctxt.loadlibfull() // XXX do it here for now + } ctxt.linksetup() ctxt.dostrdata() diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 1908d21c93..a5bd91d3ab 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -225,7 +225,10 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { } } -func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, localSymVersion int, libByPkg map[string]*sym.Library) { +// Load relocations for building the dependency graph in deadcode pass. +// For now, we load symbol types, relocations, gotype, and the contents +// of type symbols, which are needed in deadcode. +func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { // PkgIdx pkglist := r.Pkglist() @@ -262,7 +265,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, return l.Syms[i] } - pcdataBase := r.PcdataBase() for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { s := l.Syms[istart+i] if s == nil || s.Name == "" { @@ -272,28 +274,30 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, osym := goobj2.Sym{} osym.Read(r, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - if s.Name != name { + if s.Name != name { // Sanity check. We can remove it in the final version. fmt.Println("name mismatch:", lib, i, s.Name, name) panic("name mismatch") } - dupok := osym.Flag&goobj2.SymFlagDupok != 0 - local := osym.Flag&goobj2.SymFlagLocal != 0 - makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 - nreloc := r.NReloc(i) - datasize := r.DataSize(i) - size := osym.Siz - - t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] if s.Type != 0 && s.Type != sym.SXREF { fmt.Println("symbol already processed:", lib, i, s) panic("symbol already processed") } - // Symbol data - s.P = r.BytesAt(r.DataOff(i), datasize) + t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] + if t == sym.SXREF { + log.Fatalf("bad sxref") + } + if t == 0 { + log.Fatalf("missing type for %s in %s", s.Name, lib) + } + if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { + t = s.Type + } + s.Type = t // Reloc + nreloc := r.NReloc(i) s.R = make([]sym.Reloc, nreloc) for j := range s.R { rel := goobj2.Reloc{} @@ -307,9 +311,13 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, } } - // Aux symbol info - isym := -1 - funcdata := make([]goobj2.SymRef, 0, 4) + // XXX deadcode needs symbol data for type symbols. Read it now. + if strings.HasPrefix(name, "type.") { + s.P = r.BytesAt(r.DataOff(i), r.DataSize(i)) + s.Size = int64(osym.Siz) + } + + // Aux symbol naux := r.NAux(i) for j := 0; j < naux; j++ { a := goobj2.Aux{} @@ -320,13 +328,115 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if typ != nil { s.Gotype = typ } + case goobj2.AuxFuncdata: + pc := s.FuncInfo + if pc == nil { + pc = &sym.FuncInfo{Funcdata: make([]*sym.Symbol, 0, 4)} + s.FuncInfo = pc + } + pc.Funcdata = append(pc.Funcdata, resolveSymRef(a.Sym)) + } + } + + if s.Type == sym.STEXT { + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + if !dupok { + if s.Attr.OnList() { + log.Fatalf("symbol %s listed multiple times", s.Name) + } + s.Attr |= sym.AttrOnList + lib.Textp = append(lib.Textp, s) + } else { + // there may ba a dup in another package + // put into a temp list and add to text later + lib.DupTextSyms = append(lib.DupTextSyms, s) + } + } + } +} + +// Load full contents. +// TODO: For now, some contents are already load in LoadReloc. Maybe +// we should combine LoadReloc back into this, once we rewrite deadcode +// pass to use index directly. +func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { + // PkgIdx + pkglist := r.Pkglist() + + pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + istart := l.StartIndex(r) + + resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { + var rr *goobj2.Reader + switch p := s.PkgIdx; p { + case goobj2.PkgIdxInvalid: + if s.SymIdx != 0 { + panic("bad sym ref") + } + return nil + case goobj2.PkgIdxNone: + // Resolve by name + i := int(s.SymIdx) + r.NSym() + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + v := abiToVer(osym.ABI, localSymVersion) + nv := nameVer{name, v} + i = l.symsByName[nv] + return l.Syms[i] + case goobj2.PkgIdxSelf: + rr = r + default: + pkg := pkglist[p] + rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm) + } + i := l.ToGlobal(rr, int(s.SymIdx)) + return l.Syms[i] + } + + pcdataBase := r.PcdataBase() + for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { + s := l.Syms[istart+i] + if s == nil || s.Name == "" { + continue + } + if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) { + // No need to load unreachable symbols. + // XXX DWARF symbols may be used but are not marked reachable. + continue + } + + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + if s.Name != name { // Sanity check. We can remove it in the final version. + fmt.Println("name mismatch:", lib, i, s.Name, name) + panic("name mismatch") + } + + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + local := osym.Flag&goobj2.SymFlagLocal != 0 + makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 + datasize := r.DataSize(i) + size := osym.Siz + + // Symbol data + s.P = r.BytesAt(r.DataOff(i), datasize) + + // Aux symbol info + isym := -1 + naux := r.NAux(i) + for j := 0; j < naux; j++ { + a := goobj2.Aux{} + a.Read(r, r.AuxOff(i, j)) + switch a.Type { + case goobj2.AuxGotype, goobj2.AuxFuncdata: + // already loaded case goobj2.AuxFuncInfo: if a.Sym.PkgIdx != goobj2.PkgIdxSelf { panic("funcinfo symbol not defined in current package") } isym = int(a.Sym.SymIdx) - case goobj2.AuxFuncdata: - funcdata = append(funcdata, a.Sym) default: panic("unknown aux type") } @@ -336,16 +446,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if dupok { s.Attr |= sym.AttrDuplicateOK } - if t == sym.SXREF { - log.Fatalf("bad sxref") - } - if t == 0 { - log.Fatalf("missing type for %s in %s", s.Name, lib) - } - if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { - t = s.Type - } - s.Type = t if s.Size < int64(size) { s.Size = int64(size) } @@ -355,17 +455,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if s.Type != sym.STEXT { continue } - if !dupok { - if s.Attr.OnList() { - log.Fatalf("symbol %s listed multiple times", s.Name) - } - s.Attr |= sym.AttrOnList - lib.Textp = append(lib.Textp, s) - } else { - // there may ba a dup in another package - // put into a temp list and add to text later - lib.DupTextSyms = append(lib.DupTextSyms, s) - } // FuncInfo if isym == -1 { @@ -389,15 +478,16 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, } info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends - pc := &sym.FuncInfo{ - Args: int32(info.Args), - Locals: int32(info.Locals), - Pcdata: make([]sym.Pcdata, len(info.Pcdata)-1), // -1 as we appended one above - Funcdata: make([]*sym.Symbol, len(info.Funcdataoff)), - Funcdataoff: make([]int64, len(info.Funcdataoff)), - File: make([]*sym.Symbol, len(info.File)), + pc := s.FuncInfo + if pc == nil { + pc = &sym.FuncInfo{} + s.FuncInfo = pc } - s.FuncInfo = pc + pc.Args = int32(info.Args) + pc.Locals = int32(info.Locals) + pc.Pcdata = make([]sym.Pcdata, len(info.Pcdata)-1) // -1 as we appended one above + pc.Funcdataoff = make([]int64, len(info.Funcdataoff)) + pc.File = make([]*sym.Symbol, len(info.File)) pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp)) pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile)) pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline)) @@ -405,8 +495,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, for k := range pc.Pcdata { pc.Pcdata[k].P = r.BytesAt(pcdataBase+info.Pcdata[k], int(info.Pcdata[k+1]-info.Pcdata[k])) } - for k := range pc.Funcdata { - pc.Funcdata[k] = resolveSymRef(funcdata[k]) + for k := range pc.Funcdataoff { pc.Funcdataoff[k] = int64(info.Funcdataoff[k]) } for k := range pc.File { From 0d7404c8527dc45469644c0f6b4becfd59c2a4d9 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 27 Sep 2019 14:49:44 -0400 Subject: [PATCH 12/79] [dev.link] cmd/link, cmd/internal/goobj2: mmap object file in -newobj mode With the old object file format, we use mmap (if supported) to read object files and back symbol data with mapped read-only memory. Do the same with the new object file format. This also significantly reduces number of syscalls made to read object files. Currently we still do mmap in object file level, not archive level. This is probably ok, as there shouldn't be many archives that contain more than one object. If this is a problem we can change that later. Change-Id: Icae3ef14d8ed6adbee1b5b48d420e2af22fd9604 Reviewed-on: https://go-review.googlesource.com/c/go/+/197797 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj2/objfile.go | 81 +++++++++++++++-------- src/cmd/link/internal/ld/lib.go | 2 +- src/cmd/link/internal/objfile/objfile2.go | 9 ++- 3 files changed, 63 insertions(+), 29 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index eb9290a699..4c1bbe83f0 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -7,11 +7,13 @@ package goobj2 // TODO: replace the goobj package? import ( + "bytes" "cmd/internal/bio" "encoding/binary" "errors" "fmt" "io" + "unsafe" ) // New object file format. @@ -354,6 +356,9 @@ func (w *Writer) Offset() uint32 { } type Reader struct { + b []byte // mmapped bytes, if not nil + readonly bool // whether b is backed with read-only memory + rd io.ReaderAt start uint32 h Header // keep block offsets @@ -368,10 +373,25 @@ func NewReader(rd io.ReaderAt, off uint32) *Reader { return r } +func NewReaderFromBytes(b []byte, readonly bool) *Reader { + r := &Reader{b: b, readonly: readonly, rd: bytes.NewReader(b), start: 0} + err := r.h.Read(r) + if err != nil { + return nil + } + return r +} + func (r *Reader) BytesAt(off uint32, len int) []byte { - // TODO: read from mapped memory + if len == 0 { + return nil + } + if r.b != nil { + end := int(off) + len + return r.b[int(off):end:end] + } b := make([]byte, len) - _, err := r.rd.ReadAt(b[:], int64(r.start+off)) + _, err := r.rd.ReadAt(b, int64(r.start+off)) if err != nil { panic("corrupted input") } @@ -379,12 +399,8 @@ func (r *Reader) BytesAt(off uint32, len int) []byte { } func (r *Reader) uint64At(off uint32) uint64 { - var b [8]byte - n, err := r.rd.ReadAt(b[:], int64(r.start+off)) - if n != 8 || err != nil { - panic("corrupted input") - } - return binary.LittleEndian.Uint64(b[:]) + b := r.BytesAt(off, 8) + return binary.LittleEndian.Uint64(b) } func (r *Reader) int64At(off uint32) int64 { @@ -392,12 +408,8 @@ func (r *Reader) int64At(off uint32) int64 { } func (r *Reader) uint32At(off uint32) uint32 { - var b [4]byte - n, err := r.rd.ReadAt(b[:], int64(r.start+off)) - if n != 4 || err != nil { - panic("corrupted input") - } - return binary.LittleEndian.Uint32(b[:]) + b := r.BytesAt(off, 4) + return binary.LittleEndian.Uint32(b) } func (r *Reader) int32At(off uint32) int32 { @@ -405,26 +417,24 @@ func (r *Reader) int32At(off uint32) int32 { } func (r *Reader) uint16At(off uint32) uint16 { - var b [2]byte - n, err := r.rd.ReadAt(b[:], int64(r.start+off)) - if n != 2 || err != nil { - panic("corrupted input") - } - return binary.LittleEndian.Uint16(b[:]) + b := r.BytesAt(off, 2) + return binary.LittleEndian.Uint16(b) } func (r *Reader) uint8At(off uint32) uint8 { - var b [1]byte - n, err := r.rd.ReadAt(b[:], int64(r.start+off)) - if n != 1 || err != nil { - panic("corrupted input") - } + b := r.BytesAt(off, 1) return b[0] } func (r *Reader) StringAt(off uint32) string { - // TODO: have some way to construct a string without copy l := r.uint32At(off) + if r.b != nil { + b := r.b[off+4 : off+4+l] + if r.readonly { + return toString(b) // backed by RO memory, ok to make unsafe string + } + return string(b) + } b := make([]byte, l) n, err := r.rd.ReadAt(b, int64(r.start+off+4)) if n != int(l) || err != nil { @@ -433,6 +443,20 @@ func (r *Reader) StringAt(off uint32) string { return string(b) } +func toString(b []byte) string { + type stringHeader struct { + str unsafe.Pointer + len int + } + + if len(b) == 0 { + return "" + } + ss := stringHeader{str: unsafe.Pointer(&b[0]), len: len(b)} + s := *(*string)(unsafe.Pointer(&ss)) + return s +} + func (r *Reader) StringRef(off uint32) string { return r.StringAt(r.uint32At(off)) } @@ -511,3 +535,8 @@ func (r *Reader) DataSize(i int) int { func (r *Reader) PcdataBase() uint32 { return r.h.Offsets[BlkPcdata] } + +// ReadOnly returns whether r.BytesAt returns read-only bytes. +func (r *Reader) ReadOnly() bool { + return r.readonly +} diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 2ebd5d333c..d10933ae43 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -835,7 +835,7 @@ func loadobjfile(ctxt *Link, lib *sym.Library) { if err != nil { Exitf("cannot open file %s: %v", lib.File, err) } - //defer f.Close() + defer f.Close() defer func() { if pkg == "main" && !lib.Main { Exitf("%s: not package main", lib.File) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index a5bd91d3ab..252615febc 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -124,8 +124,11 @@ func (l *Loader) Lookup(name string, ver int) int { // Preload a package: add autolibs, add symbols to the symbol table. // Does not read symbol data yet. func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { - start := f.Offset() - r := goobj2.NewReader(f.File(), uint32(start)) + roObject, readonly, err := f.Slice(uint64(length)) + if err != nil { + log.Fatal("cannot read object file:", err) + } + r := goobj2.NewReaderFromBytes(roObject, readonly) if r == nil { panic("cannot read object file") } @@ -314,6 +317,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in // XXX deadcode needs symbol data for type symbols. Read it now. if strings.HasPrefix(name, "type.") { s.P = r.BytesAt(r.DataOff(i), r.DataSize(i)) + s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) s.Size = int64(osym.Siz) } @@ -422,6 +426,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int // Symbol data s.P = r.BytesAt(r.DataOff(i), datasize) + s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) // Aux symbol info isym := -1 From 65a649c565d06c14ae8e3a5042c4dec7f70af612 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 30 Sep 2019 11:43:41 -0400 Subject: [PATCH 13/79] [dev.link] cmd/link, cmd/internal/goobj2: adopt new DWARF compilation unit logic with new object file The dev.link branch was not sync'd with the new DWARF compilation unit logic change on the master branch, and the new object file format didn't support this. This CL adds the new DWARF CU and file table support to the new object file format. In the old object file, the DWARF file table is a separate section. For now, we do the same with the new object file, keeping it as a separate block. While here, also refactor the loader code so it is easier for the loader to carry per-object informations. Change-Id: I4c317941fc0a5831acbc11ce8c2a8b7421471372 Reviewed-on: https://go-review.googlesource.com/c/go/+/198198 Reviewed-by: Austin Clements --- src/cmd/internal/goobj2/objfile.go | 26 +++ src/cmd/internal/obj/objfile2.go | 9 + src/cmd/link/internal/ld/lib.go | 21 +- src/cmd/link/internal/objfile/objfile2.go | 228 ++++++++++++++-------- src/cmd/link/internal/sym/library.go | 7 - 5 files changed, 184 insertions(+), 107 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index 4c1bbe83f0..b5cc0d7bf7 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -31,6 +31,8 @@ import ( // // PkgIndex [...]stringOff // TODO: add fingerprints // +// DwarfFiles [...]stringOff // XXX as a separate block for now +// // SymbolDefs [...]struct { // Name stringOff // ABI uint16 @@ -126,6 +128,7 @@ const ( // Blocks const ( BlkPkgIdx = iota + BlkDwarfFile BlkSymdef BlkNonpkgdef BlkNonpkgref @@ -471,6 +474,24 @@ func (r *Reader) Pkglist() []string { return s } +func (r *Reader) NPkg() int { + return int(r.h.Offsets[BlkPkgIdx+1]-r.h.Offsets[BlkPkgIdx]) / 4 +} + +func (r *Reader) Pkg(i int) string { + off := r.h.Offsets[BlkPkgIdx] + uint32(i)*4 + return r.StringRef(off) +} + +func (r *Reader) NDwarfFile() int { + return int(r.h.Offsets[BlkDwarfFile+1]-r.h.Offsets[BlkDwarfFile]) / 4 +} + +func (r *Reader) DwarfFile(i int) string { + off := r.h.Offsets[BlkDwarfFile] + uint32(i)*4 + return r.StringRef(off) +} + func (r *Reader) NSym() int { symsiz := (&Sym{}).Size() return int(r.h.Offsets[BlkSymdef+1]-r.h.Offsets[BlkSymdef]) / symsiz @@ -531,6 +552,11 @@ func (r *Reader) DataSize(i int) int { return int(r.DataOff(i+1) - r.DataOff(i)) } +// Data returns the i-th symbol's data. +func (r *Reader) Data(i int) []byte { + return r.BytesAt(r.DataOff(i), r.DataSize(i)) +} + // AuxDataBase returns the base offset of the aux data block. func (r *Reader) PcdataBase() uint32 { return r.h.Offsets[BlkPcdata] diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index 42f050a940..4043e0b9fe 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -42,6 +42,12 @@ func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { w.StringRef(pkg) } + // DWARF file table + h.Offsets[goobj2.BlkDwarfFile] = w.Offset() + for _, f := range ctxt.PosTable.DebugLinesFileTable() { + w.StringRef(f) + } + // Symbol definitions h.Offsets[goobj2.BlkSymdef] = w.Offset() for _, s := range ctxt.defs { @@ -198,6 +204,9 @@ func (w *writer) StringTable() { w.AddString(f) } }) + for _, f := range w.ctxt.PosTable.DebugLinesFileTable() { + w.AddString(f) + } } func (w *writer) Sym(s *LSym) { diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index d10933ae43..b913479b72 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -403,11 +403,7 @@ func (ctxt *Link) loadlib() { if *flagNewobj { // Add references of externally defined symbols. - for _, lib := range ctxt.Library { - for _, r := range lib.Readers { - objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) - } - } + objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms) // Load cgo directives. for _, p := range ctxt.cgodata { @@ -550,11 +546,7 @@ func (ctxt *Link) loadlib() { // For now, load relocations for dead-code elimination. if *flagNewobj { - for _, lib := range ctxt.Library { - for _, r := range lib.Readers { - objfile.LoadReloc(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) - } - } + objfile.LoadReloc(ctxt.loader) } } @@ -2545,11 +2537,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library func (ctxt *Link) loadlibfull() { // Load full symbol contents, resolve indexed references. - for _, lib := range ctxt.Library { - for _, r := range lib.Readers { - objfile.LoadFull(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) - } - } + objfile.LoadFull(ctxt.loader) // For now, add all symbols to ctxt.Syms. for _, s := range ctxt.loader.Syms { @@ -2557,6 +2545,9 @@ func (ctxt *Link) loadlibfull() { ctxt.Syms.Add(s) } } + + // Drop the reference. + ctxt.loader = nil } func (ctxt *Link) dumpsyms() { diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 252615febc..e2442d8982 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -5,7 +5,9 @@ package objfile import ( + "bytes" "cmd/internal/bio" + "cmd/internal/dwarf" "cmd/internal/goobj2" "cmd/internal/obj" "cmd/internal/objabi" @@ -21,8 +23,18 @@ import ( var _ = fmt.Print +// oReader is a wrapper type of obj.Reader, along with some +// extra information. +// TODO: rename to objReader once the old one is gone? +type oReader struct { + *goobj2.Reader + unit *sym.CompilationUnit + version int // version of static symbol + pkgprefix string +} + type objIdx struct { - r *goobj2.Reader + r *oReader i int // start index } @@ -35,34 +47,40 @@ type nameVer struct { // // TODO: describe local-global index mapping. type Loader struct { - start map[*goobj2.Reader]int // map from object file to its start index - objs []objIdx // sorted by start index (i.e. objIdx.i) - max int // current max index + start map[*oReader]int // map from object file to its start index + objs []objIdx // sorted by start index (i.e. objIdx.i) + max int // current max index symsByName map[nameVer]int // map symbol name to index + objByPkg map[string]*oReader // map package path to its Go object reader + Syms []*sym.Symbol // indexed symbols. XXX we still make sym.Symbol for now. } func NewLoader() *Loader { return &Loader{ - start: make(map[*goobj2.Reader]int), + start: make(map[*oReader]int), objs: []objIdx{{nil, 0}}, symsByName: make(map[nameVer]int), + objByPkg: make(map[string]*oReader), Syms: []*sym.Symbol{nil}, } } // Return the start index in the global index space for a given object file. -func (l *Loader) StartIndex(r *goobj2.Reader) int { +func (l *Loader) StartIndex(r *oReader) int { return l.start[r] } // Add object file r, return the start index. -func (l *Loader) AddObj(r *goobj2.Reader) int { +func (l *Loader) AddObj(pkg string, r *oReader) int { if _, ok := l.start[r]; ok { panic("already added") } + if _, ok := l.objByPkg[pkg]; !ok { + l.objByPkg[pkg] = r + } n := r.NSym() + r.NNonpkgdef() i := l.max + 1 l.start[r] = i @@ -98,12 +116,12 @@ func (l *Loader) AddExtSym(name string, ver int) int { } // Convert a local index to a global index. -func (l *Loader) ToGlobal(r *goobj2.Reader, i int) int { +func (l *Loader) ToGlobal(r *oReader, i int) int { return l.StartIndex(r) + i } -// Convert a global index to a global index. Is it useful? -func (l *Loader) ToLocal(i int) (*goobj2.Reader, int) { +// Convert a global index to a local index. +func (l *Loader) ToLocal(i int) (*oReader, int) { k := sort.Search(i, func(k int) bool { return l.objs[k].i >= i }) @@ -113,6 +131,35 @@ func (l *Loader) ToLocal(i int) (*goobj2.Reader, int) { return l.objs[k].r, i - l.objs[k].i } +// Resolve a local symbol reference. Return global index. +func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) int { + var rr *oReader + switch p := s.PkgIdx; p { + case goobj2.PkgIdxInvalid: + if s.SymIdx != 0 { + panic("bad sym ref") + } + return 0 + case goobj2.PkgIdxNone: + // Resolve by name + i := int(s.SymIdx) + r.NSym() + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) + v := abiToVer(osym.ABI, r.version) + nv := nameVer{name, v} + return l.symsByName[nv] + case goobj2.PkgIdxBuiltin: + panic("PkgIdxBuiltin not used") + case goobj2.PkgIdxSelf: + rr = r + default: + pkg := r.Pkg(int(p)) + rr = l.objByPkg[pkg] + } + return l.ToGlobal(rr, int(s.SymIdx)) +} + // Look up a symbol by name, return global index, or 0 if not found. // This is more like Syms.ROLookup than Lookup -- it doesn't create // new symbol. @@ -133,17 +180,24 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s panic("cannot read object file") } localSymVersion := syms.IncVersion() - lib.Readers = append(lib.Readers, struct { - Reader *goobj2.Reader - Version int - }{r, localSymVersion}) - pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + or := &oReader{r, unit, localSymVersion, pkgprefix} // Autolib - lib.ImportStrings = append(lib.ImportStrings, r.Pkglist()[1:]...) + npkg := r.NPkg() + lib.ImportStrings = append(lib.ImportStrings, make([]string, npkg-1)...)[:len(lib.ImportStrings)] + for i := 1; i < npkg; i++ { + lib.ImportStrings = append(lib.ImportStrings, r.Pkg(i)) + } - istart := l.AddObj(r) + // DWARF file table + nfile := r.NDwarfFile() + unit.DWARFFileTable = make([]string, nfile) + for i := range unit.DWARFFileTable { + unit.DWARFFileTable[i] = r.DwarfFile(i) + } + + istart := l.AddObj(lib.Pkg, or) ndef := r.NSym() nnonpkgdef := r.NNonpkgdef() @@ -172,14 +226,21 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s // Make sure referenced symbols are added. Most of them should already be added. // This should only be needed for referenced external symbols. -func LoadRefs(l *Loader, r *goobj2.Reader, lib *sym.Library, arch *sys.Arch, syms *sym.Symbols, localSymVersion int) { +func LoadRefs(l *Loader, arch *sys.Arch, syms *sym.Symbols) { + for _, o := range l.objs[1:] { + loadObjRefs(l, o.r, arch, syms) + } +} + +func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch, syms *sym.Symbols) { + lib := r.unit.Lib pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." ndef := r.NSym() + r.NNonpkgdef() for i, n := 0, r.NNonpkgref(); i < n; i++ { osym := goobj2.Sym{} - osym.Read(r, r.SymOff(ndef+i)) + osym.Read(r.Reader, r.SymOff(ndef+i)) name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - v := abiToVer(osym.ABI, localSymVersion) + v := abiToVer(osym.ABI, r.version) if ii := l.AddExtSym(name, v); ii != 0 { s := syms.Newsym(name, v) preprocess(arch, s) // TODO: put this at a better place @@ -231,40 +292,19 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { // Load relocations for building the dependency graph in deadcode pass. // For now, we load symbol types, relocations, gotype, and the contents // of type symbols, which are needed in deadcode. -func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { - // PkgIdx - pkglist := r.Pkglist() +func LoadReloc(l *Loader) { + for _, o := range l.objs[1:] { + loadObjReloc(l, o.r) + } +} +func loadObjReloc(l *Loader, r *oReader) { + lib := r.unit.Lib pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." istart := l.StartIndex(r) resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { - var rr *goobj2.Reader - switch p := s.PkgIdx; p { - case goobj2.PkgIdxInvalid: - if s.SymIdx != 0 { - panic("bad sym ref") - } - return nil - case goobj2.PkgIdxNone: - // Resolve by name - i := int(s.SymIdx) + r.NSym() - osym := goobj2.Sym{} - osym.Read(r, r.SymOff(i)) - name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - v := abiToVer(osym.ABI, localSymVersion) - nv := nameVer{name, v} - i = l.symsByName[nv] - return l.Syms[i] - case goobj2.PkgIdxBuiltin: - panic("PkgIdxBuiltin is not used") - case goobj2.PkgIdxSelf: - rr = r - default: - pkg := pkglist[p] - rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm) - } - i := l.ToGlobal(rr, int(s.SymIdx)) + i := l.Resolve(r, s) return l.Syms[i] } @@ -275,7 +315,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in } osym := goobj2.Sym{} - osym.Read(r, r.SymOff(i)) + osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) if s.Name != name { // Sanity check. We can remove it in the final version. fmt.Println("name mismatch:", lib, i, s.Name, name) @@ -298,13 +338,14 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in t = s.Type } s.Type = t + s.Unit = r.unit // Reloc nreloc := r.NReloc(i) s.R = make([]sym.Reloc, nreloc) for j := range s.R { rel := goobj2.Reloc{} - rel.Read(r, r.RelocOff(i, j)) + rel.Read(r.Reader, r.RelocOff(i, j)) s.R[j] = sym.Reloc{ Off: rel.Off, Siz: rel.Siz, @@ -316,7 +357,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in // XXX deadcode needs symbol data for type symbols. Read it now. if strings.HasPrefix(name, "type.") { - s.P = r.BytesAt(r.DataOff(i), r.DataSize(i)) + s.P = r.Data(i) s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) s.Size = int64(osym.Siz) } @@ -325,7 +366,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in naux := r.NAux(i) for j := 0; j < naux; j++ { a := goobj2.Aux{} - a.Read(r, r.AuxOff(i, j)) + a.Read(r.Reader, r.AuxOff(i, j)) switch a.Type { case goobj2.AuxGotype: typ := resolveSymRef(a.Sym) @@ -363,38 +404,19 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in // TODO: For now, some contents are already load in LoadReloc. Maybe // we should combine LoadReloc back into this, once we rewrite deadcode // pass to use index directly. -func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { - // PkgIdx - pkglist := r.Pkglist() +func LoadFull(l *Loader) { + for _, o := range l.objs[1:] { + loadObjFull(l, o.r) + } +} +func loadObjFull(l *Loader, r *oReader) { + lib := r.unit.Lib pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." istart := l.StartIndex(r) resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { - var rr *goobj2.Reader - switch p := s.PkgIdx; p { - case goobj2.PkgIdxInvalid: - if s.SymIdx != 0 { - panic("bad sym ref") - } - return nil - case goobj2.PkgIdxNone: - // Resolve by name - i := int(s.SymIdx) + r.NSym() - osym := goobj2.Sym{} - osym.Read(r, r.SymOff(i)) - name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - v := abiToVer(osym.ABI, localSymVersion) - nv := nameVer{name, v} - i = l.symsByName[nv] - return l.Syms[i] - case goobj2.PkgIdxSelf: - rr = r - default: - pkg := pkglist[p] - rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm) - } - i := l.ToGlobal(rr, int(s.SymIdx)) + i := l.Resolve(r, s) return l.Syms[i] } @@ -411,7 +433,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int } osym := goobj2.Sym{} - osym.Read(r, r.SymOff(i)) + osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) if s.Name != name { // Sanity check. We can remove it in the final version. fmt.Println("name mismatch:", lib, i, s.Name, name) @@ -421,11 +443,10 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int dupok := osym.Flag&goobj2.SymFlagDupok != 0 local := osym.Flag&goobj2.SymFlagLocal != 0 makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 - datasize := r.DataSize(i) size := osym.Siz // Symbol data - s.P = r.BytesAt(r.DataOff(i), datasize) + s.P = r.Data(i) s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) // Aux symbol info @@ -433,7 +454,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int naux := r.NAux(i) for j := 0; j < naux; j++ { a := goobj2.Aux{} - a.Read(r, r.AuxOff(i, j)) + a.Read(r.Reader, r.AuxOff(i, j)) switch a.Type { case goobj2.AuxGotype, goobj2.AuxFuncdata: // already loaded @@ -457,6 +478,14 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int s.Attr.Set(sym.AttrLocal, local) s.Attr.Set(sym.AttrMakeTypelink, makeTypelink) + if s.Type == sym.SDWARFINFO { + // For DWARF symbols, replace `"".` to actual package prefix + // in the symbol content. + // TODO: maybe we should do this in the compiler and get rid + // of this. + patchDWARFName(s, r) + } + if s.Type != sym.STEXT { continue } @@ -465,7 +494,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int if isym == -1 { continue } - b := r.BytesAt(r.DataOff(isym), r.DataSize(isym)) + b := r.Data(isym) info := goobj2.FuncInfo{} info.Read(b) @@ -508,3 +537,32 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int } } } + +func patchDWARFName(s *sym.Symbol, r *oReader) { + // This is kind of ugly. Really the package name should not + // even be included here. + if s.Size < 1 || s.P[0] != dwarf.DW_ABRV_FUNCTION { + return + } + e := bytes.IndexByte(s.P, 0) + if e == -1 { + return + } + p := bytes.Index(s.P[:e], emptyPkg) + if p == -1 { + return + } + pkgprefix := []byte(r.pkgprefix) + patched := bytes.Replace(s.P[:e], emptyPkg, pkgprefix, -1) + + s.P = append(patched, s.P[e:]...) + s.Attr.Set(sym.AttrReadOnly, false) + delta := int64(len(s.P)) - s.Size + s.Size = int64(len(s.P)) + for i := range s.R { + r := &s.R[i] + if r.Off > int32(e) { + r.Off += int32(delta) + } + } +} diff --git a/src/cmd/link/internal/sym/library.go b/src/cmd/link/internal/sym/library.go index b319c6b54e..4f2023b8f7 100644 --- a/src/cmd/link/internal/sym/library.go +++ b/src/cmd/link/internal/sym/library.go @@ -4,8 +4,6 @@ package sym -import "cmd/internal/goobj2" - type Library struct { Objref string Srcref string @@ -20,11 +18,6 @@ type Library struct { Main bool Safe bool Units []*CompilationUnit - - Readers []struct { // TODO: probably move this to Loader - Reader *goobj2.Reader - Version int - } } func (l Library) String() string { From c455e8878f67eeeb02537cc7b3c5fc18a22a2ed7 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 3 Oct 2019 18:25:21 -0400 Subject: [PATCH 14/79] [dev.link] cmd/link: change some decodetype functions to operate on bytes Change some decodetype functions to operate on bytes nstead of Symbol. This is in preparation of implementing live method tracking in index-based deadcode pass, and reducing/eliminating sym.Symbol in general. Change-Id: Ia9809ad7b182884225e1bda577e8dbec0cd216c5 Reviewed-on: https://go-review.googlesource.com/c/go/+/199077 Reviewed-by: Austin Clements --- src/cmd/link/internal/ld/data.go | 4 +- src/cmd/link/internal/ld/deadcode.go | 2 +- src/cmd/link/internal/ld/decodesym.go | 56 +++++++++++++-------------- src/cmd/link/internal/ld/dwarf.go | 14 +++---- src/cmd/link/internal/ld/symtab.go | 2 +- 5 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 2266d301dd..1a8960e21e 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1085,13 +1085,13 @@ func (p *GCProg) AddSym(s *sym.Symbol) { } ptrsize := int64(p.ctxt.Arch.PtrSize) - nptr := decodetypePtrdata(p.ctxt.Arch, typ) / ptrsize + nptr := decodetypePtrdata(p.ctxt.Arch, typ.P) / ptrsize if debugGCProg { fmt.Fprintf(os.Stderr, "gcprog sym: %s at %d (ptr=%d+%d)\n", s.Name, s.Value, s.Value/ptrsize, nptr) } - if decodetypeUsegcprog(p.ctxt.Arch, typ) == 0 { + if decodetypeUsegcprog(p.ctxt.Arch, typ.P) == 0 { // Copy pointers from mask into program. mask := decodetypeGcmask(p.ctxt, typ) for i := int64(0); i < nptr; i++ { diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index 575fabc259..a024e40dff 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -322,7 +322,7 @@ func (d *deadcodepass) flood() { // later will give a better error than deadcode. continue } - if decodetypeKind(d.ctxt.Arch, s)&kindMask == kindInterface { + if decodetypeKind(d.ctxt.Arch, s.P)&kindMask == kindInterface { for _, sig := range decodeIfaceMethods(d.ctxt.Arch, s) { if d.ctxt.Debugvlog > 1 { d.ctxt.Logf("reached iface method: %s\n", sig) diff --git a/src/cmd/link/internal/ld/decodesym.go b/src/cmd/link/internal/ld/decodesym.go index 3afb38948f..3271c85157 100644 --- a/src/cmd/link/internal/ld/decodesym.go +++ b/src/cmd/link/internal/ld/decodesym.go @@ -65,28 +65,28 @@ func structfieldSize(arch *sys.Arch) int { return 3 * arch.PtrSize } // ru func uncommonSize() int { return 4 + 2 + 2 + 4 + 4 } // runtime.uncommontype // Type.commonType.kind -func decodetypeKind(arch *sys.Arch, s *sym.Symbol) uint8 { - return s.P[2*arch.PtrSize+7] & objabi.KindMask // 0x13 / 0x1f +func decodetypeKind(arch *sys.Arch, p []byte) uint8 { + return p[2*arch.PtrSize+7] & objabi.KindMask // 0x13 / 0x1f } // Type.commonType.kind -func decodetypeUsegcprog(arch *sys.Arch, s *sym.Symbol) uint8 { - return s.P[2*arch.PtrSize+7] & objabi.KindGCProg // 0x13 / 0x1f +func decodetypeUsegcprog(arch *sys.Arch, p []byte) uint8 { + return p[2*arch.PtrSize+7] & objabi.KindGCProg // 0x13 / 0x1f } // Type.commonType.size -func decodetypeSize(arch *sys.Arch, s *sym.Symbol) int64 { - return int64(decodeInuxi(arch, s.P, arch.PtrSize)) // 0x8 / 0x10 +func decodetypeSize(arch *sys.Arch, p []byte) int64 { + return int64(decodeInuxi(arch, p, arch.PtrSize)) // 0x8 / 0x10 } // Type.commonType.ptrdata -func decodetypePtrdata(arch *sys.Arch, s *sym.Symbol) int64 { - return int64(decodeInuxi(arch, s.P[arch.PtrSize:], arch.PtrSize)) // 0x8 / 0x10 +func decodetypePtrdata(arch *sys.Arch, p []byte) int64 { + return int64(decodeInuxi(arch, p[arch.PtrSize:], arch.PtrSize)) // 0x8 / 0x10 } // Type.commonType.tflag -func decodetypeHasUncommon(arch *sys.Arch, s *sym.Symbol) bool { - return s.P[2*arch.PtrSize+4]&tflagUncommon != 0 +func decodetypeHasUncommon(arch *sys.Arch, p []byte) bool { + return p[2*arch.PtrSize+4]&tflagUncommon != 0 } // Find the elf.Section of a given shared library that contains a given address. @@ -138,7 +138,7 @@ func decodetypeGcprogShlib(ctxt *Link, s *sym.Symbol) uint64 { func decodetypeGcmask(ctxt *Link, s *sym.Symbol) []byte { if s.Type == sym.SDYNIMPORT { addr := decodetypeGcprogShlib(ctxt, s) - ptrdata := decodetypePtrdata(ctxt.Arch, s) + ptrdata := decodetypePtrdata(ctxt.Arch, s.P) sect := findShlibSection(ctxt, s.File, addr) if sect != nil { r := make([]byte, ptrdata/int64(ctxt.Arch.PtrSize)) @@ -181,17 +181,17 @@ func decodetypeChanElem(arch *sys.Arch, s *sym.Symbol) *sym.Symbol { } // Type.FuncType.dotdotdot -func decodetypeFuncDotdotdot(arch *sys.Arch, s *sym.Symbol) bool { - return uint16(decodeInuxi(arch, s.P[commonsize(arch)+2:], 2))&(1<<15) != 0 +func decodetypeFuncDotdotdot(arch *sys.Arch, p []byte) bool { + return uint16(decodeInuxi(arch, p[commonsize(arch)+2:], 2))&(1<<15) != 0 } // Type.FuncType.inCount -func decodetypeFuncInCount(arch *sys.Arch, s *sym.Symbol) int { - return int(decodeInuxi(arch, s.P[commonsize(arch):], 2)) +func decodetypeFuncInCount(arch *sys.Arch, p []byte) int { + return int(decodeInuxi(arch, p[commonsize(arch):], 2)) } -func decodetypeFuncOutCount(arch *sys.Arch, s *sym.Symbol) int { - return int(uint16(decodeInuxi(arch, s.P[commonsize(arch)+2:], 2)) & (1<<15 - 1)) +func decodetypeFuncOutCount(arch *sys.Arch, p []byte) int { + return int(uint16(decodeInuxi(arch, p[commonsize(arch)+2:], 2)) & (1<<15 - 1)) } func decodetypeFuncInType(arch *sys.Arch, s *sym.Symbol, i int) *sym.Symbol { @@ -199,14 +199,14 @@ func decodetypeFuncInType(arch *sys.Arch, s *sym.Symbol, i int) *sym.Symbol { if arch.PtrSize == 8 { uadd += 4 } - if decodetypeHasUncommon(arch, s) { + if decodetypeHasUncommon(arch, s.P) { uadd += uncommonSize() } return decodeRelocSym(s, int32(uadd+i*arch.PtrSize)) } func decodetypeFuncOutType(arch *sys.Arch, s *sym.Symbol, i int) *sym.Symbol { - return decodetypeFuncInType(arch, s, i+decodetypeFuncInCount(arch, s)) + return decodetypeFuncInType(arch, s, i+decodetypeFuncInCount(arch, s.P)) } // Type.StructType.fields.Slice::length @@ -216,7 +216,7 @@ func decodetypeStructFieldCount(arch *sys.Arch, s *sym.Symbol) int { func decodetypeStructFieldArrayOff(arch *sys.Arch, s *sym.Symbol, i int) int { off := commonsize(arch) + 4*arch.PtrSize - if decodetypeHasUncommon(arch, s) { + if decodetypeHasUncommon(arch, s.P) { off += uncommonSize() } off += i * structfieldSize(arch) @@ -264,8 +264,8 @@ func decodetypeStructFieldOffsAnon(arch *sys.Arch, s *sym.Symbol, i int) int64 { } // InterfaceType.methods.length -func decodetypeIfaceMethodCount(arch *sys.Arch, s *sym.Symbol) int64 { - return int64(decodeInuxi(arch, s.P[commonsize(arch)+2*arch.PtrSize:], arch.PtrSize)) +func decodetypeIfaceMethodCount(arch *sys.Arch, p []byte) int64 { + return int64(decodeInuxi(arch, p[commonsize(arch)+2*arch.PtrSize:], arch.PtrSize)) } // methodsig is a fully qualified typed method signature, like @@ -299,7 +299,7 @@ func decodeMethodSig(arch *sys.Arch, s *sym.Symbol, off, size, count int) []meth mtypSym := decodeRelocSym(s, int32(off+4)) buf.WriteRune('(') - inCount := decodetypeFuncInCount(arch, mtypSym) + inCount := decodetypeFuncInCount(arch, mtypSym.P) for i := 0; i < inCount; i++ { if i > 0 { buf.WriteString(", ") @@ -307,7 +307,7 @@ func decodeMethodSig(arch *sys.Arch, s *sym.Symbol, off, size, count int) []meth buf.WriteString(decodetypeFuncInType(arch, mtypSym, i).Name) } buf.WriteString(") (") - outCount := decodetypeFuncOutCount(arch, mtypSym) + outCount := decodetypeFuncOutCount(arch, mtypSym.P) for i := 0; i < outCount; i++ { if i > 0 { buf.WriteString(", ") @@ -324,7 +324,7 @@ func decodeMethodSig(arch *sys.Arch, s *sym.Symbol, off, size, count int) []meth } func decodeIfaceMethods(arch *sys.Arch, s *sym.Symbol) []methodsig { - if decodetypeKind(arch, s)&kindMask != kindInterface { + if decodetypeKind(arch, s.P)&kindMask != kindInterface { panic(fmt.Sprintf("symbol %q is not an interface", s.Name)) } r := decodeReloc(s, int32(commonsize(arch)+arch.PtrSize)) @@ -335,17 +335,17 @@ func decodeIfaceMethods(arch *sys.Arch, s *sym.Symbol) []methodsig { panic(fmt.Sprintf("imethod slice pointer in %q leads to a different symbol", s.Name)) } off := int(r.Add) // array of reflect.imethod values - numMethods := int(decodetypeIfaceMethodCount(arch, s)) + numMethods := int(decodetypeIfaceMethodCount(arch, s.P)) sizeofIMethod := 4 + 4 return decodeMethodSig(arch, s, off, sizeofIMethod, numMethods) } func decodetypeMethods(arch *sys.Arch, s *sym.Symbol) []methodsig { - if !decodetypeHasUncommon(arch, s) { + if !decodetypeHasUncommon(arch, s.P) { panic(fmt.Sprintf("no methods on %q", s.Name)) } off := commonsize(arch) // reflect.rtype - switch decodetypeKind(arch, s) & kindMask { + switch decodetypeKind(arch, s.P) & kindMask { case kindStruct: // reflect.structType off += 4 * arch.PtrSize case kindPtr: // reflect.ptrType diff --git a/src/cmd/link/internal/ld/dwarf.go b/src/cmd/link/internal/ld/dwarf.go index e426a6ba7d..ebbfbb8ed2 100644 --- a/src/cmd/link/internal/ld/dwarf.go +++ b/src/cmd/link/internal/ld/dwarf.go @@ -422,8 +422,8 @@ func defgotype(ctxt *Link, gotype *sym.Symbol) *sym.Symbol { func newtype(ctxt *Link, gotype *sym.Symbol) *dwarf.DWDie { name := gotype.Name[5:] // could also decode from Type.string - kind := decodetypeKind(ctxt.Arch, gotype) - bytesize := decodetypeSize(ctxt.Arch, gotype) + kind := decodetypeKind(ctxt.Arch, gotype.P) + bytesize := decodetypeSize(ctxt.Arch, gotype.P) var die, typedefdie *dwarf.DWDie switch kind { @@ -488,17 +488,17 @@ func newtype(ctxt *Link, gotype *sym.Symbol) *dwarf.DWDie { die = newdie(ctxt, &dwtypes, dwarf.DW_ABRV_FUNCTYPE, name, 0) newattr(die, dwarf.DW_AT_byte_size, dwarf.DW_CLS_CONSTANT, bytesize, 0) typedefdie = dotypedef(ctxt, &dwtypes, name, die) - nfields := decodetypeFuncInCount(ctxt.Arch, gotype) + nfields := decodetypeFuncInCount(ctxt.Arch, gotype.P) for i := 0; i < nfields; i++ { s := decodetypeFuncInType(ctxt.Arch, gotype, i) fld := newdie(ctxt, die, dwarf.DW_ABRV_FUNCTYPEPARAM, s.Name[5:], 0) newrefattr(fld, dwarf.DW_AT_type, defgotype(ctxt, s)) } - if decodetypeFuncDotdotdot(ctxt.Arch, gotype) { + if decodetypeFuncDotdotdot(ctxt.Arch, gotype.P) { newdie(ctxt, die, dwarf.DW_ABRV_DOTDOTDOT, "...", 0) } - nfields = decodetypeFuncOutCount(ctxt.Arch, gotype) + nfields = decodetypeFuncOutCount(ctxt.Arch, gotype.P) for i := 0; i < nfields; i++ { s := decodetypeFuncOutType(ctxt.Arch, gotype, i) fld := newdie(ctxt, die, dwarf.DW_ABRV_FUNCTYPEPARAM, s.Name[5:], 0) @@ -508,7 +508,7 @@ func newtype(ctxt *Link, gotype *sym.Symbol) *dwarf.DWDie { case objabi.KindInterface: die = newdie(ctxt, &dwtypes, dwarf.DW_ABRV_IFACETYPE, name, 0) typedefdie = dotypedef(ctxt, &dwtypes, name, die) - nfields := int(decodetypeIfaceMethodCount(ctxt.Arch, gotype)) + nfields := int(decodetypeIfaceMethodCount(ctxt.Arch, gotype.P)) var s *sym.Symbol if nfields == 0 { s = lookupOrDiag(ctxt, "type.runtime.eface") @@ -733,7 +733,7 @@ func synthesizemaptypes(ctxt *Link, die *dwarf.DWDie) { gotype := getattr(die, dwarf.DW_AT_type).Data.(*sym.Symbol) keytype := decodetypeMapKey(ctxt.Arch, gotype) valtype := decodetypeMapValue(ctxt.Arch, gotype) - keysize, valsize := decodetypeSize(ctxt.Arch, keytype), decodetypeSize(ctxt.Arch, valtype) + keysize, valsize := decodetypeSize(ctxt.Arch, keytype.P), decodetypeSize(ctxt.Arch, valtype.P) keytype, valtype = walksymtypedef(ctxt, defgotype(ctxt, keytype)), walksymtypedef(ctxt, defgotype(ctxt, valtype)) // compute size info like hashmap.c does. diff --git a/src/cmd/link/internal/ld/symtab.go b/src/cmd/link/internal/ld/symtab.go index d686a8a476..98305c851e 100644 --- a/src/cmd/link/internal/ld/symtab.go +++ b/src/cmd/link/internal/ld/symtab.go @@ -693,7 +693,7 @@ func (ctxt *Link) symtab() { // creating the moduledata from scratch and it does not have a // compiler-provided size, so read it from the type data. moduledatatype := ctxt.Syms.ROLookup("type.runtime.moduledata", 0) - moduledata.Size = decodetypeSize(ctxt.Arch, moduledatatype) + moduledata.Size = decodetypeSize(ctxt.Arch, moduledatatype.P) moduledata.Grow(moduledata.Size) lastmoduledatap := ctxt.Syms.Lookup("runtime.lastmoduledatap", 0) From f7659d49be1839314ddc3a8606f8d6b3ce211b6b Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 9 Oct 2019 09:04:16 -0400 Subject: [PATCH 15/79] [dev.link] cmd/link/internal/objfile: new 'Sym' type for global symbol index First change of several to update the loader API to reflect the final consensus version of the loader API as described in Cherry's doc. This piece: - define new loader.Sym type to encapsulate a global symbol index (as opposed to just using 'int') Change-Id: I6f6483e269f80abfc7d373b2856b2c0d61b9ac24 Reviewed-on: https://go-review.googlesource.com/c/go/+/200417 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements Reviewed-by: Jeremy Faller --- src/cmd/link/internal/objfile/objfile2.go | 50 ++++++++++++----------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index e2442d8982..5bc7346096 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -23,6 +23,10 @@ import ( var _ = fmt.Print +// Sym encapsulates a global symbol index, used to identify a specific +// Go symbol. The 0-valued Sym is corresponds to an invalid symbol. +type Sym int + // oReader is a wrapper type of obj.Reader, along with some // extra information. // TODO: rename to objReader once the old one is gone? @@ -35,7 +39,7 @@ type oReader struct { type objIdx struct { r *oReader - i int // start index + i Sym // start index } type nameVer struct { @@ -47,11 +51,11 @@ type nameVer struct { // // TODO: describe local-global index mapping. type Loader struct { - start map[*oReader]int // map from object file to its start index + start map[*oReader]Sym // map from object file to its start index objs []objIdx // sorted by start index (i.e. objIdx.i) - max int // current max index + max Sym // current max index - symsByName map[nameVer]int // map symbol name to index + symsByName map[nameVer]Sym // map symbol name to index objByPkg map[string]*oReader // map package path to its Go object reader @@ -60,21 +64,21 @@ type Loader struct { func NewLoader() *Loader { return &Loader{ - start: make(map[*oReader]int), + start: make(map[*oReader]Sym), objs: []objIdx{{nil, 0}}, - symsByName: make(map[nameVer]int), + symsByName: make(map[nameVer]Sym), objByPkg: make(map[string]*oReader), Syms: []*sym.Symbol{nil}, } } // Return the start index in the global index space for a given object file. -func (l *Loader) StartIndex(r *oReader) int { +func (l *Loader) StartIndex(r *oReader) Sym { return l.start[r] } // Add object file r, return the start index. -func (l *Loader) AddObj(pkg string, r *oReader) int { +func (l *Loader) AddObj(pkg string, r *oReader) Sym { if _, ok := l.start[r]; ok { panic("already added") } @@ -85,12 +89,12 @@ func (l *Loader) AddObj(pkg string, r *oReader) int { i := l.max + 1 l.start[r] = i l.objs = append(l.objs, objIdx{r, i}) - l.max += n + l.max += Sym(n) return i } // Add a symbol with a given index, return if it is added. -func (l *Loader) AddSym(name string, ver int, i int, dupok bool) bool { +func (l *Loader) AddSym(name string, ver int, i Sym, dupok bool) bool { nv := nameVer{name, ver} if _, ok := l.symsByName[nv]; ok { if dupok || true { // TODO: "true" isn't quite right. need to implement "overwrite" logic. @@ -104,7 +108,7 @@ func (l *Loader) AddSym(name string, ver int, i int, dupok bool) bool { // Add an external symbol (without index). Return the index of newly added // symbol, or 0 if not added. -func (l *Loader) AddExtSym(name string, ver int) int { +func (l *Loader) AddExtSym(name string, ver int) Sym { nv := nameVer{name, ver} if _, ok := l.symsByName[nv]; ok { return 0 @@ -116,23 +120,23 @@ func (l *Loader) AddExtSym(name string, ver int) int { } // Convert a local index to a global index. -func (l *Loader) ToGlobal(r *oReader, i int) int { - return l.StartIndex(r) + i +func (l *Loader) ToGlobal(r *oReader, i int) Sym { + return l.StartIndex(r) + Sym(i) } // Convert a global index to a local index. -func (l *Loader) ToLocal(i int) (*oReader, int) { - k := sort.Search(i, func(k int) bool { +func (l *Loader) ToLocal(i Sym) (*oReader, int) { + k := sort.Search(int(i), func(k int) bool { return l.objs[k].i >= i }) if k == len(l.objs) { return nil, 0 } - return l.objs[k].r, i - l.objs[k].i + return l.objs[k].r, int(i - l.objs[k].i) } // Resolve a local symbol reference. Return global index. -func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) int { +func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) Sym { var rr *oReader switch p := s.PkgIdx; p { case goobj2.PkgIdxInvalid: @@ -163,7 +167,7 @@ func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) int { // Look up a symbol by name, return global index, or 0 if not found. // This is more like Syms.ROLookup than Lookup -- it doesn't create // new symbol. -func (l *Loader) Lookup(name string, ver int) int { +func (l *Loader) Lookup(name string, ver int) Sym { nv := nameVer{name, ver} return l.symsByName[nv] } @@ -213,10 +217,10 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s } v := abiToVer(osym.ABI, localSymVersion) dupok := osym.Flag&goobj2.SymFlagDupok != 0 - if l.AddSym(name, v, istart+i, dupok) { + if l.AddSym(name, v, istart+Sym(i), dupok) { s := syms.Newsym(name, v) preprocess(arch, s) // TODO: put this at a better place - l.Syms[istart+i] = s + l.Syms[istart+Sym(i)] = s } } @@ -244,7 +248,7 @@ func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch, syms *sym.Symbols) { if ii := l.AddExtSym(name, v); ii != 0 { s := syms.Newsym(name, v) preprocess(arch, s) // TODO: put this at a better place - if ii != len(l.Syms) { + if ii != Sym(len(l.Syms)) { panic("AddExtSym returned bad index") } l.Syms = append(l.Syms, s) @@ -309,7 +313,7 @@ func loadObjReloc(l *Loader, r *oReader) { } for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { - s := l.Syms[istart+i] + s := l.Syms[istart+Sym(i)] if s == nil || s.Name == "" { continue } @@ -422,7 +426,7 @@ func loadObjFull(l *Loader, r *oReader) { pcdataBase := r.PcdataBase() for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { - s := l.Syms[istart+i] + s := l.Syms[istart+Sym(i)] if s == nil || s.Name == "" { continue } From 8a9be4921a3cc91c80c02bb5b4cf2ad129c0c7cc Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 28 Sep 2019 22:42:35 -0400 Subject: [PATCH 16/79] [dev.link] cmd/link: use index for deadcode Switch the deadcode pass to use indices instead of Symbol structures when using new object file format. Delay loading symbol relocations and contents fully after the deadcode pass. The next step is not to create Symbol structures until deadcode is done. Method tracking logic hasn't been implemented. Currently, all methods of a reachable type are live. Change-Id: Iffcd06ff84e6e52bd9eb24d1220d94234d18ab6b Reviewed-on: https://go-review.googlesource.com/c/go/+/198199 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode.go | 9 ++ src/cmd/link/internal/ld/deadcode2.go | 144 ++++++++++++++++++ src/cmd/link/internal/ld/lib.go | 8 +- src/cmd/link/internal/objfile/objfile2.go | 176 +++++++++++++++++++--- 4 files changed, 312 insertions(+), 25 deletions(-) create mode 100644 src/cmd/link/internal/ld/deadcode2.go diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index a024e40dff..d0896fcf2c 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -50,6 +50,11 @@ func deadcode(ctxt *Link) { ctxt.Logf("%5.2f deadcode\n", Cputime()) } + if *flagNewobj { + deadcode2(ctxt) + return + } + d := &deadcodepass{ ctxt: ctxt, ifaceMethod: make(map[methodsig]bool), @@ -118,6 +123,10 @@ func deadcode(ctxt *Link) { } } + addToTextp(ctxt) +} + +func addToTextp(ctxt *Link) { // Remove dead text but keep file information (z symbols). textp := []*sym.Symbol{} for _, s := range ctxt.Textp { diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go new file mode 100644 index 0000000000..373cffc25e --- /dev/null +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -0,0 +1,144 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ld + +import ( + "cmd/internal/objabi" + "cmd/internal/sys" + "cmd/link/internal/objfile" + "cmd/link/internal/sym" + "fmt" + "strings" +) + +var _ = fmt.Print + +// TODO: +// - Live method tracking: +// Prune methods that are not directly called and cannot +// be potentially called by interface or reflect call. +// For now, all the methods from reachable type are alive. +// - Shared object support: +// It basically marks everything. We could consider using +// a different mechanism to represent it. +// - Field tracking support: +// It needs to record from where the symbol is referenced. + +type workQueue []objfile.Sym + +func (q *workQueue) push(i objfile.Sym) { *q = append(*q, i) } +func (q *workQueue) pop() objfile.Sym { i := (*q)[len(*q)-1]; *q = (*q)[:len(*q)-1]; return i } +func (q *workQueue) empty() bool { return len(*q) == 0 } + +type deadcodePass2 struct { + ctxt *Link + loader *objfile.Loader + wq workQueue +} + +func (d *deadcodePass2) init() { + d.loader.InitReachable() + + var names []string + + // In a normal binary, start at main.main and the init + // functions and mark what is reachable from there. + if d.ctxt.linkShared && (d.ctxt.BuildMode == BuildModeExe || d.ctxt.BuildMode == BuildModePIE) { + names = append(names, "main.main", "main..inittask") + } else { + // The external linker refers main symbol directly. + if d.ctxt.LinkMode == LinkExternal && (d.ctxt.BuildMode == BuildModeExe || d.ctxt.BuildMode == BuildModePIE) { + if d.ctxt.HeadType == objabi.Hwindows && d.ctxt.Arch.Family == sys.I386 { + *flagEntrySymbol = "_main" + } else { + *flagEntrySymbol = "main" + } + } + names = append(names, *flagEntrySymbol) + if d.ctxt.BuildMode == BuildModePlugin { + names = append(names, objabi.PathToPrefix(*flagPluginPath)+"..inittask", objabi.PathToPrefix(*flagPluginPath)+".main", "go.plugin.tabs") + + // We don't keep the go.plugin.exports symbol, + // but we do keep the symbols it refers to. + exportsIdx := d.loader.Lookup("go.plugin.exports", 0) + if exportsIdx != 0 { + nreloc := d.loader.NReloc(exportsIdx) + for i := 0; i < nreloc; i++ { + d.mark(d.loader.RelocSym(exportsIdx, i)) + } + } + } + } + for _, s := range dynexp { + d.mark(d.loader.Lookup(s.Name, int(s.Version))) + } + + for _, name := range names { + // Mark symbol as an data/ABI0 symbol. + d.mark(d.loader.Lookup(name, 0)) + // Also mark any Go functions (internal ABI). + d.mark(d.loader.Lookup(name, sym.SymVerABIInternal)) + } +} + +func (d *deadcodePass2) flood() { + for !d.wq.empty() { + symIdx := d.wq.pop() + nreloc := d.loader.NReloc(symIdx) + for i := 0; i < nreloc; i++ { + t := d.loader.RelocType(symIdx, i) + if t == objabi.R_WEAKADDROFF { + continue + } + if t == objabi.R_METHODOFF { + // TODO: we should do something about it + // For now, all the methods are considered live + } + d.mark(d.loader.RelocSym(symIdx, i)) + } + naux := d.loader.NAux(symIdx) + for i := 0; i < naux; i++ { + d.mark(d.loader.AuxSym(symIdx, i)) + } + } +} + +func (d *deadcodePass2) mark(symIdx objfile.Sym) { + if symIdx != 0 && !d.loader.Reachable.Has(symIdx) { + d.wq.push(symIdx) + d.loader.Reachable.Set(symIdx) + } +} + +func deadcode2(ctxt *Link) { + loader := ctxt.loader + d := deadcodePass2{ctxt: ctxt, loader: loader} + d.init() + d.flood() + + n := loader.NSym() + if ctxt.BuildMode != BuildModeShared { + // Keep a itablink if the symbol it points at is being kept. + // (When BuildModeShared, always keep itablinks.) + for i := 1; i < n; i++ { + s := objfile.Sym(i) + if strings.HasPrefix(loader.RawSymName(s), "go.itablink.") { + if d.loader.NReloc(s) > 0 && loader.Reachable.Has(loader.RelocSym(s, 0)) { + loader.Reachable.Set(s) + } + } + } + } + + // Set reachable attr for now. + for i := 1; i < n; i++ { + if loader.Reachable.Has(objfile.Sym(i)) { + s := loader.Syms[i] + if s != nil && s.Name != "" { + s.Attr.Set(sym.AttrReachable, true) + } + } + } +} diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index b913479b72..d030340cc0 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -543,11 +543,6 @@ func (ctxt *Link) loadlib() { ctxt.Loaded = true importcycles() - - // For now, load relocations for dead-code elimination. - if *flagNewobj { - objfile.LoadReloc(ctxt.loader) - } } // Set up flags and special symbols depending on the platform build mode. @@ -2537,6 +2532,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library func (ctxt *Link) loadlibfull() { // Load full symbol contents, resolve indexed references. + objfile.LoadReloc(ctxt.loader) objfile.LoadFull(ctxt.loader) // For now, add all symbols to ctxt.Syms. @@ -2548,6 +2544,8 @@ func (ctxt *Link) loadlibfull() { // Drop the reference. ctxt.loader = nil + + addToTextp(ctxt) } func (ctxt *Link) dumpsyms() { diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 5bc7346096..2be34b823e 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -47,19 +47,40 @@ type nameVer struct { v int } +type bitmap []uint32 + +// set the i-th bit. +func (bm bitmap) Set(i Sym) { + n, r := uint(i)/32, uint(i)%32 + bm[n] |= 1 << r +} + +// whether the i-th bit is set. +func (bm bitmap) Has(i Sym) bool { + n, r := uint(i)/32, uint(i)%32 + return bm[n]&(1<= i - }) - if k == len(l.objs) { - return nil, 0 + if l.extStart != 0 && i >= l.extStart { + return nil, int(i - l.extStart) } - return l.objs[k].r, int(i - l.objs[k].i) + // Search for the local object holding index i. + // Below k is the first one that has its start index > i, + // so k-1 is the one we want. + k := sort.Search(len(l.objs), func(k int) bool { + return l.objs[k].i > i + }) + return l.objs[k-1].r, int(i - l.objs[k-1].i) } // Resolve a local symbol reference. Return global index. @@ -172,6 +202,94 @@ func (l *Loader) Lookup(name string, ver int) Sym { return l.symsByName[nv] } +// Number of total symbols. +func (l *Loader) NSym() int { + return int(l.max + 1) +} + +// Returns the raw (unpatched) name of the i-th symbol. +func (l *Loader) RawSymName(i Sym) string { + r, li := l.ToLocal(i) + if r == nil { + return "" + } + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(li)) + return osym.Name +} + +// Returns the (patched) name of the i-th symbol. +func (l *Loader) SymName(i Sym) string { + r, li := l.ToLocal(i) + if r == nil { + return "" + } + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(li)) + return strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) +} + +// Returns the type of the i-th symbol. +func (l *Loader) SymType(i Sym) sym.SymKind { + r, li := l.ToLocal(i) + if r == nil { + return 0 + } + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(li)) + return sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] +} + +// Returns the number of relocations given a global index. +func (l *Loader) NReloc(i Sym) int { + r, li := l.ToLocal(i) + if r == nil { + return 0 + } + return r.NReloc(li) +} + +// Returns the referred symbol of the j-th relocation of the i-th +// symbol. +func (l *Loader) RelocSym(i Sym, j int) Sym { + r, li := l.ToLocal(i) + rel := goobj2.Reloc{} + rel.Read(r.Reader, r.RelocOff(li, j)) + return l.Resolve(r, rel.Sym) +} + +// Returns the relocation type of the j-th relocation of the i-th +// symbol. +func (l *Loader) RelocType(i Sym, j int) objabi.RelocType { + r, li := l.ToLocal(i) + rel := goobj2.Reloc{} + rel.Read(r.Reader, r.RelocOff(li, j)) + return objabi.RelocType(rel.Type) +} + +// Returns the number of aux symbols given a global index. +func (l *Loader) NAux(i Sym) int { + r, li := l.ToLocal(i) + if r == nil { + return 0 + } + return r.NAux(li) +} + +// Returns the referred symbol of the j-th aux symbol of the i-th +// symbol. +func (l *Loader) AuxSym(i Sym, j int) Sym { + r, li := l.ToLocal(i) + a := goobj2.Aux{} + a.Read(r.Reader, r.AuxOff(li, j)) + return l.Resolve(r, a.Sym) +} + +// Initialize Reachable bitmap for running deadcode pass. +func (l *Loader) InitReachable() { + l.Reachable = makeBitmap(l.NSym()) +} + // Preload a package: add autolibs, add symbols to the symbol table. // Does not read symbol data yet. func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { @@ -338,6 +456,12 @@ func loadObjReloc(l *Loader, r *oReader) { if t == 0 { log.Fatalf("missing type for %s in %s", s.Name, lib) } + if !s.Attr.Reachable() && (t < sym.SDWARFSECT || t > sym.SDWARFLINES) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) { + // No need to load unreachable symbols. + // XXX DWARF symbols may be used but are not marked reachable. + // XXX type symbol's content may be needed in DWARF code, but they are not marked. + continue + } if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { t = s.Type } @@ -350,22 +474,33 @@ func loadObjReloc(l *Loader, r *oReader) { for j := range s.R { rel := goobj2.Reloc{} rel.Read(r.Reader, r.RelocOff(i, j)) + rs := l.Resolve(r, rel.Sym) + rt := objabi.RelocType(rel.Type) + sz := rel.Siz + if rt == objabi.R_METHODOFF { + if l.Reachable.Has(rs) { + rt = objabi.R_ADDROFF + } else { + sz = 0 + rs = 0 + } + } + if rt == objabi.R_WEAKADDROFF && !l.Reachable.Has(rs) { + rs = 0 + sz = 0 + } + if rs != 0 && l.SymType(rs) == sym.SABIALIAS { + rs = l.RelocSym(rs, 0) + } s.R[j] = sym.Reloc{ Off: rel.Off, - Siz: rel.Siz, - Type: objabi.RelocType(rel.Type), + Siz: sz, + Type: rt, Add: rel.Add, - Sym: resolveSymRef(rel.Sym), + Sym: l.Syms[rs], } } - // XXX deadcode needs symbol data for type symbols. Read it now. - if strings.HasPrefix(name, "type.") { - s.P = r.Data(i) - s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) - s.Size = int64(osym.Siz) - } - // Aux symbol naux := r.NAux(i) for j := 0; j < naux; j++ { @@ -430,9 +565,10 @@ func loadObjFull(l *Loader, r *oReader) { if s == nil || s.Name == "" { continue } - if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) { + if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) && !(s.Type == sym.SRODATA && strings.HasPrefix(s.Name, "type.")) { // No need to load unreachable symbols. // XXX DWARF symbols may be used but are not marked reachable. + // XXX type symbol's content may be needed in DWARF code, but they are not marked. continue } From ffca64dcf35e6298d25c87f3ab42c182c355c268 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 8 Oct 2019 21:43:20 -0400 Subject: [PATCH 17/79] [dev.link] cmd/internal/obj: support -S flag in newobj mode When the compiler's -S flag is specified, it dumps the disassembly. Add this when writing the new style object file. Change-Id: I4cf85e57d22d0ceea1fda6d3b59fe363573659e7 Reviewed-on: https://go-review.googlesource.com/c/go/+/200100 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/obj/objfile.go | 5 ++--- src/cmd/internal/obj/objfile2.go | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index a27004a389..76fbc58f10 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -228,8 +228,7 @@ func (w *objWriter) writeRefs(s *LSym) { } } -func (w *objWriter) writeSymDebug(s *LSym) { - ctxt := w.ctxt +func (ctxt *Link) writeSymDebug(s *LSym) { fmt.Fprintf(ctxt.Bso, "%s ", s.Name) if s.Type != 0 { fmt.Fprintf(ctxt.Bso, "%v ", s.Type) @@ -309,7 +308,7 @@ func (w *objWriter) writeSymDebug(s *LSym) { func (w *objWriter) writeSym(s *LSym) { ctxt := w.ctxt if ctxt.Debugasm > 0 { - w.writeSymDebug(s) + w.ctxt.writeSymDebug(s) } w.wr.WriteByte(symPrefix) diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index 4043e0b9fe..c51be0265b 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -17,6 +17,10 @@ import ( // Entry point of writing new object file. func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { + if ctxt.Debugasm > 0 { + ctxt.traverseSyms(traverseDefs, ctxt.writeSymDebug) + } + genFuncInfoSyms(ctxt) w := writer{ From 0108b54a3d6d53d13bc5cb935e3a79121b7ee49e Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 9 Oct 2019 15:53:03 -0400 Subject: [PATCH 18/79] [dev.link] cmd/internal/goobj: replace `"".` with package prefix in newobj mode This is the behavior of the old code. Do the same. Change-Id: I3d393d754dcbdb7e76a577252a94214d2e316651 Reviewed-on: https://go-review.googlesource.com/c/go/+/200159 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/readnew.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index 442784de3a..b4b84692d5 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -8,6 +8,7 @@ import ( "cmd/internal/goobj2" "cmd/internal/objabi" "fmt" + "strings" ) // Read object file in new format. For now we still fill @@ -67,7 +68,11 @@ func (r *objReader) readNew() { if osym.Name == "" { continue // not a real symbol } - symID := SymID{Name: osym.Name, Version: abiToVer(osym.ABI)} + // In a symbol name in an object file, "". denotes the + // prefix for the package in which the object file has been found. + // Expand it. + name := strings.ReplaceAll(osym.Name, `"".`, r.pkgprefix) + symID := SymID{Name: name, Version: abiToVer(osym.ABI)} r.p.SymRefs = append(r.p.SymRefs, symID) if i >= ndef { From e44dfa1f2b322537ce3ee1f589af1082748accc5 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 9 Oct 2019 15:37:46 -0400 Subject: [PATCH 19/79] [dev.link] cmd/link: escape package path in objByPkg map The package references recorded in the object file, which are obtained from the compiler, are escaped. We should also use the escaped package paths in the linker for resolving package references. Change-Id: I42eb12df6ff24330e6dc7bed1dc8224bb3b8a106 Reviewed-on: https://go-review.googlesource.com/c/go/+/200158 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/objfile/objfile2.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 2be34b823e..96d9ad1bd7 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -103,6 +103,7 @@ func (l *Loader) AddObj(pkg string, r *oReader) Sym { if _, ok := l.start[r]; ok { panic("already added") } + pkg = objabi.PathToPrefix(pkg) // the object file contains escaped package path if _, ok := l.objByPkg[pkg]; !ok { l.objByPkg[pkg] = r } @@ -189,7 +190,11 @@ func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) Sym { rr = r default: pkg := r.Pkg(int(p)) - rr = l.objByPkg[pkg] + var ok bool + rr, ok = l.objByPkg[pkg] + if !ok { + log.Fatalf("reference of nonexisted package %s, from %v", pkg, r.unit.Lib) + } } return l.ToGlobal(rr, int(s.SymIdx)) } From 0b25ab5d96230b8ec5debbf9880527885e0c7573 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Fri, 11 Oct 2019 08:56:19 -0400 Subject: [PATCH 20/79] [dev.link] cmd/link/internal/objfile: new loader method for reloc queries Second change of several to update the loader API to reflect the final consensus version of the loader API as described in Cherry's doc. This piece: - define new loader.Relocs() method that returns a struct encapsulating a set of relocations on a global symbol Old way of examining relocations: nreloc := loader.NReloc(someGlobalSymbolIndex) for i := 0; i < nreloc; i++ { tgtIdx := loader.RelocSym(someGlobalSymbolIndex, i)) ... } New way of examining relocations: relocs := d.loader.Relocs(someGlobalSymbolIndex) for i := 0; i < relocs.Count; i++ { r := relocs.At(i).Sym ... } Change-Id: I5bead1d729655ea13b3396647e53aafcd3e60f97 Reviewed-on: https://go-review.googlesource.com/c/go/+/200717 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/objfile/objfile2.go | 76 +++++++++++++++++++---- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 96d9ad1bd7..b68c07b65c 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -27,6 +27,27 @@ var _ = fmt.Print // Go symbol. The 0-valued Sym is corresponds to an invalid symbol. type Sym int +// Relocs encapsulates the set of relocations on a given symbol; an +// instance of this type is returned by the Loader Relocs() method. +type Relocs struct { + Count int // number of relocs + + li int // local index of symbol whose relocs we're examining + r *oReader // object reader for containing package + l *Loader // loader +} + +// Reloc contains the payload for a specific relocation. +// TODO: replace this with sym.Reloc, once we change the +// relocation target from "*sym.Symbol" to "loader.Sym" in sym.Reloc. +type Reloc struct { + Off int32 // offset to rewrite + Size uint8 // number of bytes to rewrite: 0, 1, 2, or 4 + Type objabi.RelocType // the relocation type + Add int64 // addend + Sym Sym // global index of symbol the reloc addresses +} + // oReader is a wrapper type of obj.Reader, along with some // extra information. // TODO: rename to objReader once the old one is gone? @@ -295,6 +316,39 @@ func (l *Loader) InitReachable() { l.Reachable = makeBitmap(l.NSym()) } +// At method returns the j-th reloc for a global symbol. +func (relocs *Relocs) At(j int) Reloc { + rel := goobj2.Reloc{} + rel.Read(relocs.r.Reader, relocs.r.RelocOff(relocs.li, j)) + target := relocs.l.Resolve(relocs.r, rel.Sym) + return Reloc{ + Off: rel.Off, + Size: rel.Siz, + Type: objabi.RelocType(rel.Type), + Add: rel.Add, + Sym: target, + } +} + +// Relocs returns a Relocs object for the given global sym. +func (l *Loader) Relocs(i Sym) Relocs { + r, li := l.ToLocal(i) + if r == nil { + return Relocs{} + } + return l.relocs(r, li) +} + +// Relocs returns a Relocs object given a local sym index and reader. +func (l *Loader) relocs(r *oReader, li int) Relocs { + return Relocs{ + Count: r.NReloc(li), + li: li, + r: r, + l: l, + } +} + // Preload a package: add autolibs, add symbols to the symbol table. // Does not read symbol data yet. func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { @@ -473,15 +527,14 @@ func loadObjReloc(l *Loader, r *oReader) { s.Type = t s.Unit = r.unit - // Reloc - nreloc := r.NReloc(i) - s.R = make([]sym.Reloc, nreloc) + // Relocs + relocs := l.relocs(r, i) + s.R = make([]sym.Reloc, relocs.Count) for j := range s.R { - rel := goobj2.Reloc{} - rel.Read(r.Reader, r.RelocOff(i, j)) - rs := l.Resolve(r, rel.Sym) - rt := objabi.RelocType(rel.Type) - sz := rel.Siz + r := relocs.At(j) + rs := r.Sym + sz := r.Size + rt := r.Type if rt == objabi.R_METHODOFF { if l.Reachable.Has(rs) { rt = objabi.R_ADDROFF @@ -495,13 +548,14 @@ func loadObjReloc(l *Loader, r *oReader) { sz = 0 } if rs != 0 && l.SymType(rs) == sym.SABIALIAS { - rs = l.RelocSym(rs, 0) + rsrelocs := l.Relocs(rs) + rs = rsrelocs.At(0).Sym } s.R[j] = sym.Reloc{ - Off: rel.Off, + Off: r.Off, Siz: sz, Type: rt, - Add: rel.Add, + Add: r.Add, Sym: l.Syms[rs], } } From 9036351dd7245c66abec26d607e38c5131f989c6 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Fri, 11 Oct 2019 09:21:36 -0400 Subject: [PATCH 21/79] [dev.link] cmd/link/internal/objfile: update deadcode2 to use new reloc hooks Update the new deadcode pass to use the revised loader interface for querying relocations. Remove some of the previous loader relocation methods, since they are no longer used. Change-Id: I08cec4c05793a17698b2674068f64837a5bf4477 Reviewed-on: https://go-review.googlesource.com/c/go/+/200718 Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/deadcode2.go | 21 +++++++++--------- src/cmd/link/internal/objfile/objfile2.go | 27 ----------------------- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 373cffc25e..354d158371 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -64,9 +64,9 @@ func (d *deadcodePass2) init() { // but we do keep the symbols it refers to. exportsIdx := d.loader.Lookup("go.plugin.exports", 0) if exportsIdx != 0 { - nreloc := d.loader.NReloc(exportsIdx) - for i := 0; i < nreloc; i++ { - d.mark(d.loader.RelocSym(exportsIdx, i)) + relocs := d.loader.Relocs(exportsIdx) + for i := 0; i < relocs.Count; i++ { + d.mark(relocs.At(i).Sym) } } } @@ -86,17 +86,17 @@ func (d *deadcodePass2) init() { func (d *deadcodePass2) flood() { for !d.wq.empty() { symIdx := d.wq.pop() - nreloc := d.loader.NReloc(symIdx) - for i := 0; i < nreloc; i++ { - t := d.loader.RelocType(symIdx, i) - if t == objabi.R_WEAKADDROFF { + relocs := d.loader.Relocs(symIdx) + for i := 0; i < relocs.Count; i++ { + r := relocs.At(i) + if r.Type == objabi.R_WEAKADDROFF { continue } - if t == objabi.R_METHODOFF { + if r.Type == objabi.R_METHODOFF { // TODO: we should do something about it // For now, all the methods are considered live } - d.mark(d.loader.RelocSym(symIdx, i)) + d.mark(r.Sym) } naux := d.loader.NAux(symIdx) for i := 0; i < naux; i++ { @@ -125,7 +125,8 @@ func deadcode2(ctxt *Link) { for i := 1; i < n; i++ { s := objfile.Sym(i) if strings.HasPrefix(loader.RawSymName(s), "go.itablink.") { - if d.loader.NReloc(s) > 0 && loader.Reachable.Has(loader.RelocSym(s, 0)) { + relocs := loader.Relocs(s) + if relocs.Count > 0 && loader.Reachable.Has(relocs.At(0).Sym) { loader.Reachable.Set(s) } } diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index b68c07b65c..ad3ea8577d 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -266,33 +266,6 @@ func (l *Loader) SymType(i Sym) sym.SymKind { return sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] } -// Returns the number of relocations given a global index. -func (l *Loader) NReloc(i Sym) int { - r, li := l.ToLocal(i) - if r == nil { - return 0 - } - return r.NReloc(li) -} - -// Returns the referred symbol of the j-th relocation of the i-th -// symbol. -func (l *Loader) RelocSym(i Sym, j int) Sym { - r, li := l.ToLocal(i) - rel := goobj2.Reloc{} - rel.Read(r.Reader, r.RelocOff(li, j)) - return l.Resolve(r, rel.Sym) -} - -// Returns the relocation type of the j-th relocation of the i-th -// symbol. -func (l *Loader) RelocType(i Sym, j int) objabi.RelocType { - r, li := l.ToLocal(i) - rel := goobj2.Reloc{} - rel.Read(r.Reader, r.RelocOff(li, j)) - return objabi.RelocType(rel.Type) -} - // Returns the number of aux symbols given a global index. func (l *Loader) NAux(i Sym) int { r, li := l.ToLocal(i) From cc21e4d1306e080427ff4e62e735c9401a738d98 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Fri, 11 Oct 2019 13:58:43 -0400 Subject: [PATCH 22/79] [dev.link] cmd/link: move macho host files to new loader format Change-Id: I823b19c0742992dd760c6372428a1936bb7c7e70 Reviewed-on: https://go-review.googlesource.com/c/go/+/200768 Reviewed-by: Jeremy Faller Reviewed-by: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot --- src/cmd/link/internal/ld/lib.go | 25 +++++++++++---- src/cmd/link/internal/loadmacho/ldmacho.go | 37 +++++++++++++++++++--- src/cmd/link/internal/objfile/objfile2.go | 4 +-- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index d030340cc0..dd759a0ab1 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1621,15 +1621,26 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, } if magic&^1 == 0xfeedface || magic&^0x01000000 == 0xcefaedfe { - ldmacho := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { - textp, err := loadmacho.Load(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) - if err != nil { - Errorf(nil, "%v", err) - return + if *flagNewobj { + ldmacho := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + err := loadmacho.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } } - ctxt.Textp = append(ctxt.Textp, textp...) + return ldhostobj(ldmacho, ctxt.HeadType, f, pkg, length, pn, file) + } else { + ldmacho := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, err := loadmacho.LoadOld(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } + ctxt.Textp = append(ctxt.Textp, textp...) + } + return ldhostobj(ldmacho, ctxt.HeadType, f, pkg, length, pn, file) } - return ldhostobj(ldmacho, ctxt.HeadType, f, pkg, length, pn, file) } if c1 == 0x4c && c2 == 0x01 || c1 == 0x64 && c2 == 0x86 { diff --git a/src/cmd/link/internal/loadmacho/ldmacho.go b/src/cmd/link/internal/loadmacho/ldmacho.go index c303752992..7a0e18fe71 100644 --- a/src/cmd/link/internal/loadmacho/ldmacho.go +++ b/src/cmd/link/internal/loadmacho/ldmacho.go @@ -10,6 +10,7 @@ import ( "cmd/internal/bio" "cmd/internal/objabi" "cmd/internal/sys" + "cmd/link/internal/objfile" "cmd/link/internal/sym" "encoding/binary" "fmt" @@ -423,14 +424,40 @@ func macholoadsym(m *ldMachoObj, symtab *ldMachoSymtab) int { return 0 } -// Load loads the Mach-O file pn from f. +func Load(l *objfile.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) error { + lookup := func(name string, version int) *sym.Symbol { + // Check to see if we've already defined the symbol. + if i := l.Lookup(name, version); i != 0 { + return l.Syms[i] + } + // Not defined, let's make one. + if s := l.AddExtSym(name, version); s == 0 { + panic("AddExtSym returned bad index") + } else if int(s) != len(l.Syms) { + panic("unexpected length of loaded symbols") + } + newSym := syms.Newsym(name, version) + l.Syms = append(l.Syms, newSym) + return newSym + } + _, err := load(arch, syms.IncVersion(), lookup, f, pkg, length, pn) + return err +} + +func LoadOld(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { + lookup := func(name string, version int) *sym.Symbol { + return syms.Lookup(name, version) + } + return load(arch, syms.IncVersion(), lookup, f, pkg, length, pn) +} + +// load the Mach-O file pn from f. // Symbols are written into syms, and a slice of the text symbols is returned. -func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { +func load(arch *sys.Arch, localSymVersion int, lookup func(string, int) *sym.Symbol, f *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { errorf := func(str string, args ...interface{}) ([]*sym.Symbol, error) { return nil, fmt.Errorf("loadmacho: %v: %v", pn, fmt.Sprintf(str, args...)) } - localSymVersion := syms.IncVersion() base := f.Offset() var hdr [7 * 4]uint8 @@ -562,7 +589,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length i continue } name := fmt.Sprintf("%s(%s/%s)", pkg, sect.segname, sect.name) - s := syms.Lookup(name, localSymVersion) + s := lookup(name, localSymVersion) if s.Type != 0 { return errorf("duplicate %s/%s", sect.segname, sect.name) } @@ -610,7 +637,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length i if machsym.type_&N_EXT == 0 { v = localSymVersion } - s := syms.Lookup(name, v) + s := lookup(name, v) if machsym.type_&N_EXT == 0 { s.Attr |= sym.AttrDuplicateOK } diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index ad3ea8577d..8208125cba 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -477,8 +477,8 @@ func loadObjReloc(l *Loader, r *oReader) { } if s.Type != 0 && s.Type != sym.SXREF { - fmt.Println("symbol already processed:", lib, i, s) - panic("symbol already processed") + // We've already seen this symbol, it likely came from a host object. + continue } t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] From 7d6f79e61783a5ee86e39e47f2d85b3e21552c97 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 4 Oct 2019 15:08:58 -0400 Subject: [PATCH 23/79] [dev.link] cmd/link: implement live method tracking in deadcode2 This essentially replicates the logic of live method tracking and type symbol decoding, rewritten to operate on indices instead of Symbols. TODO: the special handling of reflect.Type.Method has not been implemented. TODO: the symbol name is used too much. It ought to be a better way to do it. Change-Id: I860ee7a506c00833902e4870d15aea698a705dd9 Reviewed-on: https://go-review.googlesource.com/c/go/+/199078 Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode2.go | 252 +++++++++++++++++++++- src/cmd/link/internal/objfile/objfile2.go | 32 ++- 2 files changed, 268 insertions(+), 16 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 354d158371..b1504e2e8a 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -5,26 +5,29 @@ package ld import ( + "bytes" "cmd/internal/objabi" "cmd/internal/sys" "cmd/link/internal/objfile" "cmd/link/internal/sym" "fmt" "strings" + "unicode" ) var _ = fmt.Print // TODO: // - Live method tracking: -// Prune methods that are not directly called and cannot -// be potentially called by interface or reflect call. -// For now, all the methods from reachable type are alive. +// The special handling of reflect.Type.Method has not +// been implemented. // - Shared object support: // It basically marks everything. We could consider using // a different mechanism to represent it. // - Field tracking support: // It needs to record from where the symbol is referenced. +// - Debug output: +// Emit messages about which symbols are kept or deleted. type workQueue []objfile.Sym @@ -36,10 +39,15 @@ type deadcodePass2 struct { ctxt *Link loader *objfile.Loader wq workQueue + + ifaceMethod map[methodsig]bool // methods declared in reached interfaces + markableMethods []methodref2 // methods of reached types + reflectMethod bool // TODO: this is not set for now } func (d *deadcodePass2) init() { d.loader.InitReachable() + d.ifaceMethod = make(map[methodsig]bool) var names []string @@ -86,6 +94,21 @@ func (d *deadcodePass2) init() { func (d *deadcodePass2) flood() { for !d.wq.empty() { symIdx := d.wq.pop() + + name := d.loader.RawSymName(symIdx) + if strings.HasPrefix(name, "type.") && name[5] != '.' { // TODO: use an attribute instead of checking name + p := d.loader.Data(symIdx) + if len(p) != 0 && decodetypeKind(d.ctxt.Arch, p)&kindMask == kindInterface { + for _, sig := range decodeIfaceMethods2(d.loader, d.ctxt.Arch, symIdx) { + if d.ctxt.Debugvlog > 1 { + d.ctxt.Logf("reached iface method: %s\n", sig) + } + d.ifaceMethod[sig] = true + } + } + } + + var methods []methodref2 relocs := d.loader.Relocs(symIdx) for i := 0; i < relocs.Count; i++ { r := relocs.At(i) @@ -93,8 +116,12 @@ func (d *deadcodePass2) flood() { continue } if r.Type == objabi.R_METHODOFF { - // TODO: we should do something about it - // For now, all the methods are considered live + if i+2 >= relocs.Count { + panic("expect three consecutive R_METHODOFF relocs") + } + methods = append(methods, methodref2{src: symIdx, r: i}) + i += 2 + continue } d.mark(r.Sym) } @@ -102,6 +129,20 @@ func (d *deadcodePass2) flood() { for i := 0; i < naux; i++ { d.mark(d.loader.AuxSym(symIdx, i)) } + + if len(methods) != 0 { + // Decode runtime type information for type methods + // to help work out which methods can be called + // dynamically via interfaces. + methodsigs := decodetypeMethods2(d.loader, d.ctxt.Arch, symIdx) + if len(methods) != len(methodsigs) { + panic(fmt.Sprintf("%q has %d method relocations for %d methods", d.loader.SymName(symIdx), len(methods), len(methodsigs))) + } + for i, m := range methodsigs { + methods[i].m = m + } + d.markableMethods = append(d.markableMethods, methods...) + } } } @@ -112,19 +153,67 @@ func (d *deadcodePass2) mark(symIdx objfile.Sym) { } } +func (d *deadcodePass2) markMethod(m methodref2) { + relocs := d.loader.Relocs(m.src) + d.mark(relocs.At(m.r).Sym) + d.mark(relocs.At(m.r + 1).Sym) + d.mark(relocs.At(m.r + 2).Sym) +} + func deadcode2(ctxt *Link) { loader := ctxt.loader d := deadcodePass2{ctxt: ctxt, loader: loader} d.init() d.flood() + callSym := loader.Lookup("reflect.Value.Call", sym.SymVerABIInternal) + methSym := loader.Lookup("reflect.Value.Method", sym.SymVerABIInternal) + reflectSeen := false + + if ctxt.DynlinkingGo() { + // Exported methods may satisfy interfaces we don't know + // about yet when dynamically linking. + reflectSeen = true + } + + for { + if !reflectSeen { + if d.reflectMethod || (callSym != 0 && loader.Reachable.Has(callSym)) || (methSym != 0 && loader.Reachable.Has(methSym)) { + // Methods might be called via reflection. Give up on + // static analysis, mark all exported methods of + // all reachable types as reachable. + reflectSeen = true + } + } + + // Mark all methods that could satisfy a discovered + // interface as reachable. We recheck old marked interfaces + // as new types (with new methods) may have been discovered + // in the last pass. + rem := d.markableMethods[:0] + for _, m := range d.markableMethods { + if (reflectSeen && m.isExported()) || d.ifaceMethod[m.m] { + d.markMethod(m) + } else { + rem = append(rem, m) + } + } + d.markableMethods = rem + + if d.wq.empty() { + // No new work was discovered. Done. + break + } + d.flood() + } + n := loader.NSym() if ctxt.BuildMode != BuildModeShared { // Keep a itablink if the symbol it points at is being kept. // (When BuildModeShared, always keep itablinks.) for i := 1; i < n; i++ { s := objfile.Sym(i) - if strings.HasPrefix(loader.RawSymName(s), "go.itablink.") { + if strings.HasPrefix(loader.RawSymName(s), "go.itablink.") { // TODO: use an attribute instread of checking name relocs := loader.Relocs(s) if relocs.Count > 0 && loader.Reachable.Has(relocs.At(0).Sym) { loader.Reachable.Set(s) @@ -143,3 +232,154 @@ func deadcode2(ctxt *Link) { } } } + +// methodref2 holds the relocations from a receiver type symbol to its +// method. There are three relocations, one for each of the fields in +// the reflect.method struct: mtyp, ifn, and tfn. +type methodref2 struct { + m methodsig + src objfile.Sym // receiver type symbol + r int // the index of R_METHODOFF relocations +} + +func (m methodref2) isExported() bool { + for _, r := range m.m { + return unicode.IsUpper(r) + } + panic("methodref has no signature") +} + +// decodeMethodSig2 decodes an array of method signature information. +// Each element of the array is size bytes. The first 4 bytes is a +// nameOff for the method name, and the next 4 bytes is a typeOff for +// the function type. +// +// Conveniently this is the layout of both runtime.method and runtime.imethod. +func decodeMethodSig2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, off, size, count int) []methodsig { + var buf bytes.Buffer + var methods []methodsig + for i := 0; i < count; i++ { + buf.WriteString(decodetypeName2(loader, symIdx, off)) + mtypSym := decodeRelocSym2(loader, symIdx, int32(off+4)) + mp := loader.Data(mtypSym) + + buf.WriteRune('(') + inCount := decodetypeFuncInCount(arch, mp) + for i := 0; i < inCount; i++ { + if i > 0 { + buf.WriteString(", ") + } + a := decodetypeFuncInType2(loader, arch, mtypSym, i) + buf.WriteString(loader.SymName(a)) + } + buf.WriteString(") (") + outCount := decodetypeFuncOutCount(arch, mp) + for i := 0; i < outCount; i++ { + if i > 0 { + buf.WriteString(", ") + } + a := decodetypeFuncOutType2(loader, arch, mtypSym, i) + buf.WriteString(loader.SymName(a)) + } + buf.WriteRune(')') + + off += size + methods = append(methods, methodsig(buf.String())) + buf.Reset() + } + return methods +} + +func decodeIfaceMethods2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym) []methodsig { + p := loader.Data(symIdx) + if decodetypeKind(arch, p)&kindMask != kindInterface { + panic(fmt.Sprintf("symbol %q is not an interface", loader.SymName(symIdx))) + } + rel := decodeReloc2(loader, symIdx, int32(commonsize(arch)+arch.PtrSize)) + if rel.Sym == 0 { + return nil + } + if rel.Sym != symIdx { + panic(fmt.Sprintf("imethod slice pointer in %q leads to a different symbol", loader.SymName(symIdx))) + } + off := int(rel.Add) // array of reflect.imethod values + numMethods := int(decodetypeIfaceMethodCount(arch, p)) + sizeofIMethod := 4 + 4 + return decodeMethodSig2(loader, arch, symIdx, off, sizeofIMethod, numMethods) +} + +func decodetypeMethods2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym) []methodsig { + p := loader.Data(symIdx) + if !decodetypeHasUncommon(arch, p) { + panic(fmt.Sprintf("no methods on %q", loader.SymName(symIdx))) + } + off := commonsize(arch) // reflect.rtype + switch decodetypeKind(arch, p) & kindMask { + case kindStruct: // reflect.structType + off += 4 * arch.PtrSize + case kindPtr: // reflect.ptrType + off += arch.PtrSize + case kindFunc: // reflect.funcType + off += arch.PtrSize // 4 bytes, pointer aligned + case kindSlice: // reflect.sliceType + off += arch.PtrSize + case kindArray: // reflect.arrayType + off += 3 * arch.PtrSize + case kindChan: // reflect.chanType + off += 2 * arch.PtrSize + case kindMap: // reflect.mapType + off += 4*arch.PtrSize + 8 + case kindInterface: // reflect.interfaceType + off += 3 * arch.PtrSize + default: + // just Sizeof(rtype) + } + + mcount := int(decodeInuxi(arch, p[off+4:], 2)) + moff := int(decodeInuxi(arch, p[off+4+2+2:], 4)) + off += moff // offset to array of reflect.method values + const sizeofMethod = 4 * 4 // sizeof reflect.method in program + return decodeMethodSig2(loader, arch, symIdx, off, sizeofMethod, mcount) +} + +func decodeReloc2(loader *objfile.Loader, symIdx objfile.Sym, off int32) objfile.Reloc { + relocs := loader.Relocs(symIdx) + for j := 0; j < relocs.Count; j++ { + rel := relocs.At(j) + if rel.Off == off { + return rel + } + } + return objfile.Reloc{} +} + +func decodeRelocSym2(loader *objfile.Loader, symIdx objfile.Sym, off int32) objfile.Sym { + return decodeReloc2(loader, symIdx, off).Sym +} + +// decodetypeName2 decodes the name from a reflect.name. +func decodetypeName2(loader *objfile.Loader, symIdx objfile.Sym, off int) string { + r := decodeRelocSym2(loader, symIdx, int32(off)) + if r == 0 { + return "" + } + + data := loader.Data(r) + namelen := int(uint16(data[1])<<8 | uint16(data[2])) + return string(data[3 : 3+namelen]) +} + +func decodetypeFuncInType2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, i int) objfile.Sym { + uadd := commonsize(arch) + 4 + if arch.PtrSize == 8 { + uadd += 4 + } + if decodetypeHasUncommon(arch, loader.Data(symIdx)) { + uadd += uncommonSize() + } + return decodeRelocSym2(loader, symIdx, int32(uadd+i*arch.PtrSize)) +} + +func decodetypeFuncOutType2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, i int) objfile.Sym { + return decodetypeFuncInType2(loader, arch, symIdx, i+decodetypeFuncInCount(arch, loader.Data(symIdx))) +} diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 8208125cba..4a91a97926 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -235,10 +235,10 @@ func (l *Loader) NSym() int { // Returns the raw (unpatched) name of the i-th symbol. func (l *Loader) RawSymName(i Sym) string { - r, li := l.ToLocal(i) - if r == nil { + if l.extStart != 0 && i >= l.extStart { return "" } + r, li := l.ToLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return osym.Name @@ -246,10 +246,10 @@ func (l *Loader) RawSymName(i Sym) string { // Returns the (patched) name of the i-th symbol. func (l *Loader) SymName(i Sym) string { - r, li := l.ToLocal(i) - if r == nil { + if l.extStart != 0 && i >= l.extStart { return "" } + r, li := l.ToLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) @@ -257,27 +257,39 @@ func (l *Loader) SymName(i Sym) string { // Returns the type of the i-th symbol. func (l *Loader) SymType(i Sym) sym.SymKind { - r, li := l.ToLocal(i) - if r == nil { + if l.extStart != 0 && i >= l.extStart { return 0 } + r, li := l.ToLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] } +// Returns the symbol content of the i-th symbol. i is global index. +func (l *Loader) Data(i Sym) []byte { + if l.extStart != 0 && i >= l.extStart { + return nil + } + r, li := l.ToLocal(i) + return r.Data(li) +} + // Returns the number of aux symbols given a global index. func (l *Loader) NAux(i Sym) int { - r, li := l.ToLocal(i) - if r == nil { + if l.extStart != 0 && i >= l.extStart { return 0 } + r, li := l.ToLocal(i) return r.NAux(li) } // Returns the referred symbol of the j-th aux symbol of the i-th // symbol. func (l *Loader) AuxSym(i Sym, j int) Sym { + if l.extStart != 0 && i >= l.extStart { + return 0 + } r, li := l.ToLocal(i) a := goobj2.Aux{} a.Read(r.Reader, r.AuxOff(li, j)) @@ -305,10 +317,10 @@ func (relocs *Relocs) At(j int) Reloc { // Relocs returns a Relocs object for the given global sym. func (l *Loader) Relocs(i Sym) Relocs { - r, li := l.ToLocal(i) - if r == nil { + if l.extStart != 0 && i >= l.extStart { return Relocs{} } + r, li := l.ToLocal(i) return l.relocs(r, li) } From d6535415fb3246ac5beda22bb6e557700df628ae Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 8 Oct 2019 18:21:22 -0400 Subject: [PATCH 24/79] [dev.link] cmd/link: support reflect.Type.Method tracking in deadcode2 When reflect.Type.Method is called, all exported methods from a reachable type need to be conservatively live. When such a function is called, the compiler sets an attribute to the function, and the linker needs to check that attribute. Implement this in the index-based deadcode pass. Unify symbol flags and FuncInfo flags to make things simpler. In particular, the deadcode pass can check the reflectMethod attribute without reading in and decoding FuncInfo. Change-Id: Ibb21e172f2996e899c6efa5551a29d0eca62df67 Reviewed-on: https://go-review.googlesource.com/c/go/+/200099 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/readnew.go | 4 +-- src/cmd/internal/goobj2/funcinfo.go | 13 +-------- src/cmd/internal/goobj2/objfile.go | 5 ++++ src/cmd/internal/obj/objfile2.go | 32 +++++++++++------------ src/cmd/link/internal/ld/deadcode2.go | 24 +++++++---------- src/cmd/link/internal/objfile/objfile2.go | 22 +++++++++++++--- 6 files changed, 51 insertions(+), 49 deletions(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index b4b84692d5..f33bbf73b1 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -142,8 +142,8 @@ func (r *objReader) readNew() { Args: int64(info.Args), Frame: int64(info.Locals), NoSplit: info.NoSplit != 0, - Leaf: info.Flags&goobj2.FuncFlagLeaf != 0, - TopFrame: info.Flags&goobj2.FuncFlagTopFrame != 0, + Leaf: osym.Flag&goobj2.SymFlagLeaf != 0, + TopFrame: osym.Flag&goobj2.SymFlagTopFrame != 0, PCSP: Data{int64(pcdataBase + info.Pcsp), int64(info.Pcfile - info.Pcsp)}, PCFile: Data{int64(pcdataBase + info.Pcfile), int64(info.Pcline - info.Pcfile)}, PCLine: Data{int64(pcdataBase + info.Pcline), int64(info.Pcinline - info.Pcline)}, diff --git a/src/cmd/internal/goobj2/funcinfo.go b/src/cmd/internal/goobj2/funcinfo.go index 5938b5f920..4de9b93a03 100644 --- a/src/cmd/internal/goobj2/funcinfo.go +++ b/src/cmd/internal/goobj2/funcinfo.go @@ -15,7 +15,6 @@ import ( // TODO: make each pcdata a separate symbol? type FuncInfo struct { NoSplit uint8 - Flags uint8 Args uint32 Locals uint32 @@ -32,17 +31,8 @@ type FuncInfo struct { // TODO: InlTree } -const ( - FuncFlagLeaf = 1 << iota - FuncFlagCFunc - FuncFlagReflectMethod - FuncFlagShared // This is really silly - FuncFlagTopFrame -) - func (a *FuncInfo) Write(w *bytes.Buffer) { w.WriteByte(a.NoSplit) - w.WriteByte(a.Flags) var b [4]byte writeUint32 := func(x uint32) { @@ -77,8 +67,7 @@ func (a *FuncInfo) Write(w *bytes.Buffer) { func (a *FuncInfo) Read(b []byte) { a.NoSplit = b[0] - a.Flags = b[1] - b = b[2:] + b = b[1:] readUint32 := func() uint32 { x := binary.LittleEndian.Uint32(b) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index b5cc0d7bf7..c92b9dd9af 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -191,6 +191,11 @@ const ( SymFlagDupok = 1 << iota SymFlagLocal SymFlagTypelink + SymFlagLeaf + SymFlagCFunc + SymFlagReflectMethod + SymFlagShared // This is really silly + SymFlagTopFrame ) func (s *Sym) Write(w *Writer) { diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index c51be0265b..39e2a4f224 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -228,6 +228,21 @@ func (w *writer) Sym(s *LSym) { if s.MakeTypelink() { flag |= goobj2.SymFlagTypelink } + if s.Leaf() { + flag |= goobj2.SymFlagLeaf + } + if s.CFunc() { + flag |= goobj2.SymFlagCFunc + } + if s.ReflectMethod() { + flag |= goobj2.SymFlagReflectMethod + } + if w.ctxt.Flag_shared { // This is really silly + flag |= goobj2.SymFlagShared + } + if s.TopFrame() { + flag |= goobj2.SymFlagTopFrame + } o := goobj2.Sym{ Name: s.Name, ABI: abi, @@ -299,25 +314,8 @@ func genFuncInfoSyms(ctxt *Link) { if s.NoSplit() { nosplit = 1 } - flags := uint8(0) - if s.Leaf() { - flags |= goobj2.FuncFlagLeaf - } - if s.CFunc() { - flags |= goobj2.FuncFlagCFunc - } - if s.ReflectMethod() { - flags |= goobj2.FuncFlagReflectMethod - } - if ctxt.Flag_shared { // This is really silly - flags |= goobj2.FuncFlagShared - } - if s.TopFrame() { - flags |= goobj2.FuncFlagTopFrame - } o := goobj2.FuncInfo{ NoSplit: nosplit, - Flags: flags, Args: uint32(s.Func.Args), Locals: uint32(s.Func.Locals), } diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index b1504e2e8a..a7a17d5097 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -18,9 +18,6 @@ import ( var _ = fmt.Print // TODO: -// - Live method tracking: -// The special handling of reflect.Type.Method has not -// been implemented. // - Shared object support: // It basically marks everything. We could consider using // a different mechanism to represent it. @@ -42,7 +39,7 @@ type deadcodePass2 struct { ifaceMethod map[methodsig]bool // methods declared in reached interfaces markableMethods []methodref2 // methods of reached types - reflectMethod bool // TODO: this is not set for now + reflectSeen bool // whether we have seen a reflect method call } func (d *deadcodePass2) init() { @@ -95,6 +92,8 @@ func (d *deadcodePass2) flood() { for !d.wq.empty() { symIdx := d.wq.pop() + d.reflectSeen = d.reflectSeen || d.loader.IsReflectMethod(symIdx) + name := d.loader.RawSymName(symIdx) if strings.HasPrefix(name, "type.") && name[5] != '.' { // TODO: use an attribute instead of checking name p := d.loader.Data(symIdx) @@ -168,23 +167,18 @@ func deadcode2(ctxt *Link) { callSym := loader.Lookup("reflect.Value.Call", sym.SymVerABIInternal) methSym := loader.Lookup("reflect.Value.Method", sym.SymVerABIInternal) - reflectSeen := false if ctxt.DynlinkingGo() { // Exported methods may satisfy interfaces we don't know // about yet when dynamically linking. - reflectSeen = true + d.reflectSeen = true } for { - if !reflectSeen { - if d.reflectMethod || (callSym != 0 && loader.Reachable.Has(callSym)) || (methSym != 0 && loader.Reachable.Has(methSym)) { - // Methods might be called via reflection. Give up on - // static analysis, mark all exported methods of - // all reachable types as reachable. - reflectSeen = true - } - } + // Methods might be called via reflection. Give up on + // static analysis, mark all exported methods of + // all reachable types as reachable. + d.reflectSeen = d.reflectSeen || (callSym != 0 && loader.Reachable.Has(callSym)) || (methSym != 0 && loader.Reachable.Has(methSym)) // Mark all methods that could satisfy a discovered // interface as reachable. We recheck old marked interfaces @@ -192,7 +186,7 @@ func deadcode2(ctxt *Link) { // in the last pass. rem := d.markableMethods[:0] for _, m := range d.markableMethods { - if (reflectSeen && m.isExported()) || d.ifaceMethod[m.m] { + if (d.reflectSeen && m.isExported()) || d.ifaceMethod[m.m] { d.markMethod(m) } else { rem = append(rem, m) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 4a91a97926..cc472954ab 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -266,6 +266,22 @@ func (l *Loader) SymType(i Sym) sym.SymKind { return sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] } +// Returns the attributes of the i-th symbol. +func (l *Loader) SymAttr(i Sym) uint8 { + if l.extStart != 0 && i >= l.extStart { + return 0 + } + r, li := l.ToLocal(i) + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(li)) + return osym.Flag +} + +// Returns whether the i-th symbol has ReflectMethod attribute set. +func (l *Loader) IsReflectMethod(i Sym) bool { + return l.SymAttr(i)&goobj2.SymFlagReflectMethod != 0 +} + // Returns the symbol content of the i-th symbol. i is global index. func (l *Loader) Data(i Sym) []byte { if l.extStart != 0 && i >= l.extStart { @@ -685,13 +701,13 @@ func loadObjFull(l *Loader, r *oReader) { if info.NoSplit != 0 { s.Attr |= sym.AttrNoSplit } - if info.Flags&goobj2.FuncFlagReflectMethod != 0 { + if osym.Flag&goobj2.SymFlagReflectMethod != 0 { s.Attr |= sym.AttrReflectMethod } - if info.Flags&goobj2.FuncFlagShared != 0 { + if osym.Flag&goobj2.SymFlagShared != 0 { s.Attr |= sym.AttrShared } - if info.Flags&goobj2.FuncFlagTopFrame != 0 { + if osym.Flag&goobj2.SymFlagTopFrame != 0 { s.Attr |= sym.AttrTopFrame } From c9f633487b6279ff17f1c0b50ccf6bd685e78656 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 12 Oct 2019 01:07:31 -0400 Subject: [PATCH 25/79] [dev.link] cmd/internal/obj: sort ctxt.Data on AIX On AIX, TOC symbols may be created and added to ctxt.Data concurrently. To ensure reproducible builds, sort ctxt.Data. This implements the same logic as WriteObjFile does for old object files. Change-Id: I2e6e2d7755352848981544a4fb68b828a188c2ca Reviewed-on: https://go-review.googlesource.com/c/go/+/201021 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/obj/sym.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index e72ec3e701..39d294183d 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -37,6 +37,7 @@ import ( "fmt" "log" "math" + "sort" ) func Linknew(arch *LinkArch) *Link { @@ -167,6 +168,16 @@ func (ctxt *Link) NumberSyms(asm bool) { return } + if ctxt.Headtype == objabi.Haix { + // Data must be sorted to keep a constant order in TOC symbols. + // As they are created during Progedit, two symbols can be switched between + // two different compilations. Therefore, BuildID will be different. + // TODO: find a better place and optimize to only sort TOC symbols + sort.Slice(ctxt.Data, func(i, j int) bool { + return ctxt.Data[i].Name < ctxt.Data[j].Name + }) + } + ctxt.pkgIdx = make(map[string]int32) ctxt.defs = []*LSym{} ctxt.nonpkgdefs = []*LSym{} From cfa421c9a62e171db3c0cd5f0ba9de255e0514d6 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 11 Oct 2019 21:51:01 -0400 Subject: [PATCH 26/79] [dev.link] cmd/link: on AIX, use relocation to locate target symbol from TOC symbol On AIX, a TOC symbol always has a relocation to its target symbol. Instead of using name lookup to locate the target symbol, we can just use the relocation. Using name lookup, besides being less efficient, needs to provide the right symbol version. In this particular case, we are looking for a data symbol so it is almost always version 0. But in case that it is a text symbol, we may get an ABIALIAS symbol, which does not have its Sect set. Change-Id: I1ecfd284b04a86bbbc450059ee89d99d40493e51 Reviewed-on: https://go-review.googlesource.com/c/go/+/201019 Reviewed-by: Than McIntosh Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/link/internal/ppc64/asm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/ppc64/asm.go b/src/cmd/link/internal/ppc64/asm.go index ad91be3dad..cf6a2846de 100644 --- a/src/cmd/link/internal/ppc64/asm.go +++ b/src/cmd/link/internal/ppc64/asm.go @@ -550,7 +550,7 @@ func archreloctoc(ctxt *ld.Link, r *sym.Reloc, s *sym.Symbol, val int64) int64 { const prefix = "TOC." var tarSym *sym.Symbol if strings.HasPrefix(r.Sym.Name, prefix) { - tarSym = ctxt.Syms.ROLookup(strings.TrimPrefix(r.Sym.Name, prefix), 0) + tarSym = r.Sym.R[0].Sym } else { ld.Errorf(s, "archreloctoc called for a symbol without TOC anchor") } From 928623ca953151451d7c780d3b634da3b792f038 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 9 Oct 2019 10:22:02 -0400 Subject: [PATCH 27/79] [dev.link] cmd/internal/goobj2: separate Autolib from Pkglist in new object file In CL 196030 we decided to combine the imported package list (Autolib) and referenced package list (PkgIdx, or Pkglist). However, in some cases the Autolib list may contain file name, instead of package path, e.g. https://go.googlesource.com/go/+/refs/heads/dev.link/src/cmd/compile/internal/gc/main.go#1181 And the linker needs that to locate the file. This mostly happens with direct invocation of the compiler and linker (i.e., not through "go build"). Instead of letting the linker make guess of the file name based on the package path, make Autolib a separate list. Change-Id: If195a69462d04db515346ee67cdec925f5a69e2e Reviewed-on: https://go-review.googlesource.com/c/go/+/200157 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/internal/goobj/readnew.go | 3 ++- src/cmd/internal/goobj2/objfile.go | 16 ++++++++++++++-- src/cmd/internal/obj/objfile2.go | 13 ++++++------- src/cmd/link/internal/objfile/objfile2.go | 6 +----- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index f33bbf73b1..6e6ec02f60 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -21,8 +21,9 @@ func (r *objReader) readNew() { } // Imports + r.p.Imports = rr.Autolib() + pkglist := rr.Pkglist() - r.p.Imports = pkglist[1:] // index 0 is a dummy invalid package abiToVer := func(abi uint16) int64 { var vers int64 diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index c92b9dd9af..e15dbdca69 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -29,7 +29,8 @@ import ( // Data [...]byte // } // -// PkgIndex [...]stringOff // TODO: add fingerprints +// Autolib [...]stringOff // imported packages (for file loading) // TODO: add fingerprints +// PkgIndex [...]stringOff // referenced packages by index // // DwarfFiles [...]stringOff // XXX as a separate block for now // @@ -127,7 +128,8 @@ const ( // Blocks const ( - BlkPkgIdx = iota + BlkAutolib = iota + BlkPkgIdx BlkDwarfFile BlkSymdef BlkNonpkgdef @@ -469,6 +471,16 @@ func (r *Reader) StringRef(off uint32) string { return r.StringAt(r.uint32At(off)) } +func (r *Reader) Autolib() []string { + n := (r.h.Offsets[BlkAutolib+1] - r.h.Offsets[BlkAutolib]) / 4 + s := make([]string, n) + for i := range s { + off := r.h.Offsets[BlkAutolib] + uint32(i)*4 + s[i] = r.StringRef(off) + } + return s +} + func (r *Reader) Pkglist() []string { n := (r.h.Offsets[BlkPkgIdx+1] - r.h.Offsets[BlkPkgIdx]) / 4 s := make([]string, n) diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index 39e2a4f224..caa442c0d3 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -40,6 +40,12 @@ func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { // String table w.StringTable() + // Autolib + h.Offsets[goobj2.BlkAutolib] = w.Offset() + for _, pkg := range ctxt.Imports { + w.StringRef(pkg) + } + // Package references h.Offsets[goobj2.BlkPkgIdx] = w.Offset() for _, pkg := range w.pkglist { @@ -172,13 +178,6 @@ func (w *writer) init() { for pkg, i := range w.ctxt.pkgIdx { w.pkglist[i] = pkg } - - // Also make sure imported packages appear in the list (even if no symbol is referenced). - for _, pkg := range w.ctxt.Imports { - if _, ok := w.ctxt.pkgIdx[pkg]; !ok { - w.pkglist = append(w.pkglist, pkg) - } - } } func (w *writer) StringTable() { diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index cc472954ab..00c996c341 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -366,11 +366,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s or := &oReader{r, unit, localSymVersion, pkgprefix} // Autolib - npkg := r.NPkg() - lib.ImportStrings = append(lib.ImportStrings, make([]string, npkg-1)...)[:len(lib.ImportStrings)] - for i := 1; i < npkg; i++ { - lib.ImportStrings = append(lib.ImportStrings, r.Pkg(i)) - } + lib.ImportStrings = append(lib.ImportStrings, r.Autolib()...) // DWARF file table nfile := r.NDwarfFile() From 90fe9c7c5c19cd0816c8afaa65a7f14d18cd3860 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 12 Oct 2019 14:01:57 -0400 Subject: [PATCH 28/79] [dev.link] cmd/internal/obj: convert "\" to "/" in file path The old code does this. Do the same. Change-Id: Ibf32ac347d6425e19ad0bc664c6b43ab5eba9c5e Reviewed-on: https://go-review.googlesource.com/c/go/+/201022 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/obj/objfile2.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index caa442c0d3..3f68d335ac 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -12,6 +12,7 @@ import ( "cmd/internal/goobj2" "cmd/internal/objabi" "fmt" + "path/filepath" "strings" ) @@ -200,11 +201,11 @@ func (w *writer) StringTable() { } pc := &s.Func.Pcln for _, f := range pc.File { - w.AddString(f) + w.AddString(filepath.ToSlash(f)) } for _, call := range pc.InlTree.nodes { f, _ := linkgetlineFromPos(w.ctxt, call.Pos) - w.AddString(f) + w.AddString(filepath.ToSlash(f)) } }) for _, f := range w.ctxt.PosTable.DebugLinesFileTable() { @@ -242,8 +243,12 @@ func (w *writer) Sym(s *LSym) { if s.TopFrame() { flag |= goobj2.SymFlagTopFrame } + name := s.Name + if strings.HasPrefix(name, "gofile..") { + name = filepath.ToSlash(name) + } o := goobj2.Sym{ - Name: s.Name, + Name: name, ABI: abi, Type: uint8(s.Type), Flag: flag, From cab29ebd84d3fae2092337684ba82999fe97e947 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 4 Oct 2019 22:05:41 -0400 Subject: [PATCH 29/79] [dev.link] cmd/link: create sym.Symbols after deadcode in newobj mode With the new object files, now we can run the deadcode pass on indices instead of Symbol structs, so we can delay creating Symbols after the deadcode pass. Then we only need to create reachable symbols. Not create Symbols in LoadNew and LoadRefs, and recombine LoadReloc into LoadFull. Split loadcgo into two parts: the first finds root symbols, the second create Symbols and sets attributes. The first runs before the deadcode pass, while the second runs after. TODO: currently there are still symbols that are not marked reachable but still used. This includes DWARF symbols, file symbols, and type symbols that are referenced by DWARF symbols. We still need to create them (conservatively). Change-Id: I695779c9312be9d49ab1683957ac3e72e1f65a1e Reviewed-on: https://go-review.googlesource.com/c/go/+/199643 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode.go | 10 +- src/cmd/link/internal/ld/deadcode2.go | 18 +- src/cmd/link/internal/ld/go.go | 48 +++-- src/cmd/link/internal/ld/lib.go | 118 +++++++----- src/cmd/link/internal/ld/link.go | 40 +--- src/cmd/link/internal/objfile/objfile2.go | 221 +++++++++------------- 6 files changed, 217 insertions(+), 238 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index d0896fcf2c..1ff34fec5f 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -65,8 +65,8 @@ func deadcode(ctxt *Link) { d.init() d.flood() - callSym := ctxt.Lookup("reflect.Value.Call", sym.SymVerABIInternal) - methSym := ctxt.Lookup("reflect.Value.Method", sym.SymVerABIInternal) + callSym := ctxt.Syms.ROLookup("reflect.Value.Call", sym.SymVerABIInternal) + methSym := ctxt.Syms.ROLookup("reflect.Value.Method", sym.SymVerABIInternal) reflectSeen := false if ctxt.DynlinkingGo() { @@ -292,7 +292,7 @@ func (d *deadcodepass) init() { // We don't keep the go.plugin.exports symbol, // but we do keep the symbols it refers to. - exports := d.ctxt.Lookup("go.plugin.exports", 0) + exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0) if exports != nil { for i := range exports.R { d.mark(exports.R[i].Sym, nil) @@ -307,9 +307,9 @@ func (d *deadcodepass) init() { for _, name := range names { // Mark symbol as an data/ABI0 symbol. - d.mark(d.ctxt.Lookup(name, 0), nil) + d.mark(d.ctxt.Syms.ROLookup(name, 0), nil) // Also mark any Go functions (internal ABI). - d.mark(d.ctxt.Lookup(name, sym.SymVerABIInternal), nil) + d.mark(d.ctxt.Syms.ROLookup(name, sym.SymVerABIInternal), nil) } } diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index a7a17d5097..3067d40c29 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -76,8 +76,12 @@ func (d *deadcodePass2) init() { } } } - for _, s := range dynexp { - d.mark(d.loader.Lookup(s.Name, int(s.Version))) + dynexpMap := d.ctxt.cgo_export_dynamic + if d.ctxt.LinkMode == LinkExternal { + dynexpMap = d.ctxt.cgo_export_static + } + for exp := range dynexpMap { + names = append(names, exp) } for _, name := range names { @@ -215,16 +219,6 @@ func deadcode2(ctxt *Link) { } } } - - // Set reachable attr for now. - for i := 1; i < n; i++ { - if loader.Reachable.Has(objfile.Sym(i)) { - s := loader.Syms[i] - if s != nil && s.Name != "" { - s.Attr.Set(sym.AttrReachable, true) - } - } - } } // methodref2 holds the relocations from a receiver type symbol to its diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index 13fbbed10f..15d4f9e50f 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -110,13 +110,7 @@ func ldpkg(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, filename s return } p1 += p0 - - if *flagNewobj { - // loadcgo creates sym.Symbol. Delay this until all the symbols are added. - ctxt.cgodata = append(ctxt.cgodata, [3]string{filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]}) - } else { - loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]) - } + loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]) } } @@ -128,6 +122,39 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { return } + // Find cgo_export symbols. They are roots in the deadcode pass. + for _, f := range directives { + switch f[0] { + case "cgo_export_static", "cgo_export_dynamic": + if len(f) < 2 || len(f) > 3 { + continue + } + local := f[1] + switch ctxt.BuildMode { + case BuildModeCShared, BuildModeCArchive, BuildModePlugin: + if local == "main" { + continue + } + } + local = expandpkg(local, pkg) + if f[0] == "cgo_export_static" { + ctxt.cgo_export_static[local] = true + } else { + ctxt.cgo_export_dynamic[local] = true + } + } + } + + if *flagNewobj { + // Record the directives. We'll process them later after Symbols are created. + ctxt.cgodata = append(ctxt.cgodata, cgodata{file, pkg, directives}) + } else { + setCgoAttr(ctxt, file, pkg, directives) + } +} + +// Set symbol attributes or flags based on cgo directives. +func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { for _, f := range directives { switch f[0] { case "cgo_import_dynamic": @@ -169,7 +196,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { if i := strings.Index(remote, "#"); i >= 0 { remote, q = remote[:i], remote[i+1:] } - s := ctxt.LookupOrCreate(local, 0) + s := ctxt.Syms.Lookup(local, 0) if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ { s.SetDynimplib(lib) s.SetExtname(remote) @@ -188,7 +215,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { } local := f[1] - s := ctxt.LookupOrCreate(local, 0) + s := ctxt.Syms.Lookup(local, 0) s.Type = sym.SHOSTOBJ s.Size = 0 continue @@ -209,7 +236,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { // functions. Link.loadlib will resolve any // ABI aliases we find here (since we may not // yet know it's an alias). - s := ctxt.LookupOrCreate(local, 0) + s := ctxt.Syms.Lookup(local, 0) switch ctxt.BuildMode { case BuildModeCShared, BuildModeCArchive, BuildModePlugin: @@ -228,7 +255,6 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { if !s.Attr.CgoExport() { s.SetExtname(remote) - dynexp = append(dynexp, s) } else if s.Extname() != remote { fmt.Fprintf(os.Stderr, "%s: conflicting cgo_export directives: %s as %s and %s\n", os.Args[0], s.Name, s.Extname(), remote) nerrors++ diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index dd759a0ab1..98748eb6fd 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -379,6 +379,9 @@ func (ctxt *Link) loadlib() { ctxt.loader = objfile.NewLoader() } + ctxt.cgo_export_static = make(map[string]bool) + ctxt.cgo_export_dynamic = make(map[string]bool) + loadinternal(ctxt, "runtime") if ctxt.Arch.Family == sys.ARM { loadinternal(ctxt, "math") @@ -402,60 +405,16 @@ func (ctxt *Link) loadlib() { } if *flagNewobj { - // Add references of externally defined symbols. - objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms) - - // Load cgo directives. - for _, p := range ctxt.cgodata { - loadcgo(ctxt, p[0], p[1], p[2]) - } + iscgo = ctxt.loader.Lookup("x_cgo_init", 0) != 0 + ctxt.canUsePlugins = ctxt.loader.Lookup("plugin.Open", sym.SymVerABIInternal) != 0 + } else { + iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil + ctxt.canUsePlugins = ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil } - iscgo = ctxt.Lookup("x_cgo_init", 0) != nil - - // Record whether we can use plugins. - ctxt.canUsePlugins = (ctxt.Lookup("plugin.Open", sym.SymVerABIInternal) != nil) - // We now have enough information to determine the link mode. determineLinkMode(ctxt) - // Now that we know the link mode, trim the dynexp list. - x := sym.AttrCgoExportDynamic - - if ctxt.LinkMode == LinkExternal { - x = sym.AttrCgoExportStatic - } - w := 0 - for i := range dynexp { - if dynexp[i].Attr&x != 0 { - dynexp[w] = dynexp[i] - w++ - } - } - dynexp = dynexp[:w] - - // Resolve ABI aliases in the list of cgo-exported functions. - // This is necessary because we load the ABI0 symbol for all - // cgo exports. - for i, s := range dynexp { - if s.Type != sym.SABIALIAS { - continue - } - t := resolveABIAlias(s) - t.Attr |= s.Attr - t.SetExtname(s.Extname()) - dynexp[i] = t - } - - for _, lib := range ctxt.Library { - if lib.Shlib != "" { - if ctxt.Debugvlog > 1 { - ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) - } - ldshlibsyms(ctxt, lib.Shlib) - } - } - if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { // This indicates a user requested -linkmode=external. // The startup code uses an import of runtime/cgo to decide @@ -473,6 +432,25 @@ func (ctxt *Link) loadlib() { } } + if *flagNewobj { + // Add references of externally defined symbols. + objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms) + } + + // Now that we know the link mode, set the dynexp list. + if !*flagNewobj { // set this later in newobj mode + setupdynexp(ctxt) + } + + for _, lib := range ctxt.Library { + if lib.Shlib != "" { + if ctxt.Debugvlog > 1 { + ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + } + ldshlibsyms(ctxt, lib.Shlib) + } + } + // In internal link mode, read the host object files. if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. @@ -545,6 +523,35 @@ func (ctxt *Link) loadlib() { importcycles() } +// Set up dynexp list. +func setupdynexp(ctxt *Link) { + dynexpMap := ctxt.cgo_export_dynamic + if ctxt.LinkMode == LinkExternal { + dynexpMap = ctxt.cgo_export_static + } + dynexp = make([]*sym.Symbol, 0, len(dynexpMap)) + for exp := range dynexpMap { + s := ctxt.Syms.Lookup(exp, 0) + dynexp = append(dynexp, s) + } + + // Resolve ABI aliases in the list of cgo-exported functions. + // This is necessary because we load the ABI0 symbol for all + // cgo exports. + for i, s := range dynexp { + if s.Type != sym.SABIALIAS { + continue + } + t := resolveABIAlias(s) + t.Attr |= s.Attr + t.SetExtname(s.Extname()) + dynexp[i] = t + } + + ctxt.cgo_export_static = nil + ctxt.cgo_export_dynamic = nil +} + // Set up flags and special symbols depending on the platform build mode. func (ctxt *Link) linksetup() { switch ctxt.BuildMode { @@ -1923,7 +1930,7 @@ func ldshlibsyms(ctxt *Link, shlib string) { ver = sym.SymVerABIInternal } - lsym := ctxt.LookupOrCreate(elfsym.Name, ver) + lsym := ctxt.Syms.Lookup(elfsym.Name, ver) // Because loadlib above loads all .a files before loading any shared // libraries, any non-dynimport symbols we find that duplicate symbols // already loaded should be ignored (the symbols from the .a files @@ -2543,8 +2550,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library func (ctxt *Link) loadlibfull() { // Load full symbol contents, resolve indexed references. - objfile.LoadReloc(ctxt.loader) - objfile.LoadFull(ctxt.loader) + objfile.LoadFull(ctxt.loader, ctxt.Arch, ctxt.Syms) // For now, add all symbols to ctxt.Syms. for _, s := range ctxt.loader.Syms { @@ -2553,8 +2559,16 @@ func (ctxt *Link) loadlibfull() { } } + // Load cgo directives. + for _, d := range ctxt.cgodata { + setCgoAttr(ctxt, d.file, d.pkg, d.directives) + } + + setupdynexp(ctxt) + // Drop the reference. ctxt.loader = nil + ctxt.cgodata = nil addToTextp(ctxt) } diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index dfb686f038..46bf08bb1c 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -99,7 +99,16 @@ type Link struct { relocbuf []byte // temporary buffer for applying relocations loader *objfile.Loader - cgodata [][3]string // cgo directives to load, three strings are args for loadcgo + cgodata []cgodata // cgo directives to load, three strings are args for loadcgo + + cgo_export_static map[string]bool + cgo_export_dynamic map[string]bool +} + +type cgodata struct { + file string + pkg string + directives [][]string } type unresolvedSymKey struct { @@ -176,32 +185,3 @@ func addImports(ctxt *Link, l *sym.Library, pn string) { } l.ImportStrings = nil } - -// convenient helper during the transition period. -func (ctxt *Link) Lookup(name string, ver int) *sym.Symbol { - if *flagNewobj { - i := ctxt.loader.Lookup(name, ver) - if i == 0 { - return nil - } - return ctxt.loader.Syms[i] - } else { - return ctxt.Syms.ROLookup(name, ver) - } -} - -// convenient helper during the transition period. -func (ctxt *Link) LookupOrCreate(name string, ver int) *sym.Symbol { - if *flagNewobj { - i := ctxt.loader.Lookup(name, ver) - if i != 0 { - return ctxt.loader.Syms[i] - } - ctxt.loader.AddExtSym(name, ver) - s := ctxt.Syms.Newsym(name, ver) - ctxt.loader.Syms = append(ctxt.loader.Syms, s) - return s - } else { - return ctxt.Syms.Lookup(name, ver) - } -} diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 00c996c341..a099eaba92 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -94,6 +94,7 @@ type Loader struct { objs []objIdx // sorted by start index (i.e. objIdx.i) max Sym // current max index extStart Sym // from this index on, the symbols are externally defined + extSyms []nameVer // externally defined symbols symsByName map[nameVer]Sym // map symbol name to index @@ -110,7 +111,6 @@ func NewLoader() *Loader { objs: []objIdx{{nil, 0}}, symsByName: make(map[nameVer]Sym), objByPkg: make(map[string]*oReader), - Syms: []*sym.Symbol{nil}, } } @@ -165,6 +165,7 @@ func (l *Loader) AddExtSym(name string, ver int) Sym { if l.extStart == 0 { l.extStart = i } + l.extSyms = append(l.extSyms, nv) return i } @@ -379,9 +380,6 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s ndef := r.NSym() nnonpkgdef := r.NNonpkgdef() - - // XXX add all symbols for now - l.Syms = append(l.Syms, make([]*sym.Symbol, ndef+nnonpkgdef)...) for i, n := 0, ndef+nnonpkgdef; i < n; i++ { osym := goobj2.Sym{} osym.Read(r, r.SymOff(i)) @@ -391,11 +389,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s } v := abiToVer(osym.ABI, localSymVersion) dupok := osym.Flag&goobj2.SymFlagDupok != 0 - if l.AddSym(name, v, istart+Sym(i), dupok) { - s := syms.Newsym(name, v) - preprocess(arch, s) // TODO: put this at a better place - l.Syms[istart+Sym(i)] = s - } + l.AddSym(name, v, istart+Sym(i), dupok) } // The caller expects us consuming all the data @@ -411,22 +405,13 @@ func LoadRefs(l *Loader, arch *sys.Arch, syms *sym.Symbols) { } func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch, syms *sym.Symbols) { - lib := r.unit.Lib - pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." ndef := r.NSym() + r.NNonpkgdef() for i, n := 0, r.NNonpkgref(); i < n; i++ { osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(ndef+i)) - name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) v := abiToVer(osym.ABI, r.version) - if ii := l.AddExtSym(name, v); ii != 0 { - s := syms.Newsym(name, v) - preprocess(arch, s) // TODO: put this at a better place - if ii != Sym(len(l.Syms)) { - panic("AddExtSym returned bad index") - } - l.Syms = append(l.Syms, s) - } + l.AddExtSym(name, v) } } @@ -463,45 +448,47 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { default: log.Panicf("unrecognized $-symbol: %s", s.Name) } - s.Attr.Set(sym.AttrReachable, false) } } -// Load relocations for building the dependency graph in deadcode pass. -// For now, we load symbol types, relocations, gotype, and the contents -// of type symbols, which are needed in deadcode. -func LoadReloc(l *Loader) { +// Load full contents. +func LoadFull(l *Loader, arch *sys.Arch, syms *sym.Symbols) { + // create all Symbols first. + l.Syms = make([]*sym.Symbol, l.NSym()) for _, o := range l.objs[1:] { - loadObjReloc(l, o.r) + loadObjSyms(l, syms, o.r) + } + + // external symbols + for i := l.extStart; i <= l.max; i++ { + nv := l.extSyms[i-l.extStart] + if l.Reachable.Has(i) || strings.HasPrefix(nv.name, "go.info.") || strings.HasPrefix(nv.name, "gofile..") { // XXX some go.info and file symbols are used but not marked + s := syms.Newsym(nv.name, nv.v) + preprocess(arch, s) + s.Attr.Set(sym.AttrReachable, true) + l.Syms[i] = s + } + } + + // load contents of defined symbols + for _, o := range l.objs[1:] { + loadObjFull(l, o.r) } } -func loadObjReloc(l *Loader, r *oReader) { +func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { lib := r.unit.Lib - pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." istart := l.StartIndex(r) - resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { - i := l.Resolve(r, s) - return l.Syms[i] - } - for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { - s := l.Syms[istart+Sym(i)] - if s == nil || s.Name == "" { - continue - } - osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(i)) - name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - if s.Name != name { // Sanity check. We can remove it in the final version. - fmt.Println("name mismatch:", lib, i, s.Name, name) - panic("name mismatch") + name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) + if name == "" { + continue } - - if s.Type != 0 && s.Type != sym.SXREF { - // We've already seen this symbol, it likely came from a host object. + ver := abiToVer(osym.ABI, r.version) + if l.symsByName[nameVer{name, ver}] != istart+Sym(i) { continue } @@ -510,19 +497,63 @@ func loadObjReloc(l *Loader, r *oReader) { log.Fatalf("bad sxref") } if t == 0 { - log.Fatalf("missing type for %s in %s", s.Name, lib) + log.Fatalf("missing type for %s in %s", name, lib) } - if !s.Attr.Reachable() && (t < sym.SDWARFSECT || t > sym.SDWARFLINES) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) { + if !l.Reachable.Has(istart+Sym(i)) && (t < sym.SDWARFSECT || t > sym.SDWARFLINES) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) && name != "runtime.addmoduledata" && name != "runtime.lastmoduledatap" { // No need to load unreachable symbols. // XXX DWARF symbols may be used but are not marked reachable. // XXX type symbol's content may be needed in DWARF code, but they are not marked. + // XXX reference to runtime.addmoduledata may be generated later by the linker in plugin mode. continue } + + s := syms.Newsym(name, ver) + if s.Type != 0 && s.Type != sym.SXREF { + fmt.Println("symbol already processed:", lib, i, s) + panic("symbol already processed") + } if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { t = s.Type } s.Type = t s.Unit = r.unit + s.Attr.Set(sym.AttrReachable, l.Reachable.Has(istart+Sym(i))) + l.Syms[istart+Sym(i)] = s + } +} + +func loadObjFull(l *Loader, r *oReader) { + lib := r.unit.Lib + istart := l.StartIndex(r) + + resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { + i := l.Resolve(r, s) + return l.Syms[i] + } + + pcdataBase := r.PcdataBase() + for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { + s := l.Syms[istart+Sym(i)] + if s == nil || s.Name == "" { + continue + } + + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) + if s.Name != name { // Sanity check. We can remove it in the final version. + fmt.Println("name mismatch:", lib, i, s.Name, name) + panic("name mismatch") + } + + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + local := osym.Flag&goobj2.SymFlagLocal != 0 + makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 + size := osym.Siz + + // Symbol data + s.P = r.Data(i) + s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) // Relocs relocs := l.relocs(r, i) @@ -557,7 +588,8 @@ func loadObjReloc(l *Loader, r *oReader) { } } - // Aux symbol + // Aux symbol info + isym := -1 naux := r.NAux(i) for j := 0; j < naux; j++ { a := goobj2.Aux{} @@ -575,85 +607,6 @@ func loadObjReloc(l *Loader, r *oReader) { s.FuncInfo = pc } pc.Funcdata = append(pc.Funcdata, resolveSymRef(a.Sym)) - } - } - - if s.Type == sym.STEXT { - dupok := osym.Flag&goobj2.SymFlagDupok != 0 - if !dupok { - if s.Attr.OnList() { - log.Fatalf("symbol %s listed multiple times", s.Name) - } - s.Attr |= sym.AttrOnList - lib.Textp = append(lib.Textp, s) - } else { - // there may ba a dup in another package - // put into a temp list and add to text later - lib.DupTextSyms = append(lib.DupTextSyms, s) - } - } - } -} - -// Load full contents. -// TODO: For now, some contents are already load in LoadReloc. Maybe -// we should combine LoadReloc back into this, once we rewrite deadcode -// pass to use index directly. -func LoadFull(l *Loader) { - for _, o := range l.objs[1:] { - loadObjFull(l, o.r) - } -} - -func loadObjFull(l *Loader, r *oReader) { - lib := r.unit.Lib - pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." - istart := l.StartIndex(r) - - resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { - i := l.Resolve(r, s) - return l.Syms[i] - } - - pcdataBase := r.PcdataBase() - for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { - s := l.Syms[istart+Sym(i)] - if s == nil || s.Name == "" { - continue - } - if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) && !(s.Type == sym.SRODATA && strings.HasPrefix(s.Name, "type.")) { - // No need to load unreachable symbols. - // XXX DWARF symbols may be used but are not marked reachable. - // XXX type symbol's content may be needed in DWARF code, but they are not marked. - continue - } - - osym := goobj2.Sym{} - osym.Read(r.Reader, r.SymOff(i)) - name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - if s.Name != name { // Sanity check. We can remove it in the final version. - fmt.Println("name mismatch:", lib, i, s.Name, name) - panic("name mismatch") - } - - dupok := osym.Flag&goobj2.SymFlagDupok != 0 - local := osym.Flag&goobj2.SymFlagLocal != 0 - makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 - size := osym.Siz - - // Symbol data - s.P = r.Data(i) - s.Attr.Set(sym.AttrReadOnly, r.ReadOnly()) - - // Aux symbol info - isym := -1 - naux := r.NAux(i) - for j := 0; j < naux; j++ { - a := goobj2.Aux{} - a.Read(r.Reader, r.AuxOff(i, j)) - switch a.Type { - case goobj2.AuxGotype, goobj2.AuxFuncdata: - // already loaded case goobj2.AuxFuncInfo: if a.Sym.PkgIdx != goobj2.PkgIdxSelf { panic("funcinfo symbol not defined in current package") @@ -664,7 +617,7 @@ func loadObjFull(l *Loader, r *oReader) { } } - s.File = pkgprefix[:len(pkgprefix)-1] + s.File = r.pkgprefix[:len(r.pkgprefix)-1] if dupok { s.Attr |= sym.AttrDuplicateOK } @@ -731,6 +684,18 @@ func loadObjFull(l *Loader, r *oReader) { for k := range pc.File { pc.File[k] = resolveSymRef(info.File[k]) } + + if !dupok { + if s.Attr.OnList() { + log.Fatalf("symbol %s listed multiple times", s.Name) + } + s.Attr.Set(sym.AttrOnList, true) + lib.Textp = append(lib.Textp, s) + } else { + // there may ba a dup in another package + // put into a temp list and add to text later + lib.DupTextSyms = append(lib.DupTextSyms, s) + } } } From 27111e5fec1d0e7d9c1ba4e5cf6d01ddb06b8905 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 7 Oct 2019 21:10:41 -0400 Subject: [PATCH 30/79] [dev.link] cmd/internal/obj, cmd/link: use aux symbol for DWARF symbols Use the auxiliary symbol mechanism to connect the text symbol and its associated DWARF symbols. This way, the linker can track the DWARF symbols from the text symbol, without looking up special names. Currently, in the linker this is only used in the deadcode pass to track which DWARF symbols are used and need to load. Later passes still use name lookup for now. Change-Id: I2fe49f3b1f0ecc1472ae8aa93907cff740022d8d Reviewed-on: https://go-review.googlesource.com/c/go/+/199801 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/readnew.go | 2 + src/cmd/internal/goobj2/objfile.go | 6 ++- src/cmd/internal/obj/objfile2.go | 62 ++++++++++++++++++++--- src/cmd/link/internal/ld/deadcode2.go | 6 +++ src/cmd/link/internal/objfile/objfile2.go | 11 ++-- 5 files changed, 74 insertions(+), 13 deletions(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index 6e6ec02f60..9a9123a584 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -125,6 +125,8 @@ func (r *objReader) readNew() { isym = int(a.Sym.SymIdx) case goobj2.AuxFuncdata: funcdata = append(funcdata, a.Sym) + case goobj2.AuxDwarfInfo, goobj2.AuxDwarfLoc, goobj2.AuxDwarfRanges, goobj2.AuxDwarfLines: + // nothing to do default: panic("unknown aux type") } diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index e15dbdca69..ad1b4ad3a7 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -280,8 +280,12 @@ const ( AuxGotype = iota AuxFuncInfo AuxFuncdata + AuxDwarfInfo + AuxDwarfLoc + AuxDwarfRanges + AuxDwarfLines - // TODO: more. DWARF? Pcdata? + // TODO: more. Pcdata? ) func (a *Aux) Write(w *Writer) { diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index 3f68d335ac..f7d87fd9a3 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -95,13 +95,7 @@ func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { for _, list := range lists { for _, s := range list { w.Uint32(naux) - if s.Gotype != nil { - naux++ - } - if s.Func != nil { - // FuncInfo is an aux symbol, each Funcdata is an aux symbol - naux += 1 + uint32(len(s.Func.Pcln.Funcdata)) - } + naux += uint32(nAuxSym(s)) } } w.Uint32(naux) @@ -301,9 +295,63 @@ func (w *writer) Aux(s *LSym) { } o.Write(w.Writer) } + + if s.Func.dwarfInfoSym != nil { + o := goobj2.Aux{ + Type: goobj2.AuxDwarfInfo, + Sym: makeSymRef(s.Func.dwarfInfoSym), + } + o.Write(w.Writer) + } + if s.Func.dwarfLocSym != nil { + o := goobj2.Aux{ + Type: goobj2.AuxDwarfLoc, + Sym: makeSymRef(s.Func.dwarfLocSym), + } + o.Write(w.Writer) + } + if s.Func.dwarfRangesSym != nil { + o := goobj2.Aux{ + Type: goobj2.AuxDwarfRanges, + Sym: makeSymRef(s.Func.dwarfRangesSym), + } + o.Write(w.Writer) + } + if s.Func.dwarfDebugLinesSym != nil { + o := goobj2.Aux{ + Type: goobj2.AuxDwarfLines, + Sym: makeSymRef(s.Func.dwarfDebugLinesSym), + } + o.Write(w.Writer) + } } } +// return the number of aux symbols s have. +func nAuxSym(s *LSym) int { + n := 0 + if s.Gotype != nil { + n++ + } + if s.Func != nil { + // FuncInfo is an aux symbol, each Funcdata is an aux symbol + n += 1 + len(s.Func.Pcln.Funcdata) + if s.Func.dwarfInfoSym != nil { + n++ + } + if s.Func.dwarfLocSym != nil { + n++ + } + if s.Func.dwarfRangesSym != nil { + n++ + } + if s.Func.dwarfDebugLinesSym != nil { + n++ + } + } + return n +} + // generate symbols for FuncInfo. func genFuncInfoSyms(ctxt *Link) { infosyms := make([]*LSym, 0, len(ctxt.Text)) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 3067d40c29..008285c429 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -126,6 +126,12 @@ func (d *deadcodePass2) flood() { i += 2 continue } + if r.Type == objabi.R_USETYPE { + // type symbol used for DWARF. we need to load the symbol but it may not + // be otherwise reachable in the program. + // do nothing for now as we still load all type symbols. + continue + } d.mark(r.Sym) } naux := d.loader.NAux(symIdx) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index a099eaba92..c48cf96074 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -462,10 +462,10 @@ func LoadFull(l *Loader, arch *sys.Arch, syms *sym.Symbols) { // external symbols for i := l.extStart; i <= l.max; i++ { nv := l.extSyms[i-l.extStart] - if l.Reachable.Has(i) || strings.HasPrefix(nv.name, "go.info.") || strings.HasPrefix(nv.name, "gofile..") { // XXX some go.info and file symbols are used but not marked + if l.Reachable.Has(i) || strings.HasPrefix(nv.name, "gofile..") { // XXX file symbols are used but not marked s := syms.Newsym(nv.name, nv.v) preprocess(arch, s) - s.Attr.Set(sym.AttrReachable, true) + s.Attr.Set(sym.AttrReachable, l.Reachable.Has(i)) l.Syms[i] = s } } @@ -499,10 +499,9 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { if t == 0 { log.Fatalf("missing type for %s in %s", name, lib) } - if !l.Reachable.Has(istart+Sym(i)) && (t < sym.SDWARFSECT || t > sym.SDWARFLINES) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) && name != "runtime.addmoduledata" && name != "runtime.lastmoduledatap" { + if !l.Reachable.Has(istart+Sym(i)) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) && name != "runtime.addmoduledata" && name != "runtime.lastmoduledatap" { // No need to load unreachable symbols. - // XXX DWARF symbols may be used but are not marked reachable. - // XXX type symbol's content may be needed in DWARF code, but they are not marked. + // XXX some type symbol's content may be needed in DWARF code, but they are not marked. // XXX reference to runtime.addmoduledata may be generated later by the linker in plugin mode. continue } @@ -612,6 +611,8 @@ func loadObjFull(l *Loader, r *oReader) { panic("funcinfo symbol not defined in current package") } isym = int(a.Sym.SymIdx) + case goobj2.AuxDwarfInfo, goobj2.AuxDwarfLoc, goobj2.AuxDwarfRanges, goobj2.AuxDwarfLines: + // ignored for now default: panic("unknown aux type") } From dab05a04842e75126fdf880368c2e721485f8a09 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 8 Oct 2019 15:35:36 -0400 Subject: [PATCH 31/79] [dev.link] cmd/link: implement symbol overwrite logic If two defined symbols have the same name, one contentless and one with content, the one with content "wins". This is mainly for go:linkname on data symbols. Support this logic in newobj mode. Introduce an "overwrite" mechanism, letting one symbol overwrite another. This machanism could later be used for the linker overwriting symbol contents (e.g. -X flag). Change-Id: I32ee7d4b82df275f11b38c3abefc99b878ff12d7 Reviewed-on: https://go-review.googlesource.com/c/go/+/200097 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/objfile/objfile2.go | 39 +++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index c48cf96074..b37a665f43 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -97,6 +97,7 @@ type Loader struct { extSyms []nameVer // externally defined symbols symsByName map[nameVer]Sym // map symbol name to index + overwrite map[Sym]Sym // overwrite[i]=j if symbol j overwrites symbol i objByPkg map[string]*oReader // map package path to its Go object reader @@ -111,6 +112,7 @@ func NewLoader() *Loader { objs: []objIdx{{nil, 0}}, symsByName: make(map[nameVer]Sym), objByPkg: make(map[string]*oReader), + overwrite: make(map[Sym]Sym), } } @@ -137,16 +139,34 @@ func (l *Loader) AddObj(pkg string, r *oReader) Sym { } // Add a symbol with a given index, return if it is added. -func (l *Loader) AddSym(name string, ver int, i Sym, dupok bool) bool { +func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ sym.SymKind) bool { if l.extStart != 0 { panic("AddSym called after AddExtSym is called") } nv := nameVer{name, ver} - if _, ok := l.symsByName[nv]; ok { - if dupok || true { // TODO: "true" isn't quite right. need to implement "overwrite" logic. + if oldi, ok := l.symsByName[nv]; ok { + if dupok { + return false + } + overwrite := r.DataSize(int(i-l.StartIndex(r))) != 0 + if overwrite { + // new symbol overwrites old symbol. + oldr, li := l.ToLocal(oldi) + oldsym := goobj2.Sym{} + oldsym.Read(oldr.Reader, oldr.SymOff(li)) + oldtyp := sym.AbiSymKindToSymKind[objabi.SymKind(oldsym.Type)] + if oldsym.Flag&goobj2.SymFlagDupok == 0 && !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol + log.Fatalf("duplicated definition of symbol " + name) + } + l.overwrite[oldi] = i + } else { + // old symbol overwrites new symbol. + if typ != sym.SDATA && typ != sym.SNOPTRDATA && typ != sym.SBSS && typ != sym.SNOPTRBSS { // only allow overwriting data symbol + log.Fatalf("duplicated definition of symbol " + name) + } + l.overwrite[i] = oldi return false } - panic("duplicated definition of symbol " + name) } l.symsByName[nv] = i return true @@ -171,11 +191,18 @@ func (l *Loader) AddExtSym(name string, ver int) Sym { // Convert a local index to a global index. func (l *Loader) ToGlobal(r *oReader, i int) Sym { - return l.StartIndex(r) + Sym(i) + g := l.StartIndex(r) + Sym(i) + if ov, ok := l.overwrite[g]; ok { + return ov + } + return g } // Convert a global index to a local index. func (l *Loader) ToLocal(i Sym) (*oReader, int) { + if ov, ok := l.overwrite[i]; ok { + i = ov + } if l.extStart != 0 && i >= l.extStart { return nil, int(i - l.extStart) } @@ -389,7 +416,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s } v := abiToVer(osym.ABI, localSymVersion) dupok := osym.Flag&goobj2.SymFlagDupok != 0 - l.AddSym(name, v, istart+Sym(i), dupok) + l.AddSym(name, v, istart+Sym(i), or, dupok, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) } // The caller expects us consuming all the data From 6ba3ae9ca547f6078c51f871717f48746e93271a Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 8 Oct 2019 17:22:20 -0400 Subject: [PATCH 32/79] [dev.link] cmd/internal/obj, cmd/link: add InlTree in new object files Add InlTree to the FuncInfo aux symbol in new object files. In the linker, change InlinedCall.Func from a Symbol to a string, as we only use its Name. (There was a use of Func.File, but that use is not correct anyway.) So we don't need to create a Symbol if not necessary. Change-Id: I38ce568ae0934cd9cb6d0b30599f1c8d75444fc9 Reviewed-on: https://go-review.googlesource.com/c/go/+/200098 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/readnew.go | 11 +++++ src/cmd/internal/goobj2/funcinfo.go | 54 ++++++++++++++++++++--- src/cmd/internal/obj/objfile2.go | 12 +++++ src/cmd/internal/obj/sym.go | 4 ++ src/cmd/link/internal/ld/pcln.go | 4 +- src/cmd/link/internal/objfile/objfile.go | 2 +- src/cmd/link/internal/objfile/objfile2.go | 11 +++++ src/cmd/link/internal/sym/symbol.go | 2 +- 8 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index 9a9123a584..3074f8131e 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -154,6 +154,7 @@ func (r *objReader) readNew() { PCData: make([]Data, len(info.Pcdata)-1), // -1 as we appended one above FuncData: make([]FuncData, len(info.Funcdataoff)), File: make([]string, len(info.File)), + InlTree: make([]InlinedCall, len(info.InlTree)), } sym.Func = f for k := range f.PCData { @@ -167,5 +168,15 @@ func (r *objReader) readNew() { symID := resolveSymRef(info.File[k]) f.File[k] = symID.Name } + for k := range f.InlTree { + inl := &info.InlTree[k] + f.InlTree[k] = InlinedCall{ + Parent: int64(inl.Parent), + File: resolveSymRef(inl.File).Name, + Line: int64(inl.Line), + Func: resolveSymRef(inl.Func), + ParentPC: int64(inl.ParentPC), + } + } } } diff --git a/src/cmd/internal/goobj2/funcinfo.go b/src/cmd/internal/goobj2/funcinfo.go index 4de9b93a03..8620931970 100644 --- a/src/cmd/internal/goobj2/funcinfo.go +++ b/src/cmd/internal/goobj2/funcinfo.go @@ -28,7 +28,7 @@ type FuncInfo struct { Funcdataoff []uint32 File []SymRef // TODO: just use string? - // TODO: InlTree + InlTree []InlTreeNode } func (a *FuncInfo) Write(w *bytes.Buffer) { @@ -61,8 +61,10 @@ func (a *FuncInfo) Write(w *bytes.Buffer) { writeUint32(f.PkgIdx) writeUint32(f.SymIdx) } - - // TODO: InlTree + writeUint32(uint32(len(a.InlTree))) + for i := range a.InlTree { + a.InlTree[i].Write(w) + } } func (a *FuncInfo) Read(b []byte) { @@ -98,6 +100,48 @@ func (a *FuncInfo) Read(b []byte) { for i := range a.File { a.File[i] = SymRef{readUint32(), readUint32()} } - - // TODO: InlTree + inltreelen := readUint32() + a.InlTree = make([]InlTreeNode, inltreelen) + for i := range a.InlTree { + b = a.InlTree[i].Read(b) + } +} + +// InlTreeNode is the serialized form of FileInfo.InlTree. +type InlTreeNode struct { + Parent int32 + File SymRef + Line int32 + Func SymRef + ParentPC int32 +} + +func (inl *InlTreeNode) Write(w *bytes.Buffer) { + var b [4]byte + writeUint32 := func(x uint32) { + binary.LittleEndian.PutUint32(b[:], x) + w.Write(b[:]) + } + writeUint32(uint32(inl.Parent)) + writeUint32(inl.File.PkgIdx) + writeUint32(inl.File.SymIdx) + writeUint32(uint32(inl.Line)) + writeUint32(inl.Func.PkgIdx) + writeUint32(inl.Func.SymIdx) + writeUint32(uint32(inl.ParentPC)) +} + +// Read an InlTreeNode from b, return the remaining bytes. +func (inl *InlTreeNode) Read(b []byte) []byte { + readUint32 := func() uint32 { + x := binary.LittleEndian.Uint32(b) + b = b[4:] + return x + } + inl.Parent = int32(readUint32()) + inl.File = SymRef{readUint32(), readUint32()} + inl.Line = int32(readUint32()) + inl.Func = SymRef{readUint32(), readUint32()} + inl.ParentPC = int32(readUint32()) + return b } diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index f7d87fd9a3..843f6fb5ea 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -395,6 +395,18 @@ func genFuncInfoSyms(ctxt *Link) { fsym := ctxt.Lookup(f) o.File[i] = makeSymRef(fsym) } + o.InlTree = make([]goobj2.InlTreeNode, len(pc.InlTree.nodes)) + for i, inl := range pc.InlTree.nodes { + f, l := linkgetlineFromPos(ctxt, inl.Pos) + fsym := ctxt.Lookup(f) + o.InlTree[i] = goobj2.InlTreeNode{ + Parent: int32(inl.Parent), + File: makeSymRef(fsym), + Line: l, + Func: makeSymRef(inl.Func), + ParentPC: inl.ParentPC, + } + } o.Write(&b) isym := &LSym{ diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index 39d294183d..de415695f3 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -277,6 +277,10 @@ func (ctxt *Link) traverseSyms(flag traverseFlag, fn func(*LSym)) { if call.Func != nil { fn(call.Func) } + f, _ := linkgetlineFromPos(ctxt, call.Pos) + if fsym := ctxt.Lookup(f); fsym != nil { + fn(fsym) + } } } } diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index d9904f9093..5cfd1006fe 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -294,10 +294,10 @@ func (ctxt *Link) pclntab() { // appears in the Pcfile table. In that case, this assigns // the outer file a number. numberfile(ctxt, call.File) - nameoff := nameToOffset(call.Func.Name) + nameoff := nameToOffset(call.Func) inlTreeSym.SetUint16(ctxt.Arch, int64(i*20+0), uint16(call.Parent)) - inlTreeSym.SetUint8(ctxt.Arch, int64(i*20+2), uint8(objabi.GetFuncID(call.Func.Name, call.Func.File))) + inlTreeSym.SetUint8(ctxt.Arch, int64(i*20+2), uint8(objabi.GetFuncID(call.Func, ""))) // byte 3 is unused inlTreeSym.SetUint32(ctxt.Arch, int64(i*20+4), uint32(call.File.Value)) inlTreeSym.SetUint32(ctxt.Arch, int64(i*20+8), uint32(call.Line)) diff --git a/src/cmd/link/internal/objfile/objfile.go b/src/cmd/link/internal/objfile/objfile.go index 3a4ba8224c..a15d3c3e07 100644 --- a/src/cmd/link/internal/objfile/objfile.go +++ b/src/cmd/link/internal/objfile/objfile.go @@ -369,7 +369,7 @@ overwrite: pc.InlTree[i].Parent = r.readInt32() pc.InlTree[i].File = r.readSymIndex() pc.InlTree[i].Line = r.readInt32() - pc.InlTree[i].Func = r.readSymIndex() + pc.InlTree[i].Func = r.readSymIndex().Name pc.InlTree[i].ParentPC = r.readInt32() } diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index b37a665f43..21454cf247 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -699,6 +699,7 @@ func loadObjFull(l *Loader, r *oReader) { pc.Pcdata = make([]sym.Pcdata, len(info.Pcdata)-1) // -1 as we appended one above pc.Funcdataoff = make([]int64, len(info.Funcdataoff)) pc.File = make([]*sym.Symbol, len(info.File)) + pc.InlTree = make([]sym.InlinedCall, len(info.InlTree)) pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp)) pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile)) pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline)) @@ -712,6 +713,16 @@ func loadObjFull(l *Loader, r *oReader) { for k := range pc.File { pc.File[k] = resolveSymRef(info.File[k]) } + for k := range pc.InlTree { + inl := &info.InlTree[k] + pc.InlTree[k] = sym.InlinedCall{ + Parent: inl.Parent, + File: resolveSymRef(inl.File), + Line: inl.Line, + Func: l.SymName(l.Resolve(r, inl.Func)), + ParentPC: inl.ParentPC, + } + } if !dupok { if s.Attr.OnList() { diff --git a/src/cmd/link/internal/sym/symbol.go b/src/cmd/link/internal/sym/symbol.go index 698f8ee653..e9819a064f 100644 --- a/src/cmd/link/internal/sym/symbol.go +++ b/src/cmd/link/internal/sym/symbol.go @@ -534,7 +534,7 @@ type InlinedCall struct { Parent int32 // index of parent in InlTree File *Symbol // file of the inlined call Line int32 // line number of the inlined call - Func *Symbol // function that was inlined + Func string // name of the function that was inlined ParentPC int32 // PC of the instruction just before the inlined body (offset from function start) } From 5fec7882391e839e0b6811b2d4347f0ba8b12358 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 11 Oct 2019 17:43:32 -0400 Subject: [PATCH 33/79] [dev.link] cmd/link: add dupok symbols resolved to another package to textp When a dupok symbol is resolved to another package, we still need to record its presence in the current package, as the trampoline pass expects packages are laid out in dependency order. At the point after deadcode where we populate symbol contents for reachable symbols (add relocations and read symbol data), make a note of the dupok text symbols for each package. Later in addToTextp we will visit packages in dependency order, process the dup text symbol list for each package and select a final lib for each dup text symbol. Change-Id: Ib885e0a7e2343229d853aa629e3e337111df6011 Reviewed-on: https://go-review.googlesource.com/c/go/+/200797 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/objfile/objfile2.go | 28 ++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 21454cf247..d37f73776f 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -559,20 +559,36 @@ func loadObjFull(l *Loader, r *oReader) { pcdataBase := r.PcdataBase() for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { - s := l.Syms[istart+Sym(i)] - if s == nil || s.Name == "" { - continue - } - osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) + if name == "" { + continue + } + ver := abiToVer(osym.ABI, r.version) + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + if dupsym := l.symsByName[nameVer{name, ver}]; dupsym != istart+Sym(i) { + if dupok && l.Reachable.Has(dupsym) { + // A dupok symbol is resolved to another package. We still need + // to record its presence in the current package, as the trampoline + // pass expects packages are laid out in dependency order. + s := l.Syms[dupsym] + if s.Type == sym.STEXT { + lib.DupTextSyms = append(lib.DupTextSyms, s) + } + } + continue + } + + s := l.Syms[istart+Sym(i)] + if s == nil { + continue + } if s.Name != name { // Sanity check. We can remove it in the final version. fmt.Println("name mismatch:", lib, i, s.Name, name) panic("name mismatch") } - dupok := osym.Flag&goobj2.SymFlagDupok != 0 local := osym.Flag&goobj2.SymFlagLocal != 0 makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 size := osym.Siz From 7256f50065f1d53f44c1c9562ad04184ee93279d Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 12 Oct 2019 01:05:03 -0400 Subject: [PATCH 34/79] [dev.link] cmd/link: let cgo import overwrite contentless data symbol A contentless data symbol may be a declaration of a cgo-imported variable, e.g. //go:cgo_import_dynamic xxx var xxx uintptr In this case, we want to mark the symbol imported, instead of defined with zero value. We used to load cgo directives before loading the object file, so we'll mark the symbol SDYNIMPORT first. But in newobj mode, currently we load cgo directives later. Letting SDYNIMPORT overwrite contentless data symbol makes it work in both ordering. Change-Id: I878f52086d6cdb5a347669bf8f848a49bce87b52 Reviewed-on: https://go-review.googlesource.com/c/go/+/201020 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/link/internal/ld/go.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index 15d4f9e50f..6c61869ed8 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -197,7 +197,7 @@ func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { remote, q = remote[:i], remote[i+1:] } s := ctxt.Syms.Lookup(local, 0) - if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ { + if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SBSS || s.Type == sym.SNOPTRBSS || s.Type == sym.SHOSTOBJ { s.SetDynimplib(lib) s.SetExtname(remote) s.SetDynimpvers(q) From 4a9b30cfc9b429971f03a2fbb0587aba994b3fec Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 14 Oct 2019 11:17:18 -0400 Subject: [PATCH 35/79] [dev.link] cmd/internal/goobj2: provide accessor methods for flags Per Jeremy's comment in CL 199643. This makes the code read better. Change-Id: If270aecd712a27fb52e3faf5a4339200327d9ffe Reviewed-on: https://go-review.googlesource.com/c/go/+/201023 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/internal/goobj/readnew.go | 6 +++--- src/cmd/internal/goobj2/objfile.go | 9 +++++++++ src/cmd/link/internal/objfile/objfile2.go | 16 ++++++++-------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index 3074f8131e..e5dc652800 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -87,7 +87,7 @@ func (r *objReader) readNew() { sym := Sym{ SymID: symID, Kind: objabi.SymKind(osym.Type), - DupOK: osym.Flag&goobj2.SymFlagDupok != 0, + DupOK: osym.Dupok(), Size: int64(osym.Siz), Data: Data{int64(start + dataOff), siz}, } @@ -145,8 +145,8 @@ func (r *objReader) readNew() { Args: int64(info.Args), Frame: int64(info.Locals), NoSplit: info.NoSplit != 0, - Leaf: osym.Flag&goobj2.SymFlagLeaf != 0, - TopFrame: osym.Flag&goobj2.SymFlagTopFrame != 0, + Leaf: osym.Leaf(), + TopFrame: osym.TopFrame(), PCSP: Data{int64(pcdataBase + info.Pcsp), int64(info.Pcfile - info.Pcsp)}, PCFile: Data{int64(pcdataBase + info.Pcfile), int64(info.Pcline - info.Pcfile)}, PCLine: Data{int64(pcdataBase + info.Pcline), int64(info.Pcinline - info.Pcline)}, diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index ad1b4ad3a7..bc3a0072f1 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -220,6 +220,15 @@ func (s *Sym) Size() int { return 4 + 2 + 1 + 1 + 4 } +func (s *Sym) Dupok() bool { return s.Flag&SymFlagDupok != 0 } +func (s *Sym) Local() bool { return s.Flag&SymFlagLocal != 0 } +func (s *Sym) Typelink() bool { return s.Flag&SymFlagTypelink != 0 } +func (s *Sym) Leaf() bool { return s.Flag&SymFlagLeaf != 0 } +func (s *Sym) CFunc() bool { return s.Flag&SymFlagCFunc != 0 } +func (s *Sym) ReflectMethod() bool { return s.Flag&SymFlagReflectMethod != 0 } +func (s *Sym) Shared() bool { return s.Flag&SymFlagShared != 0 } +func (s *Sym) TopFrame() bool { return s.Flag&SymFlagTopFrame != 0 } + // Symbol reference. type SymRef struct { PkgIdx uint32 diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index d37f73776f..d80ea1b379 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -155,7 +155,7 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ oldsym := goobj2.Sym{} oldsym.Read(oldr.Reader, oldr.SymOff(li)) oldtyp := sym.AbiSymKindToSymKind[objabi.SymKind(oldsym.Type)] - if oldsym.Flag&goobj2.SymFlagDupok == 0 && !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol + if !oldsym.Dupok() && !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol log.Fatalf("duplicated definition of symbol " + name) } l.overwrite[oldi] = i @@ -415,7 +415,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s continue // don't add unnamed aux symbol } v := abiToVer(osym.ABI, localSymVersion) - dupok := osym.Flag&goobj2.SymFlagDupok != 0 + dupok := osym.Dupok() l.AddSym(name, v, istart+Sym(i), or, dupok, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) } @@ -566,7 +566,7 @@ func loadObjFull(l *Loader, r *oReader) { continue } ver := abiToVer(osym.ABI, r.version) - dupok := osym.Flag&goobj2.SymFlagDupok != 0 + dupok := osym.Dupok() if dupsym := l.symsByName[nameVer{name, ver}]; dupsym != istart+Sym(i) { if dupok && l.Reachable.Has(dupsym) { // A dupok symbol is resolved to another package. We still need @@ -589,8 +589,8 @@ func loadObjFull(l *Loader, r *oReader) { panic("name mismatch") } - local := osym.Flag&goobj2.SymFlagLocal != 0 - makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 + local := osym.Local() + makeTypelink := osym.Typelink() size := osym.Siz // Symbol data @@ -694,13 +694,13 @@ func loadObjFull(l *Loader, r *oReader) { if info.NoSplit != 0 { s.Attr |= sym.AttrNoSplit } - if osym.Flag&goobj2.SymFlagReflectMethod != 0 { + if osym.ReflectMethod() { s.Attr |= sym.AttrReflectMethod } - if osym.Flag&goobj2.SymFlagShared != 0 { + if osym.Shared() { s.Attr |= sym.AttrShared } - if osym.Flag&goobj2.SymFlagTopFrame != 0 { + if osym.TopFrame() { s.Attr |= sym.AttrTopFrame } From c9470b04833332a8e2287364ccfa3e690f5a2047 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Mon, 14 Oct 2019 10:06:37 -0400 Subject: [PATCH 36/79] [dev.link] cmd/link/internal/objfile: relocate loader to new package Third change of several to update the loader API to reflect the final consensus version of the loader API as described in Cherry's doc. This piece: - move objfile.Loader into its own separate package, and update clients accordingly. This includes a few minor cleanups, including converting a couple of loader-related functions to methods, and privatizing some of the loader methods such as ToGlobal/ToLocal. Change-Id: Iae20585751a45491d8b19dcffc096aadae6bbfc6 Reviewed-on: https://go-review.googlesource.com/c/go/+/200998 Run-TryBot: Than McIntosh Reviewed-by: Jeremy Faller TryBot-Result: Gobot Gobot --- src/cmd/dist/buildtool.go | 1 + src/cmd/link/internal/ld/deadcode2.go | 143 +++++++++--------- src/cmd/link/internal/ld/lib.go | 9 +- src/cmd/link/internal/ld/link.go | 4 +- .../{objfile/objfile2.go => loader/loader.go} | 60 ++++---- src/cmd/link/internal/loadmacho/ldmacho.go | 4 +- 6 files changed, 112 insertions(+), 109 deletions(-) rename src/cmd/link/internal/{objfile/objfile2.go => loader/loader.go} (94%) diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index e85dd9a660..2458b439a8 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -73,6 +73,7 @@ var bootstrapDirs = []string{ "cmd/link/internal/arm64", "cmd/link/internal/ld", "cmd/link/internal/loadelf", + "cmd/link/internal/loader", "cmd/link/internal/loadmacho", "cmd/link/internal/loadpe", "cmd/link/internal/loadxcoff", diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 008285c429..ff5cb60a60 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -8,7 +8,7 @@ import ( "bytes" "cmd/internal/objabi" "cmd/internal/sys" - "cmd/link/internal/objfile" + "cmd/link/internal/loader" "cmd/link/internal/sym" "fmt" "strings" @@ -26,16 +26,16 @@ var _ = fmt.Print // - Debug output: // Emit messages about which symbols are kept or deleted. -type workQueue []objfile.Sym +type workQueue []loader.Sym -func (q *workQueue) push(i objfile.Sym) { *q = append(*q, i) } -func (q *workQueue) pop() objfile.Sym { i := (*q)[len(*q)-1]; *q = (*q)[:len(*q)-1]; return i } -func (q *workQueue) empty() bool { return len(*q) == 0 } +func (q *workQueue) push(i loader.Sym) { *q = append(*q, i) } +func (q *workQueue) pop() loader.Sym { i := (*q)[len(*q)-1]; *q = (*q)[:len(*q)-1]; return i } +func (q *workQueue) empty() bool { return len(*q) == 0 } type deadcodePass2 struct { - ctxt *Link - loader *objfile.Loader - wq workQueue + ctxt *Link + ldr *loader.Loader + wq workQueue ifaceMethod map[methodsig]bool // methods declared in reached interfaces markableMethods []methodref2 // methods of reached types @@ -43,7 +43,7 @@ type deadcodePass2 struct { } func (d *deadcodePass2) init() { - d.loader.InitReachable() + d.ldr.InitReachable() d.ifaceMethod = make(map[methodsig]bool) var names []string @@ -67,15 +67,16 @@ func (d *deadcodePass2) init() { // We don't keep the go.plugin.exports symbol, // but we do keep the symbols it refers to. - exportsIdx := d.loader.Lookup("go.plugin.exports", 0) + exportsIdx := d.ldr.Lookup("go.plugin.exports", 0) if exportsIdx != 0 { - relocs := d.loader.Relocs(exportsIdx) + relocs := d.ldr.Relocs(exportsIdx) for i := 0; i < relocs.Count; i++ { d.mark(relocs.At(i).Sym) } } } } + dynexpMap := d.ctxt.cgo_export_dynamic if d.ctxt.LinkMode == LinkExternal { dynexpMap = d.ctxt.cgo_export_static @@ -86,9 +87,9 @@ func (d *deadcodePass2) init() { for _, name := range names { // Mark symbol as an data/ABI0 symbol. - d.mark(d.loader.Lookup(name, 0)) + d.mark(d.ldr.Lookup(name, 0)) // Also mark any Go functions (internal ABI). - d.mark(d.loader.Lookup(name, sym.SymVerABIInternal)) + d.mark(d.ldr.Lookup(name, sym.SymVerABIInternal)) } } @@ -96,13 +97,13 @@ func (d *deadcodePass2) flood() { for !d.wq.empty() { symIdx := d.wq.pop() - d.reflectSeen = d.reflectSeen || d.loader.IsReflectMethod(symIdx) + d.reflectSeen = d.reflectSeen || d.ldr.IsReflectMethod(symIdx) - name := d.loader.RawSymName(symIdx) + name := d.ldr.RawSymName(symIdx) if strings.HasPrefix(name, "type.") && name[5] != '.' { // TODO: use an attribute instead of checking name - p := d.loader.Data(symIdx) + p := d.ldr.Data(symIdx) if len(p) != 0 && decodetypeKind(d.ctxt.Arch, p)&kindMask == kindInterface { - for _, sig := range decodeIfaceMethods2(d.loader, d.ctxt.Arch, symIdx) { + for _, sig := range decodeIfaceMethods2(d.ldr, d.ctxt.Arch, symIdx) { if d.ctxt.Debugvlog > 1 { d.ctxt.Logf("reached iface method: %s\n", sig) } @@ -112,7 +113,7 @@ func (d *deadcodePass2) flood() { } var methods []methodref2 - relocs := d.loader.Relocs(symIdx) + relocs := d.ldr.Relocs(symIdx) for i := 0; i < relocs.Count; i++ { r := relocs.At(i) if r.Type == objabi.R_WEAKADDROFF { @@ -134,18 +135,18 @@ func (d *deadcodePass2) flood() { } d.mark(r.Sym) } - naux := d.loader.NAux(symIdx) + naux := d.ldr.NAux(symIdx) for i := 0; i < naux; i++ { - d.mark(d.loader.AuxSym(symIdx, i)) + d.mark(d.ldr.AuxSym(symIdx, i)) } if len(methods) != 0 { // Decode runtime type information for type methods // to help work out which methods can be called // dynamically via interfaces. - methodsigs := decodetypeMethods2(d.loader, d.ctxt.Arch, symIdx) + methodsigs := decodetypeMethods2(d.ldr, d.ctxt.Arch, symIdx) if len(methods) != len(methodsigs) { - panic(fmt.Sprintf("%q has %d method relocations for %d methods", d.loader.SymName(symIdx), len(methods), len(methodsigs))) + panic(fmt.Sprintf("%q has %d method relocations for %d methods", d.ldr.SymName(symIdx), len(methods), len(methodsigs))) } for i, m := range methodsigs { methods[i].m = m @@ -155,29 +156,28 @@ func (d *deadcodePass2) flood() { } } -func (d *deadcodePass2) mark(symIdx objfile.Sym) { - if symIdx != 0 && !d.loader.Reachable.Has(symIdx) { +func (d *deadcodePass2) mark(symIdx loader.Sym) { + if symIdx != 0 && !d.ldr.Reachable.Has(symIdx) { d.wq.push(symIdx) - d.loader.Reachable.Set(symIdx) + d.ldr.Reachable.Set(symIdx) } } func (d *deadcodePass2) markMethod(m methodref2) { - relocs := d.loader.Relocs(m.src) + relocs := d.ldr.Relocs(m.src) d.mark(relocs.At(m.r).Sym) d.mark(relocs.At(m.r + 1).Sym) d.mark(relocs.At(m.r + 2).Sym) } func deadcode2(ctxt *Link) { - loader := ctxt.loader - d := deadcodePass2{ctxt: ctxt, loader: loader} + ldr := ctxt.loader + d := deadcodePass2{ctxt: ctxt, ldr: ldr} d.init() d.flood() - callSym := loader.Lookup("reflect.Value.Call", sym.SymVerABIInternal) - methSym := loader.Lookup("reflect.Value.Method", sym.SymVerABIInternal) - + callSym := ldr.Lookup("reflect.Value.Call", sym.SymVerABIInternal) + methSym := ldr.Lookup("reflect.Value.Method", sym.SymVerABIInternal) if ctxt.DynlinkingGo() { // Exported methods may satisfy interfaces we don't know // about yet when dynamically linking. @@ -188,7 +188,7 @@ func deadcode2(ctxt *Link) { // Methods might be called via reflection. Give up on // static analysis, mark all exported methods of // all reachable types as reachable. - d.reflectSeen = d.reflectSeen || (callSym != 0 && loader.Reachable.Has(callSym)) || (methSym != 0 && loader.Reachable.Has(methSym)) + d.reflectSeen = d.reflectSeen || (callSym != 0 && ldr.Reachable.Has(callSym)) || (methSym != 0 && ldr.Reachable.Has(methSym)) // Mark all methods that could satisfy a discovered // interface as reachable. We recheck old marked interfaces @@ -211,16 +211,17 @@ func deadcode2(ctxt *Link) { d.flood() } - n := loader.NSym() + n := ldr.NSym() + if ctxt.BuildMode != BuildModeShared { // Keep a itablink if the symbol it points at is being kept. // (When BuildModeShared, always keep itablinks.) for i := 1; i < n; i++ { - s := objfile.Sym(i) - if strings.HasPrefix(loader.RawSymName(s), "go.itablink.") { // TODO: use an attribute instread of checking name - relocs := loader.Relocs(s) - if relocs.Count > 0 && loader.Reachable.Has(relocs.At(0).Sym) { - loader.Reachable.Set(s) + s := loader.Sym(i) + if strings.HasPrefix(ldr.RawSymName(s), "go.itablink.") { // TODO: use an attribute instread of checking name + relocs := ldr.Relocs(s) + if relocs.Count > 0 && ldr.Reachable.Has(relocs.At(0).Sym) { + ldr.Reachable.Set(s) } } } @@ -232,8 +233,8 @@ func deadcode2(ctxt *Link) { // the reflect.method struct: mtyp, ifn, and tfn. type methodref2 struct { m methodsig - src objfile.Sym // receiver type symbol - r int // the index of R_METHODOFF relocations + src loader.Sym // receiver type symbol + r int // the index of R_METHODOFF relocations } func (m methodref2) isExported() bool { @@ -249,13 +250,13 @@ func (m methodref2) isExported() bool { // the function type. // // Conveniently this is the layout of both runtime.method and runtime.imethod. -func decodeMethodSig2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, off, size, count int) []methodsig { +func decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, off, size, count int) []methodsig { var buf bytes.Buffer var methods []methodsig for i := 0; i < count; i++ { - buf.WriteString(decodetypeName2(loader, symIdx, off)) - mtypSym := decodeRelocSym2(loader, symIdx, int32(off+4)) - mp := loader.Data(mtypSym) + buf.WriteString(decodetypeName2(ldr, symIdx, off)) + mtypSym := decodeRelocSym2(ldr, symIdx, int32(off+4)) + mp := ldr.Data(mtypSym) buf.WriteRune('(') inCount := decodetypeFuncInCount(arch, mp) @@ -263,8 +264,8 @@ func decodeMethodSig2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym if i > 0 { buf.WriteString(", ") } - a := decodetypeFuncInType2(loader, arch, mtypSym, i) - buf.WriteString(loader.SymName(a)) + a := decodetypeFuncInType2(ldr, arch, mtypSym, i) + buf.WriteString(ldr.SymName(a)) } buf.WriteString(") (") outCount := decodetypeFuncOutCount(arch, mp) @@ -272,8 +273,8 @@ func decodeMethodSig2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym if i > 0 { buf.WriteString(", ") } - a := decodetypeFuncOutType2(loader, arch, mtypSym, i) - buf.WriteString(loader.SymName(a)) + a := decodetypeFuncOutType2(ldr, arch, mtypSym, i) + buf.WriteString(ldr.SymName(a)) } buf.WriteRune(')') @@ -284,28 +285,28 @@ func decodeMethodSig2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym return methods } -func decodeIfaceMethods2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym) []methodsig { - p := loader.Data(symIdx) +func decodeIfaceMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) []methodsig { + p := ldr.Data(symIdx) if decodetypeKind(arch, p)&kindMask != kindInterface { - panic(fmt.Sprintf("symbol %q is not an interface", loader.SymName(symIdx))) + panic(fmt.Sprintf("symbol %q is not an interface", ldr.SymName(symIdx))) } - rel := decodeReloc2(loader, symIdx, int32(commonsize(arch)+arch.PtrSize)) + rel := decodeReloc2(ldr, symIdx, int32(commonsize(arch)+arch.PtrSize)) if rel.Sym == 0 { return nil } if rel.Sym != symIdx { - panic(fmt.Sprintf("imethod slice pointer in %q leads to a different symbol", loader.SymName(symIdx))) + panic(fmt.Sprintf("imethod slice pointer in %q leads to a different symbol", ldr.SymName(symIdx))) } off := int(rel.Add) // array of reflect.imethod values numMethods := int(decodetypeIfaceMethodCount(arch, p)) sizeofIMethod := 4 + 4 - return decodeMethodSig2(loader, arch, symIdx, off, sizeofIMethod, numMethods) + return decodeMethodSig2(ldr, arch, symIdx, off, sizeofIMethod, numMethods) } -func decodetypeMethods2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym) []methodsig { - p := loader.Data(symIdx) +func decodetypeMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) []methodsig { + p := ldr.Data(symIdx) if !decodetypeHasUncommon(arch, p) { - panic(fmt.Sprintf("no methods on %q", loader.SymName(symIdx))) + panic(fmt.Sprintf("no methods on %q", ldr.SymName(symIdx))) } off := commonsize(arch) // reflect.rtype switch decodetypeKind(arch, p) & kindMask { @@ -333,47 +334,47 @@ func decodetypeMethods2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.S moff := int(decodeInuxi(arch, p[off+4+2+2:], 4)) off += moff // offset to array of reflect.method values const sizeofMethod = 4 * 4 // sizeof reflect.method in program - return decodeMethodSig2(loader, arch, symIdx, off, sizeofMethod, mcount) + return decodeMethodSig2(ldr, arch, symIdx, off, sizeofMethod, mcount) } -func decodeReloc2(loader *objfile.Loader, symIdx objfile.Sym, off int32) objfile.Reloc { - relocs := loader.Relocs(symIdx) +func decodeReloc2(ldr *loader.Loader, symIdx loader.Sym, off int32) loader.Reloc { + relocs := ldr.Relocs(symIdx) for j := 0; j < relocs.Count; j++ { rel := relocs.At(j) if rel.Off == off { return rel } } - return objfile.Reloc{} + return loader.Reloc{} } -func decodeRelocSym2(loader *objfile.Loader, symIdx objfile.Sym, off int32) objfile.Sym { - return decodeReloc2(loader, symIdx, off).Sym +func decodeRelocSym2(ldr *loader.Loader, symIdx loader.Sym, off int32) loader.Sym { + return decodeReloc2(ldr, symIdx, off).Sym } // decodetypeName2 decodes the name from a reflect.name. -func decodetypeName2(loader *objfile.Loader, symIdx objfile.Sym, off int) string { - r := decodeRelocSym2(loader, symIdx, int32(off)) +func decodetypeName2(ldr *loader.Loader, symIdx loader.Sym, off int) string { + r := decodeRelocSym2(ldr, symIdx, int32(off)) if r == 0 { return "" } - data := loader.Data(r) + data := ldr.Data(r) namelen := int(uint16(data[1])<<8 | uint16(data[2])) return string(data[3 : 3+namelen]) } -func decodetypeFuncInType2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, i int) objfile.Sym { +func decodetypeFuncInType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, i int) loader.Sym { uadd := commonsize(arch) + 4 if arch.PtrSize == 8 { uadd += 4 } - if decodetypeHasUncommon(arch, loader.Data(symIdx)) { + if decodetypeHasUncommon(arch, ldr.Data(symIdx)) { uadd += uncommonSize() } - return decodeRelocSym2(loader, symIdx, int32(uadd+i*arch.PtrSize)) + return decodeRelocSym2(ldr, symIdx, int32(uadd+i*arch.PtrSize)) } -func decodetypeFuncOutType2(loader *objfile.Loader, arch *sys.Arch, symIdx objfile.Sym, i int) objfile.Sym { - return decodetypeFuncInType2(loader, arch, symIdx, i+decodetypeFuncInCount(arch, loader.Data(symIdx))) +func decodetypeFuncOutType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, i int) loader.Sym { + return decodetypeFuncInType2(ldr, arch, symIdx, i+decodetypeFuncInCount(arch, ldr.Data(symIdx))) } diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 98748eb6fd..7d24e650a2 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -38,6 +38,7 @@ import ( "cmd/internal/objabi" "cmd/internal/sys" "cmd/link/internal/loadelf" + "cmd/link/internal/loader" "cmd/link/internal/loadmacho" "cmd/link/internal/loadpe" "cmd/link/internal/loadxcoff" @@ -376,7 +377,7 @@ func (ctxt *Link) findLibPath(libname string) string { func (ctxt *Link) loadlib() { if *flagNewobj { - ctxt.loader = objfile.NewLoader() + ctxt.loader = loader.NewLoader() } ctxt.cgo_export_static = make(map[string]bool) @@ -434,7 +435,7 @@ func (ctxt *Link) loadlib() { if *flagNewobj { // Add references of externally defined symbols. - objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms) + ctxt.loader.LoadRefs(ctxt.Arch, ctxt.Syms) } // Now that we know the link mode, set the dynexp list. @@ -1772,7 +1773,7 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, } var c int if *flagNewobj { - objfile.LoadNew(ctxt.loader, ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) + ctxt.loader.Preload(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) } else { c = objfile.Load(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags) } @@ -2550,7 +2551,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library func (ctxt *Link) loadlibfull() { // Load full symbol contents, resolve indexed references. - objfile.LoadFull(ctxt.loader, ctxt.Arch, ctxt.Syms) + ctxt.loader.LoadFull(ctxt.Arch, ctxt.Syms) // For now, add all symbols to ctxt.Syms. for _, s := range ctxt.loader.Syms { diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index 46bf08bb1c..124f7d9001 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -35,7 +35,7 @@ import ( "cmd/internal/obj" "cmd/internal/objabi" "cmd/internal/sys" - "cmd/link/internal/objfile" + "cmd/link/internal/loader" "cmd/link/internal/sym" "debug/elf" "fmt" @@ -98,7 +98,7 @@ type Link struct { relocbuf []byte // temporary buffer for applying relocations - loader *objfile.Loader + loader *loader.Loader cgodata []cgodata // cgo directives to load, three strings are args for loadcgo cgo_export_static map[string]bool diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/loader/loader.go similarity index 94% rename from src/cmd/link/internal/objfile/objfile2.go rename to src/cmd/link/internal/loader/loader.go index d80ea1b379..e986f7e2c1 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/loader/loader.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package objfile +package loader import ( "bytes" @@ -87,8 +87,6 @@ func makeBitmap(n int) bitmap { } // A Loader loads new object files and resolves indexed symbol references. -// -// TODO: describe local-global index mapping. type Loader struct { start map[*oReader]Sym // map from object file to its start index objs []objIdx // sorted by start index (i.e. objIdx.i) @@ -117,12 +115,12 @@ func NewLoader() *Loader { } // Return the start index in the global index space for a given object file. -func (l *Loader) StartIndex(r *oReader) Sym { +func (l *Loader) startIndex(r *oReader) Sym { return l.start[r] } // Add object file r, return the start index. -func (l *Loader) AddObj(pkg string, r *oReader) Sym { +func (l *Loader) addObj(pkg string, r *oReader) Sym { if _, ok := l.start[r]; ok { panic("already added") } @@ -148,10 +146,10 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ if dupok { return false } - overwrite := r.DataSize(int(i-l.StartIndex(r))) != 0 + overwrite := r.DataSize(int(i-l.startIndex(r))) != 0 if overwrite { // new symbol overwrites old symbol. - oldr, li := l.ToLocal(oldi) + oldr, li := l.toLocal(oldi) oldsym := goobj2.Sym{} oldsym.Read(oldr.Reader, oldr.SymOff(li)) oldtyp := sym.AbiSymKindToSymKind[objabi.SymKind(oldsym.Type)] @@ -190,8 +188,8 @@ func (l *Loader) AddExtSym(name string, ver int) Sym { } // Convert a local index to a global index. -func (l *Loader) ToGlobal(r *oReader, i int) Sym { - g := l.StartIndex(r) + Sym(i) +func (l *Loader) toGlobal(r *oReader, i int) Sym { + g := l.startIndex(r) + Sym(i) if ov, ok := l.overwrite[g]; ok { return ov } @@ -199,7 +197,7 @@ func (l *Loader) ToGlobal(r *oReader, i int) Sym { } // Convert a global index to a local index. -func (l *Loader) ToLocal(i Sym) (*oReader, int) { +func (l *Loader) toLocal(i Sym) (*oReader, int) { if ov, ok := l.overwrite[i]; ok { i = ov } @@ -216,7 +214,7 @@ func (l *Loader) ToLocal(i Sym) (*oReader, int) { } // Resolve a local symbol reference. Return global index. -func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) Sym { +func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { var rr *oReader switch p := s.PkgIdx; p { case goobj2.PkgIdxInvalid: @@ -245,7 +243,7 @@ func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) Sym { log.Fatalf("reference of nonexisted package %s, from %v", pkg, r.unit.Lib) } } - return l.ToGlobal(rr, int(s.SymIdx)) + return l.toGlobal(rr, int(s.SymIdx)) } // Look up a symbol by name, return global index, or 0 if not found. @@ -266,7 +264,7 @@ func (l *Loader) RawSymName(i Sym) string { if l.extStart != 0 && i >= l.extStart { return "" } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return osym.Name @@ -277,7 +275,7 @@ func (l *Loader) SymName(i Sym) string { if l.extStart != 0 && i >= l.extStart { return "" } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) @@ -288,7 +286,7 @@ func (l *Loader) SymType(i Sym) sym.SymKind { if l.extStart != 0 && i >= l.extStart { return 0 } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] @@ -299,7 +297,7 @@ func (l *Loader) SymAttr(i Sym) uint8 { if l.extStart != 0 && i >= l.extStart { return 0 } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(li)) return osym.Flag @@ -315,7 +313,7 @@ func (l *Loader) Data(i Sym) []byte { if l.extStart != 0 && i >= l.extStart { return nil } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) return r.Data(li) } @@ -324,7 +322,7 @@ func (l *Loader) NAux(i Sym) int { if l.extStart != 0 && i >= l.extStart { return 0 } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) return r.NAux(li) } @@ -334,10 +332,10 @@ func (l *Loader) AuxSym(i Sym, j int) Sym { if l.extStart != 0 && i >= l.extStart { return 0 } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) a := goobj2.Aux{} a.Read(r.Reader, r.AuxOff(li, j)) - return l.Resolve(r, a.Sym) + return l.resolve(r, a.Sym) } // Initialize Reachable bitmap for running deadcode pass. @@ -349,7 +347,7 @@ func (l *Loader) InitReachable() { func (relocs *Relocs) At(j int) Reloc { rel := goobj2.Reloc{} rel.Read(relocs.r.Reader, relocs.r.RelocOff(relocs.li, j)) - target := relocs.l.Resolve(relocs.r, rel.Sym) + target := relocs.l.resolve(relocs.r, rel.Sym) return Reloc{ Off: rel.Off, Size: rel.Siz, @@ -364,7 +362,7 @@ func (l *Loader) Relocs(i Sym) Relocs { if l.extStart != 0 && i >= l.extStart { return Relocs{} } - r, li := l.ToLocal(i) + r, li := l.toLocal(i) return l.relocs(r, li) } @@ -380,7 +378,7 @@ func (l *Loader) relocs(r *oReader, li int) Relocs { // Preload a package: add autolibs, add symbols to the symbol table. // Does not read symbol data yet. -func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { +func (l *Loader) Preload(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64, pn string, flags int) { roObject, readonly, err := f.Slice(uint64(length)) if err != nil { log.Fatal("cannot read object file:", err) @@ -403,7 +401,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s unit.DWARFFileTable[i] = r.DwarfFile(i) } - istart := l.AddObj(lib.Pkg, or) + istart := l.addObj(lib.Pkg, or) ndef := r.NSym() nnonpkgdef := r.NNonpkgdef() @@ -425,7 +423,7 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s // Make sure referenced symbols are added. Most of them should already be added. // This should only be needed for referenced external symbols. -func LoadRefs(l *Loader, arch *sys.Arch, syms *sym.Symbols) { +func (l *Loader) LoadRefs(arch *sys.Arch, syms *sym.Symbols) { for _, o := range l.objs[1:] { loadObjRefs(l, o.r, arch, syms) } @@ -479,7 +477,7 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { } // Load full contents. -func LoadFull(l *Loader, arch *sys.Arch, syms *sym.Symbols) { +func (l *Loader) LoadFull(arch *sys.Arch, syms *sym.Symbols) { // create all Symbols first. l.Syms = make([]*sym.Symbol, l.NSym()) for _, o := range l.objs[1:] { @@ -505,7 +503,7 @@ func LoadFull(l *Loader, arch *sys.Arch, syms *sym.Symbols) { func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { lib := r.unit.Lib - istart := l.StartIndex(r) + istart := l.startIndex(r) for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { osym := goobj2.Sym{} @@ -550,10 +548,10 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { func loadObjFull(l *Loader, r *oReader) { lib := r.unit.Lib - istart := l.StartIndex(r) + istart := l.startIndex(r) resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { - i := l.Resolve(r, s) + i := l.resolve(r, s) return l.Syms[i] } @@ -735,7 +733,7 @@ func loadObjFull(l *Loader, r *oReader) { Parent: inl.Parent, File: resolveSymRef(inl.File), Line: inl.Line, - Func: l.SymName(l.Resolve(r, inl.Func)), + Func: l.SymName(l.resolve(r, inl.Func)), ParentPC: inl.ParentPC, } } @@ -754,6 +752,8 @@ func loadObjFull(l *Loader, r *oReader) { } } +var emptyPkg = []byte(`"".`) + func patchDWARFName(s *sym.Symbol, r *oReader) { // This is kind of ugly. Really the package name should not // even be included here. diff --git a/src/cmd/link/internal/loadmacho/ldmacho.go b/src/cmd/link/internal/loadmacho/ldmacho.go index 7a0e18fe71..18a0678af8 100644 --- a/src/cmd/link/internal/loadmacho/ldmacho.go +++ b/src/cmd/link/internal/loadmacho/ldmacho.go @@ -10,7 +10,7 @@ import ( "cmd/internal/bio" "cmd/internal/objabi" "cmd/internal/sys" - "cmd/link/internal/objfile" + "cmd/link/internal/loader" "cmd/link/internal/sym" "encoding/binary" "fmt" @@ -424,7 +424,7 @@ func macholoadsym(m *ldMachoObj, symtab *ldMachoSymtab) int { return 0 } -func Load(l *objfile.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) error { +func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) error { lookup := func(name string, version int) *sym.Symbol { // Check to see if we've already defined the symbol. if i := l.Lookup(name, version); i != 0 { From 5caac2f73efe7fc6919f7850d99c35cde02012b5 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 9 Oct 2019 10:22:20 -0400 Subject: [PATCH 37/79] [dev.link] cmd: default to new object files Switch the default to new object files. Internal linking cgo is disabled for now, as it does not work yet in newobj mode. Shared libraries are also broken. Disable some tests that are known broken for now. Change-Id: I8ca74793423861d607a2aa7b0d89a4f4d4ca7671 Reviewed-on: https://go-review.googlesource.com/c/go/+/200161 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- misc/cgo/testplugin/testdata/host/host.go | 13 +++++++------ src/cmd/asm/internal/flags/flags.go | 2 +- src/cmd/compile/internal/gc/main.go | 2 +- src/cmd/dist/test.go | 7 +++++-- src/cmd/link/internal/ld/config.go | 4 ++++ src/cmd/link/internal/ld/lib.go | 2 +- src/cmd/link/internal/ld/main.go | 2 +- src/cmd/nm/nm_cgo_test.go | 2 +- src/debug/pe/file_cgo_test.go | 1 + src/runtime/runtime-gdb_test.go | 2 ++ test/linkx.go | 8 ++++++++ 11 files changed, 32 insertions(+), 13 deletions(-) diff --git a/misc/cgo/testplugin/testdata/host/host.go b/misc/cgo/testplugin/testdata/host/host.go index a3799328cd..d836523da8 100644 --- a/misc/cgo/testplugin/testdata/host/host.go +++ b/misc/cgo/testplugin/testdata/host/host.go @@ -145,12 +145,13 @@ func main() { } _, err = plugin.Open("plugin-mismatch.so") - if err == nil { - log.Fatal(`plugin.Open("plugin-mismatch.so"): should have failed`) - } - if s := err.Error(); !strings.Contains(s, "different version") { - log.Fatalf(`plugin.Open("plugin-mismatch.so"): error does not mention "different version": %v`, s) - } + // TODO: newobj + //if err == nil { + // log.Fatal(`plugin.Open("plugin-mismatch.so"): should have failed`) + //} + //if s := err.Error(); !strings.Contains(s, "different version") { + // log.Fatalf(`plugin.Open("plugin-mismatch.so"): error does not mention "different version": %v`, s) + //} _, err = plugin.Open("plugin2-dup.so") if err == nil { diff --git a/src/cmd/asm/internal/flags/flags.go b/src/cmd/asm/internal/flags/flags.go index fad87b221a..95575e15a3 100644 --- a/src/cmd/asm/internal/flags/flags.go +++ b/src/cmd/asm/internal/flags/flags.go @@ -23,7 +23,7 @@ var ( Dynlink = flag.Bool("dynlink", false, "support references to Go symbols defined in other shared libraries") AllErrors = flag.Bool("e", false, "no limit on number of errors reported") SymABIs = flag.Bool("gensymabis", false, "write symbol ABI information to output file, don't assemble") - Newobj = flag.Bool("newobj", false, "use new object file format") + Newobj = flag.Bool("newobj", true, "use new object file format") ) var ( diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 9e8abbcdeb..4797912d60 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -263,7 +263,7 @@ func Main(archInit func(*Arch)) { flag.StringVar(&benchfile, "bench", "", "append benchmark times to `file`") flag.BoolVar(&smallFrames, "smallframes", false, "reduce the size limit for stack allocated objects") flag.BoolVar(&Ctxt.UseBASEntries, "dwarfbasentries", Ctxt.UseBASEntries, "use base address selection entries in DWARF") - flag.BoolVar(&Ctxt.Flag_newobj, "newobj", false, "use new object file format") + flag.BoolVar(&Ctxt.Flag_newobj, "newobj", true, "use new object file format") objabi.Flagparse(usage) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 273ef2e19a..46556f2f79 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -585,7 +585,7 @@ func (t *tester) registerTests() { }, }) // Also test a cgo package. - if t.cgoEnabled { + if t.cgoEnabled && t.internalLink() { t.tests = append(t.tests, distTest{ name: "pie_internal_cgo", heading: "internal linking of -buildmode=pie", @@ -681,7 +681,7 @@ func (t *tester) registerTests() { if t.supportedBuildmode("c-shared") { t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".") } - if t.supportedBuildmode("shared") { + if t.supportedBuildmode("shared") && false { // TODO: newobj t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".") } if t.supportedBuildmode("plugin") { @@ -904,6 +904,9 @@ func (t *tester) extLink() bool { } func (t *tester) internalLink() bool { + if true { // appease vet... + return false // TODO: newobj + } if gohostos == "dragonfly" { // linkmode=internal fails on dragonfly since errno is a TLS relocation. return false diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index 3f5b6d4fdf..cfb8c9a786 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -183,6 +183,10 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) { return true, "msan" } + if iscgo { // TODO: internal linking cgo doesn't work yet + return true, "TODO: newobj" + } + // Internally linking cgo is incomplete on some architectures. // https://golang.org/issue/14449 // https://golang.org/issue/21961 diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 7d24e650a2..424dffda97 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -816,7 +816,7 @@ func genhash(ctxt *Link, lib *sym.Library) { return } h.Write(pkgDefBytes[0:firstEOL]) - h.Write(pkgDefBytes[firstDoubleDollar : firstDoubleDollar+secondDoubleDollar]) + //h.Write(pkgDefBytes[firstDoubleDollar : firstDoubleDollar+secondDoubleDollar]) // TODO: newobj: -dynlink may change symbol numbering? which will make the export data differ lib.Hash = hex.EncodeToString(h.Sum(nil)) } diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index e667afecc1..3d8bc069af 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -86,7 +86,7 @@ var ( flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker") FlagDebugTramp = flag.Int("debugtramp", 0, "debug trampolines") FlagStrictDups = flag.Int("strictdups", 0, "sanity check duplicate symbol contents during object file reading (1=warn 2=err).") - flagNewobj = flag.Bool("newobj", false, "use new object file format") + flagNewobj = flag.Bool("newobj", true, "use new object file format") FlagRound = flag.Int("R", -1, "set address rounding `quantum`") FlagTextAddr = flag.Int64("T", -1, "set text segment `address`") diff --git a/src/cmd/nm/nm_cgo_test.go b/src/cmd/nm/nm_cgo_test.go index 475c57b4c2..63001f85c6 100644 --- a/src/cmd/nm/nm_cgo_test.go +++ b/src/cmd/nm/nm_cgo_test.go @@ -32,7 +32,7 @@ func canInternalLink() bool { } func TestInternalLinkerCgoExec(t *testing.T) { - if !canInternalLink() { + if !canInternalLink() || true { // TODO: newobj t.Skip("skipping; internal linking is not supported") } testGoExec(t, true, false) diff --git a/src/debug/pe/file_cgo_test.go b/src/debug/pe/file_cgo_test.go index 739671d73f..e89894953b 100644 --- a/src/debug/pe/file_cgo_test.go +++ b/src/debug/pe/file_cgo_test.go @@ -23,6 +23,7 @@ func TestDefaultLinkerDWARF(t *testing.T) { } func TestInternalLinkerDWARF(t *testing.T) { + t.Skip("TODO: newobj") testCgoDWARF(t, linkCgoInternal) } diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index de1bac65da..e810a59507 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -489,6 +489,8 @@ func main() { ` func TestGdbConst(t *testing.T) { + t.Skip("TODO: newobj") // XXX the constant DIEs are not referenced, so they are not pulled in. Maybe it'll be fine if we rewrite linker's dwarf pass to index? + checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) diff --git a/test/linkx.go b/test/linkx.go index 520a065182..2b5b6edd47 100644 --- a/test/linkx.go +++ b/test/linkx.go @@ -29,4 +29,12 @@ func main() { fmt.Println(overwrite) fmt.Println(overwritecopy) fmt.Println(arraycopy[1]) + + // Check non-string symbols are not overwritten. + // This also make them used. + // TODO: decide if we need to issue an error if -X + // is applied to a non-string unreachable symbol. + if b || x != 0 { + panic("b or x overwritten") + } } From 2bbf2e0233799ab90bedc82f51a87f7d7aa76920 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 16 Oct 2019 15:44:04 -0400 Subject: [PATCH 38/79] [dev.link] cmd/link: add basic shared library support in newobj mode This CL adds basic shared library support in newobj mode. This is not complete -- there are still tests in misc/cgo/testshared failing. But at least a simple program works, and some tests there pass. Add the mechanism of loading external symbols with contents. (Before, external symbols are always contentless.) This may potentially be also used for other host objects. Change-Id: I68dbf71e7949cc01ebf37ea159084e798ae16925 Reviewed-on: https://go-review.googlesource.com/c/go/+/201537 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/link/internal/ld/deadcode2.go | 16 +++- src/cmd/link/internal/ld/lib.go | 42 ++++++--- src/cmd/link/internal/loader/loader.go | 121 +++++++++++++++++++++++-- 3 files changed, 155 insertions(+), 24 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index ff5cb60a60..259199eea1 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -18,9 +18,6 @@ import ( var _ = fmt.Print // TODO: -// - Shared object support: -// It basically marks everything. We could consider using -// a different mechanism to represent it. // - Field tracking support: // It needs to record from where the symbol is referenced. // - Debug output: @@ -46,6 +43,19 @@ func (d *deadcodePass2) init() { d.ldr.InitReachable() d.ifaceMethod = make(map[methodsig]bool) + if d.ctxt.BuildMode == BuildModeShared { + // Mark all symbols defined in this library as reachable when + // building a shared library. + n := d.ldr.NDef() + for i := 1; i < n; i++ { + s := loader.Sym(i) + if !d.ldr.IsDup(s) { + d.mark(s) + } + } + return + } + var names []string // In a normal binary, start at main.main and the init diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 424dffda97..063bdded0c 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -433,6 +433,15 @@ func (ctxt *Link) loadlib() { } } + for _, lib := range ctxt.Library { + if lib.Shlib != "" { + if ctxt.Debugvlog > 1 { + ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + } + ldshlibsyms(ctxt, lib.Shlib) + } + } + if *flagNewobj { // Add references of externally defined symbols. ctxt.loader.LoadRefs(ctxt.Arch, ctxt.Syms) @@ -443,15 +452,6 @@ func (ctxt *Link) loadlib() { setupdynexp(ctxt) } - for _, lib := range ctxt.Library { - if lib.Shlib != "" { - if ctxt.Debugvlog > 1 { - ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) - } - ldshlibsyms(ctxt, lib.Shlib) - } - } - // In internal link mode, read the host object files. if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. @@ -1931,7 +1931,17 @@ func ldshlibsyms(ctxt *Link, shlib string) { ver = sym.SymVerABIInternal } - lsym := ctxt.Syms.Lookup(elfsym.Name, ver) + var lsym *sym.Symbol + if *flagNewobj { + i := ctxt.loader.AddExtSym(elfsym.Name, ver) + if i == 0 { + continue + } + lsym = ctxt.Syms.Newsym(elfsym.Name, ver) + ctxt.loader.Syms[i] = lsym + } else { + lsym = ctxt.Syms.Lookup(elfsym.Name, ver) + } // Because loadlib above loads all .a files before loading any shared // libraries, any non-dynimport symbols we find that duplicate symbols // already loaded should be ignored (the symbols from the .a files @@ -1960,7 +1970,17 @@ func ldshlibsyms(ctxt *Link, shlib string) { // mangle Go function names in the .so to include the // ABI. if elf.ST_TYPE(elfsym.Info) == elf.STT_FUNC && ver == 0 { - alias := ctxt.Syms.Lookup(elfsym.Name, sym.SymVerABIInternal) + var alias *sym.Symbol + if *flagNewobj { + i := ctxt.loader.AddExtSym(elfsym.Name, sym.SymVerABIInternal) + if i == 0 { + continue + } + alias = ctxt.Syms.Newsym(elfsym.Name, sym.SymVerABIInternal) + ctxt.loader.Syms[i] = alias + } else { + alias = ctxt.Syms.Lookup(elfsym.Name, sym.SymVerABIInternal) + } if alias.Type != 0 { continue } diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index e986f7e2c1..1cd44fd1b6 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -35,6 +35,8 @@ type Relocs struct { li int // local index of symbol whose relocs we're examining r *oReader // object reader for containing package l *Loader // loader + + ext *sym.Symbol // external symbol if not nil } // Reloc contains the payload for a specific relocation. @@ -184,9 +186,24 @@ func (l *Loader) AddExtSym(name string, ver int) Sym { l.extStart = i } l.extSyms = append(l.extSyms, nv) + l.growSyms(int(i)) return i } +// Returns whether i is an external symbol. +func (l *Loader) isExternal(i Sym) bool { + return l.extStart != 0 && i >= l.extStart +} + +// Ensure Syms slice als enough space. +func (l *Loader) growSyms(i int) { + n := len(l.Syms) + if n > i { + return + } + l.Syms = append(l.Syms, make([]*sym.Symbol, i+1-n)...) +} + // Convert a local index to a global index. func (l *Loader) toGlobal(r *oReader, i int) Sym { g := l.startIndex(r) + Sym(i) @@ -201,7 +218,7 @@ func (l *Loader) toLocal(i Sym) (*oReader, int) { if ov, ok := l.overwrite[i]; ok { i = ov } - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { return nil, int(i - l.extStart) } // Search for the local object holding index i. @@ -254,14 +271,45 @@ func (l *Loader) Lookup(name string, ver int) Sym { return l.symsByName[nv] } +// Returns whether i is a dup of another symbol, and i is not +// "primary", i.e. Lookup i by name will not return i. +func (l *Loader) IsDup(i Sym) bool { + if _, ok := l.overwrite[i]; ok { + return true + } + if l.isExternal(i) { + return false + } + r, li := l.toLocal(i) + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(li)) + if !osym.Dupok() { + return false + } + if osym.Name == "" { + return false + } + name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) + ver := abiToVer(osym.ABI, r.version) + return l.symsByName[nameVer{name, ver}] != i +} + // Number of total symbols. func (l *Loader) NSym() int { return int(l.max + 1) } +// Number of defined Go symbols. +func (l *Loader) NDef() int { + return int(l.extStart) +} + // Returns the raw (unpatched) name of the i-th symbol. func (l *Loader) RawSymName(i Sym) string { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + if s := l.Syms[i]; s != nil { + return s.Name + } return "" } r, li := l.toLocal(i) @@ -272,7 +320,10 @@ func (l *Loader) RawSymName(i Sym) string { // Returns the (patched) name of the i-th symbol. func (l *Loader) SymName(i Sym) string { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + if s := l.Syms[i]; s != nil { + return s.Name // external name should already be patched? + } return "" } r, li := l.toLocal(i) @@ -283,7 +334,10 @@ func (l *Loader) SymName(i Sym) string { // Returns the type of the i-th symbol. func (l *Loader) SymType(i Sym) sym.SymKind { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + if s := l.Syms[i]; s != nil { + return s.Type + } return 0 } r, li := l.toLocal(i) @@ -294,7 +348,8 @@ func (l *Loader) SymType(i Sym) sym.SymKind { // Returns the attributes of the i-th symbol. func (l *Loader) SymAttr(i Sym) uint8 { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + // TODO: do something? External symbols have different representation of attributes. For now, ReflectMethod is the only thing matters and it cannot be set by external symbol. return 0 } r, li := l.toLocal(i) @@ -310,7 +365,10 @@ func (l *Loader) IsReflectMethod(i Sym) bool { // Returns the symbol content of the i-th symbol. i is global index. func (l *Loader) Data(i Sym) []byte { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + if s := l.Syms[i]; s != nil { + return s.P + } return nil } r, li := l.toLocal(i) @@ -319,7 +377,7 @@ func (l *Loader) Data(i Sym) []byte { // Returns the number of aux symbols given a global index. func (l *Loader) NAux(i Sym) int { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { return 0 } r, li := l.toLocal(i) @@ -329,7 +387,7 @@ func (l *Loader) NAux(i Sym) int { // Returns the referred symbol of the j-th aux symbol of the i-th // symbol. func (l *Loader) AuxSym(i Sym, j int) Sym { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { return 0 } r, li := l.toLocal(i) @@ -345,6 +403,16 @@ func (l *Loader) InitReachable() { // At method returns the j-th reloc for a global symbol. func (relocs *Relocs) At(j int) Reloc { + if relocs.ext != nil { + rel := &relocs.ext.R[j] + return Reloc{ + Off: rel.Off, + Size: rel.Siz, + Type: rel.Type, + Add: rel.Add, + Sym: relocs.l.Lookup(rel.Sym.Name, int(rel.Sym.Version)), + } + } rel := goobj2.Reloc{} rel.Read(relocs.r.Reader, relocs.r.RelocOff(relocs.li, j)) target := relocs.l.resolve(relocs.r, rel.Sym) @@ -359,7 +427,10 @@ func (relocs *Relocs) At(j int) Reloc { // Relocs returns a Relocs object for the given global sym. func (l *Loader) Relocs(i Sym) Relocs { - if l.extStart != 0 && i >= l.extStart { + if l.isExternal(i) { + if s := l.Syms[i]; s != nil { + return Relocs{Count: len(s.R), l: l, ext: s} + } return Relocs{} } r, li := l.toLocal(i) @@ -479,13 +550,17 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { // Load full contents. func (l *Loader) LoadFull(arch *sys.Arch, syms *sym.Symbols) { // create all Symbols first. - l.Syms = make([]*sym.Symbol, l.NSym()) + l.growSyms(l.NSym()) for _, o := range l.objs[1:] { loadObjSyms(l, syms, o.r) } // external symbols for i := l.extStart; i <= l.max; i++ { + if s := l.Syms[i]; s != nil { + s.Attr.Set(sym.AttrReachable, l.Reachable.Has(i)) + continue // already loaded from external object + } nv := l.extSyms[i-l.extStart] if l.Reachable.Has(i) || strings.HasPrefix(nv.name, "gofile..") { // XXX file symbols are used but not marked s := syms.Newsym(nv.name, nv.v) @@ -782,3 +857,29 @@ func patchDWARFName(s *sym.Symbol, r *oReader) { } } } + +// For debugging. +func (l *Loader) Dump() { + fmt.Println("objs") + for _, obj := range l.objs { + if obj.r != nil { + fmt.Println(obj.i, obj.r.unit.Lib) + } + } + fmt.Println("syms") + for i, s := range l.Syms { + if i == 0 { + continue + } + if s != nil { + fmt.Println(i, s, s.Type) + } else { + fmt.Println(i, l.SymName(Sym(i)), "") + } + } + fmt.Println("overwrite:", l.overwrite) + fmt.Println("symsByName") + for nv, i := range l.symsByName { + fmt.Println(i, nv.name, nv.v) + } +} From 15634a023062ad29ba53e8a4211e15d389f86ee9 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 08:54:58 -0400 Subject: [PATCH 39/79] [dev.link] cmd: convert symbol "shared" flag to object file flag For the new object file format, don't tag individual symbols with a "shared" flag, since that characteristic is better off as an attribute of the containing object file as opposed to the individual symbol. Add a new flags field in the object file header and put a bit in the flags if the shared flags is in effect during compilation. Change-Id: I2cf6d33bf7bf2fd8a7614ae0cd6ef03914777498 Reviewed-on: https://go-review.googlesource.com/c/go/+/201398 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller Reviewed-by: Cherry Zhang --- src/cmd/internal/goobj2/objfile.go | 18 +++++++++++++++--- src/cmd/internal/obj/objfile2.go | 9 +++++---- src/cmd/link/internal/loader/loader.go | 7 ++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index bc3a0072f1..ef32c4873e 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -20,6 +20,7 @@ import ( // // Header struct { // Magic [...]byte // "\x00go114LD" +// Flags uint32 // // TODO: Fingerprint // Offsets [...]uint32 // byte offset of each block below // } @@ -148,6 +149,7 @@ const ( // TODO: probably no need to export this. type Header struct { Magic string + Flags uint32 Offsets [NBlk]uint32 } @@ -155,6 +157,7 @@ const Magic = "\x00go114LD" func (h *Header) Write(w *Writer) { w.RawString(h.Magic) + w.Uint32(h.Flags) for _, x := range h.Offsets { w.Uint32(x) } @@ -167,6 +170,8 @@ func (h *Header) Read(r *Reader) error { return errors.New("wrong magic, not a Go object file") } off := uint32(len(h.Magic)) + h.Flags = r.uint32At(off) + off += 4 for i := range h.Offsets { h.Offsets[i] = r.uint32At(off) off += 4 @@ -175,7 +180,7 @@ func (h *Header) Read(r *Reader) error { } func (h *Header) Size() int { - return len(h.Magic) + 4*len(h.Offsets) + return len(h.Magic) + 4 + 4*len(h.Offsets) } // Symbol definition. @@ -189,6 +194,10 @@ type Sym struct { const SymABIstatic = ^uint16(0) +const ( + ObjFlagShared = 1 << iota +) + const ( SymFlagDupok = 1 << iota SymFlagLocal @@ -196,7 +205,6 @@ const ( SymFlagLeaf SymFlagCFunc SymFlagReflectMethod - SymFlagShared // This is really silly SymFlagTopFrame ) @@ -226,7 +234,6 @@ func (s *Sym) Typelink() bool { return s.Flag&SymFlagTypelink != 0 } func (s *Sym) Leaf() bool { return s.Flag&SymFlagLeaf != 0 } func (s *Sym) CFunc() bool { return s.Flag&SymFlagCFunc != 0 } func (s *Sym) ReflectMethod() bool { return s.Flag&SymFlagReflectMethod != 0 } -func (s *Sym) Shared() bool { return s.Flag&SymFlagShared != 0 } func (s *Sym) TopFrame() bool { return s.Flag&SymFlagTopFrame != 0 } // Symbol reference. @@ -596,3 +603,8 @@ func (r *Reader) PcdataBase() uint32 { func (r *Reader) ReadOnly() bool { return r.readonly } + +// Flags returns the flag bits read from the object file header. +func (r *Reader) Flags() uint32 { + return r.h.Flags +} diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index 843f6fb5ea..f3389612d6 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -35,7 +35,11 @@ func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) { // Header // We just reserve the space. We'll fill in the offsets later. - h := goobj2.Header{Magic: goobj2.Magic} + flags := uint32(0) + if ctxt.Flag_shared { + flags |= goobj2.ObjFlagShared + } + h := goobj2.Header{Magic: goobj2.Magic, Flags: flags} h.Write(w.Writer) // String table @@ -231,9 +235,6 @@ func (w *writer) Sym(s *LSym) { if s.ReflectMethod() { flag |= goobj2.SymFlagReflectMethod } - if w.ctxt.Flag_shared { // This is really silly - flag |= goobj2.SymFlagShared - } if s.TopFrame() { flag |= goobj2.SymFlagTopFrame } diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 1cd44fd1b6..c155f27dcb 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -56,7 +56,8 @@ type Reloc struct { type oReader struct { *goobj2.Reader unit *sym.CompilationUnit - version int // version of static symbol + version int // version of static symbol + flags uint32 // read from object file pkgprefix string } @@ -460,7 +461,7 @@ func (l *Loader) Preload(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib * } localSymVersion := syms.IncVersion() pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." - or := &oReader{r, unit, localSymVersion, pkgprefix} + or := &oReader{r, unit, localSymVersion, r.Flags(), pkgprefix} // Autolib lib.ImportStrings = append(lib.ImportStrings, r.Autolib()...) @@ -770,7 +771,7 @@ func loadObjFull(l *Loader, r *oReader) { if osym.ReflectMethod() { s.Attr |= sym.AttrReflectMethod } - if osym.Shared() { + if r.Flags()&goobj2.ObjFlagShared != 0 { s.Attr |= sym.AttrShared } if osym.TopFrame() { From 8011051361d85ca0c062f7226e80afbad21341c8 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 09:13:59 -0400 Subject: [PATCH 40/79] [dev.link] cmd: add flag to mark gotype symbols Add a flag bit to mark symbols in the new object file as containing Go type information. The use of a flag eliminates the need to do symbol name matching as part of the new dead code elimination pass, which should produce a minor speedup. Change-Id: Iec8700e1139e2c4e310644c0766379865d2d6f82 Reviewed-on: https://go-review.googlesource.com/c/go/+/201399 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller Reviewed-by: Cherry Zhang --- src/cmd/internal/goobj2/objfile.go | 2 ++ src/cmd/internal/obj/objfile2.go | 3 +++ src/cmd/link/internal/ld/deadcode2.go | 4 +--- src/cmd/link/internal/loader/loader.go | 5 +++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index ef32c4873e..03b322da6c 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -205,6 +205,7 @@ const ( SymFlagLeaf SymFlagCFunc SymFlagReflectMethod + SymFlagGoType SymFlagTopFrame ) @@ -234,6 +235,7 @@ func (s *Sym) Typelink() bool { return s.Flag&SymFlagTypelink != 0 } func (s *Sym) Leaf() bool { return s.Flag&SymFlagLeaf != 0 } func (s *Sym) CFunc() bool { return s.Flag&SymFlagCFunc != 0 } func (s *Sym) ReflectMethod() bool { return s.Flag&SymFlagReflectMethod != 0 } +func (s *Sym) IsGoType() bool { return s.Flag&SymFlagGoType != 0 } func (s *Sym) TopFrame() bool { return s.Flag&SymFlagTopFrame != 0 } // Symbol reference. diff --git a/src/cmd/internal/obj/objfile2.go b/src/cmd/internal/obj/objfile2.go index f3389612d6..69019e033d 100644 --- a/src/cmd/internal/obj/objfile2.go +++ b/src/cmd/internal/obj/objfile2.go @@ -238,6 +238,9 @@ func (w *writer) Sym(s *LSym) { if s.TopFrame() { flag |= goobj2.SymFlagTopFrame } + if strings.HasPrefix(s.Name, "type.") && s.Name[5] != '.' && s.Type == objabi.SRODATA { + flag |= goobj2.SymFlagGoType + } name := s.Name if strings.HasPrefix(name, "gofile..") { name = filepath.ToSlash(name) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 259199eea1..818024069e 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -108,9 +108,7 @@ func (d *deadcodePass2) flood() { symIdx := d.wq.pop() d.reflectSeen = d.reflectSeen || d.ldr.IsReflectMethod(symIdx) - - name := d.ldr.RawSymName(symIdx) - if strings.HasPrefix(name, "type.") && name[5] != '.' { // TODO: use an attribute instead of checking name + if d.ldr.IsGoType(symIdx) { p := d.ldr.Data(symIdx) if len(p) != 0 && decodetypeKind(d.ctxt.Arch, p)&kindMask == kindInterface { for _, sig := range decodeIfaceMethods2(d.ldr, d.ctxt.Arch, symIdx) { diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index c155f27dcb..708e8d0d3e 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -364,6 +364,11 @@ func (l *Loader) IsReflectMethod(i Sym) bool { return l.SymAttr(i)&goobj2.SymFlagReflectMethod != 0 } +// Returns whether this is a Go type symbol. +func (l *Loader) IsGoType(i Sym) bool { + return l.SymAttr(i)&goobj2.SymFlagGoType != 0 +} + // Returns the symbol content of the i-th symbol. i is global index. func (l *Loader) Data(i Sym) []byte { if l.isExternal(i) { From 6cbf37b30b852164b1cd098e0369498ca72ede09 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 09:29:56 -0400 Subject: [PATCH 41/79] [dev.link] cmd/link: record go.itablink symbols during object file read Change the new loader to keep a note of the set of "go.itablink.*" symbols (using a small map), and add a method that clients can use to query whether a given global index corresponds to a "go.itablink.*" sym. This eliminates one instance of raw symbol name reading/matching during new deadcode, which should produce a minor speedup. Change-Id: I5915773a3f33c16099ccd68592dbba783d909bc9 Reviewed-on: https://go-review.googlesource.com/c/go/+/201400 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/deadcode2.go | 3 +-- src/cmd/link/internal/loader/loader.go | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 818024069e..82626c7a28 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -11,7 +11,6 @@ import ( "cmd/link/internal/loader" "cmd/link/internal/sym" "fmt" - "strings" "unicode" ) @@ -226,7 +225,7 @@ func deadcode2(ctxt *Link) { // (When BuildModeShared, always keep itablinks.) for i := 1; i < n; i++ { s := loader.Sym(i) - if strings.HasPrefix(ldr.RawSymName(s), "go.itablink.") { // TODO: use an attribute instread of checking name + if ldr.IsItabLink(s) { relocs := ldr.Relocs(s) if relocs.Count > 0 && ldr.Reachable.Has(relocs.At(0).Sym) { ldr.Reachable.Set(s) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 708e8d0d3e..5f631f1625 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -100,6 +100,8 @@ type Loader struct { symsByName map[nameVer]Sym // map symbol name to index overwrite map[Sym]Sym // overwrite[i]=j if symbol j overwrites symbol i + itablink map[Sym]struct{} // itablink[j] defined if j is go.itablink.* + objByPkg map[string]*oReader // map package path to its Go object reader Syms []*sym.Symbol // indexed symbols. XXX we still make sym.Symbol for now. @@ -114,6 +116,7 @@ func NewLoader() *Loader { symsByName: make(map[nameVer]Sym), objByPkg: make(map[string]*oReader), overwrite: make(map[Sym]Sym), + itablink: make(map[Sym]struct{}), } } @@ -369,6 +372,14 @@ func (l *Loader) IsGoType(i Sym) bool { return l.SymAttr(i)&goobj2.SymFlagGoType != 0 } +// Returns whether this is a "go.itablink.*" symbol. +func (l *Loader) IsItabLink(i Sym) bool { + if _, ok := l.itablink[i]; ok { + return true + } + return false +} + // Returns the symbol content of the i-th symbol. i is global index. func (l *Loader) Data(i Sym) []byte { if l.isExternal(i) { @@ -491,7 +502,10 @@ func (l *Loader) Preload(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib * } v := abiToVer(osym.ABI, localSymVersion) dupok := osym.Dupok() - l.AddSym(name, v, istart+Sym(i), or, dupok, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) + added := l.AddSym(name, v, istart+Sym(i), or, dupok, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) + if added && strings.HasPrefix(name, "go.itablink.") { + l.itablink[istart+Sym(i)] = struct{}{} + } } // The caller expects us consuming all the data From e5acb58c394c7d900d9aa948a9b601c49c97ab09 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 12:31:33 -0400 Subject: [PATCH 42/79] [dev.link] cmd/objdump: switch to using NewReaderFromBytes Convert the object file dumper to use NewReaderFromBytes when reading new object files, as opposed to NewReader. Change-Id: I9f5e0356bd21c16f545cdd70262e983a2ed38bfc Reviewed-on: https://go-review.googlesource.com/c/go/+/201441 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/internal/goobj/read.go | 9 ++++++--- src/cmd/internal/goobj/readnew.go | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/cmd/internal/goobj/read.go b/src/cmd/internal/goobj/read.go index 2a3afffeb0..e61e95dcc8 100644 --- a/src/cmd/internal/goobj/read.go +++ b/src/cmd/internal/goobj/read.go @@ -502,12 +502,15 @@ func (r *objReader) parseObject(prefix []byte) error { } // TODO: extract OS + build ID if/when we need it - r.readFull(r.tmp[:8]) - if bytes.Equal(r.tmp[:8], []byte("\x00go114LD")) { - r.offset -= 8 + p, err := r.peek(8) + if err != nil { + return err + } + if bytes.Equal(p, []byte("\x00go114LD")) { r.readNew() return nil } + r.readFull(r.tmp[:8]) if !bytes.Equal(r.tmp[:8], []byte("\x00go114ld")) { return r.error(errCorruptObject) } diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index e5dc652800..de05f37c3b 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -15,7 +15,11 @@ import ( // the data to the current goobj API. func (r *objReader) readNew() { start := uint32(r.offset) - rr := goobj2.NewReader(r.f, start) + + length := r.limit - r.offset + objbytes := make([]byte, length) + r.readFull(objbytes) + rr := goobj2.NewReaderFromBytes(objbytes, false) if rr == nil { panic("cannot read object file") } From 6ecaae032520615a89c55c1ee31f0060feab0f4a Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 10:39:58 -0400 Subject: [PATCH 43/79] [dev.link] cmd/link: remove unused slow paths from BytesAt/StringAt This change removes the NewReader function (no longer used by objdump) and prunes away the now unused code paths from Reader.BytesAt and Reader.StringAt, which helps with performance. At the moment the reader operates by always ingesting the entire object file (either via direct read or by mmap), meaning that there will always be a slice available for us to index into. Change-Id: I3af7396effe19e50ed594fe8d82fd2d15465687c Reviewed-on: https://go-review.googlesource.com/c/go/+/201437 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang Reviewed-by: Jeremy Faller --- src/cmd/internal/goobj2/objfile.go | 35 +++++------------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index 03b322da6c..e10ce43833 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -396,15 +396,6 @@ type Reader struct { h Header // keep block offsets } -func NewReader(rd io.ReaderAt, off uint32) *Reader { - r := &Reader{rd: rd, start: off} - err := r.h.Read(r) - if err != nil { - return nil - } - return r -} - func NewReaderFromBytes(b []byte, readonly bool) *Reader { r := &Reader{b: b, readonly: readonly, rd: bytes.NewReader(b), start: 0} err := r.h.Read(r) @@ -418,16 +409,8 @@ func (r *Reader) BytesAt(off uint32, len int) []byte { if len == 0 { return nil } - if r.b != nil { - end := int(off) + len - return r.b[int(off):end:end] - } - b := make([]byte, len) - _, err := r.rd.ReadAt(b, int64(r.start+off)) - if err != nil { - panic("corrupted input") - } - return b + end := int(off) + len + return r.b[int(off):end:end] } func (r *Reader) uint64At(off uint32) uint64 { @@ -460,17 +443,9 @@ func (r *Reader) uint8At(off uint32) uint8 { func (r *Reader) StringAt(off uint32) string { l := r.uint32At(off) - if r.b != nil { - b := r.b[off+4 : off+4+l] - if r.readonly { - return toString(b) // backed by RO memory, ok to make unsafe string - } - return string(b) - } - b := make([]byte, l) - n, err := r.rd.ReadAt(b, int64(r.start+off+4)) - if n != int(l) || err != nil { - panic("corrupted input") + b := r.b[off+4 : off+4+l] + if r.readonly { + return toString(b) // backed by RO memory, ok to make unsafe string } return string(b) } From b661547d94b89f6279710e4f4c58282b664e7b9c Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 16 Oct 2019 16:21:42 -0400 Subject: [PATCH 44/79] [dev.link] cmd/link: new loader method for reading relocations in batch Add a new loader.Relocs method that reads all of the relocations for a symbol into a slice. Handy in cases where the client knows in advance that it wants to visit all the relocations on a symbol (as opposed to just one or two). Change-Id: I1a420513e160c8bb4b90c9824ae8d5b5de060c15 Reviewed-on: https://go-review.googlesource.com/c/go/+/201721 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller Reviewed-by: Cherry Zhang --- src/cmd/link/internal/loader/loader.go | 49 +++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 5f631f1625..6ad37d6061 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -442,6 +442,51 @@ func (relocs *Relocs) At(j int) Reloc { } } +// ReadAll method reads all relocations for a symbol into the +// specified slice. If the slice capacity is not large enough, a new +// larger slice will be allocated. Final slice is returned. +func (relocs *Relocs) ReadAll(dst []Reloc) []Reloc { + if relocs.Count == 0 { + return dst + } + + if cap(dst) < relocs.Count { + dst = make([]Reloc, relocs.Count) + } + dst = dst[:0] + + if relocs.ext != nil { + for i := 0; i < relocs.Count; i++ { + erel := &relocs.ext.R[i] + rel := Reloc{ + Off: erel.Off, + Size: erel.Siz, + Type: erel.Type, + Add: erel.Add, + Sym: relocs.l.Lookup(erel.Sym.Name, int(erel.Sym.Version)), + } + dst = append(dst, rel) + } + return dst + } + + off := relocs.r.RelocOff(relocs.li, 0) + for i := 0; i < relocs.Count; i++ { + rel := goobj2.Reloc{} + rel.Read(relocs.r.Reader, off) + off += uint32(rel.Size()) + target := relocs.l.resolve(relocs.r, rel.Sym) + dst = append(dst, Reloc{ + Off: rel.Off, + Size: rel.Siz, + Type: objabi.RelocType(rel.Type), + Add: rel.Add, + Sym: target, + }) + } + return dst +} + // Relocs returns a Relocs object for the given global sym. func (l *Loader) Relocs(i Sym) Relocs { if l.isExternal(i) { @@ -651,6 +696,7 @@ func loadObjFull(l *Loader, r *oReader) { } pcdataBase := r.PcdataBase() + rslice := []Reloc{} for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(i)) @@ -692,9 +738,10 @@ func loadObjFull(l *Loader, r *oReader) { // Relocs relocs := l.relocs(r, i) + rslice = relocs.ReadAll(rslice) s.R = make([]sym.Reloc, relocs.Count) for j := range s.R { - r := relocs.At(j) + r := rslice[j] rs := r.Sym sz := r.Size rt := r.Type From 286d744246a84b3a19093db6e7f24a823aac50bb Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 17 Oct 2019 01:11:23 -0400 Subject: [PATCH 45/79] [dev.link] cmd/compile: pass index through when re-exporting When we re-export an imported symbol that has an index, we should pass the index through. Currently, if the symbol is not referenced in the generated machine code, it does not get assigned a package index, and the exporter will not export its symbol index. Let the exporter handle this case -- if the symbol has a symbol index but not a package index, still export its symbol index. This is safe as referenced-by-name symbols always have their package indices set to a special value. This should reduce the number of referenced-by-name symbols, and also make the export data more stable, less dependent on codegen details. Change-Id: Ic515a002ae84226e7fdbe68a53496c051b7badcc Reviewed-on: https://go-review.googlesource.com/c/go/+/201719 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/compile/internal/gc/iexport.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/gc/iexport.go b/src/cmd/compile/internal/gc/iexport.go index 9125f67586..259b70a69f 100644 --- a/src/cmd/compile/internal/gc/iexport.go +++ b/src/cmd/compile/internal/gc/iexport.go @@ -993,9 +993,14 @@ func (w *exportWriter) linkname(s *types.Sym) { func (w *exportWriter) symIdx(s *types.Sym) { if Ctxt.Flag_newobj { lsym := s.Linksym() - if lsym.PkgIdx > goobj2.PkgIdxSelf || lsym.PkgIdx == goobj2.PkgIdxInvalid || s.Linkname != "" { + if lsym.PkgIdx > goobj2.PkgIdxSelf || (lsym.PkgIdx == goobj2.PkgIdxInvalid && !lsym.Indexed()) || s.Linkname != "" { + // Don't export index for non-package symbols, linkname'd symbols, + // and symbols without an index. They can only be referenced by + // name. w.int64(-1) } else { + // For a defined symbol, export its index. + // For re-exporting an imported symbol, pass its index through. w.int64(int64(lsym.SymIdx)) } } From bd229936cf3873c552ab087232030780dc432067 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 17 Oct 2019 01:17:42 -0400 Subject: [PATCH 46/79] [dev.link] cmd/link: restore export data hash With the previous CL, the export data will not change whether it is compiled with -dynlink flag or not. Restore the export data hash, and reenable plugin version check. TODO: it may be still better to just generate a fingerprint for each package at compile time. Change-Id: I1f298ac97c3ab9b8d05d1c95e8be74d10ca7cd0e Reviewed-on: https://go-review.googlesource.com/c/go/+/201720 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- misc/cgo/testplugin/testdata/host/host.go | 13 ++++++------- src/cmd/link/internal/ld/lib.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/misc/cgo/testplugin/testdata/host/host.go b/misc/cgo/testplugin/testdata/host/host.go index d836523da8..a3799328cd 100644 --- a/misc/cgo/testplugin/testdata/host/host.go +++ b/misc/cgo/testplugin/testdata/host/host.go @@ -145,13 +145,12 @@ func main() { } _, err = plugin.Open("plugin-mismatch.so") - // TODO: newobj - //if err == nil { - // log.Fatal(`plugin.Open("plugin-mismatch.so"): should have failed`) - //} - //if s := err.Error(); !strings.Contains(s, "different version") { - // log.Fatalf(`plugin.Open("plugin-mismatch.so"): error does not mention "different version": %v`, s) - //} + if err == nil { + log.Fatal(`plugin.Open("plugin-mismatch.so"): should have failed`) + } + if s := err.Error(); !strings.Contains(s, "different version") { + log.Fatalf(`plugin.Open("plugin-mismatch.so"): error does not mention "different version": %v`, s) + } _, err = plugin.Open("plugin2-dup.so") if err == nil { diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 063bdded0c..63dcb22d98 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -816,7 +816,7 @@ func genhash(ctxt *Link, lib *sym.Library) { return } h.Write(pkgDefBytes[0:firstEOL]) - //h.Write(pkgDefBytes[firstDoubleDollar : firstDoubleDollar+secondDoubleDollar]) // TODO: newobj: -dynlink may change symbol numbering? which will make the export data differ + h.Write(pkgDefBytes[firstDoubleDollar : firstDoubleDollar+secondDoubleDollar]) lib.Hash = hex.EncodeToString(h.Sum(nil)) } From 5777ffd6e6f63647b4e4ab9a0dbfba670d2d5ced Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 17 Oct 2019 18:42:38 -0400 Subject: [PATCH 47/79] [dev.link] cmd/link: allow either of duplicated symbols being dupok If two symbols have the same name, the old code allows either one being dupok (preferably both, but either is ok). Currently, the new code only works when the new symbol being dupok (or both). Allow only old symbol being dupok as well. One example for this is the tls_g variable on ARM64 and PPC64 when the race detector is enabled. Should fix Linux/ARM64 build. Change-Id: I8dd21c017e826847f13471c30dfd71bf225d8076 Reviewed-on: https://go-review.googlesource.com/c/go/+/201642 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/loader/loader.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 6ad37d6061..6f4bc98234 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -152,14 +152,17 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ if dupok { return false } + oldr, li := l.toLocal(oldi) + oldsym := goobj2.Sym{} + oldsym.Read(oldr.Reader, oldr.SymOff(li)) + if oldsym.Dupok() { + return false + } overwrite := r.DataSize(int(i-l.startIndex(r))) != 0 if overwrite { // new symbol overwrites old symbol. - oldr, li := l.toLocal(oldi) - oldsym := goobj2.Sym{} - oldsym.Read(oldr.Reader, oldr.SymOff(li)) oldtyp := sym.AbiSymKindToSymKind[objabi.SymKind(oldsym.Type)] - if !oldsym.Dupok() && !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol + if !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol log.Fatalf("duplicated definition of symbol " + name) } l.overwrite[oldi] = i From 97e497b2537d0d9588e52bb0c20df59604e6f098 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 16 Oct 2019 21:42:18 -0400 Subject: [PATCH 48/79] [dev.link] cmd: reference symbols by name when linking against Go shared library When building a program that links against Go shared libraries, it needs to reference symbols defined in the shared library. At compile time, we don't know where the shared library boundary is. If we reference a symbol in package p by index, and package p is actually part of a shared library, we cannot resolve the index at link time, as the linker doesn't see the object file of p. So when linking against Go shared libraries, always use named reference for now. To do this, the compiler needs to know whether we will be linking against Go shared libraries. The -dynlink flag kind of indicates that (as the document says), but currently it is actually overloaded: it is also used when building a plugin or a shared library, which is self-contained (if -linkshared is not otherwise specified) and could use index for symbol reference. So we introduce another compiler flag, -linkshared, specifically for linking against Go shared libraries. The go command will pass this flag if its -linkshared flag is specified ("go build -linkshared"). There may be better way to handle this. For example, we can put the symbol indices in a special section in the shared library that the linker can read. Or we can generate some per-package description file to include the indices. (Currently we generate a .shlibname file for each package that is included in a shared library, which contains the path of the library. We could consider extending this.) That said, this CL is a stop-gap solution. And it is no worse than the old object files. If we were to redesign the build system so that the shared library boundary is known at compile time, we could use indices for symbol references that do not cross shared library boundary, as well as doing other things better. Change-Id: I9c02aad36518051cc4785dbe25c4b4cef8f3faeb Reviewed-on: https://go-review.googlesource.com/c/go/+/201818 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/compile/internal/gc/main.go | 1 + src/cmd/go/alldocs.go | 4 ++-- src/cmd/go/internal/work/build.go | 4 ++-- src/cmd/go/internal/work/init.go | 1 + src/cmd/internal/obj/link.go | 1 + src/cmd/internal/obj/sym.go | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index c21d939b4c..121342e80d 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -251,6 +251,7 @@ func Main(archInit func(*Arch)) { if supportsDynlink(thearch.LinkArch.Arch) { flag.BoolVar(&flag_shared, "shared", false, "generate code that can be linked into a shared library") flag.BoolVar(&flag_dynlink, "dynlink", false, "support references to Go symbols defined in other shared libraries") + flag.BoolVar(&Ctxt.Flag_linkshared, "linkshared", false, "generate code that will be linked against Go shared libraries") } flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to `file`") flag.StringVar(&memprofile, "memprofile", "", "write memory profile to `file`") diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 2561f5b2f8..4774ee8201 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -145,8 +145,8 @@ // -ldflags '[pattern=]arg list' // arguments to pass on each go tool link invocation. // -linkshared -// link against shared libraries previously created with -// -buildmode=shared. +// build code that will be linked against shared libraries previously +// created with -buildmode=shared. // -mod mode // module download mode to use: readonly or vendor. // See 'go help modules' for more. diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 54b049b68f..6264593c34 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -97,8 +97,8 @@ and test commands: -ldflags '[pattern=]arg list' arguments to pass on each go tool link invocation. -linkshared - link against shared libraries previously created with - -buildmode=shared. + build code that will be linked against shared libraries previously + created with -buildmode=shared. -mod mode module download mode to use: readonly or vendor. See 'go help modules' for more. diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index f3055b6293..f4ae0e11c1 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -224,6 +224,7 @@ func buildModeInit() { base.Fatalf("-linkshared not supported on %s\n", platform) } codegenArg = "-dynlink" + forcedGcflags = append(forcedGcflags, "-linkshared") // TODO(mwhudson): remove -w when that gets fixed in linker. forcedLdflags = append(forcedLdflags, "-linkshared", "-w") } diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index 2c106bab30..2e94d55225 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -648,6 +648,7 @@ type Link struct { Debugpcln string Flag_shared bool Flag_dynlink bool + Flag_linkshared bool Flag_optimize bool Flag_locationlists bool Flag_newobj bool // use new object file format diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index de415695f3..4c116d28f2 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -184,7 +184,7 @@ func (ctxt *Link) NumberSyms(asm bool) { var idx, nonpkgidx int32 = 0, 0 ctxt.traverseSyms(traverseDefs, func(s *LSym) { - if asm || s.Pkg == "_" || s.DuplicateOK() { + if asm || s.Pkg == "_" || s.DuplicateOK() || ctxt.Flag_linkshared { s.PkgIdx = goobj2.PkgIdxNone s.SymIdx = nonpkgidx if nonpkgidx != int32(len(ctxt.nonpkgdefs)) { From 4fa524b7762d01fe2f7cb1db3297ff9759410451 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 16 Oct 2019 21:43:54 -0400 Subject: [PATCH 49/79] [dev.link] cmd/dist: reenable shared library tests Change-Id: Ifa4de9333b9275d832ebf68c89d3239ed438b104 Reviewed-on: https://go-review.googlesource.com/c/go/+/201819 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/dist/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 46556f2f79..83bcc86172 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -681,7 +681,7 @@ func (t *tester) registerTests() { if t.supportedBuildmode("c-shared") { t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".") } - if t.supportedBuildmode("shared") && false { // TODO: newobj + if t.supportedBuildmode("shared") { t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".") } if t.supportedBuildmode("plugin") { From d56c8149ac3b6c8c074d46853ce8c7b4e03d4b0f Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 18 Oct 2019 10:30:04 -0400 Subject: [PATCH 50/79] [dev.link] cmd/internal/obj: use index for static symbols In assembly we always reference symbols by name. But for static symbols, as they are reachable only within the current file, we can assign them local indices and use the indices to reference them. The index is only meaningful locally, and it is fine. Change-Id: I16e011cd41575ef703ceb6f35899e5fa58fbcf1e Reviewed-on: https://go-review.googlesource.com/c/go/+/201997 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/obj/sym.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index 4c116d28f2..ab886bce36 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -184,7 +184,7 @@ func (ctxt *Link) NumberSyms(asm bool) { var idx, nonpkgidx int32 = 0, 0 ctxt.traverseSyms(traverseDefs, func(s *LSym) { - if asm || s.Pkg == "_" || s.DuplicateOK() || ctxt.Flag_linkshared { + if isNonPkgSym(ctxt, asm, s) { s.PkgIdx = goobj2.PkgIdxNone s.SymIdx = nonpkgidx if nonpkgidx != int32(len(ctxt.nonpkgdefs)) { @@ -232,6 +232,31 @@ func (ctxt *Link) NumberSyms(asm bool) { }) } +// Returns whether s is a non-package symbol, which needs to be referenced +// by name instead of by index. +func isNonPkgSym(ctxt *Link, asm bool, s *LSym) bool { + if asm && !s.Static() { + // asm symbols are referenced by name only, except static symbols + // which are file-local and can be referenced by index. + return true + } + if ctxt.Flag_linkshared { + // The referenced symbol may be in a different shared library so + // the linker cannot see its index. + return true + } + if s.Pkg == "_" { + // The frontend uses package "_" to mark symbols that should not + // be referenced by index, e.g. linkname'd symbols. + return true + } + if s.DuplicateOK() { + // Dupok symbol needs to be dedup'd by name. + return true + } + return false +} + type traverseFlag uint32 const ( From c480d32fadb438155e5d5711ec166f58c73853e9 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 18 Oct 2019 11:19:32 -0400 Subject: [PATCH 51/79] [dev.link] cmd/link: do not put static symbols into name lookup table Since the previous CL, we will not reference static symbols by name. Therefore no need to put them into the name lookup table. On Linux/ARM, in runtime/internal/atomic/sys_linux_arm.s, the kernelcas function has a definition and a reference written in two different forms, one with package prefix, one without. This way, the assembler cannot know they are the same symbol, only the linker knows. This is quite unusual, unify the names to so the assembler can resolve it to index. Change-Id: Ie7223097be6a3b65f3fa43ed4575da9972ef5b69 Reviewed-on: https://go-review.googlesource.com/c/go/+/201998 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/loader/loader.go | 33 ++++++++++++++------- src/runtime/internal/atomic/sys_linux_arm.s | 4 +-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 6f4bc98234..3f5ec829a0 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -147,6 +147,12 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ if l.extStart != 0 { panic("AddSym called after AddExtSym is called") } + if ver == r.version { + // Static symbol. Add its global index but don't + // add to name lookup table, as it cannot be + // referenced by name. + return true + } nv := nameVer{name, ver} if oldi, ok := l.symsByName[nv]; ok { if dupok { @@ -294,7 +300,10 @@ func (l *Loader) IsDup(i Sym) bool { return false } if osym.Name == "" { - return false + return false // Unnamed aux symbol cannot be dup. + } + if osym.ABI == goobj2.SymABIstatic { + return false // Static symbol cannot be dup. } name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) ver := abiToVer(osym.ABI, r.version) @@ -656,7 +665,7 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { continue } ver := abiToVer(osym.ABI, r.version) - if l.symsByName[nameVer{name, ver}] != istart+Sym(i) { + if osym.ABI != goobj2.SymABIstatic && l.symsByName[nameVer{name, ver}] != istart+Sym(i) { continue } @@ -709,17 +718,19 @@ func loadObjFull(l *Loader, r *oReader) { } ver := abiToVer(osym.ABI, r.version) dupok := osym.Dupok() - if dupsym := l.symsByName[nameVer{name, ver}]; dupsym != istart+Sym(i) { - if dupok && l.Reachable.Has(dupsym) { - // A dupok symbol is resolved to another package. We still need - // to record its presence in the current package, as the trampoline - // pass expects packages are laid out in dependency order. - s := l.Syms[dupsym] - if s.Type == sym.STEXT { - lib.DupTextSyms = append(lib.DupTextSyms, s) + if dupok { + if dupsym := l.symsByName[nameVer{name, ver}]; dupsym != istart+Sym(i) { + if l.Reachable.Has(dupsym) { + // A dupok symbol is resolved to another package. We still need + // to record its presence in the current package, as the trampoline + // pass expects packages are laid out in dependency order. + s := l.Syms[dupsym] + if s.Type == sym.STEXT { + lib.DupTextSyms = append(lib.DupTextSyms, s) + } } + continue } - continue } s := l.Syms[istart+Sym(i)] diff --git a/src/runtime/internal/atomic/sys_linux_arm.s b/src/runtime/internal/atomic/sys_linux_arm.s index df62f6c8ad..1fd3e832b7 100644 --- a/src/runtime/internal/atomic/sys_linux_arm.s +++ b/src/runtime/internal/atomic/sys_linux_arm.s @@ -29,9 +29,9 @@ TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT|NOFRAME,$0 CMP $7, R11 BLT 2(PC) JMP ·armcas(SB) - JMP ·kernelcas<>(SB) + JMP kernelcas<>(SB) -TEXT runtime∕internal∕atomic·kernelcas<>(SB),NOSPLIT,$0 +TEXT kernelcas<>(SB),NOSPLIT,$0 MOVW ptr+0(FP), R2 // trigger potential paging fault here, // because we don't know how to traceback through __kuser_cmpxchg From 4adf822fc9457b25aa951ec1f7eab94f5e471614 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 23 Oct 2019 11:04:16 -0400 Subject: [PATCH 52/79] [dev.link] cmd/link: add in change missed from code review Incorporate a change suggested by Cherry for CL 201721 that I missed accidentally. Change-Id: I65e6532e78888505573169e56bc4ace9a0f8c510 Reviewed-on: https://go-review.googlesource.com/c/go/+/202760 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/loader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 3f5ec829a0..3f61726b24 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -459,7 +459,7 @@ func (relocs *Relocs) At(j int) Reloc { // larger slice will be allocated. Final slice is returned. func (relocs *Relocs) ReadAll(dst []Reloc) []Reloc { if relocs.Count == 0 { - return dst + return dst[:0] } if cap(dst) < relocs.Count { From df01b7968bb613d30fa2bbd11fef2f3fb77803a5 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 18 Oct 2019 12:11:56 -0400 Subject: [PATCH 53/79] [dev.link] cmd/link: use string map for name lookup As we no longer include static symbols into the name lookup table, it is basically just two maps, one for ABI0, one for ABIInternal. Just use two maps instead. It may be slightly faster to use string-keyed maps than struct-keyed maps (still need performance data to confirm). For now, allow external symbols being referenced by name, as external objects don't use index. Change-Id: I60cedaa7346fce7535970780bc67f93c82160646 Reviewed-on: https://go-review.googlesource.com/c/go/+/201999 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller Reviewed-by: Than McIntosh --- src/cmd/link/internal/loader/loader.go | 65 ++++++++++++++++---------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 3f61726b24..e3f7480ac7 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -97,8 +97,9 @@ type Loader struct { extStart Sym // from this index on, the symbols are externally defined extSyms []nameVer // externally defined symbols - symsByName map[nameVer]Sym // map symbol name to index - overwrite map[Sym]Sym // overwrite[i]=j if symbol j overwrites symbol i + symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal + extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name + overwrite map[Sym]Sym // overwrite[i]=j if symbol j overwrites symbol i itablink map[Sym]struct{} // itablink[j] defined if j is go.itablink.* @@ -111,12 +112,13 @@ type Loader struct { func NewLoader() *Loader { return &Loader{ - start: make(map[*oReader]Sym), - objs: []objIdx{{nil, 0}}, - symsByName: make(map[nameVer]Sym), - objByPkg: make(map[string]*oReader), - overwrite: make(map[Sym]Sym), - itablink: make(map[Sym]struct{}), + start: make(map[*oReader]Sym), + objs: []objIdx{{nil, 0}}, + symsByName: [2]map[string]Sym{make(map[string]Sym), make(map[string]Sym)}, + objByPkg: make(map[string]*oReader), + overwrite: make(map[Sym]Sym), + itablink: make(map[Sym]struct{}), + extStaticSyms: make(map[nameVer]Sym), } } @@ -153,8 +155,7 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ // referenced by name. return true } - nv := nameVer{name, ver} - if oldi, ok := l.symsByName[nv]; ok { + if oldi, ok := l.symsByName[ver][name]; ok { if dupok { return false } @@ -181,24 +182,34 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ return false } } - l.symsByName[nv] = i + l.symsByName[ver][name] = i return true } // Add an external symbol (without index). Return the index of newly added // symbol, or 0 if not added. func (l *Loader) AddExtSym(name string, ver int) Sym { - nv := nameVer{name, ver} - if _, ok := l.symsByName[nv]; ok { - return 0 + static := ver >= sym.SymVerStatic + if static { + if _, ok := l.extStaticSyms[nameVer{name, ver}]; ok { + return 0 + } + } else { + if _, ok := l.symsByName[ver][name]; ok { + return 0 + } } i := l.max + 1 - l.symsByName[nv] = i + if static { + l.extStaticSyms[nameVer{name, ver}] = i + } else { + l.symsByName[ver][name] = i + } l.max++ if l.extStart == 0 { l.extStart = i } - l.extSyms = append(l.extSyms, nv) + l.extSyms = append(l.extSyms, nameVer{name, ver}) l.growSyms(int(i)) return i } @@ -259,8 +270,7 @@ func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) v := abiToVer(osym.ABI, r.version) - nv := nameVer{name, v} - return l.symsByName[nv] + return l.Lookup(name, v) case goobj2.PkgIdxBuiltin: panic("PkgIdxBuiltin not used") case goobj2.PkgIdxSelf: @@ -280,8 +290,10 @@ func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { // This is more like Syms.ROLookup than Lookup -- it doesn't create // new symbol. func (l *Loader) Lookup(name string, ver int) Sym { - nv := nameVer{name, ver} - return l.symsByName[nv] + if ver >= sym.SymVerStatic { + return l.extStaticSyms[nameVer{name, ver}] + } + return l.symsByName[ver][name] } // Returns whether i is a dup of another symbol, and i is not @@ -307,7 +319,7 @@ func (l *Loader) IsDup(i Sym) bool { } name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) ver := abiToVer(osym.ABI, r.version) - return l.symsByName[nameVer{name, ver}] != i + return l.symsByName[ver][name] != i } // Number of total symbols. @@ -665,7 +677,7 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { continue } ver := abiToVer(osym.ABI, r.version) - if osym.ABI != goobj2.SymABIstatic && l.symsByName[nameVer{name, ver}] != istart+Sym(i) { + if osym.ABI != goobj2.SymABIstatic && l.symsByName[ver][name] != istart+Sym(i) { continue } @@ -719,7 +731,7 @@ func loadObjFull(l *Loader, r *oReader) { ver := abiToVer(osym.ABI, r.version) dupok := osym.Dupok() if dupok { - if dupsym := l.symsByName[nameVer{name, ver}]; dupsym != istart+Sym(i) { + if dupsym := l.symsByName[ver][name]; dupsym != istart+Sym(i) { if l.Reachable.Has(dupsym) { // A dupok symbol is resolved to another package. We still need // to record its presence in the current package, as the trampoline @@ -960,7 +972,10 @@ func (l *Loader) Dump() { } fmt.Println("overwrite:", l.overwrite) fmt.Println("symsByName") - for nv, i := range l.symsByName { - fmt.Println(i, nv.name, nv.v) + for name, i := range l.symsByName[0] { + fmt.Println(i, name, 0) + } + for name, i := range l.symsByName[1] { + fmt.Println(i, name, 1) } } From fb066098171c349355f2ad1e8d40d56d64673dc9 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 18 Oct 2019 17:08:35 -0400 Subject: [PATCH 54/79] [dev.link] cmd/link: assign special indices for builtin functions Compiler-generated function references (e.g. call to runtime.newobject) appear frequently. We assign special indices for them, so they don't need to be referenced by name. Change-Id: I2072594cbc56c9e1037a26e4aae12e68c2436e9f Reviewed-on: https://go-review.googlesource.com/c/go/+/202085 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Jeremy Faller --- src/cmd/internal/goobj/readnew.go | 3 +- src/cmd/internal/goobj2/builtin.go | 45 ++++++ src/cmd/internal/goobj2/builtinlist.go | 194 +++++++++++++++++++++++++ src/cmd/internal/goobj2/mkbuiltin.go | 124 ++++++++++++++++ src/cmd/internal/obj/sym.go | 11 ++ src/cmd/link/internal/loader/loader.go | 21 ++- 6 files changed, 391 insertions(+), 7 deletions(-) create mode 100644 src/cmd/internal/goobj2/builtin.go create mode 100644 src/cmd/internal/goobj2/builtinlist.go create mode 100644 src/cmd/internal/goobj2/mkbuiltin.go diff --git a/src/cmd/internal/goobj/readnew.go b/src/cmd/internal/goobj/readnew.go index de05f37c3b..3f9d0d1db6 100644 --- a/src/cmd/internal/goobj/readnew.go +++ b/src/cmd/internal/goobj/readnew.go @@ -49,7 +49,8 @@ func (r *objReader) readNew() { case goobj2.PkgIdxNone: i = int(s.SymIdx) + rr.NSym() case goobj2.PkgIdxBuiltin: - panic("PkgIdxBuiltin is unused") + name, abi := goobj2.BuiltinName(int(s.SymIdx)) + return SymID{name, int64(abi)} case goobj2.PkgIdxSelf: i = int(s.SymIdx) default: diff --git a/src/cmd/internal/goobj2/builtin.go b/src/cmd/internal/goobj2/builtin.go new file mode 100644 index 0000000000..65f9dd5d95 --- /dev/null +++ b/src/cmd/internal/goobj2/builtin.go @@ -0,0 +1,45 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package goobj2 + +// Builtin (compiler-generated) function references appear +// frequently. We assign special indices for them, so they +// don't need to be referenced by name. + +// NBuiltin returns the number of listed builtin +// symbols. +func NBuiltin() int { + return len(builtins) +} + +// BuiltinName returns the name and ABI of the i-th +// builtin symbol. +func BuiltinName(i int) (string, int) { + return builtins[i].name, builtins[i].abi +} + +// BuiltinIdx returns the index of the builtin with the +// given name and abi, or -1 if it is not a builtin. +func BuiltinIdx(name string, abi int) int { + i, ok := builtinMap[name] + if !ok { + return -1 + } + if builtins[i].abi != abi { + return -1 + } + return i +} + +//go:generate go run mkbuiltin.go + +var builtinMap map[string]int + +func init() { + builtinMap = make(map[string]int, len(builtins)) + for i, b := range builtins { + builtinMap[b.name] = i + } +} diff --git a/src/cmd/internal/goobj2/builtinlist.go b/src/cmd/internal/goobj2/builtinlist.go new file mode 100644 index 0000000000..bcab6f2e0b --- /dev/null +++ b/src/cmd/internal/goobj2/builtinlist.go @@ -0,0 +1,194 @@ +// Code generated by mkbuiltin.go. DO NOT EDIT. + +package goobj2 + +var builtins = [...]struct { + name string + abi int +}{ + {"runtime.newobject", 1}, + {"runtime.panicdivide", 1}, + {"runtime.panicshift", 1}, + {"runtime.panicmakeslicelen", 1}, + {"runtime.throwinit", 1}, + {"runtime.panicwrap", 1}, + {"runtime.gopanic", 1}, + {"runtime.gorecover", 1}, + {"runtime.goschedguarded", 1}, + {"runtime.goPanicIndex", 1}, + {"runtime.goPanicIndexU", 1}, + {"runtime.goPanicSliceAlen", 1}, + {"runtime.goPanicSliceAlenU", 1}, + {"runtime.goPanicSliceAcap", 1}, + {"runtime.goPanicSliceAcapU", 1}, + {"runtime.goPanicSliceB", 1}, + {"runtime.goPanicSliceBU", 1}, + {"runtime.goPanicSlice3Alen", 1}, + {"runtime.goPanicSlice3AlenU", 1}, + {"runtime.goPanicSlice3Acap", 1}, + {"runtime.goPanicSlice3AcapU", 1}, + {"runtime.goPanicSlice3B", 1}, + {"runtime.goPanicSlice3BU", 1}, + {"runtime.goPanicSlice3C", 1}, + {"runtime.goPanicSlice3CU", 1}, + {"runtime.printbool", 1}, + {"runtime.printfloat", 1}, + {"runtime.printint", 1}, + {"runtime.printhex", 1}, + {"runtime.printuint", 1}, + {"runtime.printcomplex", 1}, + {"runtime.printstring", 1}, + {"runtime.printpointer", 1}, + {"runtime.printiface", 1}, + {"runtime.printeface", 1}, + {"runtime.printslice", 1}, + {"runtime.printnl", 1}, + {"runtime.printsp", 1}, + {"runtime.printlock", 1}, + {"runtime.printunlock", 1}, + {"runtime.concatstring2", 1}, + {"runtime.concatstring3", 1}, + {"runtime.concatstring4", 1}, + {"runtime.concatstring5", 1}, + {"runtime.concatstrings", 1}, + {"runtime.cmpstring", 1}, + {"runtime.intstring", 1}, + {"runtime.slicebytetostring", 1}, + {"runtime.slicebytetostringtmp", 1}, + {"runtime.slicerunetostring", 1}, + {"runtime.stringtoslicebyte", 1}, + {"runtime.stringtoslicerune", 1}, + {"runtime.slicecopy", 1}, + {"runtime.slicestringcopy", 1}, + {"runtime.decoderune", 1}, + {"runtime.countrunes", 1}, + {"runtime.convI2I", 1}, + {"runtime.convT16", 1}, + {"runtime.convT32", 1}, + {"runtime.convT64", 1}, + {"runtime.convTstring", 1}, + {"runtime.convTslice", 1}, + {"runtime.convT2E", 1}, + {"runtime.convT2Enoptr", 1}, + {"runtime.convT2I", 1}, + {"runtime.convT2Inoptr", 1}, + {"runtime.assertE2I", 1}, + {"runtime.assertE2I2", 1}, + {"runtime.assertI2I", 1}, + {"runtime.assertI2I2", 1}, + {"runtime.panicdottypeE", 1}, + {"runtime.panicdottypeI", 1}, + {"runtime.panicnildottype", 1}, + {"runtime.ifaceeq", 1}, + {"runtime.efaceeq", 1}, + {"runtime.fastrand", 1}, + {"runtime.makemap64", 1}, + {"runtime.makemap", 1}, + {"runtime.makemap_small", 1}, + {"runtime.mapaccess1", 1}, + {"runtime.mapaccess1_fast32", 1}, + {"runtime.mapaccess1_fast64", 1}, + {"runtime.mapaccess1_faststr", 1}, + {"runtime.mapaccess1_fat", 1}, + {"runtime.mapaccess2", 1}, + {"runtime.mapaccess2_fast32", 1}, + {"runtime.mapaccess2_fast64", 1}, + {"runtime.mapaccess2_faststr", 1}, + {"runtime.mapaccess2_fat", 1}, + {"runtime.mapassign", 1}, + {"runtime.mapassign_fast32", 1}, + {"runtime.mapassign_fast32ptr", 1}, + {"runtime.mapassign_fast64", 1}, + {"runtime.mapassign_fast64ptr", 1}, + {"runtime.mapassign_faststr", 1}, + {"runtime.mapiterinit", 1}, + {"runtime.mapdelete", 1}, + {"runtime.mapdelete_fast32", 1}, + {"runtime.mapdelete_fast64", 1}, + {"runtime.mapdelete_faststr", 1}, + {"runtime.mapiternext", 1}, + {"runtime.mapclear", 1}, + {"runtime.makechan64", 1}, + {"runtime.makechan", 1}, + {"runtime.chanrecv1", 1}, + {"runtime.chanrecv2", 1}, + {"runtime.chansend1", 1}, + {"runtime.closechan", 1}, + {"runtime.writeBarrier", 0}, + {"runtime.typedmemmove", 1}, + {"runtime.typedmemclr", 1}, + {"runtime.typedslicecopy", 1}, + {"runtime.selectnbsend", 1}, + {"runtime.selectnbrecv", 1}, + {"runtime.selectnbrecv2", 1}, + {"runtime.selectsetpc", 1}, + {"runtime.selectgo", 1}, + {"runtime.block", 1}, + {"runtime.makeslice", 1}, + {"runtime.makeslice64", 1}, + {"runtime.growslice", 1}, + {"runtime.memmove", 1}, + {"runtime.memclrNoHeapPointers", 1}, + {"runtime.memclrHasPointers", 1}, + {"runtime.memequal", 1}, + {"runtime.memequal0", 1}, + {"runtime.memequal8", 1}, + {"runtime.memequal16", 1}, + {"runtime.memequal32", 1}, + {"runtime.memequal64", 1}, + {"runtime.memequal128", 1}, + {"runtime.f32equal", 1}, + {"runtime.f64equal", 1}, + {"runtime.c64equal", 1}, + {"runtime.c128equal", 1}, + {"runtime.strequal", 1}, + {"runtime.interequal", 1}, + {"runtime.nilinterequal", 1}, + {"runtime.memhash", 1}, + {"runtime.memhash0", 1}, + {"runtime.memhash8", 1}, + {"runtime.memhash16", 1}, + {"runtime.memhash32", 1}, + {"runtime.memhash64", 1}, + {"runtime.memhash128", 1}, + {"runtime.f32hash", 1}, + {"runtime.f64hash", 1}, + {"runtime.c64hash", 1}, + {"runtime.c128hash", 1}, + {"runtime.strhash", 1}, + {"runtime.interhash", 1}, + {"runtime.nilinterhash", 1}, + {"runtime.int64div", 1}, + {"runtime.uint64div", 1}, + {"runtime.int64mod", 1}, + {"runtime.uint64mod", 1}, + {"runtime.float64toint64", 1}, + {"runtime.float64touint64", 1}, + {"runtime.float64touint32", 1}, + {"runtime.int64tofloat64", 1}, + {"runtime.uint64tofloat64", 1}, + {"runtime.uint32tofloat64", 1}, + {"runtime.complex128div", 1}, + {"runtime.racefuncenter", 1}, + {"runtime.racefuncenterfp", 1}, + {"runtime.racefuncexit", 1}, + {"runtime.raceread", 1}, + {"runtime.racewrite", 1}, + {"runtime.racereadrange", 1}, + {"runtime.racewriterange", 1}, + {"runtime.msanread", 1}, + {"runtime.msanwrite", 1}, + {"runtime.checkptrAlignment", 1}, + {"runtime.checkptrArithmetic", 1}, + {"runtime.x86HasPOPCNT", 0}, + {"runtime.x86HasSSE41", 0}, + {"runtime.arm64HasATOMICS", 0}, + {"runtime.gcWriteBarrier", 0}, + {"runtime.deferproc", 1}, + {"runtime.deferprocStack", 1}, + {"runtime.deferreturn", 1}, + {"runtime.newproc", 1}, + {"runtime.morestack", 0}, + {"runtime.morestackc", 0}, + {"runtime.morestack_noctxt", 0}, +} diff --git a/src/cmd/internal/goobj2/mkbuiltin.go b/src/cmd/internal/goobj2/mkbuiltin.go new file mode 100644 index 0000000000..0061aeb237 --- /dev/null +++ b/src/cmd/internal/goobj2/mkbuiltin.go @@ -0,0 +1,124 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Generate builtinlist.go from cmd/compile/internal/gc/builtin/runtime.go. + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" +) + +var stdout = flag.Bool("stdout", false, "write to stdout instead of builtinlist.go") + +func main() { + flag.Parse() + + var b bytes.Buffer + fmt.Fprintln(&b, "// Code generated by mkbuiltin.go. DO NOT EDIT.") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "package goobj2") + + mkbuiltin(&b) + + out, err := format.Source(b.Bytes()) + if err != nil { + log.Fatal(err) + } + if *stdout { + _, err = os.Stdout.Write(out) + } else { + err = ioutil.WriteFile("builtinlist.go", out, 0666) + } + if err != nil { + log.Fatal(err) + } +} + +func mkbuiltin(w io.Writer) { + pkg := "runtime" + fset := token.NewFileSet() + path := filepath.Join("..", "..", "compile", "internal", "gc", "builtin", "runtime.go") + f, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + log.Fatal(err) + } + + decls := make(map[string]bool) + + fmt.Fprintf(w, "var builtins = [...]struct{ name string; abi int }{\n") + for _, decl := range f.Decls { + switch decl := decl.(type) { + case *ast.FuncDecl: + if decl.Recv != nil { + log.Fatal("methods unsupported") + } + if decl.Body != nil { + log.Fatal("unexpected function body") + } + declName := pkg + "." + decl.Name.Name + decls[declName] = true + fmt.Fprintf(w, "{%q, 1},\n", declName) // functions are ABIInternal (1) + case *ast.GenDecl: + if decl.Tok == token.IMPORT { + continue + } + if decl.Tok != token.VAR { + log.Fatal("unhandled declaration kind", decl.Tok) + } + for _, spec := range decl.Specs { + spec := spec.(*ast.ValueSpec) + if len(spec.Values) != 0 { + log.Fatal("unexpected values") + } + for _, name := range spec.Names { + declName := pkg + "." + name.Name + decls[declName] = true + fmt.Fprintf(w, "{%q, 0},\n", declName) // variables are ABI0 + } + } + default: + log.Fatal("unhandled decl type", decl) + } + } + + // The list above only contains ones that are used by the frontend. + // The backend may create more references of builtin functions. + // Add them. + for _, b := range extra { + name := pkg + "." + b.name + if decls[name] { + log.Fatalf("%q already added -- mkbuiltin.go out of sync?", name) + } + fmt.Fprintf(w, "{%q, %d},\n", name, b.abi) + } + fmt.Fprintln(w, "}") +} + +var extra = [...]struct { + name string + abi int +}{ + {"gcWriteBarrier", 0}, // asm function, ABI0 + {"deferproc", 1}, + {"deferprocStack", 1}, + {"deferreturn", 1}, + {"newproc", 1}, + {"morestack", 0}, // asm function, ABI0 + {"morestackc", 0}, // asm function, ABI0 + {"morestack_noctxt", 0}, // asm function, ABI0 +} diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index ab886bce36..3ef886651f 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -210,6 +210,17 @@ func (ctxt *Link) NumberSyms(asm bool) { if rs.PkgIdx != goobj2.PkgIdxInvalid { return } + if !ctxt.Flag_linkshared { + // Assign special index for builtin symbols. + // Don't do it when linking against shared libraries, as the runtime + // may be in a different library. + if i := goobj2.BuiltinIdx(rs.Name, int(rs.ABI())); i != -1 { + rs.PkgIdx = goobj2.PkgIdxBuiltin + rs.SymIdx = int32(i) + rs.Set(AttrIndexed, true) + return + } + } pkg := rs.Pkg if pkg == "" || pkg == "\"\"" || pkg == "_" || !rs.Indexed() { rs.PkgIdx = goobj2.PkgIdxNone diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index e3f7480ac7..846e954aa1 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -91,11 +91,12 @@ func makeBitmap(n int) bitmap { // A Loader loads new object files and resolves indexed symbol references. type Loader struct { - start map[*oReader]Sym // map from object file to its start index - objs []objIdx // sorted by start index (i.e. objIdx.i) - max Sym // current max index - extStart Sym // from this index on, the symbols are externally defined - extSyms []nameVer // externally defined symbols + start map[*oReader]Sym // map from object file to its start index + objs []objIdx // sorted by start index (i.e. objIdx.i) + max Sym // current max index + extStart Sym // from this index on, the symbols are externally defined + extSyms []nameVer // externally defined symbols + builtinSyms []Sym // global index of builtin symbols symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name @@ -111,6 +112,7 @@ type Loader struct { } func NewLoader() *Loader { + nbuiltin := goobj2.NBuiltin() return &Loader{ start: make(map[*oReader]Sym), objs: []objIdx{{nil, 0}}, @@ -119,6 +121,7 @@ func NewLoader() *Loader { overwrite: make(map[Sym]Sym), itablink: make(map[Sym]struct{}), extStaticSyms: make(map[nameVer]Sym), + builtinSyms: make([]Sym, nbuiltin), } } @@ -272,7 +275,7 @@ func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { v := abiToVer(osym.ABI, r.version) return l.Lookup(name, v) case goobj2.PkgIdxBuiltin: - panic("PkgIdxBuiltin not used") + return l.builtinSyms[s.SymIdx] case goobj2.PkgIdxSelf: rr = r default: @@ -575,6 +578,12 @@ func (l *Loader) Preload(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib * if added && strings.HasPrefix(name, "go.itablink.") { l.itablink[istart+Sym(i)] = struct{}{} } + if added && strings.HasPrefix(name, "runtime.") { + if bi := goobj2.BuiltinIdx(name, v); bi != -1 { + // This is a definition of a builtin symbol. Record where it is. + l.builtinSyms[bi] = istart + Sym(i) + } + } } // The caller expects us consuming all the data From 376ef734a7d2cec8764ec34ab51902028101b630 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Thu, 17 Oct 2019 09:22:01 -0400 Subject: [PATCH 55/79] [dev.link] cmd/link: rework relocation handling in new deadcode Do a better job of reading relocations in the new deadcode pass. Specifically, during method type processing, read relocations for the symbol we're working on into a slice, and then pass the slice to helper functions, as opposed to rereading relocs at each stage. Change-Id: I95e3737ae91bb09b4da8e6ee68112ec255ceb0fc Reviewed-on: https://go-review.googlesource.com/c/go/+/201722 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/deadcode2.go | 81 ++++++++++++++++----------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 82626c7a28..368e151377 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -32,6 +32,7 @@ type deadcodePass2 struct { ctxt *Link ldr *loader.Loader wq workQueue + rtmp []loader.Reloc ifaceMethod map[methodsig]bool // methods declared in reached interfaces markableMethods []methodref2 // methods of reached types @@ -78,9 +79,9 @@ func (d *deadcodePass2) init() { // but we do keep the symbols it refers to. exportsIdx := d.ldr.Lookup("go.plugin.exports", 0) if exportsIdx != 0 { - relocs := d.ldr.Relocs(exportsIdx) - for i := 0; i < relocs.Count; i++ { - d.mark(relocs.At(i).Sym) + d.ReadRelocs(exportsIdx) + for i := 0; i < len(d.rtmp); i++ { + d.mark(d.rtmp[i].Sym) } } } @@ -103,14 +104,19 @@ func (d *deadcodePass2) init() { } func (d *deadcodePass2) flood() { + symRelocs := []loader.Reloc{} for !d.wq.empty() { symIdx := d.wq.pop() d.reflectSeen = d.reflectSeen || d.ldr.IsReflectMethod(symIdx) + + relocs := d.ldr.Relocs(symIdx) + symRelocs = relocs.ReadAll(symRelocs) + if d.ldr.IsGoType(symIdx) { p := d.ldr.Data(symIdx) if len(p) != 0 && decodetypeKind(d.ctxt.Arch, p)&kindMask == kindInterface { - for _, sig := range decodeIfaceMethods2(d.ldr, d.ctxt.Arch, symIdx) { + for _, sig := range d.decodeIfaceMethods2(d.ldr, d.ctxt.Arch, symIdx, symRelocs) { if d.ctxt.Debugvlog > 1 { d.ctxt.Logf("reached iface method: %s\n", sig) } @@ -120,9 +126,8 @@ func (d *deadcodePass2) flood() { } var methods []methodref2 - relocs := d.ldr.Relocs(symIdx) for i := 0; i < relocs.Count; i++ { - r := relocs.At(i) + r := symRelocs[i] if r.Type == objabi.R_WEAKADDROFF { continue } @@ -151,7 +156,7 @@ func (d *deadcodePass2) flood() { // Decode runtime type information for type methods // to help work out which methods can be called // dynamically via interfaces. - methodsigs := decodetypeMethods2(d.ldr, d.ctxt.Arch, symIdx) + methodsigs := d.decodetypeMethods2(d.ldr, d.ctxt.Arch, symIdx, symRelocs) if len(methods) != len(methodsigs) { panic(fmt.Sprintf("%q has %d method relocations for %d methods", d.ldr.SymName(symIdx), len(methods), len(methodsigs))) } @@ -171,10 +176,10 @@ func (d *deadcodePass2) mark(symIdx loader.Sym) { } func (d *deadcodePass2) markMethod(m methodref2) { - relocs := d.ldr.Relocs(m.src) - d.mark(relocs.At(m.r).Sym) - d.mark(relocs.At(m.r + 1).Sym) - d.mark(relocs.At(m.r + 2).Sym) + d.ReadRelocs(m.src) + d.mark(d.rtmp[m.r].Sym) + d.mark(d.rtmp[m.r+1].Sym) + d.mark(d.rtmp[m.r+2].Sym) } func deadcode2(ctxt *Link) { @@ -257,12 +262,15 @@ func (m methodref2) isExported() bool { // the function type. // // Conveniently this is the layout of both runtime.method and runtime.imethod. -func decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, off, size, count int) []methodsig { +func (d *deadcodePass2) decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, symRelocs []loader.Reloc, off, size, count int) []methodsig { var buf bytes.Buffer var methods []methodsig for i := 0; i < count; i++ { - buf.WriteString(decodetypeName2(ldr, symIdx, off)) - mtypSym := decodeRelocSym2(ldr, symIdx, int32(off+4)) + buf.WriteString(decodetypeName2(ldr, symIdx, symRelocs, off)) + mtypSym := decodeRelocSym2(ldr, symIdx, symRelocs, int32(off+4)) + // FIXME: add some sort of caching here, since we may see some of the + // same symbols over time for param types. + d.ReadRelocs(mtypSym) mp := ldr.Data(mtypSym) buf.WriteRune('(') @@ -271,7 +279,7 @@ func decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, off if i > 0 { buf.WriteString(", ") } - a := decodetypeFuncInType2(ldr, arch, mtypSym, i) + a := d.decodetypeFuncInType2(ldr, arch, mtypSym, d.rtmp, i) buf.WriteString(ldr.SymName(a)) } buf.WriteString(") (") @@ -280,7 +288,7 @@ func decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, off if i > 0 { buf.WriteString(", ") } - a := decodetypeFuncOutType2(ldr, arch, mtypSym, i) + a := d.decodetypeFuncOutType2(ldr, arch, mtypSym, d.rtmp, i) buf.WriteString(ldr.SymName(a)) } buf.WriteRune(')') @@ -292,12 +300,12 @@ func decodeMethodSig2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, off return methods } -func decodeIfaceMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) []methodsig { +func (d *deadcodePass2) decodeIfaceMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, symRelocs []loader.Reloc) []methodsig { p := ldr.Data(symIdx) if decodetypeKind(arch, p)&kindMask != kindInterface { panic(fmt.Sprintf("symbol %q is not an interface", ldr.SymName(symIdx))) } - rel := decodeReloc2(ldr, symIdx, int32(commonsize(arch)+arch.PtrSize)) + rel := decodeReloc2(ldr, symIdx, symRelocs, int32(commonsize(arch)+arch.PtrSize)) if rel.Sym == 0 { return nil } @@ -307,10 +315,10 @@ func decodeIfaceMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) off := int(rel.Add) // array of reflect.imethod values numMethods := int(decodetypeIfaceMethodCount(arch, p)) sizeofIMethod := 4 + 4 - return decodeMethodSig2(ldr, arch, symIdx, off, sizeofIMethod, numMethods) + return d.decodeMethodSig2(ldr, arch, symIdx, symRelocs, off, sizeofIMethod, numMethods) } -func decodetypeMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) []methodsig { +func (d *deadcodePass2) decodetypeMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, symRelocs []loader.Reloc) []methodsig { p := ldr.Data(symIdx) if !decodetypeHasUncommon(arch, p) { panic(fmt.Sprintf("no methods on %q", ldr.SymName(symIdx))) @@ -341,13 +349,12 @@ func decodetypeMethods2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym) [ moff := int(decodeInuxi(arch, p[off+4+2+2:], 4)) off += moff // offset to array of reflect.method values const sizeofMethod = 4 * 4 // sizeof reflect.method in program - return decodeMethodSig2(ldr, arch, symIdx, off, sizeofMethod, mcount) + return d.decodeMethodSig2(ldr, arch, symIdx, symRelocs, off, sizeofMethod, mcount) } -func decodeReloc2(ldr *loader.Loader, symIdx loader.Sym, off int32) loader.Reloc { - relocs := ldr.Relocs(symIdx) - for j := 0; j < relocs.Count; j++ { - rel := relocs.At(j) +func decodeReloc2(ldr *loader.Loader, symIdx loader.Sym, symRelocs []loader.Reloc, off int32) loader.Reloc { + for j := 0; j < len(symRelocs); j++ { + rel := symRelocs[j] if rel.Off == off { return rel } @@ -355,13 +362,13 @@ func decodeReloc2(ldr *loader.Loader, symIdx loader.Sym, off int32) loader.Reloc return loader.Reloc{} } -func decodeRelocSym2(ldr *loader.Loader, symIdx loader.Sym, off int32) loader.Sym { - return decodeReloc2(ldr, symIdx, off).Sym +func decodeRelocSym2(ldr *loader.Loader, symIdx loader.Sym, symRelocs []loader.Reloc, off int32) loader.Sym { + return decodeReloc2(ldr, symIdx, symRelocs, off).Sym } // decodetypeName2 decodes the name from a reflect.name. -func decodetypeName2(ldr *loader.Loader, symIdx loader.Sym, off int) string { - r := decodeRelocSym2(ldr, symIdx, int32(off)) +func decodetypeName2(ldr *loader.Loader, symIdx loader.Sym, symRelocs []loader.Reloc, off int) string { + r := decodeRelocSym2(ldr, symIdx, symRelocs, int32(off)) if r == 0 { return "" } @@ -371,7 +378,7 @@ func decodetypeName2(ldr *loader.Loader, symIdx loader.Sym, off int) string { return string(data[3 : 3+namelen]) } -func decodetypeFuncInType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, i int) loader.Sym { +func (d *deadcodePass2) decodetypeFuncInType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, symRelocs []loader.Reloc, i int) loader.Sym { uadd := commonsize(arch) + 4 if arch.PtrSize == 8 { uadd += 4 @@ -379,9 +386,17 @@ func decodetypeFuncInType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym if decodetypeHasUncommon(arch, ldr.Data(symIdx)) { uadd += uncommonSize() } - return decodeRelocSym2(ldr, symIdx, int32(uadd+i*arch.PtrSize)) + return decodeRelocSym2(ldr, symIdx, symRelocs, int32(uadd+i*arch.PtrSize)) } -func decodetypeFuncOutType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, i int) loader.Sym { - return decodetypeFuncInType2(ldr, arch, symIdx, i+decodetypeFuncInCount(arch, ldr.Data(symIdx))) +func (d *deadcodePass2) decodetypeFuncOutType2(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, symRelocs []loader.Reloc, i int) loader.Sym { + return d.decodetypeFuncInType2(ldr, arch, symIdx, symRelocs, i+decodetypeFuncInCount(arch, ldr.Data(symIdx))) +} + +// readRelocs reads the relocations for the specified symbol into the +// deadcode relocs work array. Use with care, since the work array +// is a singleton. +func (d *deadcodePass2) ReadRelocs(symIdx loader.Sym) { + relocs := d.ldr.Relocs(symIdx) + d.rtmp = relocs.ReadAll(d.rtmp) } From 2e2ef666b4d0bf0e86aaa3afbef7fc17f34232e6 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Thu, 24 Oct 2019 11:59:49 -0400 Subject: [PATCH 56/79] [dev.link] cmd/link/internal/loader: add PkgNone resolver cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a cache for the loader.Loader.resolve() method to use when looking mapping local PkgNone symbols to global symbol indices. This helps avoid repeated map lookups during deadcode and other early phases of the linker when we haven't fully read in all of object file symbols. Benchstat numbers: name old time/op new time/op delta LinkCompiler 1.97s ±13% 1.67s ± 8% -15.34% (p=0.000 n=20+20) LinkWithoutDebugCompiler 1.48s ±12% 1.21s ±11% -18.14% (p=0.000 n=20+20) name old user-time/op new user-time/op delta LinkCompiler 2.19s ± 9% 2.04s ±17% -6.98% (p=0.002 n=19+20) LinkWithoutDebugCompiler 1.29s ±13% 1.20s ±13% -7.70% (p=0.000 n=20+20) Change-Id: I4b0b05c8208ee44ee9405b24774b84443e486831 Reviewed-on: https://go-review.googlesource.com/c/go/+/203197 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/loader/loader.go | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 846e954aa1..52809c63da 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -59,6 +59,7 @@ type oReader struct { version int // version of static symbol flags uint32 // read from object file pkgprefix string + rcache []Sym // cache mapping local PkgNone symbol to resolved Sym } type objIdx struct { @@ -257,6 +258,26 @@ func (l *Loader) toLocal(i Sym) (*oReader, int) { return l.objs[k-1].r, int(i - l.objs[k-1].i) } +// rcacheGet checks for a valid entry for 's' in the readers cache, +// where 's' is a local PkgIdxNone ref or def, or zero if +// the cache is empty or doesn't contain a value for 's'. +func (or *oReader) rcacheGet(symIdx uint32) Sym { + if len(or.rcache) > 0 { + return or.rcache[symIdx] + } + return 0 +} + +// rcacheSet installs a new entry in the oReader's PkgNone +// resolver cache for the specified PkgIdxNone ref or def, +// allocating a new cache if needed. +func (or *oReader) rcacheSet(symIdx uint32, gsym Sym) { + if len(or.rcache) == 0 { + or.rcache = make([]Sym, or.NNonpkgdef()+or.NNonpkgref()) + } + or.rcache[symIdx] = gsym +} + // Resolve a local symbol reference. Return global index. func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { var rr *oReader @@ -267,13 +288,20 @@ func (l *Loader) resolve(r *oReader, s goobj2.SymRef) Sym { } return 0 case goobj2.PkgIdxNone: + // Check for cached version first + if cached := r.rcacheGet(s.SymIdx); cached != 0 { + return cached + } // Resolve by name i := int(s.SymIdx) + r.NSym() osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) v := abiToVer(osym.ABI, r.version) - return l.Lookup(name, v) + gsym := l.Lookup(name, v) + // Add to cache, then return. + r.rcacheSet(s.SymIdx, gsym) + return gsym case goobj2.PkgIdxBuiltin: return l.builtinSyms[s.SymIdx] case goobj2.PkgIdxSelf: @@ -549,7 +577,7 @@ func (l *Loader) Preload(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib * } localSymVersion := syms.IncVersion() pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." - or := &oReader{r, unit, localSymVersion, r.Flags(), pkgprefix} + or := &oReader{r, unit, localSymVersion, r.Flags(), pkgprefix, nil} // Autolib lib.ImportStrings = append(lib.ImportStrings, r.Autolib()...) From e90e6e75f8f0b06a5b0c9c68d8d98b8f37b0d3e0 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 30 Oct 2019 10:14:33 -0400 Subject: [PATCH 57/79] [dev.link] cmd/link/internal/loader: add 1-element object cache for use in toLocal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To speed up the loader.Loader.toLocal() method, cache the index of the most recently accessed object file and check that object's sym range in toLocal() before doing a full binary search over all object symbol ranges. This speeds up relink of kubernetes/hyperkube by about 2%, and improves compilebench (relative to the dev.link branch) by about 5%: name old time/op new time/op delta LinkCompiler 1.62s ± 8% 1.50s ± 9% -7.21% (p=0.000 n=20+19) LinkWithoutDebugCompiler 1.13s ± 8% 1.09s ±12% ~ (p=0.052 n=20+20) name old user-time/op new user-time/op delta LinkCompiler 1.94s ±18% 1.97s ±16% ~ (p=0.813 n=19+20) LinkWithoutDebugCompiler 1.15s ±16% 1.13s ±12% ~ (p=0.547 n=20+20) Change-Id: Id5a8a847b533858373c0462f03972d436eda6748 Reviewed-on: https://go-review.googlesource.com/c/go/+/204337 Reviewed-by: Jeremy Faller Reviewed-by: Cherry Zhang --- src/cmd/link/internal/loader/loader.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 52809c63da..95e3005af2 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -65,6 +65,7 @@ type oReader struct { type objIdx struct { r *oReader i Sym // start index + e Sym // end index } type nameVer struct { @@ -98,6 +99,7 @@ type Loader struct { extStart Sym // from this index on, the symbols are externally defined extSyms []nameVer // externally defined symbols builtinSyms []Sym // global index of builtin symbols + ocache int // index (into 'objs') of most recent lookup symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name @@ -116,7 +118,7 @@ func NewLoader() *Loader { nbuiltin := goobj2.NBuiltin() return &Loader{ start: make(map[*oReader]Sym), - objs: []objIdx{{nil, 0}}, + objs: []objIdx{{nil, 0, 0}}, symsByName: [2]map[string]Sym{make(map[string]Sym), make(map[string]Sym)}, objByPkg: make(map[string]*oReader), overwrite: make(map[Sym]Sym), @@ -143,7 +145,7 @@ func (l *Loader) addObj(pkg string, r *oReader) Sym { n := r.NSym() + r.NNonpkgdef() i := l.max + 1 l.start[r] = i - l.objs = append(l.objs, objIdx{r, i}) + l.objs = append(l.objs, objIdx{r, i, i + Sym(n) - 1}) l.max += Sym(n) return i } @@ -249,12 +251,17 @@ func (l *Loader) toLocal(i Sym) (*oReader, int) { if l.isExternal(i) { return nil, int(i - l.extStart) } + oc := l.ocache + if oc != 0 && i >= l.objs[oc].i && i <= l.objs[oc].e { + return l.objs[oc].r, int(i - l.objs[oc].i) + } // Search for the local object holding index i. // Below k is the first one that has its start index > i, // so k-1 is the one we want. k := sort.Search(len(l.objs), func(k int) bool { return l.objs[k].i > i }) + l.ocache = k - 1 return l.objs[k-1].r, int(i - l.objs[k-1].i) } From 851b1f40b48c02f6eeee160957c40dbad46df3bc Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Fri, 25 Oct 2019 12:39:20 -0400 Subject: [PATCH 58/79] [dev.link] cmd/link: add new slice interface for querying aux symbols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new loader.Loader.ReadAuxSyms method that returns a slice containing the ids of the aux symbols for a specified global symbol. This is similar to the new interface recently added that allows you to get back a slice of relocations (as opposed to making calls into the loader for each one). This was idea suggested by Cherry. Compilebench numbers: name old time/op new time/op delta LinkCompiler 1.63s ± 9% 1.57s ± 7% -3.84% (p=0.006 n=20+20) LinkWithoutDebugCompiler 1.15s ±11% 1.11s ±11% ~ (p=0.108 n=20+20) name old user-time/op new user-time/op delta LinkCompiler 1.99s ± 8% 2.00s ±12% ~ (p=0.751 n=19+19) LinkWithoutDebugCompiler 1.14s ±11% 1.19s ±21% ~ (p=0.183 n=20+20) Change-Id: Iab6cbe18419aaa61d9cadb3f626a4515c71f2686 Reviewed-on: https://go-review.googlesource.com/c/go/+/203501 Reviewed-by: Jeremy Faller Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/deadcode2.go | 8 +++++--- src/cmd/link/internal/loader/loader.go | 27 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 368e151377..04a2e925c3 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -105,6 +105,7 @@ func (d *deadcodePass2) init() { func (d *deadcodePass2) flood() { symRelocs := []loader.Reloc{} + auxSyms := []loader.Sym{} for !d.wq.empty() { symIdx := d.wq.pop() @@ -147,9 +148,10 @@ func (d *deadcodePass2) flood() { } d.mark(r.Sym) } - naux := d.ldr.NAux(symIdx) - for i := 0; i < naux; i++ { - d.mark(d.ldr.AuxSym(symIdx, i)) + + auxSyms = d.ldr.ReadAuxSyms(symIdx, auxSyms) + for i := 0; i < len(auxSyms); i++ { + d.mark(auxSyms[i]) } if len(methods) != 0 { diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 95e3005af2..42a5aa50a7 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -475,6 +475,33 @@ func (l *Loader) AuxSym(i Sym, j int) Sym { return l.resolve(r, a.Sym) } +// ReadAuxSyms reads the aux symbol ids for the specified symbol into the +// slice passed as a parameter. If the slice capacity is not large enough, a new +// larger slice will be allocated. Final slice is returned. +func (l *Loader) ReadAuxSyms(symIdx Sym, dst []Sym) []Sym { + if l.isExternal(symIdx) { + return dst[:0] + } + naux := l.NAux(symIdx) + if naux == 0 { + return dst[:0] + } + + if cap(dst) < naux { + dst = make([]Sym, naux) + } + dst = dst[:0] + + r, li := l.toLocal(symIdx) + for i := 0; i < naux; i++ { + a := goobj2.Aux{} + a.Read(r.Reader, r.AuxOff(li, i)) + dst = append(dst, l.resolve(r, a.Sym)) + } + + return dst +} + // Initialize Reachable bitmap for running deadcode pass. func (l *Loader) InitReachable() { l.Reachable = makeBitmap(l.NSym()) From 43d25a61c16bfe263c61d36237ee1b51a545b8cf Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 30 Oct 2019 18:07:55 -0400 Subject: [PATCH 59/79] [dev.link] cmd/link: ensure deterministic order in dynexp dynexp is used for generating the dynamic symbol table. It is created from a map. Sort it to ensure deterministic order. Should fix solaris build. Change-Id: I561b9da3a4136a7ea41139073f76c98fb069d4fa Reviewed-on: https://go-review.googlesource.com/c/go/+/204378 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/lib.go | 2 ++ src/cmd/link/internal/ld/util.go | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 4017ea1c79..1cbfc10ab0 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -58,6 +58,7 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strings" "sync" ) @@ -540,6 +541,7 @@ func setupdynexp(ctxt *Link) { s := ctxt.Syms.Lookup(exp, 0) dynexp = append(dynexp, s) } + sort.Sort(byName(dynexp)) // Resolve ABI aliases in the list of cgo-exported functions. // This is necessary because we load the ABI0 symbol for all diff --git a/src/cmd/link/internal/ld/util.go b/src/cmd/link/internal/ld/util.go index b5b02296a1..488386fec2 100644 --- a/src/cmd/link/internal/ld/util.go +++ b/src/cmd/link/internal/ld/util.go @@ -99,3 +99,10 @@ func contains(s []string, v string) bool { } return false } + +// implements sort.Interface, for sorting symbols by name. +type byName []*sym.Symbol + +func (s byName) Len() int { return len(s) } +func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byName) Less(i, j int) bool { return s[i].Name < s[j].Name } From 5a210b58588f9614c33e1b1e7231a9968879d9e4 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 30 Oct 2019 19:49:53 -0400 Subject: [PATCH 60/79] [dev.link] cmd/link: keep DWARF constant DIE symbols live DWARF constant DIE symbols are not referenced by any other symbol, but are needed by the DWARF pass, where they get linked to the compilation unit. Reenable gdb constant test. Change-Id: If77a0d379d9a6f1591939345bc31b027c2567f22 Reviewed-on: https://go-review.googlesource.com/c/go/+/204397 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode2.go | 9 +++++++++ src/runtime/runtime-gdb_test.go | 2 -- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 04a2e925c3..2517f7d159 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -6,6 +6,7 @@ package ld import ( "bytes" + "cmd/internal/dwarf" "cmd/internal/objabi" "cmd/internal/sys" "cmd/link/internal/loader" @@ -95,6 +96,14 @@ func (d *deadcodePass2) init() { names = append(names, exp) } + // DWARF constant DIE symbols are not referenced, but needed by + // the dwarf pass. + if !*FlagW { + for _, lib := range d.ctxt.Library { + names = append(names, dwarf.ConstInfoPrefix+lib.Pkg) + } + } + for _, name := range names { // Mark symbol as an data/ABI0 symbol. d.mark(d.ldr.Lookup(name, 0)) diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index c389b36efe..8cbc7638ca 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -489,8 +489,6 @@ func main() { ` func TestGdbConst(t *testing.T) { - t.Skip("TODO: newobj") // XXX the constant DIEs are not referenced, so they are not pulled in. Maybe it'll be fine if we rewrite linker's dwarf pass to index? - checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) From 0e76e2f4e5353fb935181fed353ed8667476c4ad Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Thu, 17 Oct 2019 11:06:11 -0400 Subject: [PATCH 61/79] [dev.link] cmd/link: elf host obj support w/ new obj files Add support for elf host objects with new object file format. Change-Id: Ic5be1953359b9b6b78d9a0b715af69763aefd227 Reviewed-on: https://go-review.googlesource.com/c/go/+/201728 Reviewed-by: Cherry Zhang Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/config.go | 9 +- src/cmd/link/internal/ld/deadcode2.go | 8 ++ src/cmd/link/internal/ld/lib.go | 52 ++++++---- src/cmd/link/internal/loadelf/ldelf.go | 52 +++++++--- src/cmd/link/internal/loader/loader.go | 126 ++++++++++++++++++++----- src/cmd/link/internal/sym/symkind.go | 5 + 6 files changed, 195 insertions(+), 57 deletions(-) diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index cfb8c9a786..43dc472230 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -164,6 +164,13 @@ func (mode *LinkMode) String() string { return fmt.Sprintf("LinkMode(%d)", uint8(*mode)) } +func canLinkHostObj(ctxt *Link) bool { + if !*flagNewobj { + return true + } + return ctxt.IsELF +} + // mustLinkExternal reports whether the program being linked requires // the external linker be used to complete the link. func mustLinkExternal(ctxt *Link) (res bool, reason string) { @@ -183,7 +190,7 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) { return true, "msan" } - if iscgo { // TODO: internal linking cgo doesn't work yet + if iscgo && !canLinkHostObj(ctxt) { return true, "TODO: newobj" } diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 2517f7d159..a1f7d2f3a4 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -162,6 +162,14 @@ func (d *deadcodePass2) flood() { for i := 0; i < len(auxSyms); i++ { d.mark(auxSyms[i]) } + // Some host object symbols have an outer object, which acts like a + // "carrier" symbol, or it holds all the symbols for a particular + // section. We need to mark all "referenced" symbols from that carrier, + // so we make sure we're pulling in all outer symbols, and their sub + // symbols. This is not ideal, and these carrier/section symbols could + // be removed. + d.mark(d.ldr.OuterSym(symIdx)) + d.mark(d.ldr.SubSym(symIdx)) if len(methods) != 0 { // Decode runtime type information for type methods diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 1cbfc10ab0..e5e0f1e0dd 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -443,6 +443,10 @@ func (ctxt *Link) loadlib() { } } + // Conditionally load host objects, or setup for external linking. + hostobjs(ctxt) + hostlinksetup(ctxt) + if *flagNewobj { // Add references of externally defined symbols. ctxt.loader.LoadRefs(ctxt.Arch, ctxt.Syms) @@ -453,7 +457,6 @@ func (ctxt *Link) loadlib() { setupdynexp(ctxt) } - // In internal link mode, read the host object files. if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. // Turns out we won't be needing them. @@ -471,8 +474,6 @@ func (ctxt *Link) loadlib() { } } - hostobjs(ctxt) - // If we have any undefined symbols in external // objects, try to read them from the libgcc file. any := false @@ -520,8 +521,6 @@ func (ctxt *Link) loadlib() { */ } } - } else if ctxt.LinkMode == LinkExternal { - hostlinksetup(ctxt) } // We've loaded all the code now. @@ -977,6 +976,9 @@ func ldhostobj(ld func(*Link, *bio.Reader, string, int64, string), headType obja } func hostobjs(ctxt *Link) { + if ctxt.LinkMode != LinkInternal { + return + } var h *Hostobj for i := 0; i < len(hostobj); i++ { @@ -1623,16 +1625,29 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, magic := uint32(c1)<<24 | uint32(c2)<<16 | uint32(c3)<<8 | uint32(c4) if magic == 0x7f454c46 { // \x7F E L F - ldelf := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { - textp, flags, err := loadelf.Load(ctxt.Arch, ctxt.Syms, f, pkg, length, pn, ehdr.flags) - if err != nil { - Errorf(nil, "%v", err) - return + if *flagNewobj { + ldelf := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, flags, err := loadelf.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn, ehdr.flags) + if err != nil { + Errorf(nil, "%v", err) + return + } + ehdr.flags = flags + ctxt.Textp = append(ctxt.Textp, textp...) } - ehdr.flags = flags - ctxt.Textp = append(ctxt.Textp, textp...) + return ldhostobj(ldelf, ctxt.HeadType, f, pkg, length, pn, file) + } else { + ldelf := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, flags, err := loadelf.LoadOld(ctxt.Arch, ctxt.Syms, f, pkg, length, pn, ehdr.flags) + if err != nil { + Errorf(nil, "%v", err) + return + } + ehdr.flags = flags + ctxt.Textp = append(ctxt.Textp, textp...) + } + return ldhostobj(ldelf, ctxt.HeadType, f, pkg, length, pn, file) } - return ldhostobj(ldelf, ctxt.HeadType, f, pkg, length, pn, file) } if magic&^1 == 0xfeedface || magic&^0x01000000 == 0xcefaedfe { @@ -2379,6 +2394,9 @@ func genasmsym(ctxt *Link, put func(*Link, *sym.Symbol, string, SymbolType, int6 put(ctxt, s, s.Name, BSSSym, Symaddr(s), s.Gotype) case sym.SHOSTOBJ: + if !s.Attr.Reachable() { + continue + } if ctxt.HeadType == objabi.Hwindows || ctxt.IsELF { put(ctxt, s, s.Name, UndefinedSym, s.Value, nil) } @@ -2580,12 +2598,8 @@ func (ctxt *Link) loadlibfull() { // Load full symbol contents, resolve indexed references. ctxt.loader.LoadFull(ctxt.Arch, ctxt.Syms) - // For now, add all symbols to ctxt.Syms. - for _, s := range ctxt.loader.Syms { - if s != nil && s.Name != "" { - ctxt.Syms.Add(s) - } - } + // Pull the symbols out. + ctxt.loader.ExtractSymbols(ctxt.Syms) // Load cgo directives. for _, d := range ctxt.cgodata { diff --git a/src/cmd/link/internal/loadelf/ldelf.go b/src/cmd/link/internal/loadelf/ldelf.go index e895056bb2..627f836835 100644 --- a/src/cmd/link/internal/loadelf/ldelf.go +++ b/src/cmd/link/internal/loadelf/ldelf.go @@ -10,6 +10,7 @@ import ( "cmd/internal/bio" "cmd/internal/objabi" "cmd/internal/sys" + "cmd/link/internal/loader" "cmd/link/internal/sym" "debug/elf" "encoding/binary" @@ -451,7 +452,37 @@ func parseArmAttributes(e binary.ByteOrder, data []byte) (found bool, ehdrFlags return found, ehdrFlags, nil } -// Load loads the ELF file pn from f. +func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string, flags uint32) ([]*sym.Symbol, uint32, error) { + newSym := func(name string, version int) *sym.Symbol { + // If we've seen the symbol, we might need to load it. + i := l.Lookup(name, version) + if i != 0 { + // Already loaded. + if l.Syms[i] != nil { + return l.Syms[i] + } + if l.IsExternal(i) { + panic("Can't load an external symbol.") + } + return l.LoadSymbol(name, version, syms) + } + if i = l.AddExtSym(name, version); i == 0 { + panic("AddExtSym returned bad index") + } + newSym := syms.Newsym(name, version) + l.Syms[i] = newSym + return newSym + } + return load(arch, syms.IncVersion(), newSym, newSym, f, pkg, length, pn, flags) +} + +func LoadOld(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string, flags uint32) ([]*sym.Symbol, uint32, error) { + return load(arch, syms.IncVersion(), syms.Newsym, syms.Lookup, f, pkg, length, pn, flags) +} + +type lookupFunc func(string, int) *sym.Symbol + +// load loads the ELF file pn from f. // Symbols are written into syms, and a slice of the text symbols is returned. // // On ARM systems, Load will attempt to determine what ELF header flags to @@ -459,12 +490,11 @@ func parseArmAttributes(e binary.ByteOrder, data []byte) (found bool, ehdrFlags // parameter initEhdrFlags contains the current header flags for the output // object, and the returned ehdrFlags contains what this Load function computes. // TODO: find a better place for this logic. -func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string, initEhdrFlags uint32) (textp []*sym.Symbol, ehdrFlags uint32, err error) { +func load(arch *sys.Arch, localSymVersion int, newSym, lookup lookupFunc, f *bio.Reader, pkg string, length int64, pn string, initEhdrFlags uint32) (textp []*sym.Symbol, ehdrFlags uint32, err error) { errorf := func(str string, args ...interface{}) ([]*sym.Symbol, uint32, error) { return nil, 0, fmt.Errorf("loadelf: %s: %v", pn, fmt.Sprintf(str, args...)) } - localSymVersion := syms.IncVersion() base := f.Offset() var hdrbuf [64]uint8 @@ -715,7 +745,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length i } sectsymNames[name] = true - s := syms.Lookup(name, localSymVersion) + s := lookup(name, localSymVersion) switch int(sect.flags) & (ElfSectFlagAlloc | ElfSectFlagWrite | ElfSectFlagExec) { default: @@ -754,7 +784,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length i for i := 1; i < elfobj.nsymtab; i++ { var elfsym ElfSym - if err := readelfsym(arch, syms, elfobj, i, &elfsym, 1, localSymVersion); err != nil { + if err := readelfsym(newSym, lookup, arch, elfobj, i, &elfsym, 1, localSymVersion); err != nil { return errorf("%s: malformed elf file: %v", pn, err) } symbols[i] = elfsym.sym @@ -925,7 +955,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length i rp.Sym = nil } else { var elfsym ElfSym - if err := readelfsym(arch, syms, elfobj, int(info>>32), &elfsym, 0, 0); err != nil { + if err := readelfsym(newSym, lookup, arch, elfobj, int(info>>32), &elfsym, 0, 0); err != nil { return errorf("malformed elf file: %v", err) } elfsym.sym = symbols[info>>32] @@ -1002,7 +1032,7 @@ func elfmap(elfobj *ElfObj, sect *ElfSect) (err error) { return nil } -func readelfsym(arch *sys.Arch, syms *sym.Symbols, elfobj *ElfObj, i int, elfsym *ElfSym, needSym int, localSymVersion int) (err error) { +func readelfsym(newSym, lookup lookupFunc, arch *sys.Arch, elfobj *ElfObj, i int, elfsym *ElfSym, needSym int, localSymVersion int) (err error) { if i >= elfobj.nsymtab || i < 0 { err = fmt.Errorf("invalid elf symbol index") return err @@ -1052,7 +1082,7 @@ func readelfsym(arch *sys.Arch, syms *sym.Symbols, elfobj *ElfObj, i int, elfsym switch elfsym.bind { case ElfSymBindGlobal: if needSym != 0 { - s = syms.Lookup(elfsym.name, 0) + s = lookup(elfsym.name, 0) // for global scoped hidden symbols we should insert it into // symbol hash table, but mark them as hidden. @@ -1077,7 +1107,7 @@ func readelfsym(arch *sys.Arch, syms *sym.Symbols, elfobj *ElfObj, i int, elfsym // We need to be able to look this up, // so put it in the hash table. if needSym != 0 { - s = syms.Lookup(elfsym.name, localSymVersion) + s = lookup(elfsym.name, localSymVersion) s.Attr |= sym.AttrVisibilityHidden } @@ -1088,14 +1118,14 @@ func readelfsym(arch *sys.Arch, syms *sym.Symbols, elfobj *ElfObj, i int, elfsym // local names and hidden global names are unique // and should only be referenced by their index, not name, so we // don't bother to add them into the hash table - s = syms.Newsym(elfsym.name, localSymVersion) + s = newSym(elfsym.name, localSymVersion) s.Attr |= sym.AttrVisibilityHidden } case ElfSymBindWeak: if needSym != 0 { - s = syms.Lookup(elfsym.name, 0) + s = lookup(elfsym.name, 0) if elfsym.other == 2 { s.Attr |= sym.AttrVisibilityHidden } diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 42a5aa50a7..ff38e7cf88 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -175,7 +175,7 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ if overwrite { // new symbol overwrites old symbol. oldtyp := sym.AbiSymKindToSymKind[objabi.SymKind(oldsym.Type)] - if !((oldtyp == sym.SDATA || oldtyp == sym.SNOPTRDATA || oldtyp == sym.SBSS || oldtyp == sym.SNOPTRBSS) && oldr.DataSize(li) == 0) { // only allow overwriting 0-sized data symbol + if !oldtyp.IsData() && r.DataSize(li) == 0 { log.Fatalf("duplicated definition of symbol " + name) } l.overwrite[oldi] = i @@ -220,8 +220,7 @@ func (l *Loader) AddExtSym(name string, ver int) Sym { return i } -// Returns whether i is an external symbol. -func (l *Loader) isExternal(i Sym) bool { +func (l *Loader) IsExternal(i Sym) bool { return l.extStart != 0 && i >= l.extStart } @@ -248,7 +247,7 @@ func (l *Loader) toLocal(i Sym) (*oReader, int) { if ov, ok := l.overwrite[i]; ok { i = ov } - if l.isExternal(i) { + if l.IsExternal(i) { return nil, int(i - l.extStart) } oc := l.ocache @@ -340,7 +339,7 @@ func (l *Loader) IsDup(i Sym) bool { if _, ok := l.overwrite[i]; ok { return true } - if l.isExternal(i) { + if l.IsExternal(i) { return false } r, li := l.toLocal(i) @@ -372,7 +371,7 @@ func (l *Loader) NDef() int { // Returns the raw (unpatched) name of the i-th symbol. func (l *Loader) RawSymName(i Sym) string { - if l.isExternal(i) { + if l.IsExternal(i) { if s := l.Syms[i]; s != nil { return s.Name } @@ -386,7 +385,7 @@ func (l *Loader) RawSymName(i Sym) string { // Returns the (patched) name of the i-th symbol. func (l *Loader) SymName(i Sym) string { - if l.isExternal(i) { + if l.IsExternal(i) { if s := l.Syms[i]; s != nil { return s.Name // external name should already be patched? } @@ -400,7 +399,7 @@ func (l *Loader) SymName(i Sym) string { // Returns the type of the i-th symbol. func (l *Loader) SymType(i Sym) sym.SymKind { - if l.isExternal(i) { + if l.IsExternal(i) { if s := l.Syms[i]; s != nil { return s.Type } @@ -414,7 +413,7 @@ func (l *Loader) SymType(i Sym) sym.SymKind { // Returns the attributes of the i-th symbol. func (l *Loader) SymAttr(i Sym) uint8 { - if l.isExternal(i) { + if l.IsExternal(i) { // TODO: do something? External symbols have different representation of attributes. For now, ReflectMethod is the only thing matters and it cannot be set by external symbol. return 0 } @@ -444,7 +443,7 @@ func (l *Loader) IsItabLink(i Sym) bool { // Returns the symbol content of the i-th symbol. i is global index. func (l *Loader) Data(i Sym) []byte { - if l.isExternal(i) { + if l.IsExternal(i) { if s := l.Syms[i]; s != nil { return s.P } @@ -456,7 +455,7 @@ func (l *Loader) Data(i Sym) []byte { // Returns the number of aux symbols given a global index. func (l *Loader) NAux(i Sym) int { - if l.isExternal(i) { + if l.IsExternal(i) { return 0 } r, li := l.toLocal(i) @@ -466,7 +465,7 @@ func (l *Loader) NAux(i Sym) int { // Returns the referred symbol of the j-th aux symbol of the i-th // symbol. func (l *Loader) AuxSym(i Sym, j int) Sym { - if l.isExternal(i) { + if l.IsExternal(i) { return 0 } r, li := l.toLocal(i) @@ -502,6 +501,26 @@ func (l *Loader) ReadAuxSyms(symIdx Sym, dst []Sym) []Sym { return dst } +// OuterSym gets the outer symbol for host object loaded symbols. +func (l *Loader) OuterSym(i Sym) Sym { + sym := l.Syms[i] + if sym != nil && sym.Outer != nil { + outer := sym.Outer + return l.Lookup(outer.Name, int(outer.Version)) + } + return 0 +} + +// SubSym gets sub symbols for a previously loaded host object symbol. +func (l *Loader) SubSym(i Sym) Sym { + sym := l.Syms[i] + if sym != nil && sym.Sub != nil { + sub := sym.Sub + return l.Lookup(sub.Name, int(sub.Version)) + } + return 0 +} + // Initialize Reachable bitmap for running deadcode pass. func (l *Loader) InitReachable() { l.Reachable = makeBitmap(l.NSym()) @@ -578,7 +597,7 @@ func (relocs *Relocs) ReadAll(dst []Reloc) []Reloc { // Relocs returns a Relocs object for the given global sym. func (l *Loader) Relocs(i Sym) Relocs { - if l.isExternal(i) { + if l.IsExternal(i) { if s := l.Syms[i]; s != nil { return Relocs{Count: len(s.R), l: l, ext: s} } @@ -736,11 +755,52 @@ func (l *Loader) LoadFull(arch *sys.Arch, syms *sym.Symbols) { } } +// ExtractSymbols grabs the symbols out of the loader for work that hasn't been +// ported to the new symbol type. +func (l *Loader) ExtractSymbols(syms *sym.Symbols) { + // Nil out overwritten symbols. + // Overwritten Go symbols aren't a problem (as they're lazy loaded), but + // symbols loaded from host object loaders are fully loaded, and we might + // have multiple symbols with the same name. This loop nils them out. + for oldI := range l.overwrite { + l.Syms[oldI] = nil + } + + // For now, add all symbols to ctxt.Syms. + for _, s := range l.Syms { + if s != nil && s.Name != "" { + syms.Add(s) + } + } + +} + +// addNewSym adds a new sym.Symbol to the i-th index in the list of symbols. +func (l *Loader) addNewSym(i Sym, syms *sym.Symbols, name string, ver int, unit *sym.CompilationUnit, t sym.SymKind) *sym.Symbol { + s := syms.Newsym(name, ver) + if s.Type != 0 && s.Type != sym.SXREF { + fmt.Println("symbol already processed:", unit.Lib, i, s) + panic("symbol already processed") + } + if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { + t = s.Type + } + s.Type = t + s.Unit = unit + l.Syms[i] = s + return s +} + func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { - lib := r.unit.Lib istart := l.startIndex(r) for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { + // If it's been previously loaded in host object loading, we don't need to do it again. + if s := l.Syms[istart+Sym(i)]; s != nil { + // Mark symbol as reachable as it wasn't marked as such before. + s.Attr.Set(sym.AttrReachable, l.Reachable.Has(istart+Sym(i))) + continue + } osym := goobj2.Sym{} osym.Read(r.Reader, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1) @@ -757,7 +817,7 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { log.Fatalf("bad sxref") } if t == 0 { - log.Fatalf("missing type for %s in %s", name, lib) + log.Fatalf("missing type for %s in %s", name, r.unit.Lib) } if !l.Reachable.Has(istart+Sym(i)) && !(t == sym.SRODATA && strings.HasPrefix(name, "type.")) && name != "runtime.addmoduledata" && name != "runtime.lastmoduledatap" { // No need to load unreachable symbols. @@ -766,21 +826,35 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { continue } - s := syms.Newsym(name, ver) - if s.Type != 0 && s.Type != sym.SXREF { - fmt.Println("symbol already processed:", lib, i, s) - panic("symbol already processed") - } - if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { - t = s.Type - } - s.Type = t - s.Unit = r.unit + s := l.addNewSym(istart+Sym(i), syms, name, ver, r.unit, t) s.Attr.Set(sym.AttrReachable, l.Reachable.Has(istart+Sym(i))) - l.Syms[istart+Sym(i)] = s } } +// LoadSymbol loads a single symbol by name. +// This function should only be used by the host object loaders. +// NB: This function does NOT set the symbol as reachable. +func (l *Loader) LoadSymbol(name string, version int, syms *sym.Symbols) *sym.Symbol { + global := l.Lookup(name, version) + + // If we're already loaded, bail. + if global != 0 && l.Syms[global] != nil { + return l.Syms[global] + } + + // Read the symbol. + r, i := l.toLocal(global) + istart := l.startIndex(r) + + osym := goobj2.Sym{} + osym.Read(r.Reader, r.SymOff(int(i))) + if l.symsByName[version][name] != istart+Sym(i) { + return nil + } + + return l.addNewSym(istart+Sym(i), syms, name, version, r.unit, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) +} + func loadObjFull(l *Loader, r *oReader) { lib := r.unit.Lib istart := l.startIndex(r) diff --git a/src/cmd/link/internal/sym/symkind.go b/src/cmd/link/internal/sym/symkind.go index 4e44d3fce1..a81070f253 100644 --- a/src/cmd/link/internal/sym/symkind.go +++ b/src/cmd/link/internal/sym/symkind.go @@ -158,3 +158,8 @@ var RelROMap = map[SymKind]SymKind{ SRODATA: SRODATARELRO, SFUNCTAB: SFUNCTABRELRO, } + +// IsData returns true if the type is a data type. +func (t SymKind) IsData() bool { + return t == SDATA || t == SNOPTRDATA || t == SBSS || t == SNOPTRBSS +} From 931845aee3fcb7efb9abe3b47a319b1422557e4c Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Thu, 31 Oct 2019 09:51:20 -0400 Subject: [PATCH 62/79] [dev.link] cmd/link: fix merge error Change-Id: Ief8384a74ac9cf303a959656f807f34a0ff9873b Reviewed-on: https://go-review.googlesource.com/c/go/+/204518 Run-TryBot: Jeremy Faller TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/loader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index ff38e7cf88..f0689032f3 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -478,7 +478,7 @@ func (l *Loader) AuxSym(i Sym, j int) Sym { // slice passed as a parameter. If the slice capacity is not large enough, a new // larger slice will be allocated. Final slice is returned. func (l *Loader) ReadAuxSyms(symIdx Sym, dst []Sym) []Sym { - if l.isExternal(symIdx) { + if l.IsExternal(symIdx) { return dst[:0] } naux := l.NAux(symIdx) From 396a9b98bac38dce31085146f8c713fbe3c0c664 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Tue, 29 Oct 2019 10:52:20 -0400 Subject: [PATCH 63/79] [dev.link] cmd/link: fix macho host objects Small updates to macho loader, fixing some misunderstandings I had about using the new object file format. Change-Id: I9224b01ca327e3a087ebfa36800bd6eef6abcc80 Reviewed-on: https://go-review.googlesource.com/c/go/+/204097 Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/config.go | 2 +- src/cmd/link/internal/ld/lib.go | 3 ++- src/cmd/link/internal/loader/loader.go | 2 +- src/cmd/link/internal/loadmacho/ldmacho.go | 25 ++++++++-------------- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index 43dc472230..0c571c30e7 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -168,7 +168,7 @@ func canLinkHostObj(ctxt *Link) bool { if !*flagNewobj { return true } - return ctxt.IsELF + return ctxt.IsELF || objabi.GOOS == "darwin" } // mustLinkExternal reports whether the program being linked requires diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index e5e0f1e0dd..679d44e8dd 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1653,11 +1653,12 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, if magic&^1 == 0xfeedface || magic&^0x01000000 == 0xcefaedfe { if *flagNewobj { ldmacho := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { - err := loadmacho.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + textp, err := loadmacho.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn) if err != nil { Errorf(nil, "%v", err) return } + ctxt.Textp = append(ctxt.Textp, textp...) } return ldhostobj(ldmacho, ctxt.HeadType, f, pkg, length, pn, file) } else { diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index f0689032f3..0ce6f54ef7 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -511,7 +511,7 @@ func (l *Loader) OuterSym(i Sym) Sym { return 0 } -// SubSym gets sub symbols for a previously loaded host object symbol. +// SubSym gets the subsymbol for host object loaded symbols. func (l *Loader) SubSym(i Sym) Sym { sym := l.Syms[i] if sym != nil && sym.Sub != nil { diff --git a/src/cmd/link/internal/loadmacho/ldmacho.go b/src/cmd/link/internal/loadmacho/ldmacho.go index 18a0678af8..f21d4bd14f 100644 --- a/src/cmd/link/internal/loadmacho/ldmacho.go +++ b/src/cmd/link/internal/loadmacho/ldmacho.go @@ -424,31 +424,24 @@ func macholoadsym(m *ldMachoObj, symtab *ldMachoSymtab) int { return 0 } -func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) error { - lookup := func(name string, version int) *sym.Symbol { - // Check to see if we've already defined the symbol. - if i := l.Lookup(name, version); i != 0 { - return l.Syms[i] +func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) ([]*sym.Symbol, error) { + newSym := func(name string, version int) *sym.Symbol { + i := l.Lookup(name, version) + if i != 0 { + return l.LoadSymbol(name, version, syms) } - // Not defined, let's make one. - if s := l.AddExtSym(name, version); s == 0 { + if i = l.AddExtSym(name, version); i == 0 { panic("AddExtSym returned bad index") - } else if int(s) != len(l.Syms) { - panic("unexpected length of loaded symbols") } newSym := syms.Newsym(name, version) - l.Syms = append(l.Syms, newSym) + l.Syms[i] = newSym return newSym } - _, err := load(arch, syms.IncVersion(), lookup, f, pkg, length, pn) - return err + return load(arch, syms.IncVersion(), newSym, f, pkg, length, pn) } func LoadOld(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { - lookup := func(name string, version int) *sym.Symbol { - return syms.Lookup(name, version) - } - return load(arch, syms.IncVersion(), lookup, f, pkg, length, pn) + return load(arch, syms.IncVersion(), syms.Lookup, f, pkg, length, pn) } // load the Mach-O file pn from f. From 4a43a5079184f9a4cff355fbf8c6d3145e253bb2 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 31 Oct 2019 19:35:55 -0400 Subject: [PATCH 64/79] [dev.link] cmd/link: process cgo_import_static before host object loading In internal linking mode, we need to process cgo_import_static directives before loading host objects, because the directive is to tell the host object loader how to deal with imported symbols. This should fix linking with old object files. I think there needs some similar logic for new object files, but I'll leave that for later. Change-Id: Icaa286de626ea1876086dbdd015047084c92caf9 Reviewed-on: https://go-review.googlesource.com/c/go/+/204697 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/lib.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 679d44e8dd..b882df29a1 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -443,20 +443,6 @@ func (ctxt *Link) loadlib() { } } - // Conditionally load host objects, or setup for external linking. - hostobjs(ctxt) - hostlinksetup(ctxt) - - if *flagNewobj { - // Add references of externally defined symbols. - ctxt.loader.LoadRefs(ctxt.Arch, ctxt.Syms) - } - - // Now that we know the link mode, set the dynexp list. - if !*flagNewobj { // set this later in newobj mode - setupdynexp(ctxt) - } - if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. // Turns out we won't be needing them. @@ -473,7 +459,23 @@ func (ctxt *Link) loadlib() { } } } + } + // Conditionally load host objects, or setup for external linking. + hostobjs(ctxt) + hostlinksetup(ctxt) + + if *flagNewobj { + // Add references of externally defined symbols. + ctxt.loader.LoadRefs(ctxt.Arch, ctxt.Syms) + } + + // Now that we know the link mode, set the dynexp list. + if !*flagNewobj { // set this later in newobj mode + setupdynexp(ctxt) + } + + if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // If we have any undefined symbols in external // objects, try to read them from the libgcc file. any := false From 63815923fc7ef8dd9b6ae04b791e2811d74f9c1b Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 31 Oct 2019 21:53:49 -0400 Subject: [PATCH 65/79] [dev.link] cmd/link: resolve ABI aliases for external symbols ABI alias references in Go symbols are resolved during loadObjFull. But for external symbols they are not resolved. If there is a reference from an external symbol to a Go ABIInternal symbol, this reference will be invalid as it is not resolved. The old code resolve ABI aliases in the deadcode pass. But the new deadcode pass doesn't do it, as it works with indices instead of Symbols. We do this in LoadFull. This makes all internal cgo linking tests pass on Mach-O. Change-Id: Iac6c084c03f5ddbcc9455527800ce7ed7313f9a7 Reviewed-on: https://go-review.googlesource.com/c/go/+/204698 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/lib.go | 2 +- src/cmd/link/internal/loader/loader.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index b882df29a1..e46457d858 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -2620,7 +2620,7 @@ func (ctxt *Link) loadlibfull() { func (ctxt *Link) dumpsyms() { for _, s := range ctxt.Syms.Allsym { - fmt.Printf("%s %s %p\n", s, s.Type, s) + fmt.Printf("%s %s %p %v %v\n", s, s.Type, s, s.Attr.Reachable(), s.Attr.OnList()) for i := range s.R { fmt.Println("\t", s.R[i].Type, s.R[i].Sym) } diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 0ce6f54ef7..67c4c9719c 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -753,6 +753,21 @@ func (l *Loader) LoadFull(arch *sys.Arch, syms *sym.Symbols) { for _, o := range l.objs[1:] { loadObjFull(l, o.r) } + + // Resolve ABI aliases for external symbols. This is only + // needed for internal cgo linking. + // (The old code does this in deadcode, but deadcode2 doesn't + // do this.) + for i := l.extStart; i <= l.max; i++ { + if s := l.Syms[i]; s != nil && s.Attr.Reachable() { + for ri := range s.R { + r := &s.R[ri] + if r.Sym != nil && r.Sym.Type == sym.SABIALIAS { + r.Sym = r.Sym.R[0].Sym + } + } + } + } } // ExtractSymbols grabs the symbols out of the loader for work that hasn't been From 219922e95b8e49cfb94da9de0c48edb22a2e7054 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 30 Oct 2019 12:31:55 -0400 Subject: [PATCH 66/79] [dev.link] cmd/link: add support to new deadcode for field tracking Fix up the new dead code pass to include support for populating the ctxt "Reachparent" map, which is needed to support field tracking. Since we don't have sym.Symbols created at the point where new dead code runs, keep track of reachability using global symbol indices, and then once loader.LoadFull is complete we can translate the index mappings into symbol mappings. The fieldtracking output is unfortunately different relative to master, due to differences in the order in which symbols are encountered in deadcode, but I have eyeballed the results to make sure they look reasonable. Change-Id: I48c7a4597f05c00f15af3bfd37fc15ab4d0017c2 Reviewed-on: https://go-review.googlesource.com/c/go/+/204342 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/deadcode2.go | 33 ++++++++++++++------------ src/cmd/link/internal/ld/lib.go | 16 +++++++++++++ src/cmd/link/internal/loader/loader.go | 5 ++++ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index a1f7d2f3a4..2fbc0a94d6 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -18,8 +18,6 @@ import ( var _ = fmt.Print // TODO: -// - Field tracking support: -// It needs to record from where the symbol is referenced. // - Debug output: // Emit messages about which symbols are kept or deleted. @@ -43,6 +41,9 @@ type deadcodePass2 struct { func (d *deadcodePass2) init() { d.ldr.InitReachable() d.ifaceMethod = make(map[methodsig]bool) + if d.ctxt.Reachparent != nil { + d.ldr.Reachparent = make([]loader.Sym, d.ldr.NSym()) + } if d.ctxt.BuildMode == BuildModeShared { // Mark all symbols defined in this library as reachable when @@ -51,7 +52,7 @@ func (d *deadcodePass2) init() { for i := 1; i < n; i++ { s := loader.Sym(i) if !d.ldr.IsDup(s) { - d.mark(s) + d.mark(s, 0) } } return @@ -82,7 +83,7 @@ func (d *deadcodePass2) init() { if exportsIdx != 0 { d.ReadRelocs(exportsIdx) for i := 0; i < len(d.rtmp); i++ { - d.mark(d.rtmp[i].Sym) + d.mark(d.rtmp[i].Sym, 0) } } } @@ -106,9 +107,9 @@ func (d *deadcodePass2) init() { for _, name := range names { // Mark symbol as an data/ABI0 symbol. - d.mark(d.ldr.Lookup(name, 0)) + d.mark(d.ldr.Lookup(name, 0), 0) // Also mark any Go functions (internal ABI). - d.mark(d.ldr.Lookup(name, sym.SymVerABIInternal)) + d.mark(d.ldr.Lookup(name, sym.SymVerABIInternal), 0) } } @@ -155,12 +156,11 @@ func (d *deadcodePass2) flood() { // do nothing for now as we still load all type symbols. continue } - d.mark(r.Sym) + d.mark(r.Sym, symIdx) } - auxSyms = d.ldr.ReadAuxSyms(symIdx, auxSyms) for i := 0; i < len(auxSyms); i++ { - d.mark(auxSyms[i]) + d.mark(auxSyms[i], symIdx) } // Some host object symbols have an outer object, which acts like a // "carrier" symbol, or it holds all the symbols for a particular @@ -168,8 +168,8 @@ func (d *deadcodePass2) flood() { // so we make sure we're pulling in all outer symbols, and their sub // symbols. This is not ideal, and these carrier/section symbols could // be removed. - d.mark(d.ldr.OuterSym(symIdx)) - d.mark(d.ldr.SubSym(symIdx)) + d.mark(d.ldr.OuterSym(symIdx), symIdx) + d.mark(d.ldr.SubSym(symIdx), symIdx) if len(methods) != 0 { // Decode runtime type information for type methods @@ -187,18 +187,21 @@ func (d *deadcodePass2) flood() { } } -func (d *deadcodePass2) mark(symIdx loader.Sym) { +func (d *deadcodePass2) mark(symIdx, parent loader.Sym) { if symIdx != 0 && !d.ldr.Reachable.Has(symIdx) { d.wq.push(symIdx) d.ldr.Reachable.Set(symIdx) + if d.ctxt.Reachparent != nil { + d.ldr.Reachparent[symIdx] = parent + } } } func (d *deadcodePass2) markMethod(m methodref2) { d.ReadRelocs(m.src) - d.mark(d.rtmp[m.r].Sym) - d.mark(d.rtmp[m.r+1].Sym) - d.mark(d.rtmp[m.r+2].Sym) + d.mark(d.rtmp[m.r].Sym, m.src) + d.mark(d.rtmp[m.r+1].Sym, m.src) + d.mark(d.rtmp[m.r+2].Sym, m.src) } func deadcode2(ctxt *Link) { diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index e46457d858..811dd0f9ef 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -2611,6 +2611,22 @@ func (ctxt *Link) loadlibfull() { setupdynexp(ctxt) + // Populate ctxt.Reachparent if appropriate. + if ctxt.Reachparent != nil { + for i := 0; i < len(ctxt.loader.Reachparent); i++ { + p := ctxt.loader.Reachparent[i] + if p == 0 { + continue + } + if p == loader.Sym(i) { + panic("self-cycle in reachparent") + } + sym := ctxt.loader.Syms[i] + psym := ctxt.loader.Syms[p] + ctxt.Reachparent[sym] = psym + } + } + // Drop the reference. ctxt.loader = nil ctxt.cgodata = nil diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 67c4c9719c..38b2c810e3 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -112,6 +112,11 @@ type Loader struct { Syms []*sym.Symbol // indexed symbols. XXX we still make sym.Symbol for now. Reachable bitmap // bitmap of reachable symbols, indexed by global index + + // Used to implement field tracking; created during deadcode if + // field tracking is enabled. Reachparent[K] contains the index of + // the symbol that triggered the marking of symbol K as live. + Reachparent []Sym } func NewLoader() *Loader { From e961b26c274498f9e9bf17a6609dbc0f542f2d40 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Tue, 29 Oct 2019 11:37:44 -0400 Subject: [PATCH 67/79] [dev.link] cmd/link: fix xcoff loader for new obj format config.go needs to be removed from this CL. Change-Id: I04a267feeae1551bb18f6a03a725adc9db593fdb Reviewed-on: https://go-review.googlesource.com/c/go/+/204099 Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/config.go | 2 +- src/cmd/link/internal/ld/lib.go | 26 ++++++++++++----- src/cmd/link/internal/loadxcoff/ldxcoff.go | 34 ++++++++++++++++++---- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index 0c571c30e7..b4ee67825f 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -168,7 +168,7 @@ func canLinkHostObj(ctxt *Link) bool { if !*flagNewobj { return true } - return ctxt.IsELF || objabi.GOOS == "darwin" + return ctxt.IsELF || objabi.GOOS == "darwin" || objabi.GOOS == "aix" } // mustLinkExternal reports whether the program being linked requires diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index a0f85b85c7..bf43ef36d0 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1707,15 +1707,27 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, } if c1 == 0x01 && (c2 == 0xD7 || c2 == 0xF7) { - ldxcoff := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { - textp, err := loadxcoff.Load(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) - if err != nil { - Errorf(nil, "%v", err) - return + if *flagNewobj { + ldxcoff := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, err := loadxcoff.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } + ctxt.Textp = append(ctxt.Textp, textp...) } - ctxt.Textp = append(ctxt.Textp, textp...) + return ldhostobj(ldxcoff, ctxt.HeadType, f, pkg, length, pn, file) + } else { + ldxcoff := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, err := loadxcoff.LoadOld(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } + ctxt.Textp = append(ctxt.Textp, textp...) + } + return ldhostobj(ldxcoff, ctxt.HeadType, f, pkg, length, pn, file) } - return ldhostobj(ldxcoff, ctxt.HeadType, f, pkg, length, pn, file) } /* check the header */ diff --git a/src/cmd/link/internal/loadxcoff/ldxcoff.go b/src/cmd/link/internal/loadxcoff/ldxcoff.go index f52b23ce6a..fc5d3cf2bf 100644 --- a/src/cmd/link/internal/loadxcoff/ldxcoff.go +++ b/src/cmd/link/internal/loadxcoff/ldxcoff.go @@ -9,6 +9,7 @@ import ( "cmd/internal/bio" "cmd/internal/objabi" "cmd/internal/sys" + "cmd/link/internal/loader" "cmd/link/internal/sym" "errors" "fmt" @@ -38,13 +39,34 @@ func (f *xcoffBiobuf) ReadAt(p []byte, off int64) (int, error) { return n, nil } -// Load loads the Xcoff file pn from f. +// Load loads xcoff files with the indexed object files. +func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { + lookup := func(name string, version int) *sym.Symbol { + i := l.Lookup(name, version) + if i != 0 { + return l.LoadSymbol(name, version, syms) + } + if i = l.AddExtSym(name, version); i == 0 { + panic("AddExtSym returned bad index") + } + newSym := syms.Newsym(name, version) + l.Syms[i] = newSym + return newSym + } + return load(arch, lookup, syms.IncVersion(), input, pkg, length, pn) +} + +// LoadOld uses the old version of object loading. +func LoadOld(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { + return load(arch, syms.Lookup, syms.IncVersion(), input, pkg, length, pn) +} + +// loads the Xcoff file pn from f. // Symbols are written into syms, and a slice of the text symbols is returned. -func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { +func load(arch *sys.Arch, lookup func(string, int) *sym.Symbol, localSymVersion int, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { errorf := func(str string, args ...interface{}) ([]*sym.Symbol, error) { return nil, fmt.Errorf("loadxcoff: %v: %v", pn, fmt.Sprintf(str, args...)) } - localSymVersion := syms.IncVersion() var ldSections []*ldSection @@ -62,7 +84,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng lds := new(ldSection) lds.Section = *sect name := fmt.Sprintf("%s(%s)", pkg, lds.Name) - s := syms.Lookup(name, localSymVersion) + s := lookup(name, localSymVersion) switch lds.Type { default: @@ -100,7 +122,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng continue } - s := syms.Lookup(sx.Name, 0) + s := lookup(sx.Name, 0) // Text symbol if s.Type == sym.STEXT { @@ -122,7 +144,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng for i, rx := range sect.Relocs { r := &rs[i] - r.Sym = syms.Lookup(rx.Symbol.Name, 0) + r.Sym = lookup(rx.Symbol.Name, 0) if uint64(int32(rx.VirtualAddress)) != rx.VirtualAddress { return errorf("virtual address of a relocation is too big: 0x%x", rx.VirtualAddress) } From 5aea597905f914bb0c8d0d986e17c5022cf9aed1 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Thu, 24 Oct 2019 11:27:47 -0400 Subject: [PATCH 68/79] [dev.link] cmd/link/internal/loader: do more bulk allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the loader to do more bulk allocation when making slices of small objects (sym.Reloc, etc) as part of creating and populating sym.Symbols in loader.LoadFull(). This replaces a large number of small allocations with a smaller number of large allocations, improving performace. Compilebench numbers (linker portion) for this change: name old time/op new time/op delta LinkCompiler 1.71s ±11% 1.57s ± 9% -8.35% (p=0.000 n=19+20) LinkWithoutDebugCompiler 1.19s ±14% 1.10s ±13% -7.93% (p=0.000 n=20+19) name old user-time/op new user-time/op delta LinkCompiler 1.86s ±15% 1.34s ±10% -28.02% (p=0.000 n=20+20) LinkWithoutDebugCompiler 1.05s ±14% 0.95s ± 9% -9.17% (p=0.000 n=19+20) Hyperkube from kubernetes doesn't show any significant benefit (which seems a little surprising). Change-Id: Ide97f78532fb60b08bb6e4cfa097e9058f7ea8ab Reviewed-on: https://go-review.googlesource.com/c/go/+/203457 Run-TryBot: Than McIntosh Reviewed-by: Cherry Zhang Reviewed-by: Jeremy Faller TryBot-Result: Gobot Gobot --- src/cmd/link/internal/loader/loader.go | 124 +++++++++++++++++++++---- 1 file changed, 104 insertions(+), 20 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 38b2c810e3..573c251058 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -117,6 +117,8 @@ type Loader struct { // field tracking is enabled. Reachparent[K] contains the index of // the symbol that triggered the marking of symbol K as live. Reachparent []Sym + + relocBatch []sym.Reloc // for bulk allocation of relocations } func NewLoader() *Loader { @@ -229,7 +231,7 @@ func (l *Loader) IsExternal(i Sym) bool { return l.extStart != 0 && i >= l.extStart } -// Ensure Syms slice als enough space. +// Ensure Syms slice has enough space. func (l *Loader) growSyms(i int) { n := len(l.Syms) if n > i { @@ -735,10 +737,15 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { func (l *Loader) LoadFull(arch *sys.Arch, syms *sym.Symbols) { // create all Symbols first. l.growSyms(l.NSym()) + + nr := 0 // total number of sym.Reloc's we'll need for _, o := range l.objs[1:] { - loadObjSyms(l, syms, o.r) + nr += loadObjSyms(l, syms, o.r) } + // allocate a single large slab of relocations for all live symbols + l.relocBatch = make([]sym.Reloc, nr) + // external symbols for i := l.extStart; i <= l.max; i++ { if s := l.Syms[i]; s != nil { @@ -811,14 +818,19 @@ func (l *Loader) addNewSym(i Sym, syms *sym.Symbols, name string, ver int, unit return s } -func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { +// loadObjSyms creates sym.Symbol objects for the live Syms in the +// object corresponding to object reader "r". Return value is the +// number of sym.Reloc entries required for all the new symbols. +func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) int { istart := l.startIndex(r) + nr := 0 for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { // If it's been previously loaded in host object loading, we don't need to do it again. if s := l.Syms[istart+Sym(i)]; s != nil { // Mark symbol as reachable as it wasn't marked as such before. s.Attr.Set(sym.AttrReachable, l.Reachable.Has(istart+Sym(i))) + nr += r.NReloc(i) continue } osym := goobj2.Sym{} @@ -848,7 +860,28 @@ func loadObjSyms(l *Loader, syms *sym.Symbols, r *oReader) { s := l.addNewSym(istart+Sym(i), syms, name, ver, r.unit, t) s.Attr.Set(sym.AttrReachable, l.Reachable.Has(istart+Sym(i))) + nr += r.NReloc(i) } + return nr +} + +// funcInfoSym records the sym.Symbol for a function, along with a copy +// of the corresponding goobj2.Sym and the index of its FuncInfo aux sym. +// We use this to delay populating FuncInfo until we can batch-allocate +// slices for their sub-objects. +type funcInfoSym struct { + s *sym.Symbol // sym.Symbol for a live function + osym goobj2.Sym // object file symbol data for that function + isym int // global symbol index of FuncInfo aux sym for func +} + +// funcAllocInfo records totals/counts for all functions in an objfile; +// used to help with bulk allocation of sym.Symbol sub-objects. +type funcAllocInfo struct { + symPtr uint32 // number of *sym.Symbol's needed in file slices + inlCall uint32 // number of sym.InlinedCall's needed in inltree slices + pcData uint32 // number of sym.Pcdata's needed in pdata slices + fdOff uint32 // number of int64's needed in all Funcdataoff slices } // LoadSymbol loads a single symbol by name. @@ -884,6 +917,9 @@ func loadObjFull(l *Loader, r *oReader) { return l.Syms[i] } + funcs := []funcInfoSym{} + fdsyms := []*sym.Symbol{} + var funcAllocCounts funcAllocInfo pcdataBase := r.PcdataBase() rslice := []Reloc{} for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { @@ -930,7 +966,9 @@ func loadObjFull(l *Loader, r *oReader) { // Relocs relocs := l.relocs(r, i) rslice = relocs.ReadAll(rslice) - s.R = make([]sym.Reloc, relocs.Count) + batch := l.relocBatch + s.R = batch[:relocs.Count:relocs.Count] + l.relocBatch = batch[relocs.Count:] for j := range s.R { r := rslice[j] rs := r.Sym @@ -974,12 +1012,7 @@ func loadObjFull(l *Loader, r *oReader) { s.Gotype = typ } case goobj2.AuxFuncdata: - pc := s.FuncInfo - if pc == nil { - pc = &sym.FuncInfo{Funcdata: make([]*sym.Symbol, 0, 4)} - s.FuncInfo = pc - } - pc.Funcdata = append(pc.Funcdata, resolveSymRef(a.Sym)) + fdsyms = append(fdsyms, resolveSymRef(a.Sym)) case goobj2.AuxFuncInfo: if a.Sym.PkgIdx != goobj2.PkgIdxSelf { panic("funcinfo symbol not defined in current package") @@ -1014,10 +1047,44 @@ func loadObjFull(l *Loader, r *oReader) { continue } - // FuncInfo if isym == -1 { continue } + + // Record function sym and associated info for additional + // processing in the loop below. + fwis := funcInfoSym{s: s, isym: isym, osym: osym} + funcs = append(funcs, fwis) + + // Read the goobj2.FuncInfo for this text symbol so that we can + // collect allocation counts. We'll read it again in the loop + // below. + b := r.Data(isym) + info := goobj2.FuncInfo{} + info.Read(b) + funcAllocCounts.symPtr += uint32(len(info.File)) + funcAllocCounts.pcData += uint32(len(info.Pcdata)) + funcAllocCounts.inlCall += uint32(len(info.InlTree)) + funcAllocCounts.fdOff += uint32(len(info.Funcdataoff)) + } + + // At this point we can do batch allocation of the sym.FuncInfo's, + // along with the slices of sub-objects they use. + fiBatch := make([]sym.FuncInfo, len(funcs)) + inlCallBatch := make([]sym.InlinedCall, funcAllocCounts.inlCall) + symPtrBatch := make([]*sym.Symbol, funcAllocCounts.symPtr) + pcDataBatch := make([]sym.Pcdata, funcAllocCounts.pcData) + fdOffBatch := make([]int64, funcAllocCounts.fdOff) + + // Populate FuncInfo contents for func symbols. + for fi := 0; fi < len(funcs); fi++ { + s := funcs[fi].s + isym := funcs[fi].isym + osym := funcs[fi].osym + + s.FuncInfo = &fiBatch[0] + fiBatch = fiBatch[1:] + b := r.Data(isym) info := goobj2.FuncInfo{} info.Read(b) @@ -1035,18 +1102,34 @@ func loadObjFull(l *Loader, r *oReader) { s.Attr |= sym.AttrTopFrame } - info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends pc := s.FuncInfo - if pc == nil { - pc = &sym.FuncInfo{} - s.FuncInfo = pc + + if len(info.Funcdataoff) != 0 { + nfd := len(info.Funcdataoff) + pc.Funcdata = fdsyms[:nfd:nfd] + fdsyms = fdsyms[nfd:] } + + info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends pc.Args = int32(info.Args) pc.Locals = int32(info.Locals) - pc.Pcdata = make([]sym.Pcdata, len(info.Pcdata)-1) // -1 as we appended one above - pc.Funcdataoff = make([]int64, len(info.Funcdataoff)) - pc.File = make([]*sym.Symbol, len(info.File)) - pc.InlTree = make([]sym.InlinedCall, len(info.InlTree)) + + npc := len(info.Pcdata) - 1 // -1 as we appended one above + pc.Pcdata = pcDataBatch[:npc:npc] + pcDataBatch = pcDataBatch[npc:] + + nfd := len(info.Funcdataoff) + pc.Funcdataoff = fdOffBatch[:nfd:nfd] + fdOffBatch = fdOffBatch[nfd:] + + nsp := len(info.File) + pc.File = symPtrBatch[:nsp:nsp] + symPtrBatch = symPtrBatch[nsp:] + + nic := len(info.InlTree) + pc.InlTree = inlCallBatch[:nic:nic] + inlCallBatch = inlCallBatch[nic:] + pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp)) pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile)) pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline)) @@ -1071,6 +1154,7 @@ func loadObjFull(l *Loader, r *oReader) { } } + dupok := osym.Dupok() if !dupok { if s.Attr.OnList() { log.Fatalf("symbol %s listed multiple times", s.Name) @@ -1078,7 +1162,7 @@ func loadObjFull(l *Loader, r *oReader) { s.Attr.Set(sym.AttrOnList, true) lib.Textp = append(lib.Textp, s) } else { - // there may ba a dup in another package + // there may be a dup in another package // put into a temp list and add to text later lib.DupTextSyms = append(lib.DupTextSyms, s) } From dfd8de10040eb64dc255d1f6ed98728b169af885 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 1 Nov 2019 15:15:58 -0400 Subject: [PATCH 69/79] [dev.link] all: clean up some TODOs Change-Id: Iae1ca888729014b6fec97d7bd7ae082dbceb9fe5 Reviewed-on: https://go-review.googlesource.com/c/go/+/204837 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/internal/goobj2/objfile.go | 2 +- src/cmd/internal/obj/sizeof_test.go | 2 +- test/linkx.go | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cmd/internal/goobj2/objfile.go b/src/cmd/internal/goobj2/objfile.go index e10ce43833..4c364b0c54 100644 --- a/src/cmd/internal/goobj2/objfile.go +++ b/src/cmd/internal/goobj2/objfile.go @@ -33,7 +33,7 @@ import ( // Autolib [...]stringOff // imported packages (for file loading) // TODO: add fingerprints // PkgIndex [...]stringOff // referenced packages by index // -// DwarfFiles [...]stringOff // XXX as a separate block for now +// DwarfFiles [...]stringOff // // SymbolDefs [...]struct { // Name stringOff diff --git a/src/cmd/internal/obj/sizeof_test.go b/src/cmd/internal/obj/sizeof_test.go index ac65143b6b..b5e170c694 100644 --- a/src/cmd/internal/obj/sizeof_test.go +++ b/src/cmd/internal/obj/sizeof_test.go @@ -21,7 +21,7 @@ func TestSizeof(t *testing.T) { _64bit uintptr // size on 64bit platforms }{ {Addr{}, 32, 48}, - //{LSym{}, 56, 104}, // TODO: re-enable + {LSym{}, 76, 128}, {Prog{}, 132, 200}, } diff --git a/test/linkx.go b/test/linkx.go index 2b5b6edd47..4f85b241a9 100644 --- a/test/linkx.go +++ b/test/linkx.go @@ -32,8 +32,6 @@ func main() { // Check non-string symbols are not overwritten. // This also make them used. - // TODO: decide if we need to issue an error if -X - // is applied to a non-string unreachable symbol. if b || x != 0 { panic("b or x overwritten") } From 48a0b979022d604771767cd03634986669c064ab Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 1 Nov 2019 10:58:27 -0400 Subject: [PATCH 70/79] [dev.link] cmd/link: set cgo attributes early when internal cgo linking In newobj mode, cgo attributes are typically set later, as we create sym.Symbols later. But when internal cgo linking, the host object loaders still work with sym.Symbols, and the cgo attributes need to be set for them to work properly. Therefore, set them early. This will cause creating some Symbols eagerly, but they are mostly host object symbols and will need to be created anyway. Now all cgo internal linking tests pass on ELF systems. Change-Id: I023a4df4429acc8ebf5e185f62e6809198497a78 Reviewed-on: https://go-review.googlesource.com/c/go/+/204857 Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/go.go | 12 +++++------ src/cmd/link/internal/ld/lib.go | 15 ++++++++++++- src/cmd/link/internal/loadelf/ldelf.go | 19 +--------------- src/cmd/link/internal/loader/loader.go | 25 +++++++++++++++++++++- src/cmd/link/internal/loadmacho/ldmacho.go | 11 +--------- src/cmd/link/internal/loadxcoff/ldxcoff.go | 11 +--------- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index 3246747bb7..21457fdfc8 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -149,12 +149,12 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { // Record the directives. We'll process them later after Symbols are created. ctxt.cgodata = append(ctxt.cgodata, cgodata{file, pkg, directives}) } else { - setCgoAttr(ctxt, file, pkg, directives) + setCgoAttr(ctxt, ctxt.Syms.Lookup, file, pkg, directives) } } // Set symbol attributes or flags based on cgo directives. -func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { +func setCgoAttr(ctxt *Link, lookup func(string, int) *sym.Symbol, file string, pkg string, directives [][]string) { for _, f := range directives { switch f[0] { case "cgo_import_dynamic": @@ -196,7 +196,7 @@ func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { if i := strings.Index(remote, "#"); i >= 0 { remote, q = remote[:i], remote[i+1:] } - s := ctxt.Syms.Lookup(local, 0) + s := lookup(local, 0) if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SBSS || s.Type == sym.SNOPTRBSS || s.Type == sym.SHOSTOBJ { s.SetDynimplib(lib) s.SetExtname(remote) @@ -215,7 +215,7 @@ func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { } local := f[1] - s := ctxt.Syms.Lookup(local, 0) + s := lookup(local, 0) s.Type = sym.SHOSTOBJ s.Size = 0 continue @@ -236,11 +236,11 @@ func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) { // functions. Link.loadlib will resolve any // ABI aliases we find here (since we may not // yet know it's an alias). - s := ctxt.Syms.Lookup(local, 0) + s := lookup(local, 0) switch ctxt.BuildMode { case BuildModeCShared, BuildModeCArchive, BuildModePlugin: - if s == ctxt.Syms.Lookup("main", 0) { + if s == lookup("main", 0) { continue } } diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index bf43ef36d0..8bf943575f 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -444,6 +444,19 @@ func (ctxt *Link) loadlib() { } if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { + if *flagNewobj { + // In newobj mode, we typically create sym.Symbols later therefore + // also set cgo attributes later. However, for internal cgo linking, + // the host object loaders still work with sym.Symbols (for now), + // and they need cgo attributes set to work properly. So process + // them now. + lookup := func(name string, ver int) *sym.Symbol { return ctxt.loader.LookupOrCreate(name, ver, ctxt.Syms) } + for _, d := range ctxt.cgodata { + setCgoAttr(ctxt, lookup, d.file, d.pkg, d.directives) + } + ctxt.cgodata = nil + } + // Drop all the cgo_import_static declarations. // Turns out we won't be needing them. for _, s := range ctxt.Syms.Allsym { @@ -2638,7 +2651,7 @@ func (ctxt *Link) loadlibfull() { // Load cgo directives. for _, d := range ctxt.cgodata { - setCgoAttr(ctxt, d.file, d.pkg, d.directives) + setCgoAttr(ctxt, ctxt.Syms.Lookup, d.file, d.pkg, d.directives) } setupdynexp(ctxt) diff --git a/src/cmd/link/internal/loadelf/ldelf.go b/src/cmd/link/internal/loadelf/ldelf.go index 627f836835..072eaf00c8 100644 --- a/src/cmd/link/internal/loadelf/ldelf.go +++ b/src/cmd/link/internal/loadelf/ldelf.go @@ -454,24 +454,7 @@ func parseArmAttributes(e binary.ByteOrder, data []byte) (found bool, ehdrFlags func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string, flags uint32) ([]*sym.Symbol, uint32, error) { newSym := func(name string, version int) *sym.Symbol { - // If we've seen the symbol, we might need to load it. - i := l.Lookup(name, version) - if i != 0 { - // Already loaded. - if l.Syms[i] != nil { - return l.Syms[i] - } - if l.IsExternal(i) { - panic("Can't load an external symbol.") - } - return l.LoadSymbol(name, version, syms) - } - if i = l.AddExtSym(name, version); i == 0 { - panic("AddExtSym returned bad index") - } - newSym := syms.Newsym(name, version) - l.Syms[i] = newSym - return newSym + return l.LookupOrCreate(name, version, syms) } return load(arch, syms.IncVersion(), newSym, newSym, f, pkg, length, pn, flags) } diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 573c251058..46d93c5124 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -814,6 +814,7 @@ func (l *Loader) addNewSym(i Sym, syms *sym.Symbols, name string, ver int, unit } s.Type = t s.Unit = unit + l.growSyms(int(i)) l.Syms[i] = s return s } @@ -891,7 +892,7 @@ func (l *Loader) LoadSymbol(name string, version int, syms *sym.Symbols) *sym.Sy global := l.Lookup(name, version) // If we're already loaded, bail. - if global != 0 && l.Syms[global] != nil { + if global != 0 && int(global) < len(l.Syms) && l.Syms[global] != nil { return l.Syms[global] } @@ -908,6 +909,28 @@ func (l *Loader) LoadSymbol(name string, version int, syms *sym.Symbols) *sym.Sy return l.addNewSym(istart+Sym(i), syms, name, version, r.unit, sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]) } +// LookupOrCreate looks up a symbol by name, and creates one if not found. +// Either way, it will also create a sym.Symbol for it, if not already. +// This should only be called when interacting with parts of the linker +// that still works on sym.Symbols (i.e. internal cgo linking, for now). +func (l *Loader) LookupOrCreate(name string, version int, syms *sym.Symbols) *sym.Symbol { + i := l.Lookup(name, version) + if i != 0 { + // symbol exists + if int(i) < len(l.Syms) && l.Syms[i] != nil { + return l.Syms[i] // already loaded + } + if l.IsExternal(i) { + panic("Can't load an external symbol.") + } + return l.LoadSymbol(name, version, syms) + } + i = l.AddExtSym(name, version) + s := syms.Newsym(name, version) + l.Syms[i] = s + return s +} + func loadObjFull(l *Loader, r *oReader) { lib := r.unit.Lib istart := l.startIndex(r) diff --git a/src/cmd/link/internal/loadmacho/ldmacho.go b/src/cmd/link/internal/loadmacho/ldmacho.go index f21d4bd14f..85a1ebc631 100644 --- a/src/cmd/link/internal/loadmacho/ldmacho.go +++ b/src/cmd/link/internal/loadmacho/ldmacho.go @@ -426,16 +426,7 @@ func macholoadsym(m *ldMachoObj, symtab *ldMachoSymtab) int { func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, pkg string, length int64, pn string) ([]*sym.Symbol, error) { newSym := func(name string, version int) *sym.Symbol { - i := l.Lookup(name, version) - if i != 0 { - return l.LoadSymbol(name, version, syms) - } - if i = l.AddExtSym(name, version); i == 0 { - panic("AddExtSym returned bad index") - } - newSym := syms.Newsym(name, version) - l.Syms[i] = newSym - return newSym + return l.LookupOrCreate(name, version, syms) } return load(arch, syms.IncVersion(), newSym, f, pkg, length, pn) } diff --git a/src/cmd/link/internal/loadxcoff/ldxcoff.go b/src/cmd/link/internal/loadxcoff/ldxcoff.go index fc5d3cf2bf..759b1769dd 100644 --- a/src/cmd/link/internal/loadxcoff/ldxcoff.go +++ b/src/cmd/link/internal/loadxcoff/ldxcoff.go @@ -42,16 +42,7 @@ func (f *xcoffBiobuf) ReadAt(p []byte, off int64) (int, error) { // Load loads xcoff files with the indexed object files. func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, err error) { lookup := func(name string, version int) *sym.Symbol { - i := l.Lookup(name, version) - if i != 0 { - return l.LoadSymbol(name, version, syms) - } - if i = l.AddExtSym(name, version); i == 0 { - panic("AddExtSym returned bad index") - } - newSym := syms.Newsym(name, version) - l.Syms[i] = newSym - return newSym + return l.LookupOrCreate(name, version, syms) } return load(arch, lookup, syms.IncVersion(), input, pkg, length, pn) } From 496e4273e44d666372b28cb5b5df40de5bf12c22 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Tue, 29 Oct 2019 19:37:56 -0400 Subject: [PATCH 71/79] [dev.link] cmd/link: fix loadpe to work with new obj file Change-Id: I0fe88df182f13e7f04c8de0b82e111db441a26e2 Reviewed-on: https://go-review.googlesource.com/c/go/+/204341 Run-TryBot: Cherry Zhang Reviewed-by: Cherry Zhang Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/config.go | 11 ---------- src/cmd/link/internal/ld/lib.go | 33 ++++++++++++++++++++-------- src/cmd/link/internal/loadpe/ldpe.go | 30 ++++++++++++++++--------- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index b4ee67825f..3f5b6d4fdf 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -164,13 +164,6 @@ func (mode *LinkMode) String() string { return fmt.Sprintf("LinkMode(%d)", uint8(*mode)) } -func canLinkHostObj(ctxt *Link) bool { - if !*flagNewobj { - return true - } - return ctxt.IsELF || objabi.GOOS == "darwin" || objabi.GOOS == "aix" -} - // mustLinkExternal reports whether the program being linked requires // the external linker be used to complete the link. func mustLinkExternal(ctxt *Link) (res bool, reason string) { @@ -190,10 +183,6 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) { return true, "msan" } - if iscgo && !canLinkHostObj(ctxt) { - return true, "TODO: newobj" - } - // Internally linking cgo is incomplete on some architectures. // https://golang.org/issue/14449 // https://golang.org/issue/21961 diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 8bf943575f..9f939e5c82 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1705,18 +1705,33 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string, } if c1 == 0x4c && c2 == 0x01 || c1 == 0x64 && c2 == 0x86 { - ldpe := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { - textp, rsrc, err := loadpe.Load(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) - if err != nil { - Errorf(nil, "%v", err) - return + if *flagNewobj { + ldpe := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, rsrc, err := loadpe.Load(ctxt.loader, ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } + if rsrc != nil { + setpersrc(ctxt, rsrc) + } + ctxt.Textp = append(ctxt.Textp, textp...) } - if rsrc != nil { - setpersrc(ctxt, rsrc) + return ldhostobj(ldpe, ctxt.HeadType, f, pkg, length, pn, file) + } else { + ldpe := func(ctxt *Link, f *bio.Reader, pkg string, length int64, pn string) { + textp, rsrc, err := loadpe.LoadOld(ctxt.Arch, ctxt.Syms, f, pkg, length, pn) + if err != nil { + Errorf(nil, "%v", err) + return + } + if rsrc != nil { + setpersrc(ctxt, rsrc) + } + ctxt.Textp = append(ctxt.Textp, textp...) } - ctxt.Textp = append(ctxt.Textp, textp...) + return ldhostobj(ldpe, ctxt.HeadType, f, pkg, length, pn, file) } - return ldhostobj(ldpe, ctxt.HeadType, f, pkg, length, pn, file) } if c1 == 0x01 && (c2 == 0xD7 || c2 == 0xF7) { diff --git a/src/cmd/link/internal/loadpe/ldpe.go b/src/cmd/link/internal/loadpe/ldpe.go index a41a7901a9..353f6e0863 100644 --- a/src/cmd/link/internal/loadpe/ldpe.go +++ b/src/cmd/link/internal/loadpe/ldpe.go @@ -9,6 +9,7 @@ import ( "cmd/internal/bio" "cmd/internal/objabi" "cmd/internal/sys" + "cmd/link/internal/loader" "cmd/link/internal/sym" "debug/pe" "encoding/binary" @@ -144,12 +145,21 @@ func (f *peBiobuf) ReadAt(p []byte, off int64) (int, error) { return n, nil } -// Load loads the PE file pn from input. +func Load(l *loader.Loader, arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, rsrc *sym.Symbol, err error) { + lookup := func(name string, version int) *sym.Symbol { + return l.LookupOrCreate(name, version, syms) + } + return load(arch, lookup, syms.IncVersion(), input, pkg, length, pn) +} + +func LoadOld(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, rsrc *sym.Symbol, err error) { + return load(arch, syms.Lookup, syms.IncVersion(), input, pkg, length, pn) +} + +// load loads the PE file pn from input. // Symbols are written into syms, and a slice of the text symbols is returned. // If an .rsrc section is found, its symbol is returned as rsrc. -func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, rsrc *sym.Symbol, err error) { - localSymVersion := syms.IncVersion() - +func load(arch *sys.Arch, lookup func(string, int) *sym.Symbol, localSymVersion int, input *bio.Reader, pkg string, length int64, pn string) (textp []*sym.Symbol, rsrc *sym.Symbol, err error) { sectsyms := make(map[*pe.Section]*sym.Symbol) sectdata := make(map[*pe.Section][]byte) @@ -181,7 +191,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng } name := fmt.Sprintf("%s(%s)", pkg, sect.Name) - s := syms.Lookup(name, localSymVersion) + s := lookup(name, localSymVersion) switch sect.Characteristics & (IMAGE_SCN_CNT_UNINITIALIZED_DATA | IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_WRITE | IMAGE_SCN_CNT_CODE | IMAGE_SCN_MEM_EXECUTE) { case IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ: //.rdata @@ -239,7 +249,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng return nil, nil, fmt.Errorf("relocation number %d symbol index idx=%d cannot be large then number of symbols %d", j, r.SymbolTableIndex, len(f.COFFSymbols)) } pesym := &f.COFFSymbols[r.SymbolTableIndex] - gosym, err := readpesym(arch, syms, f, pesym, sectsyms, localSymVersion) + gosym, err := readpesym(arch, lookup, f, pesym, sectsyms, localSymVersion) if err != nil { return nil, nil, err } @@ -351,7 +361,7 @@ func Load(arch *sys.Arch, syms *sym.Symbols, input *bio.Reader, pkg string, leng } } - s, err := readpesym(arch, syms, f, pesym, sectsyms, localSymVersion) + s, err := readpesym(arch, lookup, f, pesym, sectsyms, localSymVersion) if err != nil { return nil, nil, err } @@ -435,7 +445,7 @@ func issect(s *pe.COFFSymbol) bool { return s.StorageClass == IMAGE_SYM_CLASS_STATIC && s.Type == 0 && s.Name[0] == '.' } -func readpesym(arch *sys.Arch, syms *sym.Symbols, f *pe.File, pesym *pe.COFFSymbol, sectsyms map[*pe.Section]*sym.Symbol, localSymVersion int) (*sym.Symbol, error) { +func readpesym(arch *sys.Arch, lookup func(string, int) *sym.Symbol, f *pe.File, pesym *pe.COFFSymbol, sectsyms map[*pe.Section]*sym.Symbol, localSymVersion int) (*sym.Symbol, error) { symname, err := pesym.FullName(f.StringTable) if err != nil { return nil, err @@ -481,10 +491,10 @@ func readpesym(arch *sys.Arch, syms *sym.Symbols, f *pe.File, pesym *pe.COFFSymb case IMAGE_SYM_DTYPE_FUNCTION, IMAGE_SYM_DTYPE_NULL: switch pesym.StorageClass { case IMAGE_SYM_CLASS_EXTERNAL: //global - s = syms.Lookup(name, 0) + s = lookup(name, 0) case IMAGE_SYM_CLASS_NULL, IMAGE_SYM_CLASS_STATIC, IMAGE_SYM_CLASS_LABEL: - s = syms.Lookup(name, localSymVersion) + s = lookup(name, localSymVersion) s.Attr |= sym.AttrDuplicateOK default: From 5597e3d38968beb41f78971d523aef0a49605410 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 31 Oct 2019 16:08:56 -0400 Subject: [PATCH 72/79] [dev.link] all: reenable cgo internal linking tests Cgo internal linking was disabled when switching to new object files. Reeanble the tests, as they get supported. Change-Id: I11d2ac8785cce73f3a42f6935c10d9f067bc90a4 Reviewed-on: https://go-review.googlesource.com/c/go/+/204520 Reviewed-by: Than McIntosh --- src/cmd/dist/test.go | 3 --- src/cmd/nm/nm_cgo_test.go | 2 +- src/debug/pe/file_cgo_test.go | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index eeddd9474c..cc54554a1d 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -904,9 +904,6 @@ func (t *tester) extLink() bool { } func (t *tester) internalLink() bool { - if true { // appease vet... - return false // TODO: newobj - } if gohostos == "dragonfly" { // linkmode=internal fails on dragonfly since errno is a TLS relocation. return false diff --git a/src/cmd/nm/nm_cgo_test.go b/src/cmd/nm/nm_cgo_test.go index dde24a0b72..9a257e0ed2 100644 --- a/src/cmd/nm/nm_cgo_test.go +++ b/src/cmd/nm/nm_cgo_test.go @@ -37,7 +37,7 @@ func canInternalLink() bool { } func TestInternalLinkerCgoExec(t *testing.T) { - if !canInternalLink() || true { // TODO: newobj + if !canInternalLink() { t.Skip("skipping; internal linking is not supported") } testGoExec(t, true, false) diff --git a/src/debug/pe/file_cgo_test.go b/src/debug/pe/file_cgo_test.go index e89894953b..739671d73f 100644 --- a/src/debug/pe/file_cgo_test.go +++ b/src/debug/pe/file_cgo_test.go @@ -23,7 +23,6 @@ func TestDefaultLinkerDWARF(t *testing.T) { } func TestInternalLinkerDWARF(t *testing.T) { - t.Skip("TODO: newobj") testCgoDWARF(t, linkCgoInternal) } From 24ea07d5925af3528c44e85655a35e17e2de57de Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 30 Oct 2019 21:24:22 -0400 Subject: [PATCH 73/79] [dev.link] cmd/link: add internal packages at the end Currently in the linker we load internal packges first, then the main package, and then load imported packages following the dependency graph. As a result, packages are loaded mostly in the dependency order, except the internal packages. The global symbol indices are assigned the same way. By loading the internal packages at the end, the packages are loaded in the dependency order, so are the global indices. This way, a relocation edge is mostly either within a packge or a forward edge from a smaller index to a larger one. This allows us to use a min-heap work queue in the deadcode pass, to achieve better spatial locality (in the next CL). Change-Id: I01fa9b3cf0c9e9e66006040f6378a51fd78f0f39 Reviewed-on: https://go-review.googlesource.com/c/go/+/204437 Reviewed-by: Jeremy Faller --- src/cmd/link/internal/ld/lib.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 9f939e5c82..4c7451a114 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -384,7 +384,19 @@ func (ctxt *Link) loadlib() { ctxt.cgo_export_static = make(map[string]bool) ctxt.cgo_export_dynamic = make(map[string]bool) - loadinternal(ctxt, "runtime") + // ctxt.Library grows during the loop, so not a range loop. + i := 0 + for ; i < len(ctxt.Library); i++ { + lib := ctxt.Library[i] + if lib.Shlib == "" { + if ctxt.Debugvlog > 1 { + ctxt.Logf("autolib: %s (from %s)\n", lib.File, lib.Objref) + } + loadobjfile(ctxt, lib) + } + } + + // load internal packages, if not already if ctxt.Arch.Family == sys.ARM { loadinternal(ctxt, "math") } @@ -394,14 +406,10 @@ func (ctxt *Link) loadlib() { if *flagMsan { loadinternal(ctxt, "runtime/msan") } - - // ctxt.Library grows during the loop, so not a range loop. - for i := 0; i < len(ctxt.Library); i++ { + loadinternal(ctxt, "runtime") + for ; i < len(ctxt.Library); i++ { lib := ctxt.Library[i] if lib.Shlib == "" { - if ctxt.Debugvlog > 1 { - ctxt.Logf("autolib: %s (from %s)\n", lib.File, lib.Objref) - } loadobjfile(ctxt, lib) } } From 36dbdbe9bd086126209bcd26057ec14e4a12e953 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 30 Oct 2019 22:24:27 -0400 Subject: [PATCH 74/79] [dev.link] cmd/link: use min-heap for work queue for better locality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the deadcode pass, we use a work queue for the flood algorithm. Currently this is a simple LIFO queue. In this order, there is poor locality in accessing object files. Since the global indices are assigned in package DAG order, edges are mostly either within a package or from a smaller index to a larger one. (With named symbols, there can be backward edges, but shouldn't be too many.) Using a min-heap for the work queue, we access all symbols in one object, then move to next one. It rarely needs to revisit an object that is already visted. This should result in better locality. Benchmark result from Than (thanks!): name old time/op new time/op delta LinkCompiler 1.74s ±11% 1.61s ± 9% -7.80% (p=0.000 n=20+19) LinkWithoutDebugCompiler 1.27s ±11% 1.15s ± 9% -9.02% (p=0.000 n=20+20) Currently this uses the container/heap package, which uses interface elements. If this allocates too much, we may consider to hand-code the min heap. Change-Id: I216d5291c432fe1f40b0b8f4f1b9d388807bf6c5 Reviewed-on: https://go-review.googlesource.com/c/go/+/204438 Reviewed-by: Jeremy Faller --- src/cmd/link/internal/ld/deadcode2.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index 2fbc0a94d6..a138bc97fa 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -11,6 +11,7 @@ import ( "cmd/internal/sys" "cmd/link/internal/loader" "cmd/link/internal/sym" + "container/heap" "fmt" "unicode" ) @@ -23,8 +24,17 @@ var _ = fmt.Print type workQueue []loader.Sym -func (q *workQueue) push(i loader.Sym) { *q = append(*q, i) } -func (q *workQueue) pop() loader.Sym { i := (*q)[len(*q)-1]; *q = (*q)[:len(*q)-1]; return i } +// Implement container/heap.Interface. +func (q *workQueue) Len() int { return len(*q) } +func (q *workQueue) Less(i, j int) bool { return (*q)[i] < (*q)[j] } +func (q *workQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } +func (q *workQueue) Push(i interface{}) { *q = append(*q, i.(loader.Sym)) } +func (q *workQueue) Pop() interface{} { i := (*q)[len(*q)-1]; *q = (*q)[:len(*q)-1]; return i } + +// Functions for deadcode pass to use. +// Deadcode pass should call push/pop, not Push/Pop. +func (q *workQueue) push(i loader.Sym) { heap.Push(q, i) } +func (q *workQueue) pop() loader.Sym { return heap.Pop(q).(loader.Sym) } func (q *workQueue) empty() bool { return len(*q) == 0 } type deadcodePass2 struct { @@ -44,6 +54,7 @@ func (d *deadcodePass2) init() { if d.ctxt.Reachparent != nil { d.ldr.Reachparent = make([]loader.Sym, d.ldr.NSym()) } + heap.Init(&d.wq) if d.ctxt.BuildMode == BuildModeShared { // Mark all symbols defined in this library as reachable when From c6621d9241c1835806fbc85ca9893d90af035fc4 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 2 Nov 2019 10:24:33 -0400 Subject: [PATCH 75/79] [dev.link] cmd/link: restore -dumpdep in new deadcode pass Change-Id: I2e52206a95c9463df5661664726a8b3bbf3ad1fc Reviewed-on: https://go-review.googlesource.com/c/go/+/204825 Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode2.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode2.go b/src/cmd/link/internal/ld/deadcode2.go index a138bc97fa..cb6bb05492 100644 --- a/src/cmd/link/internal/ld/deadcode2.go +++ b/src/cmd/link/internal/ld/deadcode2.go @@ -18,10 +18,6 @@ import ( var _ = fmt.Print -// TODO: -// - Debug output: -// Emit messages about which symbols are kept or deleted. - type workQueue []loader.Sym // Implement container/heap.Interface. @@ -205,6 +201,16 @@ func (d *deadcodePass2) mark(symIdx, parent loader.Sym) { if d.ctxt.Reachparent != nil { d.ldr.Reachparent[symIdx] = parent } + if *flagDumpDep { + to := d.ldr.SymName(symIdx) + if to != "" { + from := "_" + if parent != 0 { + from = d.ldr.SymName(parent) + } + fmt.Printf("%s -> %s\n", from, to) + } + } } } From 32190b0a492f92871d39dd252c1d2a11cbfbadc8 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 2 Nov 2019 10:42:50 -0400 Subject: [PATCH 76/79] [dev.link] cmd/internal/dwarf: expand import path in function DIE Currently, at compile time we emit a function DIE with '"".' in the function name, and we expand it at link time, with a really ugly function. We can just expand it at compile time instead. This way, we don't need to modify the symbol content at link time, and also no need to allocate memory for that. Keep the linker expansion, in case the compiler is invoked without the import path. Change-Id: Id53cd2e2d3eb61efceb8d44479c4b6ef890baa43 Reviewed-on: https://go-review.googlesource.com/c/go/+/204826 Reviewed-by: Than McIntosh --- src/cmd/internal/dwarf/dwarf.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cmd/internal/dwarf/dwarf.go b/src/cmd/internal/dwarf/dwarf.go index 740b04f606..56b44a1ab5 100644 --- a/src/cmd/internal/dwarf/dwarf.go +++ b/src/cmd/internal/dwarf/dwarf.go @@ -1372,7 +1372,13 @@ func PutDefaultFunc(ctxt Context, s *FnState) error { abbrev := DW_ABRV_FUNCTION Uleb128put(ctxt, s.Info, int64(abbrev)) - putattr(ctxt, s.Info, DW_ABRV_FUNCTION, DW_FORM_string, DW_CLS_STRING, int64(len(s.Name)), s.Name) + // Expand '"".' to import path. + name := s.Name + if s.Importpath != "" { + name = strings.Replace(name, "\"\".", objabi.PathToPrefix(s.Importpath)+".", -1) + } + + putattr(ctxt, s.Info, DW_ABRV_FUNCTION, DW_FORM_string, DW_CLS_STRING, int64(len(name)), name) putattr(ctxt, s.Info, abbrev, DW_FORM_addr, DW_CLS_ADDRESS, 0, s.StartPC) putattr(ctxt, s.Info, abbrev, DW_FORM_addr, DW_CLS_ADDRESS, s.Size, s.StartPC) putattr(ctxt, s.Info, abbrev, DW_FORM_block1, DW_CLS_BLOCK, 1, []byte{DW_OP_call_frame_cfa}) From ab4a71fca7de1db1fd30e9a6c6d96fc84f8fffa4 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 2 Nov 2019 17:25:39 -0400 Subject: [PATCH 77/79] [dev.link] cmd/link: use the start of compilation unit for R_ADDRCUOFF R_ADDRCUOFF refers to the offset from the start of the compilation unit. Now that we could have multiple compilation units per package, we should use the start of the compilation unit, instead of the start of the package. Change-Id: I3d6a3c5a5c47c9ccf88091ed05413ced826f9934 Reviewed-on: https://go-review.googlesource.com/c/go/+/204828 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index ea238f7916..2cf4c726da 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -402,7 +402,7 @@ func relocsym(ctxt *Link, s *sym.Symbol) { case objabi.R_ADDRCUOFF: // debug_range and debug_loc elements use this relocation type to get an // offset from the start of the compile unit. - o = Symaddr(r.Sym) + r.Add - Symaddr(r.Sym.Unit.Lib.Textp[0]) + o = Symaddr(r.Sym) + r.Add - Symaddr(r.Sym.Unit.Textp[0]) // r->sym can be null when CALL $(constant) is transformed from absolute PC to relative PC call. case objabi.R_GOTPCREL: From db75205a9b8c4d630fef38aa88ddb66a9b63f487 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Sat, 2 Nov 2019 00:38:21 -0400 Subject: [PATCH 78/79] [dev.link] cmd/link: restore -strictdups flag in newobj mode Change-Id: I93ad769595fa343400afa342af12e1445abff084 Reviewed-on: https://go-review.googlesource.com/c/go/+/204918 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/lib.go | 15 ++++- src/cmd/link/internal/loader/loader.go | 81 +++++++++++++++++++++----- src/cmd/link/link_test.go | 65 +++++++++++++++++++++ 3 files changed, 147 insertions(+), 14 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 4c7451a114..39ed53e38c 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -378,7 +378,16 @@ func (ctxt *Link) findLibPath(libname string) string { func (ctxt *Link) loadlib() { if *flagNewobj { - ctxt.loader = loader.NewLoader() + var flags uint32 + switch *FlagStrictDups { + case 0: + // nothing to do + case 1, 2: + flags = loader.FlagStrictDups + default: + log.Fatalf("invalid -strictdups flag value %d", *FlagStrictDups) + } + ctxt.loader = loader.NewLoader(flags) } ctxt.cgo_export_static = make(map[string]bool) @@ -550,6 +559,10 @@ func (ctxt *Link) loadlib() { ctxt.Loaded = true importcycles() + + if *flagNewobj { + strictDupMsgCount = ctxt.loader.NStrictDupMsgs() + } } // Set up dynexp list. diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 46d93c5124..c0fa5fa7ce 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -119,9 +119,18 @@ type Loader struct { Reachparent []Sym relocBatch []sym.Reloc // for bulk allocation of relocations + + flags uint32 + + strictDupMsgs int // number of strict-dup warning/errors, when FlagStrictDups is enabled } -func NewLoader() *Loader { +const ( + // Loader.flags + FlagStrictDups = 1 << iota +) + +func NewLoader(flags uint32) *Loader { nbuiltin := goobj2.NBuiltin() return &Loader{ start: make(map[*oReader]Sym), @@ -132,6 +141,7 @@ func NewLoader() *Loader { itablink: make(map[Sym]struct{}), extStaticSyms: make(map[nameVer]Sym), builtinSyms: make([]Sym, nbuiltin), + flags: flags, } } @@ -170,6 +180,9 @@ func (l *Loader) AddSym(name string, ver int, i Sym, r *oReader, dupok bool, typ } if oldi, ok := l.symsByName[ver][name]; ok { if dupok { + if l.flags&FlagStrictDups != 0 { + l.checkdup(name, i, r, oldi) + } return false } oldr, li := l.toLocal(oldi) @@ -366,6 +379,42 @@ func (l *Loader) IsDup(i Sym) bool { return l.symsByName[ver][name] != i } +// Check that duplicate symbols have same contents. +func (l *Loader) checkdup(name string, i Sym, r *oReader, dup Sym) { + li := int(i - l.startIndex(r)) + p := r.Data(li) + if strings.HasPrefix(name, "go.info.") { + p, _ = patchDWARFName1(p, r) + } + rdup, ldup := l.toLocal(dup) + pdup := rdup.Data(ldup) + if strings.HasPrefix(name, "go.info.") { + pdup, _ = patchDWARFName1(pdup, rdup) + } + if bytes.Equal(p, pdup) { + return + } + reason := "same length but different contents" + if len(p) != len(pdup) { + reason = fmt.Sprintf("new length %d != old length %d", len(p), len(pdup)) + } + fmt.Fprintf(os.Stderr, "cmd/link: while reading object for '%v': duplicate symbol '%s', previous def at '%v', with mismatched payload: %s\n", r.unit.Lib, name, rdup.unit.Lib, reason) + + // For the moment, whitelist DWARF subprogram DIEs for + // auto-generated wrapper functions. What seems to happen + // here is that we get different line numbers on formal + // params; I am guessing that the pos is being inherited + // from the spot where the wrapper is needed. + whitelist := strings.HasPrefix(name, "go.info.go.interface") || + strings.HasPrefix(name, "go.info.go.builtin") || + strings.HasPrefix(name, "go.debuglines") + if !whitelist { + l.strictDupMsgs++ + } +} + +func (l *Loader) NStrictDupMsgs() int { return l.strictDupMsgs } + // Number of total symbols. func (l *Loader) NSym() int { return int(l.max + 1) @@ -1194,24 +1243,30 @@ func loadObjFull(l *Loader, r *oReader) { var emptyPkg = []byte(`"".`) -func patchDWARFName(s *sym.Symbol, r *oReader) { +func patchDWARFName1(p []byte, r *oReader) ([]byte, int) { // This is kind of ugly. Really the package name should not // even be included here. - if s.Size < 1 || s.P[0] != dwarf.DW_ABRV_FUNCTION { - return + if len(p) < 1 || p[0] != dwarf.DW_ABRV_FUNCTION { + return p, -1 } - e := bytes.IndexByte(s.P, 0) + e := bytes.IndexByte(p, 0) + if e == -1 { + return p, -1 + } + if !bytes.Contains(p[:e], emptyPkg) { + return p, -1 + } + pkgprefix := []byte(r.pkgprefix) + patched := bytes.Replace(p[:e], emptyPkg, pkgprefix, -1) + return append(patched, p[e:]...), e +} + +func patchDWARFName(s *sym.Symbol, r *oReader) { + patched, e := patchDWARFName1(s.P, r) if e == -1 { return } - p := bytes.Index(s.P[:e], emptyPkg) - if p == -1 { - return - } - pkgprefix := []byte(r.pkgprefix) - patched := bytes.Replace(s.P[:e], emptyPkg, pkgprefix, -1) - - s.P = append(patched, s.P[e:]...) + s.P = patched s.Attr.Set(sym.AttrReadOnly, false) delta := int64(len(s.P)) - s.Size s.Size = int64(len(s.P)) diff --git a/src/cmd/link/link_test.go b/src/cmd/link/link_test.go index 155fd8bce3..92830fe8b3 100644 --- a/src/cmd/link/link_test.go +++ b/src/cmd/link/link_test.go @@ -376,3 +376,68 @@ func TestIssue34788Android386TLSSequence(t *testing.T) { } } } + +const testStrictDupGoSrc = ` +package main +func f() +func main() { f() } +` + +const testStrictDupAsmSrc1 = ` +#include "textflag.h" +TEXT ·f(SB), NOSPLIT|DUPOK, $0-0 + RET +` + +const testStrictDupAsmSrc2 = ` +#include "textflag.h" +TEXT ·f(SB), NOSPLIT|DUPOK, $0-0 + JMP 0(PC) +` + +func TestStrictDup(t *testing.T) { + // Check that -strictdups flag works. + testenv.MustHaveGoBuild(t) + + tmpdir, err := ioutil.TempDir("", "TestStrictDup") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + src := filepath.Join(tmpdir, "x.go") + err = ioutil.WriteFile(src, []byte(testStrictDupGoSrc), 0666) + if err != nil { + t.Fatal(err) + } + src = filepath.Join(tmpdir, "a.s") + err = ioutil.WriteFile(src, []byte(testStrictDupAsmSrc1), 0666) + if err != nil { + t.Fatal(err) + } + src = filepath.Join(tmpdir, "b.s") + err = ioutil.WriteFile(src, []byte(testStrictDupAsmSrc2), 0666) + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(testenv.GoToolPath(t), "build", "-ldflags=-strictdups=1") + cmd.Dir = tmpdir + out, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("linking with -strictdups=1 failed: %v", err) + } + if !bytes.Contains(out, []byte("mismatched payload")) { + t.Errorf("unexpected output:\n%s", out) + } + + cmd = exec.Command(testenv.GoToolPath(t), "build", "-ldflags=-strictdups=2") + cmd.Dir = tmpdir + out, err = cmd.CombinedOutput() + if err == nil { + t.Errorf("linking with -strictdups=2 did not fail") + } + if !bytes.Contains(out, []byte("mismatched payload")) { + t.Errorf("unexpected output:\n%s", out) + } +} From 9cf6c65ca302ef5300ec970640dfa446d45ac0b8 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 31 Oct 2019 16:11:20 -0400 Subject: [PATCH 79/79] [dev.link] cmd: default to old object file format Flip back to the old object files for Go 1.14. Change-Id: I4ad499460fb7156b63fc63e9c6ea4f7099e20af2 Reviewed-on: https://go-review.googlesource.com/c/go/+/204098 Run-TryBot: Cherry Zhang Reviewed-by: Than McIntosh TryBot-Result: Gobot Gobot --- src/cmd/asm/internal/flags/flags.go | 2 +- src/cmd/compile/internal/gc/main.go | 2 +- src/cmd/link/internal/ld/main.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmd/asm/internal/flags/flags.go b/src/cmd/asm/internal/flags/flags.go index 95575e15a3..fad87b221a 100644 --- a/src/cmd/asm/internal/flags/flags.go +++ b/src/cmd/asm/internal/flags/flags.go @@ -23,7 +23,7 @@ var ( Dynlink = flag.Bool("dynlink", false, "support references to Go symbols defined in other shared libraries") AllErrors = flag.Bool("e", false, "no limit on number of errors reported") SymABIs = flag.Bool("gensymabis", false, "write symbol ABI information to output file, don't assemble") - Newobj = flag.Bool("newobj", true, "use new object file format") + Newobj = flag.Bool("newobj", false, "use new object file format") ) var ( diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 4684dfb88f..ed8cfeb40f 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -273,7 +273,7 @@ func Main(archInit func(*Arch)) { flag.StringVar(&benchfile, "bench", "", "append benchmark times to `file`") flag.BoolVar(&smallFrames, "smallframes", false, "reduce the size limit for stack allocated objects") flag.BoolVar(&Ctxt.UseBASEntries, "dwarfbasentries", Ctxt.UseBASEntries, "use base address selection entries in DWARF") - flag.BoolVar(&Ctxt.Flag_newobj, "newobj", true, "use new object file format") + flag.BoolVar(&Ctxt.Flag_newobj, "newobj", false, "use new object file format") objabi.Flagparse(usage) diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 10717f603d..e42705e3ef 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -86,7 +86,7 @@ var ( flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker") FlagDebugTramp = flag.Int("debugtramp", 0, "debug trampolines") FlagStrictDups = flag.Int("strictdups", 0, "sanity check duplicate symbol contents during object file reading (1=warn 2=err).") - flagNewobj = flag.Bool("newobj", true, "use new object file format") + flagNewobj = flag.Bool("newobj", false, "use new object file format") FlagRound = flag.Int("R", -1, "set address rounding `quantum`") FlagTextAddr = flag.Int64("T", -1, "set text segment `address`")