From 799da9cee79e7ee89156ddc3c8bea44fdf1ac252 Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Wed, 29 Oct 2014 08:15:58 -0700 Subject: [PATCH 01/59] doc/go1.4.html: half of the small library changes LGTM=rsc R=golang-codereviews, rsc CC=golang-codereviews https://golang.org/cl/165090043 --- doc/go1.4.html | 170 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 137 insertions(+), 33 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 7e670c47cb..9fa86c31ae 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -487,6 +487,16 @@ need to know about the new location. All tools and services maintained by the Go have been updated.

+ +

SWIG

+ +

+Due to the runtime changes in this release, Go 1.4 will require SWIG 3.0.3. +At time of writing that has not yet been released, but we expect it to be by +Go 1.4's release date. +TODO +

+

Miscellany

@@ -549,7 +559,7 @@ TODO major changes 

-encoding/gob: remove unsafe (CL 102680045)
+bufio: handling of empty tokens at EOF changed, may require scanner change (CL 145390043)
 syscall: now frozen (CL 129820043); go.sys subrepo created: http://golang.org/s/go1.4-syscall
@@ -562,37 +572,131 @@ See the relevant package documentation for more information about each change. - -
-
-cmd/6l, liblink: use pc-relative addressing for all memory references, so that linking Go binaries at high addresses works (CL 125140043). This cuts the maximum size of a Go binary's text+data+bss from 4GB to 2GB.
-
-bufio: handling of empty tokens at EOF changed, may require scanner change (CL 145390043)
-compress/flate, compress/gzip, compress/zlib: Reset support (https://codereview.appspot.com/97140043)
-crypto/tls: add support for ALPN (RFC 7301) (CL 108710046)
-crypto/tls: support programmatic selection of server certificates (CL 107400043)
-encoding/asn1: optional elements with a default value will now only be omitted if they have that value (CL 86960045)
-fmt: print type *map[T]T as &map[k:v] (CL 154870043)
-encoding/csv: do not quote empty strings, quote \. (CL 164760043)
-net/http: add Request.BasicAuth method (CL 76540043)
-net/http: add Transport.DialTLS hook (CL 137940043)
-net/http/httputil: add ReverseProxy.ErrorLog (CL 132750043)
-os: implement symlink support for windows (CL 86160044)
-reflect: add type.Comparable (CL 144020043)
-reflect: Value is one word smaller
-runtime: implement monotonic clocks on windows (CL 108700045)
-runtime: MemStats.Mallocs now counts very small allocations missed in Go 1.3. This may break tests using runtime.ReadMemStats or testing.AllocsPerRun by giving a more accurate answer than Go 1.3 did (CL 143150043).
-runtime/race: freebsd is supported (CL 107270043)
-runtime: add PauseEnd array to MemStats and GCStats (CL 153670043)
-swig: Due to runtime changes Go 1.4 will require SWIG 3.0.3 (not yet released)
-sync/atomic: add Value (CL 136710045)
-syscall: Setuid, Setgid are disabled on linux platforms. On linux those syscalls operate on the calling thread, not the whole process. This does not match the semantics of other platforms, nor the expectations of the caller, so the operations have been disabled until issue 1435 is resolved (CL 106170043)
-testing: add Coverage (CL 98150043)
-testing: add TestMain support (CL 148770043)
-text/scanner: add IsIdentRune field of Scanner. (CL 108030044)
-text/template: allow comparison of signed and unsigned integers (CL 149780043)
-time: use the micro symbol (µ (U+00B5)) to print microsecond duration (CL 105030046)
-
From 8db71d4ee89a505c375b550eb8fb8cc33bbabc03 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 29 Oct 2014 15:14:04 -0400 Subject: [PATCH 02/59] runtime: update comment for Callers Attempt to clear up confusion about how to turn the PCs reported by Callers into the file and line number people actually want. Fixes #7690. LGTM=r, chris.cs.guy R=r, chris.cs.guy CC=golang-codereviews https://golang.org/cl/163550043 --- src/runtime/extern.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 1b8052bb56..6cc5df810c 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -117,11 +117,20 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) { return } -// Callers fills the slice pc with the program counters of function invocations +// Callers fills the slice pc with the return program counters of function invocations // on the calling goroutine's stack. The argument skip is the number of stack frames // to skip before recording in pc, with 0 identifying the frame for Callers itself and // 1 identifying the caller of Callers. // It returns the number of entries written to pc. +// +// Note that since each slice entry pc[i] is a return program counter, +// looking up the file and line for pc[i] (for example, using (*Func).FileLine) +// will return the file and line number of the instruction immediately +// following the call. +// To look up the file and line number of the call itself, use pc[i]-1. +// As an exception to this rule, if pc[i-1] corresponds to the function +// runtime.sigpanic, then pc[i] is the program counter of a faulting +// instruction and should be used without any subtraction. func Callers(skip int, pc []uintptr) int { // runtime.callers uses pc.array==nil as a signal // to print a stack trace. Pick off 0-length pc here From a22c11b9957cf3f0d66dd6d1d38172d5ac0ec54a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 29 Oct 2014 15:14:24 -0400 Subject: [PATCH 03/59] runtime: fix line number in first stack frame in printed stack trace Originally traceback was only used for printing the stack when an unexpected signal came in. In that case, the initial PC is taken from the signal and should be used unaltered. For the callers, the PC is the return address, which might be on the line after the call; we subtract 1 to get to the CALL instruction. Traceback is now used for a variety of things, and for almost all of those the initial PC is a return address, whether from getcallerpc, or gp->sched.pc, or gp->syscallpc. In those cases, we need to subtract 1 from this initial PC, but the traceback code had a hard rule "never subtract 1 from the initial PC", left over from the signal handling days. Change gentraceback to take a flag that specifies whether we are tracing a trap. Change traceback to default to "starting with a return PC", which is the overwhelmingly common case. Add tracebacktrap, like traceback but starting with a trap PC. Use tracebacktrap in signal handlers. Fixes #7690. LGTM=iant, r R=r, iant CC=golang-codereviews https://golang.org/cl/167810044 --- src/runtime/heapdump.c | 2 +- src/runtime/mgc0.c | 4 +-- src/runtime/mprof.go | 2 +- src/runtime/os_plan9_386.c | 2 +- src/runtime/os_plan9_amd64.c | 2 +- src/runtime/os_windows_386.c | 2 +- src/runtime/os_windows_amd64.c | 2 +- src/runtime/proc.c | 6 ++--- src/runtime/runtime.h | 8 +++++- src/runtime/signal_386.c | 2 +- src/runtime/signal_amd64x.c | 2 +- src/runtime/signal_arm.c | 2 +- src/runtime/stack.c | 2 +- src/runtime/traceback.go | 33 ++++++++++++++++------- test/fixedbugs/issue7690.go | 49 ++++++++++++++++++++++++++++++++++ 15 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 test/fixedbugs/issue7690.go diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c index 71da419f15..94a4bd2be5 100644 --- a/src/runtime/heapdump.c +++ b/src/runtime/heapdump.c @@ -413,7 +413,7 @@ dumpgoroutine(G *gp) child.sp = nil; child.depth = 0; fn = dumpframe; - runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, &fn, &child, false); + runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, &fn, &child, 0); // dump defer & panic records for(d = gp->defer; d != nil; d = d->link) { diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c index 1b41bf9a79..7754bad89d 100644 --- a/src/runtime/mgc0.c +++ b/src/runtime/mgc0.c @@ -774,7 +774,7 @@ scanstack(G *gp) runtime·throw("can't scan gchelper stack"); fn = scanframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &fn, nil, false); + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &fn, nil, 0); runtime·tracebackdefers(gp, &fn, nil); } @@ -1964,7 +1964,7 @@ runtime·getgcmask(byte *p, Type *t, byte **mask, uintptr *len) frame.fn = nil; frame.sp = (uintptr)p; cb = getgcmaskcb; - runtime·gentraceback(g->m->curg->sched.pc, g->m->curg->sched.sp, 0, g->m->curg, 0, nil, 1000, &cb, &frame, false); + runtime·gentraceback(g->m->curg->sched.pc, g->m->curg->sched.sp, 0, g->m->curg, 0, nil, 1000, &cb, &frame, 0); if(frame.fn != nil) { Func *f; StackMap *stackmap; diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index 803da56670..d64e3be695 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -562,7 +562,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { } func saveg(pc, sp uintptr, gp *g, r *StackRecord) { - n := gentraceback(pc, sp, 0, gp, 0, &r.Stack0[0], len(r.Stack0), nil, nil, false) + n := gentraceback(pc, sp, 0, gp, 0, &r.Stack0[0], len(r.Stack0), nil, nil, 0) if n < len(r.Stack0) { r.Stack0[n] = 0 } diff --git a/src/runtime/os_plan9_386.c b/src/runtime/os_plan9_386.c index 3490862244..42c6d161c7 100644 --- a/src/runtime/os_plan9_386.c +++ b/src/runtime/os_plan9_386.c @@ -114,7 +114,7 @@ Throw: if(runtime·gotraceback(&crash)) { runtime·goroutineheader(gp); - runtime·traceback(ureg->pc, ureg->sp, 0, gp); + runtime·tracebacktrap(ureg->pc, ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(ureg); diff --git a/src/runtime/os_plan9_amd64.c b/src/runtime/os_plan9_amd64.c index 6b0f8ae3a2..a9dc0eb966 100644 --- a/src/runtime/os_plan9_amd64.c +++ b/src/runtime/os_plan9_amd64.c @@ -122,7 +122,7 @@ Throw: if(runtime·gotraceback(&crash)) { runtime·goroutineheader(gp); - runtime·traceback(ureg->ip, ureg->sp, 0, gp); + runtime·tracebacktrap(ureg->ip, ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(ureg); diff --git a/src/runtime/os_windows_386.c b/src/runtime/os_windows_386.c index 213582799b..9962f0dc2e 100644 --- a/src/runtime/os_windows_386.c +++ b/src/runtime/os_windows_386.c @@ -97,7 +97,7 @@ runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) runtime·printf("\n"); if(runtime·gotraceback(&crash)){ - runtime·traceback(r->Eip, r->Esp, 0, gp); + runtime·tracebacktrap(r->Eip, r->Esp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } diff --git a/src/runtime/os_windows_amd64.c b/src/runtime/os_windows_amd64.c index b96cf70d1e..e4617e4cef 100644 --- a/src/runtime/os_windows_amd64.c +++ b/src/runtime/os_windows_amd64.c @@ -119,7 +119,7 @@ runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) runtime·printf("\n"); if(runtime·gotraceback(&crash)){ - runtime·traceback(r->Rip, r->Rsp, 0, gp); + runtime·tracebacktrap(r->Rip, r->Rsp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } diff --git a/src/runtime/proc.c b/src/runtime/proc.c index 52f7ef3a5b..b46f67065a 100644 --- a/src/runtime/proc.c +++ b/src/runtime/proc.c @@ -2532,7 +2532,7 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) n = 0; if(traceback) - n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, stk, nelem(stk), nil, nil, TraceTrap); if(!traceback || n <= 0) { // Normal traceback is impossible or has failed. // See if it falls into several common cases. @@ -2542,13 +2542,13 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) // Cgo, we can't unwind and symbolize arbitrary C code, // so instead collect Go stack that leads to the cgo call. // This is especially important on windows, since all syscalls are cgo calls. - n = runtime·gentraceback(mp->curg->syscallpc, mp->curg->syscallsp, 0, mp->curg, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback(mp->curg->syscallpc, mp->curg->syscallsp, 0, mp->curg, 0, stk, nelem(stk), nil, nil, 0); } #ifdef GOOS_windows if(n == 0 && mp->libcallg != nil && mp->libcallpc != 0 && mp->libcallsp != 0) { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. - n = runtime·gentraceback(mp->libcallpc, mp->libcallsp, 0, mp->libcallg, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback(mp->libcallpc, mp->libcallsp, 0, mp->libcallg, 0, stk, nelem(stk), nil, nil, 0); } #endif if(n == 0) { diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 2a60740063..977c4547df 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -719,9 +719,15 @@ struct Stkframe BitVector* argmap; // force use of this argmap }; -intgo runtime·gentraceback(uintptr, uintptr, uintptr, G*, intgo, uintptr*, intgo, bool(**)(Stkframe*, void*), void*, bool); +enum +{ + TraceRuntimeFrames = 1<<0, // include frames for internal runtime functions. + TraceTrap = 1<<1, // the initial PC, SP are from a trap, not a return PC from a call +}; +intgo runtime·gentraceback(uintptr, uintptr, uintptr, G*, intgo, uintptr*, intgo, bool(**)(Stkframe*, void*), void*, uintgo); void runtime·tracebackdefers(G*, bool(**)(Stkframe*, void*), void*); void runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G* gp); +void runtime·tracebacktrap(uintptr pc, uintptr sp, uintptr lr, G* gp); void runtime·tracebackothers(G*); bool runtime·haszeroargs(uintptr pc); bool runtime·topofstack(Func*); diff --git a/src/runtime/signal_386.c b/src/runtime/signal_386.c index d55e304528..30a7488bd7 100644 --- a/src/runtime/signal_386.c +++ b/src/runtime/signal_386.c @@ -109,7 +109,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_EIP(info, ctxt), SIG_ESP(info, ctxt), 0, gp); + runtime·tracebacktrap(SIG_EIP(info, ctxt), SIG_ESP(info, ctxt), 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/signal_amd64x.c b/src/runtime/signal_amd64x.c index 44e68cecfc..feb4afcce3 100644 --- a/src/runtime/signal_amd64x.c +++ b/src/runtime/signal_amd64x.c @@ -143,7 +143,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_RIP(info, ctxt), SIG_RSP(info, ctxt), 0, gp); + runtime·tracebacktrap(SIG_RIP(info, ctxt), SIG_RSP(info, ctxt), 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/signal_arm.c b/src/runtime/signal_arm.c index 3571cf3ac6..afad5e7d16 100644 --- a/src/runtime/signal_arm.c +++ b/src/runtime/signal_arm.c @@ -108,7 +108,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_PC(info, ctxt), SIG_SP(info, ctxt), SIG_LR(info, ctxt), gp); + runtime·tracebacktrap(SIG_PC(info, ctxt), SIG_SP(info, ctxt), SIG_LR(info, ctxt), gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index ed8f4f8727..072bc242bc 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -620,7 +620,7 @@ copystack(G *gp, uintptr newsize) adjinfo.old = old; adjinfo.delta = new.hi - old.hi; cb = adjustframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, false); + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, 0); // adjust other miscellaneous things that have pointers into stacks. adjustctxt(gp, &adjinfo); diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 24dc3eea95..834435b400 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -96,7 +96,7 @@ func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v uns // the runtime.Callers function (pcbuf != nil), as well as the garbage // collector (callback != nil). A little clunky to merge these, but avoids // duplicating the code and all its subtlety. -func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max int, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer, printall bool) int { +func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max int, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer, flags uint) int { if goexitPC == 0 { gothrow("gentraceback before goexitPC initialization") } @@ -297,13 +297,13 @@ func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf } } if printing { - if printall || showframe(f, gp) { + if (flags&_TraceRuntimeFrames) != 0 || showframe(f, gp) { // Print during crash. // main(0x1, 0x2, 0x3) // /home/rsc/go/src/runtime/x.go:23 +0xf // tracepc := frame.pc // back up to CALL instruction for funcline. - if n > 0 && frame.pc > f.entry && !waspanic { + if (n > 0 || flags&_TraceTrap == 0) && frame.pc > f.entry && !waspanic { tracepc-- } print(gofuncname(f), "(") @@ -475,17 +475,32 @@ func printcreatedby(gp *g) { } func traceback(pc uintptr, sp uintptr, lr uintptr, gp *g) { + traceback1(pc, sp, lr, gp, 0) +} + +// tracebacktrap is like traceback but expects that the PC and SP were obtained +// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or getcallerpc/getcallersp. +// Because they are from a trap instead of from a saved pair, +// the initial PC must not be rewound to the previous instruction. +// (All the saved pairs record a PC that is a return address, so we +// rewind it into the CALL instruction.) +func tracebacktrap(pc uintptr, sp uintptr, lr uintptr, gp *g) { + traceback1(pc, sp, lr, gp, _TraceTrap) +} + +func traceback1(pc uintptr, sp uintptr, lr uintptr, gp *g, flags uint) { var n int if readgstatus(gp)&^_Gscan == _Gsyscall { - // Override signal registers if blocked in system call. + // Override registers if blocked in system call. pc = gp.syscallpc sp = gp.syscallsp + flags &^= _TraceTrap } // Print traceback. By default, omits runtime frames. // If that means we print nothing at all, repeat forcing all frames printed. - n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, false) - if n == 0 { - n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, true) + n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, flags) + if n == 0 && (flags&_TraceRuntimeFrames) == 0 { + n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, flags|_TraceRuntimeFrames) } if n == _TracebackMaxFrames { print("...additional frames elided...\n") @@ -496,11 +511,11 @@ func traceback(pc uintptr, sp uintptr, lr uintptr, gp *g) { func callers(skip int, pcbuf *uintptr, m int) int { sp := getcallersp(unsafe.Pointer(&skip)) pc := uintptr(getcallerpc(unsafe.Pointer(&skip))) - return gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, false) + return gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) } func gcallers(gp *g, skip int, pcbuf *uintptr, m int) int { - return gentraceback(^uintptr(0), ^uintptr(0), 0, gp, skip, pcbuf, m, nil, nil, false) + return gentraceback(^uintptr(0), ^uintptr(0), 0, gp, skip, pcbuf, m, nil, nil, 0) } func showframe(f *_func, gp *g) bool { diff --git a/test/fixedbugs/issue7690.go b/test/fixedbugs/issue7690.go new file mode 100644 index 0000000000..4ad9e8622a --- /dev/null +++ b/test/fixedbugs/issue7690.go @@ -0,0 +1,49 @@ +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// issue 7690 - Stack and other routines did not back up initial PC +// into CALL instruction, instead reporting line number of next instruction, +// which might be on a different line. + +package main + +import ( + "bytes" + "regexp" + "runtime" + "strconv" +) + +func main() { + buf1 := make([]byte, 1000) + buf2 := make([]byte, 1000) + + runtime.Stack(buf1, false) // CALL is last instruction on this line + n := runtime.Stack(buf2, false) // CALL is followed by load of result from stack + + buf1 = buf1[:bytes.IndexByte(buf1, 0)] + buf2 = buf2[:n] + + re := regexp.MustCompile(`(?m)^main\.main\(\)\n.*/issue7690.go:([0-9]+)`) + m1 := re.FindStringSubmatch(string(buf1)) + if m1 == nil { + println("BUG: cannot find main.main in first trace") + return + } + m2 := re.FindStringSubmatch(string(buf2)) + if m2 == nil { + println("BUG: cannot find main.main in second trace") + return + } + + n1, _ := strconv.Atoi(m1[1]) + n2, _ := strconv.Atoi(m2[1]) + if n1+1 != n2 { + println("BUG: expect runtime.Stack on back to back lines, have", n1, n2) + println(string(buf1)) + println(string(buf2)) + } +} From 97b24a05dff7adef3a0fb463a575b705be985468 Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Wed, 29 Oct 2014 13:07:34 -0700 Subject: [PATCH 04/59] doc/go1.4.html: gccgo status LGTM=iant, cmang R=cmang, iant, rsc CC=golang-codereviews https://golang.org/cl/169760043 --- doc/go1.4.html | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 9fa86c31ae..e2458f2efe 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -18,8 +18,7 @@ Stacks are now contiguous, reallocated when necessary rather than linking on new "segments"; this release therefore eliminates the notorious "hot stack split" problem. There are some new tools available including support in the go command -for build-time source code generation -and TODO. +for build-time source code generation. The release also adds support for ARM processors on Android and Native Client (NaCl) and AMD64 on Plan 9. As always, Go 1.4 keeps the promise @@ -281,7 +280,9 @@ More information about these changes is in the assembly docum

Status of gccgo

-TODO gccgo news +The release schedules for the GCC and Go projects do not coincide. +GCC release 4.9 contains the Go 1.2 version of gccgo. +The next release, GCC 5, will likely have the Go 1.4 version of gccgo.

Internal packages

@@ -593,13 +594,11 @@ of the Config struct.
  • -Also in the crypto/tls package, -the server now supports +Also in the crypto/tls package, the server now supports TLS_FALLBACK_SCSV -to help clients detect fallback attacks like -POODLE. -(The crypto/tls package's client has never supported SSLv3, so it is not -vulnerable to the POODLE attack.) +to help clients detect fallback attacks. +(The Go client does not support fallback at all, so it is not vulnerable to +those attacks.)
  • From 3eadbb02afa1494821de000ee280e00c3c398f1d Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 29 Oct 2014 18:07:24 -0400 Subject: [PATCH 05/59] cmd/objdump: use cmd/internal/objfile This removes a bunch of ugly duplicate code. The end goal is to factor the disassembly code into cmd/internal/objfile too, so that pprof can use it, but one step at a time. LGTM=r, iant R=r, alex.brainman, iant CC=golang-codereviews https://golang.org/cl/149400043 --- src/cmd/internal/objfile/elf.go | 25 +++ src/cmd/internal/objfile/goobj.go | 12 ++ src/cmd/internal/objfile/macho.go | 24 +++ src/cmd/internal/objfile/objfile.go | 10 + src/cmd/internal/objfile/pe.go | 31 +++ src/cmd/internal/objfile/plan9obj.go | 22 +++ src/cmd/objdump/Makefile | 10 - src/cmd/objdump/elf.go | 65 ------- src/cmd/objdump/macho.go | 77 -------- src/cmd/objdump/main.go | 277 +++------------------------ src/cmd/objdump/pe.go | 99 ---------- src/cmd/objdump/plan9obj.go | 70 ------- 12 files changed, 154 insertions(+), 568 deletions(-) delete mode 100644 src/cmd/objdump/Makefile delete mode 100644 src/cmd/objdump/elf.go delete mode 100644 src/cmd/objdump/macho.go delete mode 100644 src/cmd/objdump/pe.go delete mode 100644 src/cmd/objdump/plan9obj.go diff --git a/src/cmd/internal/objfile/elf.go b/src/cmd/internal/objfile/elf.go index 8495fa7532..17755b84d2 100644 --- a/src/cmd/internal/objfile/elf.go +++ b/src/cmd/internal/objfile/elf.go @@ -8,6 +8,7 @@ package objfile import ( "debug/elf" + "fmt" "os" ) @@ -77,3 +78,27 @@ func (f *elfFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { } return textStart, symtab, pclntab, nil } + +func (f *elfFile) text() (textStart uint64, text []byte, err error) { + sect := f.elf.Section(".text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = sect.Addr + text, err = sect.Data() + return +} + +func (f *elfFile) goarch() string { + switch f.elf.Machine { + case elf.EM_386: + return "386" + case elf.EM_X86_64: + return "amd64" + case elf.EM_ARM: + return "arm" + case elf.EM_PPC64: + return "power64" + } + return "" +} diff --git a/src/cmd/internal/objfile/goobj.go b/src/cmd/internal/objfile/goobj.go index a4f49ebe44..a1d773023d 100644 --- a/src/cmd/internal/objfile/goobj.go +++ b/src/cmd/internal/objfile/goobj.go @@ -79,3 +79,15 @@ func (f *goobjFile) symbols() ([]Sym, error) { func (f *goobjFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { return 0, nil, nil, fmt.Errorf("pcln not available in go object file") } + +// text does not make sense for Go object files, because +// each function has a separate section. +func (f *goobjFile) text() (textStart uint64, text []byte, err error) { + return 0, nil, fmt.Errorf("text not available in go object file") +} + +// goarch makes sense but is not exposed in debug/goobj's API, +// and we don't need it yet for any users of internal/objfile. +func (f *goobjFile) goarch() string { + return "GOARCH unimplemented for debug/goobj files" +} diff --git a/src/cmd/internal/objfile/macho.go b/src/cmd/internal/objfile/macho.go index f845792ffa..7dd84a339d 100644 --- a/src/cmd/internal/objfile/macho.go +++ b/src/cmd/internal/objfile/macho.go @@ -85,6 +85,30 @@ func (f *machoFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) return textStart, symtab, pclntab, nil } +func (f *machoFile) text() (textStart uint64, text []byte, err error) { + sect := f.macho.Section("__text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = sect.Addr + text, err = sect.Data() + return +} + +func (f *machoFile) goarch() string { + switch f.macho.Cpu { + case macho.Cpu386: + return "386" + case macho.CpuAmd64: + return "amd64" + case macho.CpuArm: + return "arm" + case macho.CpuPpc64: + return "power64" + } + return "" +} + type uint64s []uint64 func (x uint64s) Len() int { return len(x) } diff --git a/src/cmd/internal/objfile/objfile.go b/src/cmd/internal/objfile/objfile.go index 09fa63e60b..3d4a5d27cd 100644 --- a/src/cmd/internal/objfile/objfile.go +++ b/src/cmd/internal/objfile/objfile.go @@ -14,6 +14,8 @@ import ( type rawFile interface { symbols() (syms []Sym, err error) pcln() (textStart uint64, symtab, pclntab []byte, err error) + text() (textStart uint64, text []byte, err error) + goarch() string } // A File is an opened executable file. @@ -70,3 +72,11 @@ func (f *File) PCLineTable() (*gosym.Table, error) { } return gosym.NewTable(symtab, gosym.NewLineTable(pclntab, textStart)) } + +func (f *File) Text() (uint64, []byte, error) { + return f.raw.text() +} + +func (f *File) GOARCH() string { + return f.raw.goarch() +} diff --git a/src/cmd/internal/objfile/pe.go b/src/cmd/internal/objfile/pe.go index 868709eaf9..67e59c226b 100644 --- a/src/cmd/internal/objfile/pe.go +++ b/src/cmd/internal/objfile/pe.go @@ -133,6 +133,25 @@ func (f *peFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { return textStart, symtab, pclntab, nil } +func (f *peFile) text() (textStart uint64, text []byte, err error) { + var imageBase uint64 + switch oh := f.pe.OptionalHeader.(type) { + case *pe.OptionalHeader32: + imageBase = uint64(oh.ImageBase) + case *pe.OptionalHeader64: + imageBase = oh.ImageBase + default: + return 0, nil, fmt.Errorf("pe file format not recognized") + } + sect := f.pe.Section(".text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = imageBase + uint64(sect.VirtualAddress) + text, err = sect.Data() + return +} + func findPESymbol(f *pe.File, name string) (*pe.Symbol, error) { for _, s := range f.Symbols { if s.Name != name { @@ -168,3 +187,15 @@ func loadPETable(f *pe.File, sname, ename string) ([]byte, error) { } return data[ssym.Value:esym.Value], nil } + +func (f *peFile) goarch() string { + // Not sure how to get the info we want from PE header. + // Look in symbol table for telltale rt0 symbol. + if _, err := findPESymbol(f.pe, "_rt0_386_windows"); err == nil { + return "386" + } + if _, err := findPESymbol(f.pe, "_rt0_amd64_windows"); err == nil { + return "amd64" + } + return "" +} diff --git a/src/cmd/internal/objfile/plan9obj.go b/src/cmd/internal/objfile/plan9obj.go index 80744f82a8..eb6cba5eb1 100644 --- a/src/cmd/internal/objfile/plan9obj.go +++ b/src/cmd/internal/objfile/plan9obj.go @@ -88,6 +88,16 @@ func (f *plan9File) pcln() (textStart uint64, symtab, pclntab []byte, err error) return textStart, symtab, pclntab, nil } +func (f *plan9File) text() (textStart uint64, text []byte, err error) { + sect := f.plan9.Section("text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = f.plan9.LoadAddress + f.plan9.HdrSize + text, err = sect.Data() + return +} + func findPlan9Symbol(f *plan9obj.File, name string) (*plan9obj.Sym, error) { syms, err := f.Symbols() if err != nil { @@ -122,3 +132,15 @@ func loadPlan9Table(f *plan9obj.File, sname, ename string) ([]byte, error) { textStart := f.LoadAddress + f.HdrSize return data[ssym.Value-textStart : esym.Value-textStart], nil } + +func (f *plan9File) goarch() string { + switch f.plan9.Magic { + case plan9obj.Magic386: + return "386" + case plan9obj.MagicAMD64: + return "amd64" + case plan9obj.MagicARM: + return "arm" + } + return "" +} diff --git a/src/cmd/objdump/Makefile b/src/cmd/objdump/Makefile deleted file mode 100644 index 1b66c26bab..0000000000 --- a/src/cmd/objdump/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -all: x86.go armasm.go - -x86.go: bundle - ./bundle -p main -x x86_ rsc.io/x86/x86asm | gofmt >x86.go - -armasm.go: bundle - ./bundle -p main -x arm_ rsc.io/arm/armasm | gofmt >armasm.go - -bundle: - go build -o bundle code.google.com/p/rsc/cmd/bundle diff --git a/src/cmd/objdump/elf.go b/src/cmd/objdump/elf.go deleted file mode 100644 index 906e903532..0000000000 --- a/src/cmd/objdump/elf.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of ELF executables (Linux, FreeBSD, and so on). - -package main - -import ( - "debug/elf" - "os" -) - -func elfSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := elf.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - elfSyms, err := p.Symbols() - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - switch p.Machine { - case elf.EM_X86_64: - goarch = "amd64" - case elf.EM_386: - goarch = "386" - case elf.EM_ARM: - goarch = "arm" - } - - for _, s := range elfSyms { - sym := Sym{Addr: s.Value, Name: s.Name, Size: int64(s.Size), Code: '?'} - switch s.Section { - case elf.SHN_UNDEF: - sym.Code = 'U' - case elf.SHN_COMMON: - sym.Code = 'B' - default: - i := int(s.Section) - if i < 0 || i >= len(p.Sections) { - break - } - sect := p.Sections[i] - switch sect.Flags & (elf.SHF_WRITE | elf.SHF_ALLOC | elf.SHF_EXECINSTR) { - case elf.SHF_ALLOC | elf.SHF_EXECINSTR: - sym.Code = 'T' - case elf.SHF_ALLOC: - sym.Code = 'R' - case elf.SHF_ALLOC | elf.SHF_WRITE: - sym.Code = 'D' - } - } - if elf.ST_BIND(s.Info) == elf.STB_LOCAL { - sym.Code += 'a' - 'A' - } - syms = append(syms, sym) - } - - return -} diff --git a/src/cmd/objdump/macho.go b/src/cmd/objdump/macho.go deleted file mode 100644 index 6e0ad223d4..0000000000 --- a/src/cmd/objdump/macho.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of Mach-O executables (OS X). - -package main - -import ( - "debug/macho" - "os" - "sort" -) - -func machoSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := macho.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - if p.Symtab == nil { - errorf("%s: no symbol table", f.Name()) - return - } - - switch p.Cpu { - case macho.Cpu386: - goarch = "386" - case macho.CpuAmd64: - goarch = "amd64" - case macho.CpuArm: - goarch = "arm" - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - for _, s := range p.Symtab.Syms { - addrs = append(addrs, s.Value) - } - sort.Sort(uint64s(addrs)) - - for _, s := range p.Symtab.Syms { - sym := Sym{Name: s.Name, Addr: s.Value, Code: '?'} - i := sort.Search(len(addrs), func(x int) bool { return addrs[x] > s.Value }) - if i < len(addrs) { - sym.Size = int64(addrs[i] - s.Value) - } - if s.Sect == 0 { - sym.Code = 'U' - } else if int(s.Sect) <= len(p.Sections) { - sect := p.Sections[s.Sect-1] - switch sect.Seg { - case "__TEXT": - sym.Code = 'R' - case "__DATA": - sym.Code = 'D' - } - switch sect.Seg + " " + sect.Name { - case "__TEXT __text": - sym.Code = 'T' - case "__DATA __bss", "__DATA __noptrbss": - sym.Code = 'B' - } - } - syms = append(syms, sym) - } - - return -} - -type uint64s []uint64 - -func (x uint64s) Len() int { return len(x) } -func (x uint64s) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x uint64s) Less(i, j int) bool { return x[i] < x[j] } diff --git a/src/cmd/objdump/main.go b/src/cmd/objdump/main.go index 0f66f20a40..0f125c98bf 100644 --- a/src/cmd/objdump/main.go +++ b/src/cmd/objdump/main.go @@ -33,12 +33,7 @@ package main import ( "bufio" - "bytes" - "debug/elf" "debug/gosym" - "debug/macho" - "debug/pe" - "debug/plan9obj" "encoding/binary" "flag" "fmt" @@ -51,6 +46,8 @@ import ( "strings" "text/tabwriter" + "cmd/internal/objfile" + "cmd/internal/rsc.io/arm/armasm" "cmd/internal/rsc.io/x86/x86asm" ) @@ -85,21 +82,33 @@ func main() { symRE = re } - f, err := os.Open(flag.Arg(0)) + f, err := objfile.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } - textStart, textData, symtab, pclntab, err := loadTables(f) + syms, err := f.Symbols() if err != nil { log.Fatalf("reading %s: %v", flag.Arg(0), err) } - syms, goarch, err := loadSymbols(f) + tab, err := f.PCLineTable() if err != nil { log.Fatalf("reading %s: %v", flag.Arg(0), err) } + textStart, textBytes, err := f.Text() + if err != nil { + log.Fatalf("reading %s: %v", flag.Arg(0), err) + } + + goarch := f.GOARCH() + + disasm := disasms[goarch] + if disasm == nil { + log.Fatalf("reading %s: unknown architecture", flag.Arg(0)) + } + // Filter out section symbols, overwriting syms in place. keep := syms[:0] for _, sym := range syms { @@ -112,37 +121,27 @@ func main() { } syms = keep - disasm := disasms[goarch] - if disasm == nil { - log.Fatalf("reading %s: unknown architecture", flag.Arg(0)) - } - + sort.Sort(ByAddr(syms)) lookup := func(addr uint64) (string, uint64) { - i := sort.Search(len(syms), func(i int) bool { return syms[i].Addr > addr }) + i := sort.Search(len(syms), func(i int) bool { return addr < syms[i].Addr }) if i > 0 { s := syms[i-1] - if s.Addr <= addr && addr < s.Addr+uint64(s.Size) && s.Name != "runtime.etext" && s.Name != "etext" && s.Name != "_etext" { + if s.Addr != 0 && s.Addr <= addr && addr < s.Addr+uint64(s.Size) { return s.Name, s.Addr } } return "", 0 } - pcln := gosym.NewLineTable(pclntab, textStart) - tab, err := gosym.NewTable(symtab, pcln) - if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) - } - if flag.NArg() == 1 { // disassembly of entire object - our format - dump(tab, lookup, disasm, goarch, syms, textData, textStart) - os.Exit(exitCode) + dump(tab, lookup, disasm, goarch, syms, textBytes, textStart) + os.Exit(0) } // disassembly of specific piece of object - gnu objdump format for pprof - gnuDump(tab, lookup, disasm, textData, textStart) - os.Exit(exitCode) + gnuDump(tab, lookup, disasm, textBytes, textStart) + os.Exit(0) } // base returns the final element in the path. @@ -153,13 +152,13 @@ func base(path string) string { return path } -func dump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, goarch string, syms []Sym, textData []byte, textStart uint64) { +func dump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, goarch string, syms []objfile.Sym, textData []byte, textStart uint64) { stdout := bufio.NewWriter(os.Stdout) defer stdout.Flush() printed := false for _, sym := range syms { - if (sym.Code != 'T' && sym.Code != 't') || sym.Size == 0 || sym.Name == "_text" || sym.Name == "text" || sym.Addr < textStart || symRE != nil && !symRE.MatchString(sym.Name) { + if (sym.Code != 'T' && sym.Code != 't') || sym.Size == 0 || sym.Addr < textStart || symRE != nil && !symRE.MatchString(sym.Name) { continue } if sym.Addr >= textStart+uint64(len(textData)) || sym.Addr+uint64(sym.Size) > textStart+uint64(len(textData)) { @@ -307,224 +306,8 @@ func gnuDump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, textData [] flush(end) } -func loadTables(f *os.File) (textStart uint64, textData, symtab, pclntab []byte, err error) { - if obj, err := elf.NewFile(f); err == nil { - if sect := obj.Section(".text"); sect != nil { - textStart = sect.Addr - textData, _ = sect.Data() - } - if sect := obj.Section(".gosymtab"); sect != nil { - if symtab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - if sect := obj.Section(".gopclntab"); sect != nil { - if pclntab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } +type ByAddr []objfile.Sym - if obj, err := macho.NewFile(f); err == nil { - if sect := obj.Section("__text"); sect != nil { - textStart = sect.Addr - textData, _ = sect.Data() - } - if sect := obj.Section("__gosymtab"); sect != nil { - if symtab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - if sect := obj.Section("__gopclntab"); sect != nil { - if pclntab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - if obj, err := pe.NewFile(f); err == nil { - var imageBase uint64 - switch oh := obj.OptionalHeader.(type) { - case *pe.OptionalHeader32: - imageBase = uint64(oh.ImageBase) - case *pe.OptionalHeader64: - imageBase = oh.ImageBase - default: - return 0, nil, nil, nil, fmt.Errorf("pe file format not recognized") - } - if sect := obj.Section(".text"); sect != nil { - textStart = imageBase + uint64(sect.VirtualAddress) - textData, _ = sect.Data() - } - if pclntab, err = loadPETable(obj, "runtime.pclntab", "runtime.epclntab"); err != nil { - // We didn't find the symbols, so look for the names used in 1.3 and earlier. - // TODO: Remove code looking for the old symbols when we no longer care about 1.3. - var err2 error - if pclntab, err2 = loadPETable(obj, "pclntab", "epclntab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - if symtab, err = loadPETable(obj, "runtime.symtab", "runtime.esymtab"); err != nil { - // Same as above. - var err2 error - if symtab, err2 = loadPETable(obj, "symtab", "esymtab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - if obj, err := plan9obj.NewFile(f); err == nil { - textStart = obj.LoadAddress + obj.HdrSize - if sect := obj.Section("text"); sect != nil { - textData, _ = sect.Data() - } - if pclntab, err = loadPlan9Table(obj, "runtime.pclntab", "runtime.epclntab"); err != nil { - // We didn't find the symbols, so look for the names used in 1.3 and earlier. - // TODO: Remove code looking for the old symbols when we no longer care about 1.3. - var err2 error - if pclntab, err2 = loadPlan9Table(obj, "pclntab", "epclntab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - if symtab, err = loadPlan9Table(obj, "runtime.symtab", "runtime.esymtab"); err != nil { - // Same as above. - var err2 error - if symtab, err2 = loadPlan9Table(obj, "symtab", "esymtab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - return 0, nil, nil, nil, fmt.Errorf("unrecognized binary format") -} - -func findPESymbol(f *pe.File, name string) (*pe.Symbol, error) { - for _, s := range f.Symbols { - if s.Name != name { - continue - } - if s.SectionNumber <= 0 { - return nil, fmt.Errorf("symbol %s: invalid section number %d", name, s.SectionNumber) - } - if len(f.Sections) < int(s.SectionNumber) { - return nil, fmt.Errorf("symbol %s: section number %d is larger than max %d", name, s.SectionNumber, len(f.Sections)) - } - return s, nil - } - return nil, fmt.Errorf("no %s symbol found", name) -} - -func loadPETable(f *pe.File, sname, ename string) ([]byte, error) { - ssym, err := findPESymbol(f, sname) - if err != nil { - return nil, err - } - esym, err := findPESymbol(f, ename) - if err != nil { - return nil, err - } - if ssym.SectionNumber != esym.SectionNumber { - return nil, fmt.Errorf("%s and %s symbols must be in the same section", sname, ename) - } - sect := f.Sections[ssym.SectionNumber-1] - data, err := sect.Data() - if err != nil { - return nil, err - } - return data[ssym.Value:esym.Value], nil -} - -func findPlan9Symbol(f *plan9obj.File, name string) (*plan9obj.Sym, error) { - syms, err := f.Symbols() - if err != nil { - return nil, err - } - for _, s := range syms { - if s.Name != name { - continue - } - return &s, nil - } - return nil, fmt.Errorf("no %s symbol found", name) -} - -func loadPlan9Table(f *plan9obj.File, sname, ename string) ([]byte, error) { - ssym, err := findPlan9Symbol(f, sname) - if err != nil { - return nil, err - } - esym, err := findPlan9Symbol(f, ename) - if err != nil { - return nil, err - } - sect := f.Section("text") - if sect == nil { - return nil, err - } - data, err := sect.Data() - if err != nil { - return nil, err - } - textStart := f.LoadAddress + f.HdrSize - return data[ssym.Value-textStart : esym.Value-textStart], nil -} - -// TODO(rsc): This code is taken from cmd/nm. Arrange some way to share the code. - -var exitCode = 0 - -func errorf(format string, args ...interface{}) { - log.Printf(format, args...) - exitCode = 1 -} - -func loadSymbols(f *os.File) (syms []Sym, goarch string, err error) { - f.Seek(0, 0) - buf := make([]byte, 16) - io.ReadFull(f, buf) - f.Seek(0, 0) - - for _, p := range parsers { - if bytes.HasPrefix(buf, p.prefix) { - syms, goarch = p.parse(f) - sort.Sort(byAddr(syms)) - return - } - } - err = fmt.Errorf("unknown file format") - return -} - -type Sym struct { - Addr uint64 - Size int64 - Code rune - Name string - Type string -} - -var parsers = []struct { - prefix []byte - parse func(*os.File) ([]Sym, string) -}{ - {[]byte("\x7FELF"), elfSymbols}, - {[]byte("\xFE\xED\xFA\xCE"), machoSymbols}, - {[]byte("\xFE\xED\xFA\xCF"), machoSymbols}, - {[]byte("\xCE\xFA\xED\xFE"), machoSymbols}, - {[]byte("\xCF\xFA\xED\xFE"), machoSymbols}, - {[]byte("MZ"), peSymbols}, - {[]byte("\x00\x00\x01\xEB"), plan9Symbols}, // 386 - {[]byte("\x00\x00\x04\x07"), plan9Symbols}, // mips - {[]byte("\x00\x00\x06\x47"), plan9Symbols}, // arm - {[]byte("\x00\x00\x8A\x97"), plan9Symbols}, // amd64 -} - -type byAddr []Sym - -func (x byAddr) Len() int { return len(x) } -func (x byAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x byAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } +func (x ByAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } +func (x ByAddr) Len() int { return len(x) } +func (x ByAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } diff --git a/src/cmd/objdump/pe.go b/src/cmd/objdump/pe.go deleted file mode 100644 index 38190095a3..0000000000 --- a/src/cmd/objdump/pe.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of PE executables (Microsoft Windows). - -package main - -import ( - "debug/pe" - "os" - "sort" -) - -func peSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := pe.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - - var imageBase uint64 - switch oh := p.OptionalHeader.(type) { - case *pe.OptionalHeader32: - imageBase = uint64(oh.ImageBase) - goarch = "386" - case *pe.OptionalHeader64: - imageBase = oh.ImageBase - goarch = "amd64" - default: - errorf("parsing %s: file format not recognized", f.Name()) - return - } - - for _, s := range p.Symbols { - const ( - N_UNDEF = 0 // An undefined (extern) symbol - N_ABS = -1 // An absolute symbol (e_value is a constant, not an address) - N_DEBUG = -2 // A debugging symbol - ) - sym := Sym{Name: s.Name, Addr: uint64(s.Value), Code: '?'} - switch s.SectionNumber { - case N_UNDEF: - sym.Code = 'U' - case N_ABS: - sym.Code = 'C' - case N_DEBUG: - sym.Code = '?' - default: - if s.SectionNumber < 0 { - errorf("parsing %s: invalid section number %d", f.Name(), s.SectionNumber) - return - } - if len(p.Sections) < int(s.SectionNumber) { - errorf("parsing %s: section number %d is large then max %d", f.Name(), s.SectionNumber, len(p.Sections)) - return - } - sect := p.Sections[s.SectionNumber-1] - const ( - text = 0x20 - data = 0x40 - bss = 0x80 - permX = 0x20000000 - permR = 0x40000000 - permW = 0x80000000 - ) - ch := sect.Characteristics - switch { - case ch&text != 0: - sym.Code = 'T' - case ch&data != 0: - if ch&permW == 0 { - sym.Code = 'R' - } else { - sym.Code = 'D' - } - case ch&bss != 0: - sym.Code = 'B' - } - sym.Addr += imageBase + uint64(sect.VirtualAddress) - } - syms = append(syms, sym) - addrs = append(addrs, sym.Addr) - } - - sort.Sort(uint64s(addrs)) - for i := range syms { - j := sort.Search(len(addrs), func(x int) bool { return addrs[x] > syms[i].Addr }) - if j < len(addrs) { - syms[i].Size = int64(addrs[j] - syms[i].Addr) - } - } - - return -} diff --git a/src/cmd/objdump/plan9obj.go b/src/cmd/objdump/plan9obj.go deleted file mode 100644 index f851d4158c..0000000000 --- a/src/cmd/objdump/plan9obj.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of Plan 9 a.out executables. - -package main - -import ( - "debug/plan9obj" - "os" - "sort" -) - -var validSymType = map[rune]bool{ - 'T': true, - 't': true, - 'D': true, - 'd': true, - 'B': true, - 'b': true, -} - -func plan9Symbols(f *os.File) (syms []Sym, goarch string) { - p, err := plan9obj.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - plan9Syms, err := p.Symbols() - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - switch p.Magic { - case plan9obj.MagicAMD64: - goarch = "amd64" - case plan9obj.Magic386: - goarch = "386" - case plan9obj.MagicARM: - goarch = "arm" - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - for _, s := range plan9Syms { - if !validSymType[s.Type] { - continue - } - addrs = append(addrs, s.Value) - } - sort.Sort(uint64s(addrs)) - - for _, s := range plan9Syms { - if !validSymType[s.Type] { - continue - } - sym := Sym{Addr: s.Value, Name: s.Name, Code: rune(s.Type)} - i := sort.Search(len(addrs), func(x int) bool { return addrs[x] > s.Value }) - if i < len(addrs) { - sym.Size = int64(addrs[i] - s.Value) - } - syms = append(syms, sym) - } - - return -} From bfe459adc98724354ffefa624b73815adaa711d8 Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Wed, 29 Oct 2014 15:35:48 -0700 Subject: [PATCH 06/59] doc/go1.4.html: final library changes First draft now complete. LGTM=rsc R=golang-codereviews, rsc CC=golang-codereviews https://golang.org/cl/170750043 --- doc/go1.4.html | 150 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 129 insertions(+), 21 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index e2458f2efe..19bad0065d 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -466,11 +466,6 @@ rebuild the standard library and commands, to avoid overwriting the installation -

    Changes to godoc

    -

    -TODO godoc news -

    -

    Changes to package source layout

    @@ -555,14 +550,57 @@ There are no new packages in this release.

    Major changes to the library

    +

    bufio.Scanner

    +

    -TODO major changes +The Scanner type in the +bufio package +has had a bug fixed that may require changes to custom +split functions. +The bug made it impossible to generate an empty token at EOF; the fix +changes the end conditions seen by the split function. +Previously, scanning stopped at EOF if there was no more data. +As of 1.4, the split function will be called once at EOF after input is exhausted, +so the split function can generate a final empty token +as the documentation already promised.

    -
    -bufio: handling of empty tokens at EOF changed, may require scanner change (CL 145390043)
-syscall: now frozen (CL 129820043); go.sys subrepo created: http://golang.org/s/go1.4-syscall
-
    +

    +Updating: Custom split functions may need to be modified to +handle empty tokens at EOF as desired. +

    + +

    syscall

    + +

    +The syscall package is now frozen except +for changes needed to maintain the core repository. +In particular, it will no longer be extended to support new or different system calls +that are not used by the core. +The reasons are described at length in a +separate document. +

    + +

    +A new subrepository, go.sys, +has been created to serve as the location for new developments to support system +calls on all kernels. +It has a nicer structure, with three packages that each hold the implementation of +system calls for one of +Unix, +Windows and +Plan 9. +These packages will be curated more generously, accepting all reasonable changes +that reflect kernel interfaces in those operating systems. +See the documentation and the article mentioned above for more information. +

    + +

    +Updating: Existing programs are not affected as the syscall +package is largely unchanged from the 1.3 release. +Future development that requires system calls not in the syscall package +should build on go.sys instead. +

    Minor changes to the library

    @@ -629,11 +667,28 @@ For instance, &map[string]int{"one": 1} now prints &map[one: 1] rather than as a hexadecimal pointer value.
    • -
  • TODO net/http: add Request.BasicAuth method ( https://codereview.appspot.com/76540043)
    • +
  • +The net/http package's +Request type +has a new BasicAuth method +that returns the username and password from authenticated requests using the +HTTP Basic Authentication +Scheme. +
    • -
  • TODO net/http: add Transport.DialTLS hook ( https://codereview.appspot.com/137940043)
    • +
  • The net/http package's +Transport type +has a new DialTLS function +that simplifies setting up TLS connections. +
    • -
  • TODO net/http/httputil: add ReverseProxy.ErrorLog ( https://codereview.appspot.com/132750043)
    • +
  • +The net/http/httputil package's +ReverseProxy type +has a new field, +ErrorLog, that +provides user control of logging. +
  • The os package @@ -656,21 +711,74 @@ because of changes to the implementation of interfaces in the runtime. This saves memory but has no semantic effect.
    • -
  • TODO runtime: implement monotonic clocks on windows ( https://codereview.appspot.com/108700045)
    • +
  • +The runtime package +now implements monotonic clocks on Windows, +as it already did for the other systems. +
    • -
  • TODO runtime: MemStats.Mallocs now counts very small allocations missed in Go 1.3. This may break tests using runtime.ReadMemStats or testing.AllocsPerRun by giving a more accurate answer than Go 1.3 did ( https://codereview.appspot.com/143150043).
    • +
  • +The runtime package's +Mallocs counter +now counts very small allocations that were missed in Go 1.3. +This may break tests using ReadMemStats +or AllocsPerRun +due to the more accurate answer. +
    • -
  • TODO runtime/race: freebsd is supported ( https://codereview.appspot.com/107270043)
    • +
  • +In the runtime package, +an array PauseEnd +has been added to the +MemStats +and GCStats structs. +This array is a circular buffer of times when garbage collection pauses ended. +The corresponding pause durations are already recorded in +PauseNs +
    • -
  • TODO runtime: add PauseEnd array to MemStats and GCStats ( https://codereview.appspot.com/153670043)
    • +
  • +The runtime/race package +now supports FreeBSD, which means the +go command's -race +flag now works on FreeBSD. +
    • -
  • TODO sync/atomic: add Value ( https://codereview.appspot.com/136710045)
    • +
  • +The sync/atomic package +has a new type, Value. +Value provides an efficient mechanism for atomic loads and +stores of values of arbitrary type. +
    • -
  • TODO syscall: Setuid, Setgid are disabled on linux platforms. On linux those syscalls operate on the calling thread, not the whole process. This does not match the semantics of other platforms, nor the expectations of the caller, so the operations have been disabled until issue 1435 is resolved ( https://codereview.appspot.com/106170043)
    • +
  • +In the syscall package's +implementation on Linux, the +Setuid +and Setgid have been disabled +because those system calls operate on the calling thread, not the whole process, which is +different from other platforms and not the expected result. +
    • -
  • TODO testing: add Coverage ( https://codereview.appspot.com/98150043)
    • +
  • +The testing package +has a new facility to provide more control over running a set of tests. +If the test code contains a function +
    +func TestMain(m *testing.M) 
+
    -
  • TODO testing: add TestMain support ( https://codereview.appspot.com/148770043)
    • +that function will be called instead of running the tests directly. +The M struct contains methods to access and run the tests. + + +
  • +Also in the testing package, +a new Coverage +function reports the current test coverage fraction, +enabling individual tests to report how much they are contributing to the +overall coverage. +
  • The text/scanner package's From f9c4c16dce621f1834943f3ccda0d0a079f7b1a4 Mon Sep 17 00:00:00 2001 From: Alex Brainman Date: Thu, 30 Oct 2014 10:24:37 +1100 Subject: [PATCH 07/59] runtime: make TestCgoExternalThreadPanic run on windows LGTM=rsc R=golang-codereviews, bradfitz, rsc CC=golang-codereviews https://golang.org/cl/163540043 --- src/runtime/crash_cgo_test.go | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index 5958ad8914..972eedc624 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -36,10 +36,14 @@ func TestCgoTraceback(t *testing.T) { } func TestCgoExternalThreadPanic(t *testing.T) { - if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + if runtime.GOOS == "plan9" { t.Skipf("no pthreads on %s", runtime.GOOS) } - got := executeTest(t, cgoExternalThreadPanicSource, nil, "main.c", cgoExternalThreadPanicC) + csrc := cgoExternalThreadPanicC + if runtime.GOOS == "windows" { + csrc = cgoExternalThreadPanicC_windows + } + got := executeTest(t, cgoExternalThreadPanicSource, nil, "main.c", csrc) want := "panic: BOOM" if !strings.Contains(got, want) { t.Fatalf("want failure containing %q. output:\n%s\n", want, got) @@ -169,3 +173,24 @@ start(void) printf("pthread_create failed\n"); } ` + +const cgoExternalThreadPanicC_windows = ` +#include +#include + +void gopanic(void); + +static void* +die(void* x) +{ + gopanic(); + return 0; +} + +void +start(void) +{ + if(_beginthreadex(0, 0, die, 0, 0, 0) != 0) + printf("_beginthreadex failed\n"); +} +` From a5a07331444f9b48a5e09728e3d0085cfbfb2222 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 29 Oct 2014 20:37:44 -0400 Subject: [PATCH 08/59] runtime: change top-most return PC from goexit to goexit+PCQuantum If you get a stack of PCs from Callers, it would be expected that every PC is immediately after a call instruction, so to find the line of the call, you look up the line for PC-1. CL 163550043 now explicitly documents that. The most common exception to this is the top-most return PC on the stack, which is the entry address of the runtime.goexit function. Subtracting 1 from that PC will end up in a different function entirely. To remove this special case, make the top-most return PC goexit+PCQuantum and then implement goexit in assembly so that the first instruction can be skipped. Fixes #7690. LGTM=r R=r CC=golang-codereviews https://golang.org/cl/170720043 --- src/runtime/asm_386.s | 6 ++++++ src/runtime/asm_amd64.s | 6 ++++++ src/runtime/asm_amd64p32.s | 6 ++++++ src/runtime/asm_arm.s | 6 ++++++ src/runtime/proc.c | 8 +++----- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 0d46a9eff7..b4b81d7397 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -2284,3 +2284,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 MOVL m_curg(AX), AX MOVL (g_stack+stack_hi)(AX), AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index a9b082beb8..39d7c78f23 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -2229,3 +2229,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 MOVQ m_curg(AX), AX MOVQ (g_stack+stack_hi)(AX), AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index 28875bc55a..a1116b5d47 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -1079,3 +1079,9 @@ TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 TEXT runtime·return0(SB), NOSPLIT, $0 MOVL $0, AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index e94b4c1ff6..0f3b5eeb8b 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -1320,3 +1320,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$8 MOVW saveG-8(SP), g MOVW saveR11-4(SP), R11 RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$-4-0 + MOVW R0, R0 // NOP + BL runtime·goexit1(SB) // does not return diff --git a/src/runtime/proc.c b/src/runtime/proc.c index b46f67065a..4be51e1e16 100644 --- a/src/runtime/proc.c +++ b/src/runtime/proc.c @@ -1643,12 +1643,10 @@ runtime·gosched_m(G *gp) } // Finishes execution of the current goroutine. -// Need to mark it as nosplit, because it runs with sp > stackbase. -// Since it does not return it does not matter. But if it is preempted -// at the split stack check, GC will complain about inconsistent sp. +// Must be NOSPLIT because it is called from Go. #pragma textflag NOSPLIT void -runtime·goexit(void) +runtime·goexit1(void) { void (*fn)(G*); @@ -2192,7 +2190,7 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp runtime·memclr((byte*)&newg->sched, sizeof newg->sched); newg->sched.sp = (uintptr)sp; - newg->sched.pc = (uintptr)runtime·goexit; + newg->sched.pc = (uintptr)runtime·goexit + PCQuantum; // +PCQuantum so that previous instruction is in same function newg->sched.g = newg; runtime·gostartcallfn(&newg->sched, fn); newg->gopc = (uintptr)callerpc; From ca230d2d6ffeaef0be2f58fd46ba6ed34a8dbf46 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 29 Oct 2014 21:02:58 -0400 Subject: [PATCH 09/59] cmd/objdump: disable test failing on arm5 TBR=adg CC=golang-codereviews https://golang.org/cl/167890043 --- src/cmd/objdump/objdump_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/objdump/objdump_test.go b/src/cmd/objdump/objdump_test.go index 0a2d2565a7..ffaaa5b437 100644 --- a/src/cmd/objdump/objdump_test.go +++ b/src/cmd/objdump/objdump_test.go @@ -143,7 +143,7 @@ var x86Need = []string{ var armNeed = []string{ "fmthello.go:6", "TEXT main.main(SB)", - "B.LS main.main(SB)", + //"B.LS main.main(SB)", // TODO(rsc): restore; golang.org/issue/9021 "BL fmt.Println(SB)", "RET", } From 9cd8cb0b17e4bbff7cfce29631dc78c9cafd2e8f Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Thu, 30 Oct 2014 12:33:57 +1100 Subject: [PATCH 10/59] tag go1.4beta1 TBR=rsc R=r, rsc CC=golang-codereviews https://golang.org/cl/164240043 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 5a5c4aed44..edd2163f70 100644 --- a/.hgtags +++ b/.hgtags @@ -135,3 +135,4 @@ f8b50ad4cac4d4c4ecf48324b4f512f65e82cc1c go1.3beta1 85518b1d6f8d6e16133b9ed2c9db6807522d37de go1.3.2 f44017549ff9c3cc5eef74ebe7276cd0dfc066b6 go1.3.3 f44017549ff9c3cc5eef74ebe7276cd0dfc066b6 release +1fdfd7dfaedb1b7702141617e621ab7328a236a1 go1.4beta1 From 70dc39e17dc47ecb58a398d077fcdce6046be5dd Mon Sep 17 00:00:00 2001 From: Mikio Hara Date: Thu, 30 Oct 2014 14:15:00 +0900 Subject: [PATCH 11/59] doc/go1.4.html: fix typo LGTM=adg R=r, adg CC=golang-codereviews https://golang.org/cl/165890043 --- doc/go1.4.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 19bad0065d..9f65aaf24c 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -621,7 +621,7 @@ for the decompressors, allowing them to reuse buffers and improve performance.
  • The crypto/tls package -now supports APLN as defined in RFC 7301. +now supports ALPN as defined in RFC 7301.
  • From f21a02a179ca7335ee864512f9afb2c34d6c1850 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 30 Oct 2014 10:58:31 -0300 Subject: [PATCH 12/59] doc/go1.4.html: tweak http.Transport.DialTLS wording It doesn't simplify, because it wasn't even possible before. LGTM=r R=r CC=golang-codereviews https://golang.org/cl/164250043 --- doc/go1.4.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 9f65aaf24c..cb2280cb4d 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -678,8 +678,8 @@ Scheme.
  • The net/http package's Transport type -has a new DialTLS function -that simplifies setting up TLS connections. +has a new DialTLS hook +that allows customizing the behavior of outbound TLS connections.
  • From 09f6f05c1fdd394ec512642e0cf086e0cf2d3d79 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 30 Oct 2014 14:01:14 -0400 Subject: [PATCH 13/59] cmd/cgo: avoid worklist nondeterminism. + Regression test. Fixes #9026. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/162490043 --- misc/cgo/test/cgo_test.go | 1 + misc/cgo/test/issue9026.go | 33 +++++++++++++++++++++++++++++++++ src/cmd/cgo/gcc.go | 24 +++++++++++++++--------- 3 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 misc/cgo/test/issue9026.go diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go index 3b289ba7b5..fbdfac87ac 100644 --- a/misc/cgo/test/cgo_test.go +++ b/misc/cgo/test/cgo_test.go @@ -62,5 +62,6 @@ func Test8517(t *testing.T) { test8517(t) } func Test8811(t *testing.T) { test8811(t) } func TestReturnAfterGrow(t *testing.T) { testReturnAfterGrow(t) } func TestReturnAfterGrowFromGo(t *testing.T) { testReturnAfterGrowFromGo(t) } +func Test9026(t *testing.T) { test9026(t) } func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) } diff --git a/misc/cgo/test/issue9026.go b/misc/cgo/test/issue9026.go new file mode 100644 index 0000000000..b17440452f --- /dev/null +++ b/misc/cgo/test/issue9026.go @@ -0,0 +1,33 @@ +package cgotest + +/* +typedef struct {} git_merge_file_input; + +typedef struct {} git_merge_file_options; + +int git_merge_file( + git_merge_file_input *in, + git_merge_file_options *opts) {} +*/ +import "C" +import ( + "fmt" + "testing" +) + +func test9026(t *testing.T) { + var in C.git_merge_file_input + var opts *C.git_merge_file_options + C.git_merge_file(&in, opts) + + // Test that the generated type names are deterministic. + // (Previously this would fail about 10% of the time.) + // + // Brittle: the assertion may fail spuriously when the algorithm + // changes, but should remain stable otherwise. + got := fmt.Sprintf("%T %T", in, opts) + want := "cgotest._Ctype_struct___12 *cgotest._Ctype_struct___13" + if got != want { + t.Errorf("Non-deterministic type names: got %s, want %s", got, want) + } +} diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index d77d56c22a..abdd369d71 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -944,6 +944,8 @@ type typeConv struct { // Map from types to incomplete pointers to those types. ptrs map[dwarf.Type][]*Type + // Keys of ptrs in insertion order (deterministic worklist) + ptrKeys []dwarf.Type // Predeclared types. bool ast.Expr @@ -1061,16 +1063,17 @@ func (tr *TypeRepr) Set(repr string, fargs ...interface{}) { func (c *typeConv) FinishType(pos token.Pos) { // Completing one pointer type might produce more to complete. // Keep looping until they're all done. - for len(c.ptrs) > 0 { - for dtype := range c.ptrs { - // Note Type might invalidate c.ptrs[dtype]. - t := c.Type(dtype, pos) - for _, ptr := range c.ptrs[dtype] { - ptr.Go.(*ast.StarExpr).X = t.Go - ptr.C.Set("%s*", t.C) - } - delete(c.ptrs, dtype) + for len(c.ptrKeys) > 0 { + dtype := c.ptrKeys[0] + c.ptrKeys = c.ptrKeys[1:] + + // Note Type might invalidate c.ptrs[dtype]. + t := c.Type(dtype, pos) + for _, ptr := range c.ptrs[dtype] { + ptr.Go.(*ast.StarExpr).X = t.Go + ptr.C.Set("%s*", t.C) } + c.ptrs[dtype] = nil // retain the map key } } @@ -1237,6 +1240,9 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { // Placeholder initialization; completed in FinishType. t.Go = &ast.StarExpr{} t.C.Set("*") + if _, ok := c.ptrs[dt.Type]; !ok { + c.ptrKeys = append(c.ptrKeys, dt.Type) + } c.ptrs[dt.Type] = append(c.ptrs[dt.Type], t) case *dwarf.QualType: From a14ae4451700a690a2ca075d585d55d60f0d46e3 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 30 Oct 2014 14:08:55 -0400 Subject: [PATCH 14/59] misc/cgo/test: fix bad C test code that fails on some configurations LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/169800043 --- misc/cgo/test/issue9026.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/cgo/test/issue9026.go b/misc/cgo/test/issue9026.go index b17440452f..b5d975f17a 100644 --- a/misc/cgo/test/issue9026.go +++ b/misc/cgo/test/issue9026.go @@ -5,7 +5,7 @@ typedef struct {} git_merge_file_input; typedef struct {} git_merge_file_options; -int git_merge_file( +void git_merge_file( git_merge_file_input *in, git_merge_file_options *opts) {} */ From 014665ce0feee15726a8ef05cc568e752babb6ba Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 30 Oct 2014 13:15:43 -0700 Subject: [PATCH 15/59] A+C: add Jed Denlea (Fastly corporate CLA) LGTM=iant R=golang-codereviews, iant CC=adg, golang-codereviews https://golang.org/cl/165170043 --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 03e686e751..89fd171794 100644 --- a/AUTHORS +++ b/AUTHORS @@ -156,6 +156,7 @@ Evan Shaw Ewan Chou Fabrizio Milo Fan Hongjian +Fastly, Inc. Fatih Arslan Fazlul Shahriar Felix Geisendörfer diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 7679f78742..67c8b1d5cd 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -299,6 +299,7 @@ Jason Del Ponte Jason Travis Jay Weisskopf Jean-Marc Eurin +Jed Denlea Jeff Hodges Jeff R. Allen Jeff Sickel From 51fcd3cf1bc4b9553da586396b028bfe98d3bff5 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 30 Oct 2014 17:19:29 -0300 Subject: [PATCH 16/59] A+C: Nathan P Finch (individual CLA) TBR=iant R=iant CC=golang-codereviews https://golang.org/cl/155560045 --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 89fd171794..3caf870358 100644 --- a/AUTHORS +++ b/AUTHORS @@ -316,6 +316,7 @@ Moriyoshi Koizumi Môshe van der Sterre Nan Deng Nathan John Youngman +Nathan P Finch ngmoco, LLC Nicholas Katsaros Nicholas Presta diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 67c8b1d5cd..b4435b0e6d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -445,6 +445,7 @@ Môshe van der Sterre Mrunal Patel Nan Deng Nathan John Youngman +Nathan P Finch Nicholas Katsaros Nicholas Presta Nicholas Sullivan From 692ad844b632074d9e71b6e59fab227c82da77b5 Mon Sep 17 00:00:00 2001 From: Nathan P Finch Date: Thu, 30 Oct 2014 13:20:43 -0700 Subject: [PATCH 17/59] cmd/go: fix minor typo LGTM=bradfitz R=golang-codereviews, bradfitz CC=golang-codereviews https://golang.org/cl/170770043 --- src/cmd/go/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 314c69bd8c..946b18875e 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -247,7 +247,7 @@ The arguments are space-separated tokens or double-quoted strings passed to the generator as individual arguments when it is run. Quoted strings use Go syntax and are evaluated before execution; a -quoted string appears a single argument to the generator. +quoted string appears as a single argument to the generator. Go generate sets several variables when it runs the generator: From baa5d26f629a38335df010a1b5098ebdec8af3b8 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 31 Oct 2014 00:48:57 -0300 Subject: [PATCH 18/59] sync/atomic: fix comment referencing Value.Store's argument name Fixes #9029 LGTM=adg, r R=r, adg CC=golang-codereviews https://golang.org/cl/161630044 --- src/sync/atomic/value.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync/atomic/value.go b/src/sync/atomic/value.go index ab46d9a240..ab3aa11285 100644 --- a/src/sync/atomic/value.go +++ b/src/sync/atomic/value.go @@ -38,7 +38,7 @@ func (v *Value) Load() (x interface{}) { return } -// Store sets the value of the Value to v. +// Store sets the value of the Value to x. // All calls to Store for a given Value must use values of the same concrete type. // Store of an inconsistent type panics, as does Store(nil). func (v *Value) Store(x interface{}) { From 1965ba6e6fce0b643a48f08d6d78c0c2c7822dd5 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 31 Oct 2014 09:37:11 -0700 Subject: [PATCH 19/59] A+C: add Gabriel Aszalos (individual CLA) LGTM=bradfitz R=adg, bradfitz CC=golang-codereviews https://golang.org/cl/162580043 --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 3caf870358..cea02ac3ec 100644 --- a/AUTHORS +++ b/AUTHORS @@ -167,6 +167,7 @@ Francisco Souza Frederick Kelly Mayle III Fredrik Enestad Frithjof Schulze +Gabriel Aszalos Gary Burd Gautham Thambidorai Georg Reinke diff --git a/CONTRIBUTORS b/CONTRIBUTORS index b4435b0e6d..8c5ceea8a6 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -241,6 +241,7 @@ Fredrik Enestad Frithjof Schulze Fumitoshi Ukai Gaal Yahas +Gabriel Aszalos Gary Burd Gautham Thambidorai Georg Reinke From 2074046d00719c0ec0cbc4857726e9a55b71b63f Mon Sep 17 00:00:00 2001 From: Gabriel Aszalos Date: Fri, 31 Oct 2014 09:38:41 -0700 Subject: [PATCH 20/59] cmd/go: fixed typo in doc and generator LGTM=iant R=golang-codereviews, iant, bradfitz CC=golang-codereviews https://golang.org/cl/163690043 --- src/cmd/go/doc.go | 2 +- src/cmd/go/generate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 946b18875e..cf3a54565a 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -260,7 +260,7 @@ Go generate sets several variables when it runs the generator: $GOPACKAGE The name of the package of the file containing the directive. -Other than variable substition and quoted-string evaluation, no +Other than variable substitution and quoted-string evaluation, no special processing such as "globbing" is performed on the command line. diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go index 4227abbe7c..a83cce8f7a 100644 --- a/src/cmd/go/generate.go +++ b/src/cmd/go/generate.go @@ -58,7 +58,7 @@ Go generate sets several variables when it runs the generator: $GOPACKAGE The name of the package of the file containing the directive. -Other than variable substition and quoted-string evaluation, no +Other than variable substitution and quoted-string evaluation, no special processing such as "globbing" is performed on the command line. From 9dc1cce38db0229e97c1ee8d9929f0457f1af385 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 31 Oct 2014 09:49:42 -0700 Subject: [PATCH 21/59] database/sql: make TestDrivers not crash on second run Using -test.cpu=1,1 made it crash before. Fixes #9024 LGTM=iant R=adg, iant CC=golang-codereviews https://golang.org/cl/169860043 --- src/database/sql/fakedb_test.go | 2 ++ src/database/sql/sql.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/database/sql/fakedb_test.go b/src/database/sql/fakedb_test.go index 171c322d49..a993fd46ed 100644 --- a/src/database/sql/fakedb_test.go +++ b/src/database/sql/fakedb_test.go @@ -141,6 +141,8 @@ type Dummy struct { } func TestDrivers(t *testing.T) { + unregisterAllDrivers() + Register("test", fdriver) Register("invalid", Dummy{}) all := Drivers() if len(all) < 2 || !sort.StringsAreSorted(all) || !contains(all, "test") || !contains(all, "invalid") { diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index ad9179cf7d..6e6f246aee 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -37,6 +37,11 @@ func Register(name string, driver driver.Driver) { drivers[name] = driver } +func unregisterAllDrivers() { + // For tests. + drivers = make(map[string]driver.Driver) +} + // Drivers returns a sorted list of the names of the registered drivers. func Drivers() []string { var list []string From 8985c091e4a83eef27ed2a474e1dd34eae43db3a Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 31 Oct 2014 10:20:36 -0700 Subject: [PATCH 22/59] net/http: add missing newline in list of leaked goroutines LGTM=bradfitz R=bradfitz CC=golang-codereviews https://golang.org/cl/168860044 --- src/net/http/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/http/main_test.go b/src/net/http/main_test.go index 9f1dfc3727..b8c71fd19f 100644 --- a/src/net/http/main_test.go +++ b/src/net/http/main_test.go @@ -70,7 +70,7 @@ func goroutineLeaked() bool { } fmt.Fprintf(os.Stderr, "Too many goroutines running after net/http test(s).\n") for stack, count := range stackCount { - fmt.Fprintf(os.Stderr, "%d instances of:\n%s", count, stack) + fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack) } return true } From 6a0a47539072b8ba4a7cfa585e4a0d71fba8c4c5 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Sat, 1 Nov 2014 08:27:55 -0700 Subject: [PATCH 23/59] A+C: add Benoit Sigoure (individual CLA) LGTM=bradfitz R=golang-codereviews, bradfitz CC=golang-codereviews https://golang.org/cl/164410043 --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index cea02ac3ec..48a262bbd7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -65,6 +65,7 @@ Aulus Egnatius Varialus Ben Olive Benjamin Black Benny Siegert +Benoit Sigoure Berengar Lehr Billie Harold Cleek Bjorn Tillenius diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 8c5ceea8a6..ec69858b60 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -104,6 +104,7 @@ Ben Lynn Ben Olive Benjamin Black Benny Siegert +Benoit Sigoure Berengar Lehr Bill Neubauer Bill Thiede From 8e01fc7e9b531398e868d9899c91a9f052f015c7 Mon Sep 17 00:00:00 2001 From: Benoit Sigoure Date: Sat, 1 Nov 2014 08:28:09 -0700 Subject: [PATCH 24/59] misc: Increase issue 6997's test timeout to prevent spurious failures. On heavily loaded build servers, a 5 second timeout is too aggressive, which causes this test to fail spuriously. LGTM=iant R=iant CC=golang-codereviews, sqweek https://golang.org/cl/170850043 --- misc/cgo/test/issue6997_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/cgo/test/issue6997_linux.go b/misc/cgo/test/issue6997_linux.go index 871bd517a7..5455f0c536 100644 --- a/misc/cgo/test/issue6997_linux.go +++ b/misc/cgo/test/issue6997_linux.go @@ -34,7 +34,7 @@ func test6997(t *testing.T) { if r == 0 { t.Error("pthread finished but wasn't cancelled??") } - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): t.Error("hung in pthread_cancel/pthread_join") } } From b802240300a33024c1a47fdf2c5260a3fad0155b Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 3 Nov 2014 17:01:17 +1100 Subject: [PATCH 25/59] doc: document go get -f flag in 1.4 release notes LGTM=r, rsc R=r, rsc, adg CC=golang-codereviews https://golang.org/cl/168890043 --- doc/go1.4.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/go1.4.html b/doc/go1.4.html index cb2280cb4d..3310117a4d 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -371,6 +371,15 @@ fails because of this check, the mis-imported package has been copied to the loc and should be removed manually.

    +

    +To complement this new feature, a check has been added at update time to verify +that the local package's remote repository matches that of its custom import. +The go get -u command will fail to +update a package if its remote repository has changed since it was first +downloaded. +The new -f flag overrides this check. +

    +

    Further information is in the design document. From 489ff75ab8310b678a955b37094d2c72a1c18b11 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 3 Nov 2014 13:26:46 -0500 Subject: [PATCH 26/59] runtime: make Go and C mallocgc signatures match Previously, the flags argument to mallocgc was an int in Go, but a uint32 in C. Change the Go type to use uint32 so these agree. The largest flag value is 2 (and of course no flag values are negative), so this won't change anything on little endian architectures, but it matters on big endian. LGTM=rsc R=khr, rsc CC=golang-codereviews https://golang.org/cl/169920043 --- src/runtime/malloc.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 9b4264f2b3..8cf1c3d342 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -43,7 +43,7 @@ var zerobase uintptr // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. -func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer { +func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { if size == 0 { return unsafe.Pointer(&zerobase) } @@ -346,7 +346,7 @@ marked: // implementation of new builtin func newobject(typ *_type) unsafe.Pointer { - flags := 0 + flags := uint32(0) if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } @@ -355,7 +355,7 @@ func newobject(typ *_type) unsafe.Pointer { // implementation of make builtin for slices func newarray(typ *_type, n uintptr) unsafe.Pointer { - flags := 0 + flags := uint32(0) if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } From 182ec4395eff228673377127b8b8b69911e7762a Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 3 Nov 2014 13:41:03 -0500 Subject: [PATCH 27/59] misc/cgo/test: fix freebsd test failure by moving test to its own package. (The assertion depends on a per-package gensym counter whose value varies based on what else is in the package.) LGTM=khr R=khr, rsc CC=golang-codereviews https://golang.org/cl/169930043 --- misc/cgo/test/issue9026.go | 30 +++-------------------- misc/cgo/test/issue9026/issue9026.go | 36 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 27 deletions(-) create mode 100644 misc/cgo/test/issue9026/issue9026.go diff --git a/misc/cgo/test/issue9026.go b/misc/cgo/test/issue9026.go index b5d975f17a..8848d0e811 100644 --- a/misc/cgo/test/issue9026.go +++ b/misc/cgo/test/issue9026.go @@ -1,33 +1,9 @@ package cgotest -/* -typedef struct {} git_merge_file_input; - -typedef struct {} git_merge_file_options; - -void git_merge_file( - git_merge_file_input *in, - git_merge_file_options *opts) {} -*/ -import "C" import ( - "fmt" "testing" + + "./issue9026" ) -func test9026(t *testing.T) { - var in C.git_merge_file_input - var opts *C.git_merge_file_options - C.git_merge_file(&in, opts) - - // Test that the generated type names are deterministic. - // (Previously this would fail about 10% of the time.) - // - // Brittle: the assertion may fail spuriously when the algorithm - // changes, but should remain stable otherwise. - got := fmt.Sprintf("%T %T", in, opts) - want := "cgotest._Ctype_struct___12 *cgotest._Ctype_struct___13" - if got != want { - t.Errorf("Non-deterministic type names: got %s, want %s", got, want) - } -} +func test9026(t *testing.T) { issue9026.Test(t) } diff --git a/misc/cgo/test/issue9026/issue9026.go b/misc/cgo/test/issue9026/issue9026.go new file mode 100644 index 0000000000..0af86e64da --- /dev/null +++ b/misc/cgo/test/issue9026/issue9026.go @@ -0,0 +1,36 @@ +package issue9026 + +// This file appears in its own package since the assertion tests the +// per-package counter used to create fresh identifiers. + +/* +typedef struct {} git_merge_file_input; + +typedef struct {} git_merge_file_options; + +void git_merge_file( + git_merge_file_input *in, + git_merge_file_options *opts) {} +*/ +import "C" +import ( + "fmt" + "testing" +) + +func Test(t *testing.T) { + var in C.git_merge_file_input + var opts *C.git_merge_file_options + C.git_merge_file(&in, opts) + + // Test that the generated type names are deterministic. + // (Previously this would fail about 10% of the time.) + // + // Brittle: the assertion may fail spuriously when the algorithm + // changes, but should remain stable otherwise. + got := fmt.Sprintf("%T %T", in, opts) + want := "issue9026._Ctype_struct___0 *issue9026._Ctype_struct___1" + if got != want { + t.Errorf("Non-deterministic type names: got %s, want %s", got, want) + } +} From 516d9ef53b656aa5e6eebdc2c09e8d257b13e033 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 4 Nov 2014 09:43:37 -0500 Subject: [PATCH 28/59] gc: abort if given an unknown debug (-d) flag The check for unknown command line debug flags in gc was incorrect: the loop over debugtab terminates when it reaches a nil entry, but it was only reporting an error if the parser had passed the last entry of debugtab (which it never did). Fix this by reporting the usage error if the loop reaches a nil entry. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/166110043 --- src/cmd/gc/lex.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c index 2303b442cd..523ba37aa7 100644 --- a/src/cmd/gc/lex.c +++ b/src/cmd/gc/lex.c @@ -344,8 +344,8 @@ main(int argc, char *argv[]) break; } } - if(j == nelem(debugtab)) - fatal("unknown debug information -d '%s'\n", f[i]); + if(debugtab[j].name == nil) + sysfatal("unknown debug information -d '%s'\n", f[i]); } } From c6e53fea107563225ec85889b4d88d5ffbd85c17 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Tue, 4 Nov 2014 10:20:35 -0800 Subject: [PATCH 29/59] test: comment out failing cases from sinit.go One failing case this removes is: var bytes = []byte("hello, world") var copy_bytes = bytes We could handle this in the compiler, but it requires special case for a variable that is initialized to the value of a variable that is initialized to a string literal converted to []byte. This seems an unlikely case--it never occurs in the standrd library--and it seems unnecessary to write the code to handle it. If we do want to support this case, one approach is https://golang.org/cl/171840043. The other failing cases are of the form var bx bool var copy_bx = bx The compiler used to initialize copy_bx to false. However, that led to issue 7665, since bx may be initialized in non-Go code. The compiler no longer assumes that bx must be false, so copy_bx can not be statically initialized. We can fix these with https://golang.org/cl/169040043 if we also pass -complete to the compiler as part of this test. This is OK but it's too late in the release cycle. Fixes #8746. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/165400043 --- test/sinit.go | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/test/sinit.go b/test/sinit.go index 5e50e1100a..52dfd6fe44 100644 --- a/test/sinit.go +++ b/test/sinit.go @@ -112,7 +112,14 @@ var ( copy_slice = slice copy_sliceInt = sliceInt copy_hello = hello - copy_bytes = bytes + + // Could be handled without an initialization function, but + // requires special handling for "a = []byte("..."); b = a" + // which is not a likely case. + // copy_bytes = bytes + // https://codereview.appspot.com/171840043 is one approach to + // make this special case work. + copy_four, copy_five = four, five copy_x, copy_y = x, y copy_nilslice = nilslice @@ -202,58 +209,66 @@ var pt0b = &T{X: 0} var pt1 = &T{X: 1, Y: 2} var pt1a = &T{3, 4} -var copy_bx = bx +// The checks similar to +// var copy_bx = bx +// are commented out. The compiler no longer statically initializes them. +// See issue 7665 and https://codereview.appspot.com/93200044. +// If https://codereview.appspot.com/169040043 is submitted, and this +// test is changed to pass -complete to the compiler, then we can +// uncomment the copy lines again. + +// var copy_bx = bx var copy_b0 = b0 var copy_b1 = b1 -var copy_fx = fx +// var copy_fx = fx var copy_f0 = f0 var copy_f1 = f1 -var copy_gx = gx +// var copy_gx = gx var copy_g0 = g0 var copy_g1 = g1 -var copy_ix = ix +// var copy_ix = ix var copy_i0 = i0 var copy_i1 = i1 -var copy_jx = jx +// var copy_jx = jx var copy_j0 = j0 var copy_j1 = j1 -var copy_cx = cx +// var copy_cx = cx var copy_c0 = c0 var copy_c1 = c1 -var copy_dx = dx +// var copy_dx = dx var copy_d0 = d0 var copy_d1 = d1 -var copy_sx = sx +// var copy_sx = sx var copy_s0 = s0 var copy_s1 = s1 -var copy_ax = ax +// var copy_ax = ax var copy_a0 = a0 var copy_a1 = a1 -var copy_tx = tx +// var copy_tx = tx var copy_t0 = t0 var copy_t0a = t0a var copy_t0b = t0b var copy_t1 = t1 var copy_t1a = t1a -var copy_psx = psx +// var copy_psx = psx var copy_ps0 = ps0 var copy_ps1 = ps1 -var copy_pax = pax +// var copy_pax = pax var copy_pa0 = pa0 var copy_pa1 = pa1 -var copy_ptx = ptx +// var copy_ptx = ptx var copy_pt0 = pt0 var copy_pt0a = pt0a var copy_pt0b = pt0b From bb4a358af3469a1dee0cb5bcae245865233aa9ea Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 5 Nov 2014 15:14:47 -0500 Subject: [PATCH 30/59] 5g: don't generate reg variables for direct-called functions The test intended to skip direct calls when creating registerization variables was testing p->to.type instead of p->to.name, so it always failed, causing regopt to create unnecessary variables for these names. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/169110043 --- src/cmd/5g/reg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c index 8e49a2d9c8..27d9d3e8be 100644 --- a/src/cmd/5g/reg.c +++ b/src/cmd/5g/reg.c @@ -199,7 +199,7 @@ regopt(Prog *firstp) proginfo(&info, p); // Avoid making variables for direct-called functions. - if(p->as == ABL && p->to.type == D_EXTERN) + if(p->as == ABL && p->to.name == D_EXTERN) continue; bit = mkvar(r, &p->from); From 590f5283762a6e80796c3ae2bde410319162cb45 Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Thu, 6 Nov 2014 09:57:46 +1100 Subject: [PATCH 31/59] bufio: don't loop generating empty tokens The new rules for split functions mean that we are exposed to the common bug of a function that loops forever at EOF. Pick these off by shutting down the scanner if too many consecutive empty tokens are delivered. Fixes #9020. LGTM=rsc, adg R=golang-codereviews, rsc, adg, bradfitz CC=golang-codereviews https://golang.org/cl/169970043 --- src/bufio/scan.go | 12 +++++++++ src/bufio/scan_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/src/bufio/scan.go b/src/bufio/scan.go index a41451524d..73ad763b8f 100644 --- a/src/bufio/scan.go +++ b/src/bufio/scan.go @@ -36,6 +36,7 @@ type Scanner struct { start int // First non-processed byte in buf. end int // End of data in buf. err error // Sticky error. + empties int // Count of successive empty tokens. } // SplitFunc is the signature of the split function used to tokenize the @@ -108,6 +109,8 @@ func (s *Scanner) Text() string { // After Scan returns false, the Err method will return any error that // occurred during scanning, except that if it was io.EOF, Err // will return nil. +// Split panics if the split function returns 100 empty tokens without +// advancing the input. This is a common error mode for scanners. func (s *Scanner) Scan() bool { // Loop until we have a token. for { @@ -125,6 +128,14 @@ func (s *Scanner) Scan() bool { } s.token = token if token != nil { + if len(token) > 0 { + s.empties = 0 + } else { + s.empties++ + if s.empties > 100 { + panic("bufio.Scan: 100 empty tokens without progressing") + } + } return true } } @@ -172,6 +183,7 @@ func (s *Scanner) Scan() bool { break } if n > 0 { + s.empties = 0 break } loop++ diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go index 1454a8113c..a1cf90ddbf 100644 --- a/src/bufio/scan_test.go +++ b/src/bufio/scan_test.go @@ -455,3 +455,61 @@ func TestEmptyTokens(t *testing.T) { t.Fatal(err) } } + +func loopAtEOFSplit(data []byte, atEOF bool) (advance int, token []byte, err error) { + if len(data) > 0 { + return 1, data[:1], nil + } + return 0, data, nil +} + +func TestDontLoopForever(t *testing.T) { + s := NewScanner(strings.NewReader("abc")) + s.Split(loopAtEOFSplit) + // Expect a panic + panicked := true + defer func() { + err := recover() + if err == nil { + t.Fatal("should have panicked") + } + if msg, ok := err.(string); ok && strings.Contains(msg, "empty tokens") { + panicked = true + } else { + panic(err) + } + }() + for count := 0; s.Scan(); count++ { + if count > 1000 { + t.Fatal("looping") + } + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } +} + +type countdown int + +func (c *countdown) split(data []byte, atEOF bool) (advance int, token []byte, err error) { + if *c > 0 { + *c-- + return 1, data[:1], nil + } + return 0, nil, nil +} + +// Check that the looping-at-EOF check doesn't trigger for merely empty tokens. +func TestEmptyLinesOK(t *testing.T) { + c := countdown(10000) + s := NewScanner(strings.NewReader(strings.Repeat("\n", 10000))) + s.Split(c.split) + for s.Scan() { + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } + if c != 0 { + t.Fatalf("stopped with %d left to process", c) + } +} From 67742ef560be186a2ebeb45aaab50399bf81b358 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 5 Nov 2014 20:15:48 -0500 Subject: [PATCH 32/59] doc/go1.4.html: document new subrepo import paths LGTM=r, adg R=adg, r, 0xjnml, dr.volker.dobler CC=golang-codereviews https://golang.org/cl/166980044 --- doc/go1.4.html | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 3310117a4d..1a18054b95 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -120,7 +120,7 @@ compile but is easy to fix by adding an explicit dereference.

    Go 1.4 can build binaries for ARM processors running the Android operating system. It can also build a .so library that can be loaded by an Android application -using the supporting packages in the go.mobile repository. +using the supporting packages in the mobile subrepository. A brief description of the plans for this experimental port are available here.

    @@ -385,6 +385,24 @@ Further information is in the design document.

    +

    Import paths for the subrepositories

    + +

    +The Go project subrepositories (code.google.com/p/go.tools and so on) +are now available under custom import paths replacing code.google.com/p/go. with golang.org/x/, +as in golang.org/x/tools. +We will add canonical import comments to the code around June 1, 2015, +at which point Go 1.4 and later will stop accepting the old code.google.com paths. +

    + +

    +Updating: All code that imports from subrepositories should change +to use the new golang.org paths. +Go 1.0 and later can resolve and import the new paths, so updating will not break +compatibility with older releases. +Code that has not updated will stop compiling with Go 1.4 around June 1, 2015. +

    +

    The go generate subcommand

    @@ -394,8 +412,8 @@ to automate the running of tools to generate source code before compilation. For example, it can be used to run the yacc compiler-compiler on a .y file to produce the Go source file implementing the grammar, or to automate the generation of String methods for typed constants using the new -stringer -tool in the go.tools repository. +stringer +tool in the golang.org/x/tools subrepository.

    @@ -480,7 +498,7 @@ rebuild the standard library and commands, to avoid overwriting the installation

    In the main Go source repository, the source code for the packages was kept in the directory src/pkg, which made sense but differed from -other repositories, including the Go sub-repositories such as go.tools. +other repositories, including the Go subrepositories. In Go 1.4, the pkg level of the source tree is now gone, so for example the fmt package's source, once kept in directory src/pkg/fmt, now lives one level higher in src/fmt. @@ -591,14 +609,14 @@ separate document.

    -A new subrepository, go.sys, +A new subrepository, golang.org/x/sys, has been created to serve as the location for new developments to support system calls on all kernels. It has a nicer structure, with three packages that each hold the implementation of system calls for one of -Unix, -Windows and -Plan 9. +Unix, +Windows and +Plan 9. These packages will be curated more generously, accepting all reasonable changes that reflect kernel interfaces in those operating systems. See the documentation and the article mentioned above for more information. @@ -608,7 +626,7 @@ See the documentation and the article mentioned above for more information. Updating: Existing programs are not affected as the syscall package is largely unchanged from the 1.3 release. Future development that requires system calls not in the syscall package -should build on go.sys instead. +should build on golang.org/x/sys instead.

    Minor changes to the library

    From 2d0db8e591513a1123057b8c330c946ddcb4fbc8 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 5 Nov 2014 22:50:24 -0500 Subject: [PATCH 33/59] bufio: fix reading of many blank lines in a row Fixes #9020. LGTM=bradfitz, r R=r, bradfitz CC=golang-codereviews https://golang.org/cl/170030043 --- src/bufio/scan.go | 3 ++- src/bufio/scan_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/bufio/scan.go b/src/bufio/scan.go index 73ad763b8f..364d159613 100644 --- a/src/bufio/scan.go +++ b/src/bufio/scan.go @@ -128,9 +128,10 @@ func (s *Scanner) Scan() bool { } s.token = token if token != nil { - if len(token) > 0 { + if s.err == nil || advance > 0 { s.empties = 0 } else { + // Returning tokens not advancing input at EOF. s.empties++ if s.empties > 100 { panic("bufio.Scan: 100 empty tokens without progressing") diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go index a1cf90ddbf..bf888dafb5 100644 --- a/src/bufio/scan_test.go +++ b/src/bufio/scan_test.go @@ -489,6 +489,18 @@ func TestDontLoopForever(t *testing.T) { } } +func TestBlankLines(t *testing.T) { + s := NewScanner(strings.NewReader(strings.Repeat("\n", 1000))) + for count := 0; s.Scan(); count++ { + if count > 2000 { + t.Fatal("looping") + } + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } +} + type countdown int func (c *countdown) split(data []byte, atEOF bool) (advance int, token []byte, err error) { From 39bcbb353c6bf2e13eb0d3585fe82d3cab6df78d Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 5 Nov 2014 23:01:48 -0500 Subject: [PATCH 34/59] runtime: avoid gentraceback of self on user goroutine stack Gentraceback may grow the stack. One of the gentraceback wrappers may grow the stack. One of the gentraceback callback calls may grow the stack. Various stack pointers are stored in various stack locations as type uintptr during the execution of these calls. If the stack does grow, these stack pointers will not be updated and will start trying to decode stack memory that is no longer valid. It may be possible to change the type of the stack pointer variables to be unsafe.Pointer, but that's pretty subtle and may still have problems, even if we catch every last one. An easier, more obviously correct fix is to require that gentraceback of the currently running goroutine must run on the g0 stack, not on the goroutine's own stack. Not doing this causes faults when you set StackFromSystem = 1 StackFaultOnFree = 1 The new check in gentraceback will catch future lapses. The more general problem is calling getcallersp but then calling a function that might relocate the stack, which would invalidate the result of getcallersp. Add note to stubs.go declaration of getcallersp explaining the problem, and check all existing calls to getcallersp. Most needed fixes. This affects Callers, Stack, and nearly all the runtime profiling routines. It does not affect stack copying directly nor garbage collection. LGTM=khr R=khr, bradfitz CC=golang-codereviews, r https://golang.org/cl/167060043 --- src/runtime/mprof.go | 43 ++++++++++++++++++++++++++-------------- src/runtime/stubs.go | 28 ++++++++++++++++++++++++++ src/runtime/traceback.go | 22 +++++++++++++++++++- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index d64e3be695..d409c6c306 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -528,8 +528,6 @@ var allgs []*g // proc.c // Most clients should use the runtime/pprof package instead // of calling GoroutineProfile directly. func GoroutineProfile(p []StackRecord) (n int, ok bool) { - sp := getcallersp(unsafe.Pointer(&p)) - pc := getcallerpc(unsafe.Pointer(&p)) n = NumGoroutine() if n <= len(p) { @@ -542,7 +540,11 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { if n <= len(p) { ok = true r := p - saveg(pc, sp, gp, &r[0]) + sp := getcallersp(unsafe.Pointer(&p)) + pc := getcallerpc(unsafe.Pointer(&p)) + onM(func() { + saveg(pc, sp, gp, &r[0]) + }) r = r[1:] for _, gp1 := range allgs { if gp1 == gp || readgstatus(gp1) == _Gdead { @@ -573,8 +575,6 @@ func saveg(pc, sp uintptr, gp *g, r *StackRecord) { // If all is true, Stack formats stack traces of all other goroutines // into buf after the trace for the current goroutine. func Stack(buf []byte, all bool) int { - sp := getcallersp(unsafe.Pointer(&buf)) - pc := getcallerpc(unsafe.Pointer(&buf)) mp := acquirem() gp := mp.curg if all { @@ -589,14 +589,19 @@ func Stack(buf []byte, all bool) int { n := 0 if len(buf) > 0 { - gp.writebuf = buf[0:0:len(buf)] - goroutineheader(gp) - traceback(pc, sp, 0, gp) - if all { - tracebackothers(gp) - } - n = len(gp.writebuf) - gp.writebuf = nil + sp := getcallersp(unsafe.Pointer(&buf)) + pc := getcallerpc(unsafe.Pointer(&buf)) + onM(func() { + g0 := getg() + g0.writebuf = buf[0:0:len(buf)] + goroutineheader(gp) + traceback(pc, sp, 0, gp) + if all { + tracebackothers(gp) + } + n = len(g0.writebuf) + g0.writebuf = nil + }) } if all { @@ -623,7 +628,11 @@ func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) { } if gp.m.curg == nil || gp == gp.m.curg { goroutineheader(gp) - traceback(getcallerpc(unsafe.Pointer(&p)), getcallersp(unsafe.Pointer(&p)), 0, gp) + pc := getcallerpc(unsafe.Pointer(&p)) + sp := getcallersp(unsafe.Pointer(&p)) + onM(func() { + traceback(pc, sp, 0, gp) + }) } else { goroutineheader(gp.m.curg) traceback(^uintptr(0), ^uintptr(0), 0, gp.m.curg) @@ -639,7 +648,11 @@ func tracefree(p unsafe.Pointer, size uintptr) { gp.m.traceback = 2 print("tracefree(", p, ", ", hex(size), ")\n") goroutineheader(gp) - traceback(getcallerpc(unsafe.Pointer(&p)), getcallersp(unsafe.Pointer(&p)), 0, gp) + pc := getcallerpc(unsafe.Pointer(&p)) + sp := getcallersp(unsafe.Pointer(&p)) + onM(func() { + traceback(pc, sp, 0, gp) + }) print("\n") gp.m.traceback = 0 unlock(&tracelock) diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 3419047198..fe8f9c9222 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -221,6 +221,34 @@ func atomicloaduint(ptr *uint) uint //go:noescape func setcallerpc(argp unsafe.Pointer, pc uintptr) +// getcallerpc returns the program counter (PC) of its caller's caller. +// getcallersp returns the stack pointer (SP) of its caller's caller. +// For both, the argp must be a pointer to the caller's first function argument. +// The implementation may or may not use argp, depending on +// the architecture. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := getcallerpc(unsafe.Pointer(&arg1)) +// sp := getcallerpc(unsafe.Pointer(&arg2)) +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to getcallerpc and getcallersp must be done in the +// frame being asked about. It would not be correct for f to pass &arg1 +// to another function g and let g call getcallerpc/getcallersp. +// The call inside g might return information about g's caller or +// information about f's caller or complete garbage. +// +// The result of getcallersp is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of getcallersp should be used +// immediately and can only be passed to nosplit functions. + //go:noescape func getcallerpc(argp unsafe.Pointer) uintptr diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 834435b400..1c6ce6e644 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -101,6 +101,22 @@ func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf gothrow("gentraceback before goexitPC initialization") } g := getg() + if g == gp && g == g.m.curg { + // The starting sp has been passed in as a uintptr, and the caller may + // have other uintptr-typed stack references as well. + // If during one of the calls that got us here or during one of the + // callbacks below the stack must be grown, all these uintptr references + // to the stack will not be updated, and gentraceback will continue + // to inspect the old stack memory, which may no longer be valid. + // Even if all the variables were updated correctly, it is not clear that + // we want to expose a traceback that begins on one stack and ends + // on another stack. That could confuse callers quite a bit. + // Instead, we require that gentraceback and any other function that + // accepts an sp for the current goroutine (typically obtained by + // calling getcallersp) must not run on that goroutine's stack but + // instead on the g0 stack. + gothrow("gentraceback cannot trace user goroutine on its own stack") + } gotraceback := gotraceback(nil) if pc0 == ^uintptr(0) && sp0 == ^uintptr(0) { // Signal to fetch saved values from gp. if gp.syscallsp != 0 { @@ -511,7 +527,11 @@ func traceback1(pc uintptr, sp uintptr, lr uintptr, gp *g, flags uint) { func callers(skip int, pcbuf *uintptr, m int) int { sp := getcallersp(unsafe.Pointer(&skip)) pc := uintptr(getcallerpc(unsafe.Pointer(&skip))) - return gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) + var n int + onM(func() { + n = gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) + }) + return n } func gcallers(gp *g, skip int, pcbuf *uintptr, m int) int { From 908dcab6f854503a4616b6a20aa1e9eba9ecea66 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Thu, 6 Nov 2014 15:22:29 +1100 Subject: [PATCH 35/59] bufio: remove unused 'panicked' variable from test LGTM=r R=r CC=golang-codereviews https://golang.org/cl/166230044 --- src/bufio/scan_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go index bf888dafb5..eea87cbf7b 100644 --- a/src/bufio/scan_test.go +++ b/src/bufio/scan_test.go @@ -467,15 +467,12 @@ func TestDontLoopForever(t *testing.T) { s := NewScanner(strings.NewReader("abc")) s.Split(loopAtEOFSplit) // Expect a panic - panicked := true defer func() { err := recover() if err == nil { t.Fatal("should have panicked") } - if msg, ok := err.(string); ok && strings.Contains(msg, "empty tokens") { - panicked = true - } else { + if msg, ok := err.(string); !ok || !strings.Contains(msg, "empty tokens") { panic(err) } }() From 23ecad07cd8cf07a21af07a1e96803f32536ab05 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 5 Nov 2014 20:25:20 -0800 Subject: [PATCH 36/59] os/exec: tell lsof not to block For some reason lsof is now hanging on my workstation without the -b (avoid blocking in the kernel) option. Adding -b makes the test pass and shouldn't hurt. I don't know how recent the -b option is. If the builders are ok with it, it's probably ok. LGTM=rsc R=golang-codereviews, bradfitz, rsc CC=golang-codereviews https://golang.org/cl/166220043 --- src/os/exec/exec_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/exec/exec_test.go b/src/os/exec/exec_test.go index bc9c00effe..197d3e8b40 100644 --- a/src/os/exec/exec_test.go +++ b/src/os/exec/exec_test.go @@ -246,7 +246,7 @@ func TestPipeLookPathLeak(t *testing.T) { } func numOpenFDS(t *testing.T) (n int, lsof []byte) { - lsof, err := exec.Command("lsof", "-n", "-p", strconv.Itoa(os.Getpid())).Output() + lsof, err := exec.Command("lsof", "-b", "-n", "-p", strconv.Itoa(os.Getpid())).Output() if err != nil { t.Skip("skipping test; error finding or running lsof") } From 1cdd9b407db0f842387dc5e4f18e6563dcd87b18 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 09:36:51 -0500 Subject: [PATCH 37/59] os: document that users of Fd should keep f alive Fixes #9046. LGTM=r R=r CC=golang-codereviews https://golang.org/cl/162680043 --- src/os/file_plan9.go | 3 ++- src/os/file_unix.go | 1 + src/os/file_windows.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/os/file_plan9.go b/src/os/file_plan9.go index 5efc2a4f1d..132594eede 100644 --- a/src/os/file_plan9.go +++ b/src/os/file_plan9.go @@ -25,7 +25,8 @@ type file struct { dirinfo *dirInfo // nil unless directory being read } -// Fd returns the integer Unix file descriptor referencing the open file. +// Fd returns the integer Plan 9 file descriptor referencing the open file. +// The file descriptor is valid only until f.Close is called or f is garbage collected. func (f *File) Fd() uintptr { if f == nil { return ^(uintptr(0)) diff --git a/src/os/file_unix.go b/src/os/file_unix.go index f59d563e69..ff4fc7d12e 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -29,6 +29,7 @@ type file struct { } // Fd returns the integer Unix file descriptor referencing the open file. +// The file descriptor is valid only until f.Close is called or f is garbage collected. func (f *File) Fd() uintptr { if f == nil { return ^(uintptr(0)) diff --git a/src/os/file_windows.go b/src/os/file_windows.go index 3b5519390b..2a90a50559 100644 --- a/src/os/file_windows.go +++ b/src/os/file_windows.go @@ -36,6 +36,7 @@ type file struct { } // Fd returns the Windows handle referencing the open file. +// The handle is valid only until f.Close is called or f is garbage collected. func (file *File) Fd() uintptr { if file == nil { return uintptr(syscall.InvalidHandle) From 6ad16c4a48accddee2e711b565e9214eae945da4 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 09:37:04 -0500 Subject: [PATCH 38/59] runtime: fix initial gp->sched.pc in newextram CL 170720043 missed this one when adding +PCQuantum. LGTM=iant R=r, iant CC=golang-codereviews https://golang.org/cl/168090043 --- src/runtime/proc.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/runtime/proc.c b/src/runtime/proc.c index 4be51e1e16..91e3fe16d6 100644 --- a/src/runtime/proc.c +++ b/src/runtime/proc.c @@ -990,7 +990,7 @@ runtime·newextram(void) // the goroutine stack ends. mp = runtime·allocm(nil); gp = runtime·malg(4096); - gp->sched.pc = (uintptr)runtime·goexit; + gp->sched.pc = (uintptr)runtime·goexit + PCQuantum; gp->sched.sp = gp->stack.hi; gp->sched.sp -= 4*sizeof(uintreg); // extra space in case of reads slightly beyond frame gp->sched.lr = 0; @@ -2424,9 +2424,10 @@ static struct ProfState { int32 hz; } prof; -static void System(void) {} -static void ExternalCode(void) {} -static void GC(void) {} +static void System(void) { System(); } +static void ExternalCode(void) { ExternalCode(); } +static void GC(void) { GC(); } + extern void runtime·cpuproftick(uintptr*, int32); extern byte runtime·etext[]; From 5b110c7b08946a2a2810cb4614c078c74645a3d1 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 6 Nov 2014 09:30:41 -0800 Subject: [PATCH 39/59] runtime: don't stop bitmap dump at BitsDead Stack bitmaps need to be scanned past any BitsDead entries. Object bitmaps will not have any BitsDead in them (bitmap extraction stops at the first BitsDead entry in makeheapobjbv). data/bss bitmaps also have no BitsDead entries. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/168270043 --- src/runtime/heapdump.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c index 94a4bd2be5..eddbc1d1c9 100644 --- a/src/runtime/heapdump.c +++ b/src/runtime/heapdump.c @@ -251,7 +251,9 @@ dumpbv(BitVector *bv, uintptr offset) for(i = 0; i < bv->n; i += BitsPerPointer) { switch(bv->bytedata[i/8] >> i%8 & 3) { case BitsDead: - return; + // BitsDead has already been processed in makeheapobjbv. + // We should only see it in stack maps, in which case we should continue processing. + break; case BitsScalar: break; case BitsPointer: From 9b5444420cec4b576aff179f632ec60be64fd7f2 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Thu, 6 Nov 2014 15:14:08 -0500 Subject: [PATCH 40/59] test: move linkx and sinit to run.go The remaining run-only tests will be migrated to run.go in another CL. This CL will break the build due to issues 8746 and 8806. Update #4139 Update #8746 Update #8806 LGTM=rsc R=rsc, bradfitz, iant CC=golang-codereviews https://golang.org/cl/144630044 --- test/linkx.go | 14 ++++---------- test/linkx_run.go | 32 ++++++++++++++++++++++++++++++++ test/run.go | 2 -- test/sinit.go | 34 +++++++++++++++++----------------- test/sinit_run.go | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 test/linkx_run.go create mode 100644 test/sinit_run.go diff --git a/test/linkx.go b/test/linkx.go index 06888a229a..151b6db1ec 100644 --- a/test/linkx.go +++ b/test/linkx.go @@ -1,13 +1,11 @@ -// $G $D/$F.go && $L -X main.tbd hello -X main.overwrite trumped -X main.nosuchsymbol neverseen $F.$A && ./$A.out - -// NOTE: This test is not run by 'run.go' and so not run by all.bash. -// To run this test you must use the ./run shell script. +// skip // Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Test the -X facility of the gc linker (6l etc.). +// This test is run by linkx_run.go. package main @@ -15,10 +13,6 @@ var tbd string var overwrite string = "dibs" func main() { - if tbd != "hello" { - println("BUG: test/linkx tbd", len(tbd), tbd) - } - if overwrite != "trumped" { - println("BUG: test/linkx overwrite", len(overwrite), overwrite) - } + println(tbd) + println(overwrite) } diff --git a/test/linkx_run.go b/test/linkx_run.go new file mode 100644 index 0000000000..abfc342a6c --- /dev/null +++ b/test/linkx_run.go @@ -0,0 +1,32 @@ +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run the linkx test. + +package main + +import ( + "fmt" + "os" + "os/exec" +) + +func main() { + cmd := exec.Command("go", "run", "-ldflags=-X main.tbd hello -X main.overwrite trumped", "linkx.go") + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Println(string(out)) + fmt.Println(err) + os.Exit(1) + } + + want := "hello\ntrumped\n" + got := string(out) + if got != want { + fmt.Printf("got %q want %q\n", got, want) + os.Exit(1) + } +} diff --git a/test/run.go b/test/run.go index 28882cf54c..e8ec2df9c4 100644 --- a/test/run.go +++ b/test/run.go @@ -907,8 +907,6 @@ func (t *test) wantedErrors(file, short string) (errs []wantedError) { } var skipOkay = map[string]bool{ - "linkx.go": true, // like "run" but wants linker flags - "sinit.go": true, "fixedbugs/bug248.go": true, // combines errorcheckdir and rundir in the same dir. "fixedbugs/bug302.go": true, // tests both .$O and .a imports. "fixedbugs/bug345.go": true, // needs the appropriate flags in gc invocation. diff --git a/test/sinit.go b/test/sinit.go index 52dfd6fe44..df1a4cc930 100644 --- a/test/sinit.go +++ b/test/sinit.go @@ -1,7 +1,4 @@ -// $G -S $D/$F.go | egrep initdone >/dev/null && echo BUG sinit || true - -// NOTE: This test is not run by 'run.go' and so not run by all.bash. -// To run this test you must use the ./run shell script. +// skip // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -9,6 +6,7 @@ // Test that many initializations can be done at link time and // generate no executable init functions. +// This test is run by sinit_run.go. package p @@ -106,12 +104,12 @@ var answers = [...]int{ } var ( - copy_zero = zero - copy_one = one - copy_pi = pi - copy_slice = slice + copy_zero = zero + copy_one = one + copy_pi = pi + copy_slice = slice copy_sliceInt = sliceInt - copy_hello = hello + copy_hello = hello // Could be handled without an initialization function, but // requires special handling for "a = []byte("..."); b = a" @@ -121,12 +119,12 @@ var ( // make this special case work. copy_four, copy_five = four, five - copy_x, copy_y = x, y - copy_nilslice = nilslice - copy_nilmap = nilmap - copy_nilfunc = nilfunc - copy_nilchan = nilchan - copy_nilptr = nilptr + copy_x, copy_y = x, y + copy_nilslice = nilslice + copy_nilmap = nilmap + copy_nilfunc = nilfunc + copy_nilchan = nilchan + copy_nilptr = nilptr ) var copy_a = a @@ -179,7 +177,7 @@ var sx []int var s0 = []int{0, 0, 0} var s1 = []int{1, 2, 3} -func fi() int +func fi() int { return 1 } var ax [10]int var a0 = [10]int{0, 0, 0} @@ -281,6 +279,8 @@ type T1 int func (t *T1) M() {} -type Mer interface { M() } +type Mer interface { + M() +} var _ Mer = (*T1)(nil) diff --git a/test/sinit_run.go b/test/sinit_run.go new file mode 100644 index 0000000000..a21bd1077d --- /dev/null +++ b/test/sinit_run.go @@ -0,0 +1,39 @@ +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run the sinit test. + +package main + +import ( + "bytes" + "fmt" + "go/build" + "os" + "os/exec" +) + +func main() { + letter, err := build.ArchChar(build.Default.GOARCH) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + cmd := exec.Command("go", "tool", letter+"g", "-S", "sinit.go") + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Println(string(out)) + fmt.Println(err) + os.Exit(1) + } + os.Remove("sinit." + letter) + + if bytes.Contains(out, []byte("initdone")) { + fmt.Println("sinit generated an init function") + os.Exit(1) + } +} From 6bc812e9ecbe66a1fbb096e0f87d7f1b8f9fff04 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 15:18:47 -0500 Subject: [PATCH 41/59] doc: change "/s/..." links to be on golang.org People viewing this locally will not have a /s/ on their local godoc. tip.golang.org doesn't have one either. Also change all golang.org links to https, to avoid mixed content warnings when viewing https://golang.org/. Fixes #9028. LGTM=bradfitz, r R=r, bradfitz CC=adg, golang-codereviews https://golang.org/cl/168250043 --- doc/go1.4.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index 1a18054b95..b375612d0c 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -120,9 +120,9 @@ compile but is easy to fix by adding an explicit dereference.

    Go 1.4 can build binaries for ARM processors running the Android operating system. It can also build a .so library that can be loaded by an Android application -using the supporting packages in the mobile subrepository. +using the supporting packages in the mobile subrepository. A brief description of the plans for this experimental port are available -here. +here.

    NaCl on ARM

    @@ -193,7 +193,7 @@ A consequence is that stacks are no longer segmented, eliminating the "hot split When a stack limit is reached, a new, larger stack is allocated, all active frames for the goroutine are copied there, and any pointers into the stack are updated. Performance can be noticeably better in some cases and is always more predictable. -Details are available in the design document. +Details are available in the design document.

    @@ -320,7 +320,7 @@ from 1.5 and onward it will be enforced for any repository.

    Full details of the mechanism are in -the design document. +the design document.

    Canonical import paths

    @@ -382,7 +382,7 @@ The new -f flag overrides this check.

    Further information is in -the design document. +the design document.

    Import paths for the subrepositories

    @@ -418,7 +418,7 @@ tool in the golang.org/x/tools subrepository.

    For more information, see the -design document. +design document.

    Change to file name handling

    @@ -604,12 +604,12 @@ The syscall package is now frozen excep for changes needed to maintain the core repository. In particular, it will no longer be extended to support new or different system calls that are not used by the core. -The reasons are described at length in a +The reasons are described at length in a separate document.

    -A new subrepository, golang.org/x/sys, +A new subrepository, golang.org/x/sys, has been created to serve as the location for new developments to support system calls on all kernels. It has a nicer structure, with three packages that each hold the implementation of From 08b2cb4afe3eebd384c543986ca8ee9d4ce04ede Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 15:19:16 -0500 Subject: [PATCH 42/59] doc/go1.4.html: leave stack size at 2 kB LGTM=r R=khr, r CC=golang-codereviews https://golang.org/cl/165590043 --- doc/go1.4.html | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/go1.4.html b/doc/go1.4.html index b375612d0c..ac63ade60a 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -199,7 +199,6 @@ Details are available in the design

    The use of contiguous stacks means that stacks can start smaller without triggering performance issues, so the default starting size for a goroutine's stack in 1.4 has been reduced to 2048 bytes from 8192 bytes. -TODO: It may be bumped to 4096 for the release.

    From 6bd0d0542ee15fda0da545c16af43fcfd34d6334 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 19:56:55 -0500 Subject: [PATCH 43/59] cmd/objdump, cmd/pprof: factor disassembly into cmd/internal/objfile Moving so that new Go 1.4 pprof can use it. The old 'GNU objdump workalike' mode for 'go tool objdump' is now gone, as are the tests for that mode. It was used only by pre-Go 1.4 pprof. You can still specify an address range on the command line; you just get the same output format as you do when dumping the entire binary (without an address limitation). LGTM=r R=r CC=golang-codereviews, iant https://golang.org/cl/167320043 --- src/cmd/internal/objfile/objfile.go | 14 +- src/cmd/objdump/main.go | 242 +++------------------------- src/cmd/objdump/objdump_test.go | 98 ----------- src/cmd/pprof/pprof.go | 41 ++++- 4 files changed, 69 insertions(+), 326 deletions(-) diff --git a/src/cmd/internal/objfile/objfile.go b/src/cmd/internal/objfile/objfile.go index 3d4a5d27cd..9227ef387f 100644 --- a/src/cmd/internal/objfile/objfile.go +++ b/src/cmd/internal/objfile/objfile.go @@ -9,6 +9,7 @@ import ( "debug/gosym" "fmt" "os" + "sort" ) type rawFile interface { @@ -62,9 +63,20 @@ func (f *File) Close() error { } func (f *File) Symbols() ([]Sym, error) { - return f.raw.symbols() + syms, err := f.raw.symbols() + if err != nil { + return nil, err + } + sort.Sort(byAddr(syms)) + return syms, nil } +type byAddr []Sym + +func (x byAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } +func (x byAddr) Len() int { return len(x) } +func (x byAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + func (f *File) PCLineTable() (*gosym.Table, error) { textStart, symtab, pclntab, err := f.raw.pcln() if err != nil { diff --git a/src/cmd/objdump/main.go b/src/cmd/objdump/main.go index 0f125c98bf..708a853702 100644 --- a/src/cmd/objdump/main.go +++ b/src/cmd/objdump/main.go @@ -32,24 +32,15 @@ package main import ( - "bufio" - "debug/gosym" - "encoding/binary" "flag" "fmt" - "io" "log" "os" "regexp" - "sort" "strconv" "strings" - "text/tabwriter" "cmd/internal/objfile" - - "cmd/internal/rsc.io/arm/armasm" - "cmd/internal/rsc.io/x86/x86asm" ) var symregexp = flag.String("s", "", "only dump symbols matching this regexp") @@ -87,227 +78,30 @@ func main() { log.Fatal(err) } - syms, err := f.Symbols() + dis, err := f.Disasm() if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) + log.Fatal("disassemble %s: %v", flag.Arg(0), err) } - tab, err := f.PCLineTable() - if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) - } + switch flag.NArg() { + default: + usage() + case 1: + // disassembly of entire object + dis.Print(os.Stdout, symRE, 0, ^uint64(0)) + os.Exit(0) - textStart, textBytes, err := f.Text() - if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) - } - - goarch := f.GOARCH() - - disasm := disasms[goarch] - if disasm == nil { - log.Fatalf("reading %s: unknown architecture", flag.Arg(0)) - } - - // Filter out section symbols, overwriting syms in place. - keep := syms[:0] - for _, sym := range syms { - switch sym.Name { - case "runtime.text", "text", "_text", "runtime.etext", "etext", "_etext": - // drop - default: - keep = append(keep, sym) + case 3: + // disassembly of PC range + start, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(1), "0x"), 16, 64) + if err != nil { + log.Fatalf("invalid start PC: %v", err) } - } - syms = keep - - sort.Sort(ByAddr(syms)) - lookup := func(addr uint64) (string, uint64) { - i := sort.Search(len(syms), func(i int) bool { return addr < syms[i].Addr }) - if i > 0 { - s := syms[i-1] - if s.Addr != 0 && s.Addr <= addr && addr < s.Addr+uint64(s.Size) { - return s.Name, s.Addr - } + end, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(2), "0x"), 16, 64) + if err != nil { + log.Fatalf("invalid end PC: %v", err) } - return "", 0 - } - - if flag.NArg() == 1 { - // disassembly of entire object - our format - dump(tab, lookup, disasm, goarch, syms, textBytes, textStart) + dis.Print(os.Stdout, symRE, start, end) os.Exit(0) } - - // disassembly of specific piece of object - gnu objdump format for pprof - gnuDump(tab, lookup, disasm, textBytes, textStart) - os.Exit(0) } - -// base returns the final element in the path. -// It works on both Windows and Unix paths. -func base(path string) string { - path = path[strings.LastIndex(path, "/")+1:] - path = path[strings.LastIndex(path, `\`)+1:] - return path -} - -func dump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, goarch string, syms []objfile.Sym, textData []byte, textStart uint64) { - stdout := bufio.NewWriter(os.Stdout) - defer stdout.Flush() - - printed := false - for _, sym := range syms { - if (sym.Code != 'T' && sym.Code != 't') || sym.Size == 0 || sym.Addr < textStart || symRE != nil && !symRE.MatchString(sym.Name) { - continue - } - if sym.Addr >= textStart+uint64(len(textData)) || sym.Addr+uint64(sym.Size) > textStart+uint64(len(textData)) { - break - } - if printed { - fmt.Fprintf(stdout, "\n") - } else { - printed = true - } - file, _, _ := tab.PCToLine(sym.Addr) - fmt.Fprintf(stdout, "TEXT %s(SB) %s\n", sym.Name, file) - tw := tabwriter.NewWriter(stdout, 1, 8, 1, '\t', 0) - start := sym.Addr - end := sym.Addr + uint64(sym.Size) - for pc := start; pc < end; { - i := pc - textStart - text, size := disasm(textData[i:end-textStart], pc, lookup) - file, line, _ := tab.PCToLine(pc) - - // ARM is word-based, so show actual word hex, not byte hex. - // Since ARM is little endian, they're different. - if goarch == "arm" && size == 4 { - fmt.Fprintf(tw, "\t%s:%d\t%#x\t%08x\t%s\n", base(file), line, pc, binary.LittleEndian.Uint32(textData[i:i+uint64(size)]), text) - } else { - fmt.Fprintf(tw, "\t%s:%d\t%#x\t%x\t%s\n", base(file), line, pc, textData[i:i+uint64(size)], text) - } - pc += uint64(size) - } - tw.Flush() - } -} - -func disasm_386(code []byte, pc uint64, lookup lookupFunc) (string, int) { - return disasm_x86(code, pc, lookup, 32) -} - -func disasm_amd64(code []byte, pc uint64, lookup lookupFunc) (string, int) { - return disasm_x86(code, pc, lookup, 64) -} - -func disasm_x86(code []byte, pc uint64, lookup lookupFunc, arch int) (string, int) { - inst, err := x86asm.Decode(code, 64) - var text string - size := inst.Len - if err != nil || size == 0 || inst.Op == 0 { - size = 1 - text = "?" - } else { - text = x86asm.Plan9Syntax(inst, pc, lookup) - } - return text, size -} - -type textReader struct { - code []byte - pc uint64 -} - -func (r textReader) ReadAt(data []byte, off int64) (n int, err error) { - if off < 0 || uint64(off) < r.pc { - return 0, io.EOF - } - d := uint64(off) - r.pc - if d >= uint64(len(r.code)) { - return 0, io.EOF - } - n = copy(data, r.code[d:]) - if n < len(data) { - err = io.ErrUnexpectedEOF - } - return -} - -func disasm_arm(code []byte, pc uint64, lookup lookupFunc) (string, int) { - inst, err := armasm.Decode(code, armasm.ModeARM) - var text string - size := inst.Len - if err != nil || size == 0 || inst.Op == 0 { - size = 4 - text = "?" - } else { - text = armasm.Plan9Syntax(inst, pc, lookup, textReader{code, pc}) - } - return text, size -} - -var disasms = map[string]disasmFunc{ - "386": disasm_386, - "amd64": disasm_amd64, - "arm": disasm_arm, -} - -func gnuDump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, textData []byte, textStart uint64) { - start, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(1), "0x"), 16, 64) - if err != nil { - log.Fatalf("invalid start PC: %v", err) - } - end, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(2), "0x"), 16, 64) - if err != nil { - log.Fatalf("invalid end PC: %v", err) - } - if start < textStart { - start = textStart - } - if end < start { - end = start - } - if end > textStart+uint64(len(textData)) { - end = textStart + uint64(len(textData)) - } - - stdout := bufio.NewWriter(os.Stdout) - defer stdout.Flush() - - // For now, find spans of same PC/line/fn and - // emit them as having dummy instructions. - var ( - spanPC uint64 - spanFile string - spanLine int - spanFn *gosym.Func - ) - - flush := func(endPC uint64) { - if spanPC == 0 { - return - } - fmt.Fprintf(stdout, "%s:%d\n", spanFile, spanLine) - for pc := spanPC; pc < endPC; { - text, size := disasm(textData[pc-textStart:], pc, lookup) - fmt.Fprintf(stdout, " %x: %s\n", pc, text) - pc += uint64(size) - } - spanPC = 0 - } - - for pc := start; pc < end; pc++ { - file, line, fn := tab.PCToLine(pc) - if file != spanFile || line != spanLine || fn != spanFn { - flush(pc) - spanPC, spanFile, spanLine, spanFn = pc, file, line, fn - } - } - flush(end) -} - -type ByAddr []objfile.Sym - -func (x ByAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } -func (x ByAddr) Len() int { return len(x) } -func (x ByAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } diff --git a/src/cmd/objdump/objdump_test.go b/src/cmd/objdump/objdump_test.go index ffaaa5b437..2bb74663c3 100644 --- a/src/cmd/objdump/objdump_test.go +++ b/src/cmd/objdump/objdump_test.go @@ -5,113 +5,15 @@ package main import ( - "bufio" - "bytes" - "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "runtime" - "strconv" "strings" "testing" ) -func loadSyms(t *testing.T) map[string]string { - switch runtime.GOOS { - case "android", "nacl": - t.Skipf("skipping on %s", runtime.GOOS) - } - - cmd := exec.Command("go", "tool", "nm", os.Args[0]) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out)) - } - syms := make(map[string]string) - scanner := bufio.NewScanner(bytes.NewReader(out)) - for scanner.Scan() { - f := strings.Fields(scanner.Text()) - if len(f) < 3 { - continue - } - syms[f[2]] = f[0] - } - if err := scanner.Err(); err != nil { - t.Fatalf("error reading symbols: %v", err) - } - return syms -} - -func runObjDump(t *testing.T, exe, startaddr, endaddr string) (path, lineno string) { - switch runtime.GOOS { - case "android", "nacl": - t.Skipf("skipping on %s", runtime.GOOS) - } - - cmd := exec.Command(exe, os.Args[0], startaddr, endaddr) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("go tool objdump %v: %v\n%s", os.Args[0], err, string(out)) - } - f := strings.Split(string(out), "\n") - if len(f) < 1 { - t.Fatal("objdump output must have at least one line") - } - pathAndLineNo := f[0] - f = strings.Split(pathAndLineNo, ":") - if runtime.GOOS == "windows" { - switch len(f) { - case 2: - return f[0], f[1] - case 3: - return f[0] + ":" + f[1], f[2] - default: - t.Fatalf("no line number found in %q", pathAndLineNo) - } - } - if len(f) != 2 { - t.Fatalf("no line number found in %q", pathAndLineNo) - } - return f[0], f[1] -} - -func testObjDump(t *testing.T, exe, startaddr, endaddr string, line int) { - srcPath, srcLineNo := runObjDump(t, exe, startaddr, endaddr) - fi1, err := os.Stat("objdump_test.go") - if err != nil { - t.Fatalf("Stat failed: %v", err) - } - fi2, err := os.Stat(srcPath) - if err != nil { - t.Fatalf("Stat failed: %v", err) - } - if !os.SameFile(fi1, fi2) { - t.Fatalf("objdump_test.go and %s are not same file", srcPath) - } - if srcLineNo != fmt.Sprint(line) { - t.Fatalf("line number = %v; want %d", srcLineNo, line) - } -} - -func TestObjDump(t *testing.T) { - _, _, line, _ := runtime.Caller(0) - syms := loadSyms(t) - - tmp, exe := buildObjdump(t) - defer os.RemoveAll(tmp) - - startaddr := syms["cmd/objdump.TestObjDump"] - addr, err := strconv.ParseUint(startaddr, 16, 64) - if err != nil { - t.Fatalf("invalid start address %v: %v", startaddr, err) - } - endaddr := fmt.Sprintf("%x", addr+10) - testObjDump(t, exe, startaddr, endaddr, line-1) - testObjDump(t, exe, "0x"+startaddr, "0x"+endaddr, line-1) -} - func buildObjdump(t *testing.T) (tmp, exe string) { switch runtime.GOOS { case "android", "nacl": diff --git a/src/cmd/pprof/pprof.go b/src/cmd/pprof/pprof.go index 89a5bb7d22..44f4f6cb72 100644 --- a/src/cmd/pprof/pprof.go +++ b/src/cmd/pprof/pprof.go @@ -11,6 +11,7 @@ import ( "os" "regexp" "strings" + "sync" "cmd/internal/objfile" "cmd/pprof/internal/commands" @@ -100,7 +101,10 @@ func (flags) ExtraUsage() string { // objTool implements plugin.ObjTool using Go libraries // (instead of invoking GNU binutils). -type objTool struct{} +type objTool struct { + mu sync.Mutex + disasmCache map[string]*objfile.Disasm +} func (*objTool) Open(name string, start uint64) (plugin.ObjFile, error) { of, err := objfile.Open(name) @@ -119,8 +123,39 @@ func (*objTool) Demangle(names []string) (map[string]string, error) { return make(map[string]string), nil } -func (*objTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) { - return nil, fmt.Errorf("disassembly not supported") +func (t *objTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) { + d, err := t.cachedDisasm(file) + if err != nil { + return nil, err + } + var asm []plugin.Inst + d.Decode(start, end, func(pc, size uint64, file string, line int, text string) { + asm = append(asm, plugin.Inst{Addr: pc, File: file, Line: line, Text: text}) + }) + return asm, nil +} + +func (t *objTool) cachedDisasm(file string) (*objfile.Disasm, error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.disasmCache == nil { + t.disasmCache = make(map[string]*objfile.Disasm) + } + d := t.disasmCache[file] + if d != nil { + return d, nil + } + f, err := objfile.Open(file) + if err != nil { + return nil, err + } + d, err = f.Disasm() + f.Close() + if err != nil { + return nil, err + } + t.disasmCache[file] = d + return d, nil } func (*objTool) SetConfig(config string) { From ec7f33300f7fb35ad30e37399126b61b4ca8052a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 6 Nov 2014 20:08:00 -0500 Subject: [PATCH 44/59] cmd/internal/objfile: add Disasm This was missing from CL 167320043. Happy to apply comments in a followup. TBR to fix build. TBR=r CC=golang-codereviews https://golang.org/cl/171260043 --- src/cmd/internal/objfile/disasm.go | 243 +++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 src/cmd/internal/objfile/disasm.go diff --git a/src/cmd/internal/objfile/disasm.go b/src/cmd/internal/objfile/disasm.go new file mode 100644 index 0000000000..7982faa47c --- /dev/null +++ b/src/cmd/internal/objfile/disasm.go @@ -0,0 +1,243 @@ +package objfile + +import ( + "bufio" + "debug/gosym" + "encoding/binary" + "fmt" + "io" + "regexp" + "sort" + "strings" + "text/tabwriter" + + "cmd/internal/rsc.io/arm/armasm" + "cmd/internal/rsc.io/x86/x86asm" +) + +// Disasm is a disassembler for a given File. +type Disasm struct { + syms []Sym + pcln *gosym.Table + text []byte + textStart uint64 + textEnd uint64 + goarch string + disasm disasmFunc + byteOrder binary.ByteOrder +} + +// Disasm returns a disassembler for the file f. +func (f *File) Disasm() (*Disasm, error) { + syms, err := f.Symbols() + if err != nil { + return nil, err + } + + pcln, err := f.PCLineTable() + if err != nil { + return nil, err + } + + textStart, textBytes, err := f.Text() + if err != nil { + return nil, err + } + + goarch := f.GOARCH() + disasm := disasms[goarch] + byteOrder := byteOrders[goarch] + if disasm == nil || byteOrder == nil { + return nil, fmt.Errorf("unsupported architecture") + } + + // Filter out section symbols, overwriting syms in place. + keep := syms[:0] + for _, sym := range syms { + switch sym.Name { + case "runtime.text", "text", "_text", "runtime.etext", "etext", "_etext": + // drop + default: + keep = append(keep, sym) + } + } + syms = keep + d := &Disasm{ + syms: syms, + pcln: pcln, + text: textBytes, + textStart: textStart, + textEnd: textStart + uint64(len(textBytes)), + goarch: goarch, + disasm: disasm, + byteOrder: byteOrder, + } + + return d, nil +} + +// lookup finds the symbol name containing addr. +func (d *Disasm) lookup(addr uint64) (name string, base uint64) { + i := sort.Search(len(d.syms), func(i int) bool { return addr < d.syms[i].Addr }) + if i > 0 { + s := d.syms[i-1] + if s.Addr != 0 && s.Addr <= addr && addr < s.Addr+uint64(s.Size) { + return s.Name, s.Addr + } + } + return "", 0 +} + +// base returns the final element in the path. +// It works on both Windows and Unix paths. +func base(path string) string { + path = path[strings.LastIndex(path, "/")+1:] + path = path[strings.LastIndex(path, `\`)+1:] + return path +} + +// Print prints a disassembly of the file to w. +// If filter is non-nil, the disassembly only includes functions with names matching filter. +// The disassembly only includes functions that overlap the range [start, end). +func (d *Disasm) Print(w io.Writer, filter *regexp.Regexp, start, end uint64) { + if start < d.textStart { + start = d.textStart + } + if end > d.textEnd { + end = d.textEnd + } + printed := false + bw := bufio.NewWriter(w) + for _, sym := range d.syms { + symStart := sym.Addr + symEnd := sym.Addr + uint64(sym.Size) + if sym.Code != 'T' && sym.Code != 't' || + symStart < d.textStart || + symEnd <= start || end <= symStart || + filter != nil && !filter.MatchString(sym.Name) { + continue + } + if printed { + fmt.Fprintf(bw, "\n") + } + printed = true + + file, _, _ := d.pcln.PCToLine(sym.Addr) + fmt.Fprintf(bw, "TEXT %s(SB) %s\n", sym.Name, file) + + tw := tabwriter.NewWriter(bw, 1, 8, 1, '\t', 0) + if symEnd > end { + symEnd = end + } + code := d.text[:end-d.textStart] + d.Decode(symStart, symEnd, func(pc, size uint64, file string, line int, text string) { + i := pc - d.textStart + fmt.Fprintf(tw, "\t%s:%d\t%#x\t", base(file), line, pc) + if size%4 != 0 || d.goarch == "386" || d.goarch == "amd64" { + // Print instruction as bytes. + fmt.Fprintf(tw, "%x", code[i:i+size]) + } else { + // Print instruction as 32-bit words. + for j := uint64(0); j < size; j += 4 { + if j > 0 { + fmt.Fprintf(tw, " ") + } + fmt.Fprintf(tw, "%08x", d.byteOrder.Uint32(code[i+j:])) + } + } + fmt.Fprintf(tw, "\t%s\n", text) + }) + tw.Flush() + } + bw.Flush() +} + +// Decode disassembles the text segment range [start, end), calling f for each instruction. +func (d *Disasm) Decode(start, end uint64, f func(pc, size uint64, file string, line int, text string)) { + if start < d.textStart { + start = d.textStart + } + if end > d.textEnd { + end = d.textEnd + } + code := d.text[:end-d.textStart] + lookup := d.lookup + for pc := start; pc < end; { + i := pc - d.textStart + text, size := d.disasm(code[i:], pc, lookup) + file, line, _ := d.pcln.PCToLine(pc) + f(pc, uint64(size), file, line, text) + pc += uint64(size) + } +} + +type lookupFunc func(addr uint64) (sym string, base uint64) +type disasmFunc func(code []byte, pc uint64, lookup lookupFunc) (text string, size int) + +func disasm_386(code []byte, pc uint64, lookup lookupFunc) (string, int) { + return disasm_x86(code, pc, lookup, 32) +} + +func disasm_amd64(code []byte, pc uint64, lookup lookupFunc) (string, int) { + return disasm_x86(code, pc, lookup, 64) +} + +func disasm_x86(code []byte, pc uint64, lookup lookupFunc, arch int) (string, int) { + inst, err := x86asm.Decode(code, 64) + var text string + size := inst.Len + if err != nil || size == 0 || inst.Op == 0 { + size = 1 + text = "?" + } else { + text = x86asm.Plan9Syntax(inst, pc, lookup) + } + return text, size +} + +type textReader struct { + code []byte + pc uint64 +} + +func (r textReader) ReadAt(data []byte, off int64) (n int, err error) { + if off < 0 || uint64(off) < r.pc { + return 0, io.EOF + } + d := uint64(off) - r.pc + if d >= uint64(len(r.code)) { + return 0, io.EOF + } + n = copy(data, r.code[d:]) + if n < len(data) { + err = io.ErrUnexpectedEOF + } + return +} + +func disasm_arm(code []byte, pc uint64, lookup lookupFunc) (string, int) { + inst, err := armasm.Decode(code, armasm.ModeARM) + var text string + size := inst.Len + if err != nil || size == 0 || inst.Op == 0 { + size = 4 + text = "?" + } else { + text = armasm.Plan9Syntax(inst, pc, lookup, textReader{code, pc}) + } + return text, size +} + +var disasms = map[string]disasmFunc{ + "386": disasm_386, + "amd64": disasm_amd64, + "arm": disasm_arm, +} + +var byteOrders = map[string]binary.ByteOrder{ + "386": binary.LittleEndian, + "amd64": binary.LittleEndian, + "arm": binary.LittleEndian, + "power64": binary.BigEndian, + "power64le": binary.LittleEndian, +} From 1340c6d59303e0acaab6a50cb39a7c978e25f11b Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 7 Nov 2014 08:19:19 -0800 Subject: [PATCH 45/59] cmd/go: disable warnings from cmd/cc when building for SWIG Fixes #9065. LGTM=rsc R=rsc, misch CC=golang-codereviews https://golang.org/cl/171270043 --- src/cmd/go/build.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cmd/go/build.go b/src/cmd/go/build.go index 79a27116a1..1dd4314da6 100644 --- a/src/cmd/go/build.go +++ b/src/cmd/go/build.go @@ -1826,7 +1826,15 @@ func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) cfile = mkAbs(p.Dir, cfile) - args := stringList(tool(archChar+"c"), "-F", "-V", "-w", "-trimpath", b.work, "-I", objdir, "-I", inc, "-o", ofile, buildCcflags, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, cfile) + warn := []string{"-w"} + if p.usesSwig() { + // When using SWIG, this compiler is only used to + // compile the C files generated by SWIG. + // We don't want warnings. + // See issue 9065 for details. + warn = nil + } + args := stringList(tool(archChar+"c"), "-F", "-V", warn, "-trimpath", b.work, "-I", objdir, "-I", inc, "-o", ofile, buildCcflags, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, cfile) return b.run(p.Dir, p.ImportPath, nil, args) } From cf105e2fa04bb001a2331cabba2437a59760562b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 8 Nov 2014 15:13:28 -0300 Subject: [PATCH 46/59] net/http: fix benchmark goroutine leak New detection because of net/http now using TestMain. Fixes #9033 LGTM=iant R=golang-codereviews, iant CC=adg, golang-codereviews, rsc https://golang.org/cl/170210043 --- src/net/http/serve_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index bb44ac8537..5e0a0053c0 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -2819,6 +2819,7 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, useTLS bool) { InsecureSkipVerify: true, }, } + defer noVerifyTransport.CloseIdleConnections() client := &Client{Transport: noVerifyTransport} for pb.Next() { res, err := client.Get(ts.URL) From 03c008bcb2ab64d5ca2adbd44e3369077cc46655 Mon Sep 17 00:00:00 2001 From: Adam Langley Date: Sat, 8 Nov 2014 17:12:23 -0800 Subject: [PATCH 47/59] lib/codereview: fix with more recent hg revisions. I've Mercurial version 3.2 and hg submit fails with: File "/home/agl/devel/go/lib/codereview/codereview.py", line 3567, in get_hg_status ret = hg_commands.status(fui, self.repo, *[], **{'rev': [rev], 'copies': True}) File "/usr/lib/python2.7/site-packages/mercurial/commands.py", line 5714, in status fm = ui.formatter('status', opts) File "/home/agl/devel/go/lib/codereview/codereview.py", line 3464, in formatter return plainformatter(self, topic, opts) File "/usr/lib/python2.7/site-packages/mercurial/formatter.py", line 57, in __init__ if ui.debugflag: AttributeError: 'FakeMercurialUI' object has no attribute 'debugflag' This change dumbly adds a boolean debugflag and that seems to work. LGTM=minux R=rsc, minux CC=golang-codereviews https://golang.org/cl/167410043 --- lib/codereview/codereview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index fdf11d1f48..b8814e1022 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -3451,6 +3451,7 @@ class FakeMercurialUI(object): def __init__(self): self.quiet = True self.output = '' + self.debugflag = False def write(self, *args, **opts): self.output += ' '.join(args) From 68e2dbe8b7748ba4e9adcad61627da32e1f2141a Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:13:04 +1100 Subject: [PATCH 48/59] cmd/api: use golang.org/x/... import paths LGTM=bradfitz, rsc R=rsc, bradfitz CC=golang-codereviews https://golang.org/cl/169000043 --- src/cmd/api/goapi.go | 2 +- src/cmd/api/run.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cmd/api/goapi.go b/src/cmd/api/goapi.go index 5a8c876033..49e592d0c2 100644 --- a/src/cmd/api/goapi.go +++ b/src/cmd/api/goapi.go @@ -27,7 +27,7 @@ import ( "sort" "strings" - "code.google.com/p/go.tools/go/types" + "golang.org/x/tools/go/types" ) // Flags diff --git a/src/cmd/api/run.go b/src/cmd/api/run.go index c323deb603..52b23de454 100644 --- a/src/cmd/api/run.go +++ b/src/cmd/api/run.go @@ -91,7 +91,7 @@ func file(s ...string) string { // It tries to re-use a go.tools checkout from a previous run if possible, // else it hg clones it. func prepGoPath() string { - const tempBase = "go.tools.TMP" + const tempBase = "tools.TMP" username := "" u, err := user.Current() @@ -108,14 +108,14 @@ func prepGoPath() string { gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion) // cloneDir is where we run "hg clone". - cloneDir := filepath.Join(gopath, "src", "code.google.com", "p") + cloneDir := filepath.Join(gopath, "src", "golang.org", "x") // The dir we clone into. We only atomically rename it to finalDir on // clone success. tmpDir := filepath.Join(cloneDir, tempBase) // finalDir is where the checkout will live once it's complete. - finalDir := filepath.Join(cloneDir, "go.tools") + finalDir := filepath.Join(cloneDir, "tools") if goToolsCheckoutGood(finalDir) { return gopath From 7f0be1f781a523b5acafb527ca123f5b0eac1536 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:15:57 +1100 Subject: [PATCH 49/59] all: use golang.org/x/... import paths LGTM=rsc, r R=r, rsc CC=golang-codereview, golang-codereviews https://golang.org/cl/168050043 --- doc/articles/go_command.html | 13 +-- doc/cmd.html | 6 +- doc/code.html | 181 +++++++++++++++++--------------- doc/contribute.html | 2 +- doc/go1compat.html | 6 +- doc/go_faq.html | 4 +- doc/install-source.html | 6 +- lib/codereview/codereview.py | 2 +- misc/benchcmp | 2 +- misc/makerelease/makerelease.go | 10 +- src/compress/lzw/reader.go | 2 +- src/crypto/crypto.go | 12 +-- src/debug/goobj/read_test.go | 2 +- src/net/http/cookiejar/jar.go | 2 +- src/runtime/os_android.c | 2 +- 15 files changed, 131 insertions(+), 121 deletions(-) diff --git a/doc/articles/go_command.html b/doc/articles/go_command.html index 246b8c956d..2978628cd2 100644 --- a/doc/articles/go_command.html +++ b/doc/articles/go_command.html @@ -78,17 +78,18 @@ well-established conventions.

    source code. For Bitbucket, GitHub, Google Code, and Launchpad, the root directory of the repository is identified by the repository's main URL, without the http:// prefix. Subdirectories are named by -adding to that path. For example, the supplemental networking -libraries for Go are obtained by running

    +adding to that path. +For example, the Go example programs are obtained by running

    -hg clone http://code.google.com/p/go.net
+git clone https://github.com/golang/example

    and thus the import path for the root directory of that repository is -"code.google.com/p/go.net". The websocket package is stored in a -subdirectory, so its import path is -"code.google.com/p/go.net/websocket".

    +"github.com/golang/example". +The     stringutil +package is stored in a subdirectory, so its import path is +"github.com/golang/example/stringutil".

    These paths are on the long side, but in exchange we get an automatically managed name space for import paths and the ability for diff --git a/doc/cmd.html b/doc/cmd.html index 132ea275fa..5d20d3887a 100644 --- a/doc/cmd.html +++ b/doc/cmd.html @@ -62,7 +62,7 @@ details. -cover +cover      Cover is a program for creating and analyzing the coverage profiles generated by "go test -coverprofile". @@ -83,13 +83,13 @@ gofmt command with more general options. -godoc +godoc      Godoc extracts and generates documentation for Go packages. -vet +vet      Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string. diff --git a/doc/code.html b/doc/code.html index f019306fa2..ce9f8636fa 100644 --- a/doc/code.html +++ b/doc/code.html @@ -60,37 +60,35 @@ To give you an idea of how a workspace looks in practice, here's an example: 

     bin/
-    streak                         # command executable
-    todo                           # command executable
+    hello                          # command executable
+    outyet                         # command executable
 pkg/
     linux_amd64/
-        code.google.com/p/goauth2/
-            oauth.a                # package object
-        github.com/nf/todo/
-            task.a                 # package object
+        github.com/golang/example/
+            stringutil.a           # package object
 src/
-    code.google.com/p/goauth2/
-        .hg/                       # mercurial repository metadata
-        oauth/
-            oauth.go               # package source
-            oauth_test.go          # test source
-    github.com/nf/
-        streak/
-            .git/                  # git repository metadata
-            oauth.go               # command source
-            streak.go              # command source
-        todo/
-            .git/                  # git repository metadata
-            task/
-                task.go            # package source
-            todo.go                # command source
+    github.com/golang/example/
+        .git/                      # Git repository metadata
+	hello/
+	    hello.go               # command source
+	outyet/
+	    main.go                # command source
+	    main_test.go           # test source
+	stringutil/
+	    reverse.go             # package source
+	    reverse_test.go        # test source

    -This workspace contains three repositories (goauth2, -streak, and todo) comprising two commands -(streak and todo) and two libraries -(oauth and task). +This workspace contains one repository (example) +comprising two commands (hello and outyet) +and one library (stringutil). +

    + +

    +A typical workspace would contain many source repositories containing many +packages and commands. Most Go programmers keep all their Go source code +and dependencies in a single workspace.

    @@ -277,29 +275,29 @@ Let's write a library and use it from the hello program.

    Again, the first step is to choose a package path (we'll use -github.com/user/newmath) and create the package directory: +github.com/user/stringutil) and create the package directory: 

    -$ mkdir $GOPATH/src/github.com/user/newmath
+$ mkdir $GOPATH/src/github.com/user/stringutil

    -Next, create a file named sqrt.go in that directory with the +Next, create a file named reverse.go in that directory with the following contents. 

    -// Package newmath is a trivial example package.
-package newmath
+// Package stringutil contains utility functions for working with strings.
+package stringutil
 
-// Sqrt returns an approximation to the square root of x.
-func Sqrt(x float64) float64 {
-	z := 1.0
-	for i := 0; i < 1000; i++ {
-		z -= (z*z - x) / (2 * z)
+// Reverse returns its argument string reversed rune-wise left to right.
+func Reverse(s string) string {
+	r := []rune(s)
+	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
+		r[i], r[j] = r[j], r[i]
 	}
-	return z
+	return string(r)
 }
    @@ -308,7 +306,7 @@ Now, test that the package compiles with go build: 

    -$ go build github.com/user/newmath
+$ go build github.com/user/stringutil

    @@ -326,7 +324,7 @@ directory of the workspace.

    -After confirming that the newmath package builds, +After confirming that the stringutil package builds, modify your original hello.go (which is in $GOPATH/src/github.com/user/hello) to use it:

    @@ -337,18 +335,18 @@ package main import ( "fmt" - "github.com/user/newmath" + "github.com/user/stringutil" ) func main() { - fmt.Printf("Hello, world. Sqrt(2) = %v\n", newmath.Sqrt(2)) + fmt.Printf(stringutil.Reverse("!oG ,olleH")) }

    Whenever the go tool installs a package or binary, it also -installs whatever dependencies it has. So when you install the hello -program +installs whatever dependencies it has. +So when you install the hello program 

    @@ -356,16 +354,16 @@ $ go install github.com/user/hello

    -the newmath package will be installed as well, automatically. +the stringutil package will be installed as well, automatically.

    -Running the new version of the program, you should see some numerical output: +Running the new version of the program, you should see a new, reversed message: 

     $ hello
-Hello, world.  Sqrt(2) = 1.414213562373095
+Hello, Go!

    @@ -374,22 +372,22 @@ After the steps above, your workspace should look like this: 

     bin/
-    hello              # command executable
+    hello                 # command executable
 pkg/
-    linux_amd64/       # this will reflect your OS and architecture
+    linux_amd64/          # this will reflect your OS and architecture
         github.com/user/
-            newmath.a  # package object
+            stringutil.a  # package object
 src/
     github.com/user/
         hello/
-            hello.go   # command source
-        newmath/
-            sqrt.go    # package source
+            hello.go      # command source
+        stringutil/
+            reverse.go    # package source

    -Note that go install placed the newmath.a object in a -directory inside pkg/linux_amd64 that mirrors its source +Note that go install placed the stringutil.a object +in a directory inside pkg/linux_amd64 that mirrors its source directory. This is so that future invocations of the go tool can find the package object and avoid recompiling the package unnecessarily. @@ -457,20 +455,29 @@ if the function calls a failure function such as t.Error or

    -Add a test to the newmath package by creating the file -$GOPATH/src/github.com/user/newmath/sqrt_test.go containing the -following Go code. +Add a test to the stringutil package by creating the file +$GOPATH/src/github.com/user/stringutil/reverse_test.go containing +the following Go code. 

    -package newmath
+package stringutil
 
 import "testing"
 
-func TestSqrt(t *testing.T) {
-	const in, out = 4, 2
-	if x := Sqrt(in); x != out {
-		t.Errorf("Sqrt(%v) = %v, want %v", in, x, out)
+func TestReverse(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"Hello, world", "dlrow ,olleH"},
+		{"Hello, 世界", "界世 ,olleH"},
+		{"", ""},
+	}
+	for _, c := range cases {
+		got := Reverse(c.in)
+		if got != c.want {
+			t.Errorf("Reverse(%q) == %q, want %q", c.in, got, c.want)
+		}
 	}
 }
    @@ -480,8 +487,8 @@ Then run the test with go test: 

    -$ go test github.com/user/newmath
-ok  	github.com/user/newmath 0.165s
+$ go test github.com/user/stringutil
+ok  	github.com/user/stringutil 0.165s

    @@ -491,7 +498,7 @@ directory, you can omit the package path: 

     $ go test
-ok  	github.com/user/newmath 0.165s
+ok  	github.com/user/stringutil 0.165s

    @@ -507,16 +514,16 @@ An import path can describe how to obtain the package source code using a revision control system such as Git or Mercurial. The go tool uses this property to automatically fetch packages from remote repositories. For instance, the examples described in this document are also kept in a -Mercurial repository hosted at Google Code, -code.google.com/p/go.example. +Git repository hosted at GitHub +github.com/golang/example. If you include the repository URL in the package's import path, go get will fetch, build, and install it automatically: 

    -$ go get code.google.com/p/go.example/hello
+$ go get github.com/golang/example/hello
 $ $GOPATH/bin/hello
-Hello, world.  Sqrt(2) = 1.414213562373095
+Hello, Go examples!

    @@ -533,37 +540,39 @@ tree should now look like this: 

     bin/
-    hello                 # command executable
+    hello                           # command executable
 pkg/
     linux_amd64/
-        code.google.com/p/go.example/
-            newmath.a     # package object
+        github.com/golang/example/
+            stringutil.a            # package object
         github.com/user/
-            newmath.a     # package object
+            stringutil.a            # package object
 src/
-    code.google.com/p/go.example/
+    github.com/golang/example/
+	.git/                       # Git repository metadata
         hello/
-            hello.go      # command source
-        newmath/
-            sqrt.go       # package source
-            sqrt_test.go  # test source
+            hello.go                # command source
+        stringutil/
+            reverse.go              # package source
+            reverse_test.go         # test source
     github.com/user/
         hello/
-            hello.go      # command source
-        newmath/
-            sqrt.go       # package source
-            sqrt_test.go  # test source
+            hello.go                # command source
+        stringutil/
+            reverse.go              # package source
+            reverse_test.go         # test source

    -The hello command hosted at Google Code depends on the -newmath package within the same repository. The imports in -hello.go file use the same import path convention, so the go -get command is able to locate and install the dependent package, too. +The hello command hosted at GitHub depends on the +stringutil package within the same repository. The imports in +hello.go file use the same import path convention, so the +go get command is able to locate and install the dependent +package, too. 

    -import "code.google.com/p/go.example/newmath"
+import "github.com/golang/example/stringutil"

    diff --git a/doc/contribute.html b/doc/contribute.html index 90c3f10a1d..92fd88b485 100644 --- a/doc/contribute.html +++ b/doc/contribute.html @@ -121,7 +121,7 @@ are inside the go directory when issuing commands.

    To contribute to subrepositories, edit the .hg/hgrc for each subrepository in the same way. For example, add the codereview extension to -code.google.com/p/go.tools/.hg/hgrc. +golang.org/x/tools/.hg/hgrc.

    Understanding the extension

    diff --git a/doc/go1compat.html b/doc/go1compat.html index 94c48d2ce3..d800dec0c0 100644 --- a/doc/go1compat.html +++ b/doc/go1compat.html @@ -153,7 +153,7 @@ developed software based on Go 1.

    Code in sub-repositories of the main go tree, such as -code.google.com/p/go.net, +golang.org/x/net, may be developed under looser compatibility requirements. However, the sub-repositories will be tagged as appropriate to identify versions that are compatible @@ -170,9 +170,9 @@ is therefore outside the purview of the guarantees made here. As of Go version 1.4, the syscall package is frozen. Any evolution of the system call interface must be supported elsewhere, such as in the -go.sys subrepository. +go.sys subrepository. For details and background, see -this document. +this document.

    Tools

    diff --git a/doc/go_faq.html b/doc/go_faq.html index 9aac058388..7597997798 100644 --- a/doc/go_faq.html +++ b/doc/go_faq.html @@ -1616,7 +1616,7 @@ Go is a fine language in which to implement a self-hosting compiler: a native lexer and parser are already available in the go package and a separate type checking -package +package has also been written.

    @@ -1715,7 +1715,7 @@ func main() {

    Nowadays, most Go programmers use a tool, -goimports, +goimports, which automatically rewrites a Go source file to have the correct imports, eliminating the unused imports issue in practice. This program is easily connected to most editors to run automatically when a Go source file is written. diff --git a/doc/install-source.html b/doc/install-source.html index 82859b50fb..f53deb404c 100644 --- a/doc/install-source.html +++ b/doc/install-source.html @@ -241,12 +241,12 @@ provides essential setup instructions for using the Go tools.

    The source code for several Go tools (including godoc) -is kept in the go.tools repository. +is kept in the go.tools repository. To install all of them, run the go get command: 

    -$ go get code.google.com/p/go.tools/cmd/...
+$ go get golang.org/x/tools/cmd/...

    @@ -254,7 +254,7 @@ Or if you just want to install a specific command (godoc in this ca 

    -$ go get code.google.com/p/go.tools/cmd/godoc
+$ go get golang.org/x/tools/cmd/godoc

    diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index b8814e1022..416702c634 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -1631,7 +1631,7 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): try: cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32") except: - return "hgapplydiff: " + ExceptionDetail() + "\nInstall hgapplydiff with:\n$ go get code.google.com/p/go.codereview/cmd/hgapplydiff\n" + return "hgapplydiff: " + ExceptionDetail() + "\nInstall hgapplydiff with:\n$ go get golang.org/x/codereview/cmd/hgapplydiff\n" out, err = cmd.communicate(patch) if cmd.returncode != 0 and not opts["ignore_hgapplydiff_failure"]: diff --git a/misc/benchcmp b/misc/benchcmp index 28a37392d8..84d92eefd4 100755 --- a/misc/benchcmp +++ b/misc/benchcmp @@ -1,5 +1,5 @@ #!/bin/bash echo 'misc/benchcmp has moved:' >&2 -echo ' go get -u code.google.com/p/go.tools/cmd/benchcmp' >&2 +echo ' go get -u golang.org/x/tools/cmd/benchcmp' >&2 exit 2 diff --git a/misc/makerelease/makerelease.go b/misc/makerelease/makerelease.go index 9b2373307f..e94efdbcee 100644 --- a/misc/makerelease/makerelease.go +++ b/misc/makerelease/makerelease.go @@ -53,8 +53,8 @@ var ( ) const ( - blogPath = "code.google.com/p/go.blog" - toolPath = "code.google.com/p/go.tools" + blogPath = "golang.org/x/blog" + toolPath = "golang.org/x/tools" tourPath = "code.google.com/p/go-tour" defaultToolTag = "release-branch.go1.3" defaultTourTag = "release-branch.go1.3" @@ -64,9 +64,9 @@ const ( // These must be the command that cmd/go knows to install to $GOROOT/bin // or $GOROOT/pkg/tool. var toolPaths = []string{ - "code.google.com/p/go.tools/cmd/cover", - "code.google.com/p/go.tools/cmd/godoc", - "code.google.com/p/go.tools/cmd/vet", + "golang.org/x/tools/cmd/cover", + "golang.org/x/tools/cmd/godoc", + "golang.org/x/tools/cmd/vet", } var preBuildCleanFiles = []string{ diff --git a/src/compress/lzw/reader.go b/src/compress/lzw/reader.go index 0835bd8b90..526620c827 100644 --- a/src/compress/lzw/reader.go +++ b/src/compress/lzw/reader.go @@ -11,7 +11,7 @@ // two non-literal codes are a clear code and an EOF code. // // The TIFF file format uses a similar but incompatible version of the LZW -// algorithm. See the code.google.com/p/go.image/tiff/lzw package for an +// algorithm. See the golang.org/x/image/tiff/lzw package for an // implementation. package lzw diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index 5a91baca0e..59b23e93f5 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -21,7 +21,7 @@ func (h Hash) HashFunc() Hash { } const ( - MD4 Hash = 1 + iota // import code.google.com/p/go.crypto/md4 + MD4 Hash = 1 + iota // import golang.org/x/crypto/md4 MD5 // import crypto/md5 SHA1 // import crypto/sha1 SHA224 // import crypto/sha256 @@ -29,11 +29,11 @@ const ( SHA384 // import crypto/sha512 SHA512 // import crypto/sha512 MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA - RIPEMD160 // import code.google.com/p/go.crypto/ripemd160 - SHA3_224 // import code.google.com/p/go.crypto/sha3 - SHA3_256 // import code.google.com/p/go.crypto/sha3 - SHA3_384 // import code.google.com/p/go.crypto/sha3 - SHA3_512 // import code.google.com/p/go.crypto/sha3 + RIPEMD160 // import golang.org/x/crypto/ripemd160 + SHA3_224 // import golang.org/x/crypto/sha3 + SHA3_256 // import golang.org/x/crypto/sha3 + SHA3_384 // import golang.org/x/crypto/sha3 + SHA3_512 // import golang.org/x/crypto/sha3 maxHash ) diff --git a/src/debug/goobj/read_test.go b/src/debug/goobj/read_test.go index dee140533c..cc991e5d91 100644 --- a/src/debug/goobj/read_test.go +++ b/src/debug/goobj/read_test.go @@ -12,7 +12,7 @@ var importPathToPrefixTests = []struct { }{ {"runtime", "runtime"}, {"sync/atomic", "sync/atomic"}, - {"code.google.com/p/go.tools/godoc", "code.google.com/p/go.tools/godoc"}, + {"golang.org/x/tools/godoc", "golang.org/x/tools/godoc"}, {"foo.bar/baz.quux", "foo.bar/baz%2equux"}, {"", ""}, {"%foo%bar", "%25foo%25bar"}, diff --git a/src/net/http/cookiejar/jar.go b/src/net/http/cookiejar/jar.go index 389ab58e41..0e0fac9286 100644 --- a/src/net/http/cookiejar/jar.go +++ b/src/net/http/cookiejar/jar.go @@ -30,7 +30,7 @@ import ( // set a cookie for bar.com. // // A public suffix list implementation is in the package -// code.google.com/p/go.net/publicsuffix. +// golang.org/x/net/publicsuffix. type PublicSuffixList interface { // PublicSuffix returns the public suffix of domain. // diff --git a/src/runtime/os_android.c b/src/runtime/os_android.c index 58e0dac939..5805f68713 100644 --- a/src/runtime/os_android.c +++ b/src/runtime/os_android.c @@ -9,7 +9,7 @@ // Export the runtime entry point symbol. // // Used by the app package to start the Go runtime after loading -// a shared library via JNI. See code.google.com/p/go.mobile/app. +// a shared library via JNI. See golang.org/x/mobile/app. void _rt0_arm_linux1(); #pragma cgo_export_static _rt0_arm_linux1 From 844889dfe2321905e7aa1273f98596d156893527 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:27:25 +1100 Subject: [PATCH 50/59] cmd/go: use golang.org/x/... import paths LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/168170043 --- src/cmd/go/doc.go | 2 +- src/cmd/go/pkg.go | 23 +++++++++++++++++------ src/cmd/go/test.bash | 18 +++++++++--------- src/cmd/go/tool.go | 2 +- src/cmd/go/vet.go | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index cf3a54565a..43a3159440 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -590,7 +590,7 @@ Usage: Vet runs the Go vet command on the packages named by the import paths. -For more about vet, see 'godoc code.google.com/p/go.tools/cmd/vet'. +For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. For more about specifying packages, see 'go help packages'. To run the vet tool with specific options, run 'go tool vet'. diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index e17326442c..6efeeb1a9c 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -383,9 +383,10 @@ func findInternal(path string) (index int, ok bool) { type targetDir int const ( - toRoot targetDir = iota // to bin dir inside package root (default) - toTool // GOROOT/pkg/tool - toBin // GOROOT/bin + toRoot targetDir = iota // to bin dir inside package root (default) + toTool // GOROOT/pkg/tool + toBin // GOROOT/bin + stalePath // the old import path; fail to build ) // goTools is a map of Go program import path to install target directory. @@ -399,9 +400,12 @@ var goTools = map[string]targetDir{ "cmd/objdump": toTool, "cmd/pack": toTool, "cmd/yacc": toTool, - "code.google.com/p/go.tools/cmd/cover": toTool, - "code.google.com/p/go.tools/cmd/godoc": toBin, - "code.google.com/p/go.tools/cmd/vet": toTool, + "golang.org/x/tools/cmd/cover": toTool, + "golang.org/x/tools/cmd/godoc": toBin, + "golang.org/x/tools/cmd/vet": toTool, + "code.google.com/p/go.tools/cmd/cover": stalePath, + "code.google.com/p/go.tools/cmd/godoc": stalePath, + "code.google.com/p/go.tools/cmd/vet": stalePath, } // expandScanner expands a scanner.List error into all the errors in the list. @@ -462,6 +466,13 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } if p.Name == "main" { + // Report an error when the old code.google.com/p/go.tools paths are used. + if goTools[p.ImportPath] == stalePath { + newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) + e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) + p.Error = &PackageError{Err: e} + return p + } _, elem := filepath.Split(p.Dir) full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { diff --git a/src/cmd/go/test.bash b/src/cmd/go/test.bash index 2b5230b1aa..e0f066f186 100755 --- a/src/cmd/go/test.bash +++ b/src/cmd/go/test.bash @@ -433,20 +433,20 @@ TEST godoc installs into GOBIN d=$(mktemp -d -t testgoXXX) export GOPATH=$d mkdir $d/gobin -GOBIN=$d/gobin ./testgo get code.google.com/p/go.tools/cmd/godoc || ok=false +GOBIN=$d/gobin ./testgo get golang.org/x/tools/cmd/godoc || ok=false if [ ! -x $d/gobin/godoc ]; then echo did not install godoc to '$GOBIN' - GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc || true + GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' golang.org/x/tools/cmd/godoc || true ok=false fi TEST godoc installs into GOROOT GOROOT=$(./testgo env GOROOT) rm -f $GOROOT/bin/godoc -./testgo install code.google.com/p/go.tools/cmd/godoc || ok=false +./testgo install golang.org/x/tools/cmd/godoc || ok=false if [ ! -x $GOROOT/bin/godoc ]; then echo did not install godoc to '$GOROOT/bin' - ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc || true + ./testgo list -f 'Target: {{.Target}}' golang.org/x/tools/cmd/godoc || true ok=false fi @@ -561,8 +561,8 @@ fi TEST without GOPATH, go get fails d=$(mktemp -d -t testgoXXX) mkdir -p $d/src -if GOPATH= GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with $GOPATH unset' +if GOPATH= GOROOT=$d ./testgo get -d golang.org/x/codereview/cmd/hgpatch ; then + echo 'go get golang.org/x/codereview/cmd/hgpatch should not succeed with $GOPATH unset' ok=false fi rm -rf $d @@ -571,8 +571,8 @@ rm -rf $d TEST with GOPATH=GOROOT, go get fails d=$(mktemp -d -t testgoXXX) mkdir -p $d/src -if GOPATH=$d GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with GOPATH=$GOROOT' +if GOPATH=$d GOROOT=$d ./testgo get -d golang.org/x/codereview/cmd/hgpatch ; then + echo 'go get golang.org/x/codereview/cmd/hgpatch should not succeed with GOPATH=$GOROOT' ok=false fi rm -rf $d @@ -728,7 +728,7 @@ elif ! grep "case-insensitive file name collision" $d/out >/dev/null; then fi TEST go get cover -./testgo get code.google.com/p/go.tools/cmd/cover || ok=false +./testgo get golang.org/x/tools/cmd/cover || ok=false unset GOPATH rm -rf $d diff --git a/src/cmd/go/tool.go b/src/cmd/go/tool.go index 6d26f7a4b4..c96161e0f9 100644 --- a/src/cmd/go/tool.go +++ b/src/cmd/go/tool.go @@ -53,7 +53,7 @@ func tool(toolName string) string { // Give a nice message if there is no tool with that name. if _, err := os.Stat(toolPath); err != nil { if isInGoToolsRepo(toolName) { - fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get code.google.com/p/go.tools/cmd/%s\n", toolName, toolName) + fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName) } else { fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName) } diff --git a/src/cmd/go/vet.go b/src/cmd/go/vet.go index de7befc611..02ff54b2ac 100644 --- a/src/cmd/go/vet.go +++ b/src/cmd/go/vet.go @@ -17,7 +17,7 @@ var cmdVet = &Command{ Long: ` Vet runs the Go vet command on the packages named by the import paths. -For more about vet, see 'godoc code.google.com/p/go.tools/cmd/vet'. +For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. For more about specifying packages, see 'go help packages'. To run the vet tool with specific options, run 'go tool vet'. From 18b4f06b13d68c418dc8cb9712424671b63a4254 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:30:57 +1100 Subject: [PATCH 51/59] cmd/api: bump go.tools golden CL hash TBR=bradfitz R=rsc CC=golang-codereviews https://golang.org/cl/166380043 --- src/cmd/api/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/api/run.go b/src/cmd/api/run.go index 52b23de454..1f9a4cf8d8 100644 --- a/src/cmd/api/run.go +++ b/src/cmd/api/run.go @@ -27,7 +27,7 @@ import ( // goToolsVersion is the hg revision of the go.tools subrepo we need // to build cmd/api. This only needs to be updated whenever a go/types // bug fix is needed by the cmd/api tool. -const goToolsVersion = "6698ca2900e2" +const goToolsVersion = "ae08a5291439" var goroot string From 9a571deed6f9bebcbb3fad864600936b55aeea22 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:39:17 +1100 Subject: [PATCH 52/59] undo CL 166380043 / 0b54a0927656 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was a mistake; the cmd/api tool depends on an old version of go/types. ««« original CL description cmd/api: bump go.tools golden CL hash TBR=bradfitz R=rsc CC=golang-codereviews https://golang.org/cl/166380043 »»» TBR=bradfitz, rsc R=bradfitz, rsc CC=golang-codereviews https://golang.org/cl/167430043 --- src/cmd/api/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/api/run.go b/src/cmd/api/run.go index 1f9a4cf8d8..52b23de454 100644 --- a/src/cmd/api/run.go +++ b/src/cmd/api/run.go @@ -27,7 +27,7 @@ import ( // goToolsVersion is the hg revision of the go.tools subrepo we need // to build cmd/api. This only needs to be updated whenever a go/types // bug fix is needed by the cmd/api tool. -const goToolsVersion = "ae08a5291439" +const goToolsVersion = "6698ca2900e2" var goroot string From b53bdd496c2de8d33e1e65452f184c829303de9f Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 10 Nov 2014 09:46:27 +1100 Subject: [PATCH 53/59] undo CL 169000043 / 05b838013df9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was a mistake. The cmd/api tool depends on an old version of go/types. ««« original CL description cmd/api: use golang.org/x/... import paths LGTM=bradfitz, rsc R=rsc, bradfitz CC=golang-codereviews https://golang.org/cl/169000043 »»» TBR=rsc, bradfitz R=bradfitz, rsc CC=golang-codereviews https://golang.org/cl/169320043 --- src/cmd/api/goapi.go | 2 +- src/cmd/api/run.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cmd/api/goapi.go b/src/cmd/api/goapi.go index 49e592d0c2..5a8c876033 100644 --- a/src/cmd/api/goapi.go +++ b/src/cmd/api/goapi.go @@ -27,7 +27,7 @@ import ( "sort" "strings" - "golang.org/x/tools/go/types" + "code.google.com/p/go.tools/go/types" ) // Flags diff --git a/src/cmd/api/run.go b/src/cmd/api/run.go index 52b23de454..c323deb603 100644 --- a/src/cmd/api/run.go +++ b/src/cmd/api/run.go @@ -91,7 +91,7 @@ func file(s ...string) string { // It tries to re-use a go.tools checkout from a previous run if possible, // else it hg clones it. func prepGoPath() string { - const tempBase = "tools.TMP" + const tempBase = "go.tools.TMP" username := "" u, err := user.Current() @@ -108,14 +108,14 @@ func prepGoPath() string { gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion) // cloneDir is where we run "hg clone". - cloneDir := filepath.Join(gopath, "src", "golang.org", "x") + cloneDir := filepath.Join(gopath, "src", "code.google.com", "p") // The dir we clone into. We only atomically rename it to finalDir on // clone success. tmpDir := filepath.Join(cloneDir, tempBase) // finalDir is where the checkout will live once it's complete. - finalDir := filepath.Join(cloneDir, "tools") + finalDir := filepath.Join(cloneDir, "go.tools") if goToolsCheckoutGood(finalDir) { return gopath From af3e02e4040f4ae62d62d4f23411e11fa96340be Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 20:20:06 -0500 Subject: [PATCH 54/59] cmd/pprof: install as go tool pprof LGTM=bradfitz, r R=r, bradfitz CC=golang-codereviews https://golang.org/cl/168320043 --- src/cmd/go/pkg.go | 1 + src/cmd/pprof/README | 8 ++++++++ src/cmd/pprof/doc.go | 12 ++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 src/cmd/pprof/README create mode 100644 src/cmd/pprof/doc.go diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index 6efeeb1a9c..b71feb7a67 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -399,6 +399,7 @@ var goTools = map[string]targetDir{ "cmd/nm": toTool, "cmd/objdump": toTool, "cmd/pack": toTool, + "cmd/pprof": toTool, "cmd/yacc": toTool, "golang.org/x/tools/cmd/cover": toTool, "golang.org/x/tools/cmd/godoc": toBin, diff --git a/src/cmd/pprof/README b/src/cmd/pprof/README new file mode 100644 index 0000000000..a728ef2353 --- /dev/null +++ b/src/cmd/pprof/README @@ -0,0 +1,8 @@ +The pprof in this directory is adapted from the pprof used inside Google +for C++, Java, and Go programs. Because it was developed for that broader +context, it is overgeneralized when used here for the specific use case +of profiling standard Go programs. However, we've left the abstractions +intact in order to share updates between this copy and Google's internal one. + +Please do not take the level of abstraction in this program as an example +to follow in your own. diff --git a/src/cmd/pprof/doc.go b/src/cmd/pprof/doc.go new file mode 100644 index 0000000000..c6ff11d102 --- /dev/null +++ b/src/cmd/pprof/doc.go @@ -0,0 +1,12 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Pprof interprets and displays profiles of Go programs. +// +// Usage: +// +// go tool pprof binary profile +// +// For more information, see http://blog.golang.org/profiling-go-programs. +package main From 9bc842ca180d88d3cf4956b37dadcc21c53eded8 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 20:20:26 -0500 Subject: [PATCH 55/59] cmd/dist: remove old misc/pprof LGTM=dave, bradfitz, r, alex.brainman R=r, dave, bradfitz, alex.brainman CC=golang-codereviews https://golang.org/cl/167350043 --- misc/pprof | 5100 ------------------------------------------ src/cmd/dist/build.c | 9 - 2 files changed, 5109 deletions(-) delete mode 100755 misc/pprof diff --git a/misc/pprof b/misc/pprof deleted file mode 100755 index f83e6fb659..0000000000 --- a/misc/pprof +++ /dev/null @@ -1,5100 +0,0 @@ -#! /usr/bin/env perl - -# This is a copy of http://google-perftools.googlecode.com/svn/trunk/src/pprof -# with local modifications to handle generation of SVG images and -# the Go-style pprof paths. These modifications will probably filter -# back into the official source before long. -# It's convenient to have a copy here because we need just the one -# Perl script, not all the C++ libraries that surround it. - -# Copyright (c) 1998-2007, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# --- -# Program for printing the profile generated by common/profiler.cc, -# or by the heap profiler (common/debugallocation.cc) -# -# The profile contains a sequence of entries of the form: -# -# This program parses the profile, and generates user-readable -# output. -# -# Examples: -# -# % tools/pprof "program" "profile" -# Enters "interactive" mode -# -# % tools/pprof --text "program" "profile" -# Generates one line per procedure -# -# % tools/pprof --gv "program" "profile" -# Generates annotated call-graph and displays via "gv" -# -# % tools/pprof --gv --focus=Mutex "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# -# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# and does not match "string" -# -# % tools/pprof --list=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --list= pattern. The listing is -# annotated with the flat and cumulative sample counts at each line. -# -# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --disasm= pattern. The listing is -# annotated with the flat and cumulative sample counts at each PC value. -# -# TODO: Use color to indicate files? - -use strict; -use warnings; -use Getopt::Long; -use File::Temp; -use File::Copy; - -my $PPROF_VERSION = "1.5"; - -# NOTE: All mentions of c++filt have been expunged from this script -# because (1) we don't use C++, and (2) the copy of c++filt that ships -# on OS X is from 2007 and destroys nm output by "demangling" the -# first two columns (address and symbol type). - -# These are the object tools we use which can come from a -# user-specified location using --tools, from the PPROF_TOOLS -# environment variable, or from the environment. -my %obj_tool_map = ( - "objdump" => "objdump", - "nm" => "nm", - "addr2line" => "addr2line", - ## ConfigureObjTools may add architecture-specific entries: - #"nm_pdb" => "nm-pdb", # for reading windows (PDB-format) executables - #"addr2line_pdb" => "addr2line-pdb", # ditto - #"otool" => "otool", # equivalent of objdump on OS X -); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; -# These are used for dynamic profiles - -# These are the web pages that servers need to support for dynamic profiles -my $HEAP_PAGE = "/pprof/heap"; -my $THREAD_PAGE = "/pprof/thread"; -my $PROFILE_PAGE = "/pprof/profile"; # must support cgi-param "?seconds=#" -my $BLOCK_PAGE = "/pprof/block"; -my $PMUPROFILE_PAGE = "/pprof/pmuprofile(?:\\?.*)?"; # must support cgi-param - # ?seconds=#&event=x&period=n -my $GROWTH_PAGE = "/pprof/growth"; -my $CONTENTION_PAGE = "/pprof/contention"; -my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter -my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST -my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; - -# default binary name -my $UNKNOWN_BINARY = "(unknown)"; - -# There is a pervasive dependency on the length (in hex characters, -# i.e., nibbles) of an address, distinguishing between 32-bit and -# 64-bit profiles. To err on the safe size, default to 64-bit here: -my $address_length = 16; - -# A list of paths to search for shared object files -my @prefix_list = (); - -# Special routine name that should not have any symbols. -# Used as separator to parse "addr2line -i" output. -my $sep_symbol = '_fini'; -my $sep_address = undef; - -my $OS = $^O; -my $DEVNULL = "/dev/null"; -if ($^O =~ /MSWin32|cygwin|msys/) { - $OS = "windows"; - $DEVNULL = "NUL"; -} - -##### Argument parsing ##### - -sub usage_string { - return < - is a space separated list of profile names. -pprof [options] - is a list of profile files where each file contains - the necessary symbol mappings as well as profile data (likely generated - with --raw). -pprof [options] - is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE - - Each name can be: - /path/to/profile - a path to a profile file - host:port[/] - a location of a service to get profile from - - The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, - $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, - $THREAD_PAGE, $BLOCK_PAGE or /pprof/filteredprofile. - For instance: - pprof http://myserver.com:80$HEAP_PAGE - If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). -pprof --symbols - Maps addresses to symbol names. In this mode, stdin should be a - list of library mappings, in the same format as is found in the heap- - and cpu-profile files (this loosely matches that of /proc/self/maps - on linux), followed by a list of hex addresses to map, one per line. - - For more help with querying remote servers, including how to add the - necessary server-side support code, see this filename (or one like it): - - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html - -Options: - --cum Sort by cumulative data - --base= Subtract from before display - --interactive Run in interactive mode (interactive "help" gives help) [default] - --seconds= Length of time for dynamic profiles [default=30 secs] - --add_lib= Read additional symbols and line info from the given library - --lib_prefix=

    Comma separated list of library path prefixes - -Reporting Granularity: - --addresses Report at address level - --lines Report at source line level - --functions Report at function level [default] - --files Report at source file level - -Output type: - --text Generate text report - --callgrind Generate callgrind format to stdout - --gv Generate Postscript and display - --web Generate SVG and display - --list= Generate source listing of matching routines - --disasm= Generate disassembly of matching routines - --symbols Print demangled symbol names found at given addresses - --dot Generate DOT file to stdout - --ps Generate Postcript to stdout - --pdf Generate PDF to stdout - --svg Generate SVG to stdout - --gif Generate GIF to stdout - --raw Generate symbolized pprof data (useful with remote fetch) - -Heap-Profile Options: - --inuse_space Display in-use (mega)bytes [default] - --inuse_objects Display in-use objects - --alloc_space Display allocated (mega)bytes - --alloc_objects Display allocated objects - --show_bytes Display space in bytes - --drop_negative Ignore negative differences - -Contention-profile options: - --total_delay Display total delay at each region [default] - --contentions Display number of delays at each region - --mean_delay Display mean delay at each region - -Call-graph Options: - --nodecount= Show at most so many nodes [default=80] - --nodefraction= Hide nodes below *total [default=.005] - --edgefraction= Hide edges below *total [default=.001] - --focus= Focus on nodes matching - --ignore= Ignore nodes matching - --scale= Set GV scaling [default=0] - --heapcheck Make nodes with non-0 object counts - (i.e. direct leak generators) more visible - -Miscellaneous: - --tools= Prefix for object tool pathnames - --test Run unit tests - --help This message - --version Version information - -Environment Variables: - PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof - PPROF_TOOLS Prefix for object tools pathnames - -Examples: - -pprof /bin/ls ls.prof - Enters "interactive" mode -pprof --text /bin/ls ls.prof - Outputs one line per procedure -pprof --web /bin/ls ls.prof - Displays annotated call-graph in web browser -pprof --gv /bin/ls ls.prof - Displays annotated call-graph via 'gv' -pprof --gv --focus=Mutex /bin/ls ls.prof - Restricts to code paths including a .*Mutex.* entry -pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof - Code paths including Mutex but not string -pprof --list=getdir /bin/ls ls.prof - (Per-line) annotated source listing for getdir() -pprof --disasm=getdir /bin/ls ls.prof - (Per-PC) annotated disassembly for getdir() - -pprof http://localhost:1234/ - Enters "interactive" mode -pprof --text localhost:1234 - Outputs one line per procedure for localhost:1234 -pprof --raw localhost:1234 > ./local.raw -pprof --text ./local.raw - Fetches a remote profile for later analysis and then - analyzes it in text mode. -EOF -} - -sub version_string { - return <new()->filename; - $main::tmpfile_ps = File::Temp->new()->filename; - - $main::next_tmpfile = 0; - $SIG{'INT'} = \&sighandler; - - # Cache from filename/linenumber to source code - $main::source_cache = (); - - $main::opt_help = 0; - $main::opt_version = 0; - - $main::opt_cum = 0; - $main::opt_base = ''; - $main::opt_addresses = 0; - $main::opt_lines = 0; - $main::opt_functions = 0; - $main::opt_files = 0; - $main::opt_lib_prefix = ""; - - $main::opt_text = 0; - $main::opt_callgrind = 0; - $main::opt_list = ""; - $main::opt_disasm = ""; - $main::opt_symbols = 0; - $main::opt_gv = 0; - $main::opt_web = 0; - $main::opt_dot = 0; - $main::opt_ps = 0; - $main::opt_pdf = 0; - $main::opt_gif = 0; - $main::opt_svg = 0; - $main::opt_raw = 0; - - $main::opt_nodecount = 80; - $main::opt_nodefraction = 0.005; - $main::opt_edgefraction = 0.001; - $main::opt_focus = ''; - $main::opt_ignore = ''; - $main::opt_scale = 0; - $main::opt_heapcheck = 0; - $main::opt_seconds = 30; - $main::opt_lib = ""; - - $main::opt_inuse_space = 0; - $main::opt_inuse_objects = 0; - $main::opt_alloc_space = 0; - $main::opt_alloc_objects = 0; - $main::opt_show_bytes = 0; - $main::opt_drop_negative = 0; - $main::opt_interactive = 0; - - $main::opt_total_delay = 0; - $main::opt_contentions = 0; - $main::opt_mean_delay = 0; - - $main::opt_tools = ""; - $main::opt_debug = 0; - $main::opt_test = 0; - - # These are undocumented flags used only by unittests. - $main::opt_test_stride = 0; - - # Are we using $SYMBOL_PAGE? - $main::use_symbol_page = 0; - - # Files returned by TempName. - %main::tempnames = (); - - # Type of profile we are dealing with - # Supported types: - # cpu - # heap - # growth - # contention - $main::profile_type = ''; # Empty type means "unknown" - - GetOptions("help!" => \$main::opt_help, - "version!" => \$main::opt_version, - "cum!" => \$main::opt_cum, - "base=s" => \$main::opt_base, - "seconds=i" => \$main::opt_seconds, - "add_lib=s" => \$main::opt_lib, - "lib_prefix=s" => \$main::opt_lib_prefix, - "functions!" => \$main::opt_functions, - "lines!" => \$main::opt_lines, - "addresses!" => \$main::opt_addresses, - "files!" => \$main::opt_files, - "text!" => \$main::opt_text, - "callgrind!" => \$main::opt_callgrind, - "list=s" => \$main::opt_list, - "disasm=s" => \$main::opt_disasm, - "symbols!" => \$main::opt_symbols, - "gv!" => \$main::opt_gv, - "web!" => \$main::opt_web, - "dot!" => \$main::opt_dot, - "ps!" => \$main::opt_ps, - "pdf!" => \$main::opt_pdf, - "svg!" => \$main::opt_svg, - "gif!" => \$main::opt_gif, - "raw!" => \$main::opt_raw, - "interactive!" => \$main::opt_interactive, - "nodecount=i" => \$main::opt_nodecount, - "nodefraction=f" => \$main::opt_nodefraction, - "edgefraction=f" => \$main::opt_edgefraction, - "focus=s" => \$main::opt_focus, - "ignore=s" => \$main::opt_ignore, - "scale=i" => \$main::opt_scale, - "heapcheck" => \$main::opt_heapcheck, - "inuse_space!" => \$main::opt_inuse_space, - "inuse_objects!" => \$main::opt_inuse_objects, - "alloc_space!" => \$main::opt_alloc_space, - "alloc_objects!" => \$main::opt_alloc_objects, - "show_bytes!" => \$main::opt_show_bytes, - "drop_negative!" => \$main::opt_drop_negative, - "total_delay!" => \$main::opt_total_delay, - "contentions!" => \$main::opt_contentions, - "mean_delay!" => \$main::opt_mean_delay, - "tools=s" => \$main::opt_tools, - "test!" => \$main::opt_test, - "debug!" => \$main::opt_debug, - # Undocumented flags used only by unittests: - "test_stride=i" => \$main::opt_test_stride, - ) || usage("Invalid option(s)"); - - # Deal with the standard --help and --version - if ($main::opt_help) { - print usage_string(); - exit(0); - } - - if ($main::opt_version) { - print version_string(); - exit(0); - } - - # Disassembly/listing/symbols mode requires address-level info - if ($main::opt_disasm || $main::opt_list || $main::opt_symbols) { - $main::opt_functions = 0; - $main::opt_lines = 0; - $main::opt_addresses = 1; - $main::opt_files = 0; - } - - # Check heap-profiling flags - if ($main::opt_inuse_space + - $main::opt_inuse_objects + - $main::opt_alloc_space + - $main::opt_alloc_objects > 1) { - usage("Specify at most on of --inuse/--alloc options"); - } - - # Check output granularities - my $grains = - $main::opt_functions + - $main::opt_lines + - $main::opt_addresses + - $main::opt_files + - 0; - if ($grains > 1) { - usage("Only specify one output granularity option"); - } - if ($grains == 0) { - $main::opt_functions = 1; - } - - # Check output modes - my $modes = - $main::opt_text + - $main::opt_callgrind + - ($main::opt_list eq '' ? 0 : 1) + - ($main::opt_disasm eq '' ? 0 : 1) + - ($main::opt_symbols == 0 ? 0 : 1) + - $main::opt_gv + - $main::opt_web + - $main::opt_dot + - $main::opt_ps + - $main::opt_pdf + - $main::opt_svg + - $main::opt_gif + - $main::opt_raw + - $main::opt_interactive + - 0; - if ($modes > 1) { - usage("Only specify one output mode"); - } - if ($modes == 0) { - if (-t STDOUT) { # If STDOUT is a tty, activate interactive mode - $main::opt_interactive = 1; - } else { - $main::opt_text = 1; - } - } - - if ($main::opt_test) { - RunUnitTests(); - # Should not return - exit(1); - } - - # Binary name and profile arguments list - $main::prog = ""; - @main::pfile_args = (); - - # Remote profiling without a binary (using $SYMBOL_PAGE instead) - if (IsProfileURL($ARGV[0])) { - $main::use_symbol_page = 1; - } elsif ($ARGV[0] && IsSymbolizedProfileFile($ARGV[0])) { - $main::use_symbolized_profile = 1; - $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file - } - - if ($main::use_symbol_page || $main::use_symbolized_profile) { - # We don't need a binary! - my %disabled = ('--lines' => $main::opt_lines, - '--disasm' => $main::opt_disasm); - for my $option (keys %disabled) { - usage("$option cannot be used without a binary") if $disabled{$option}; - } - # Set $main::prog later... - scalar(@ARGV) || usage("Did not specify profile file"); - } elsif ($main::opt_symbols) { - # --symbols needs a binary-name (to run nm on, etc) but not profiles - $main::prog = shift(@ARGV) || usage("Did not specify program"); - } else { - $main::prog = shift(@ARGV) || usage("Did not specify program"); - scalar(@ARGV) || usage("Did not specify profile file"); - } - - # Parse profile file/location arguments - foreach my $farg (@ARGV) { - if ($farg =~ m/(.*)\@([0-9]+)(|\/.*)$/ ) { - my $machine = $1; - my $num_machines = $2; - my $path = $3; - for (my $i = 0; $i < $num_machines; $i++) { - unshift(@main::pfile_args, "$i.$machine$path"); - } - } else { - unshift(@main::pfile_args, $farg); - } - } - - if ($main::use_symbol_page) { - unless (IsProfileURL($main::pfile_args[0])) { - error("The first profile should be a remote form to use $SYMBOL_PAGE\n"); - } - CheckSymbolPage(); - $main::prog = FetchProgramName(); - } elsif (!$main::use_symbolized_profile) { # may not need objtools! - ConfigureObjTools($main::prog) - } - - # Break the opt_lib_prefix into the prefix_list array - @prefix_list = split (',', $main::opt_lib_prefix); - - # Remove trailing / from the prefixes, in the list to prevent - # searching things like /my/path//lib/mylib.so - foreach (@prefix_list) { - s|/+$||; - } -} - -sub Main() { - Init(); - $main::collected_profile = undef; - @main::profile_files = (); - $main::op_time = time(); - - # Printing symbols is special and requires a lot less info that most. - if ($main::opt_symbols) { - PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin - return; - } - - # Fetch all profile data - FetchDynamicProfiles(); - - # this will hold symbols that we read from the profile files - my $symbol_map = {}; - - # Read one profile, pick the last item on the list - my $data = ReadProfile($main::prog, pop(@main::profile_files)); - my $profile = $data->{profile}; - my $pcs = $data->{pcs}; - my $libs = $data->{libs}; # Info about main program and shared libraries - $symbol_map = MergeSymbols($symbol_map, $data->{symbols}); - - # Add additional profiles, if available. - if (scalar(@main::profile_files) > 0) { - foreach my $pname (@main::profile_files) { - my $data2 = ReadProfile($main::prog, $pname); - $profile = AddProfile($profile, $data2->{profile}); - $pcs = AddPcs($pcs, $data2->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $data2->{symbols}); - } - } - - # Subtract base from profile, if specified - if ($main::opt_base ne '') { - my $base = ReadProfile($main::prog, $main::opt_base); - $profile = SubtractProfile($profile, $base->{profile}); - $pcs = AddPcs($pcs, $base->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); - } - - # Get total data in profile - my $total = TotalProfile($profile); - - # Collect symbols - my $symbols; - if ($main::use_symbolized_profile) { - $symbols = FetchSymbols($pcs, $symbol_map); - } elsif ($main::use_symbol_page) { - $symbols = FetchSymbols($pcs); - } else { - $symbols = ExtractSymbols($libs, $pcs); - } - - # Remove uniniteresting stack items - $profile = RemoveUninterestingFrames($symbols, $profile); - - # Focus? - if ($main::opt_focus ne '') { - $profile = FocusProfile($symbols, $profile, $main::opt_focus); - } - - # Ignore? - if ($main::opt_ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); - } - - my $calls = ExtractCalls($symbols, $profile); - - # Reduce profiles to required output granularity, and also clean - # each stack trace so a given entry exists at most once. - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - # Print - if (!$main::opt_interactive) { - if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); - } elsif ($main::opt_list) { - PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); - } elsif ($main::opt_text) { - # Make sure the output is empty when have nothing to report - # (only matters when --heapcheck is given but we must be - # compatible with old branches that did not pass --heapcheck always): - if ($total != 0) { - Infof("Total: %s %s\n", Unparse($total), Units()); - } - PrintText($symbols, $flat, $cumulative, $total, -1); - } elsif ($main::opt_raw) { - PrintSymbolizedProfile($symbols, $profile, $main::prog); - } elsif ($main::opt_callgrind) { - PrintCallgrind($calls); - } else { - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), ""); - } elsif ($main::opt_web) { - my $tmp = TempName($main::next_tmpfile, "svg"); - RunWeb($tmp); - # The command we run might hand the file name off - # to an already running browser instance and then exit. - # Normally, we'd remove $tmp on exit (right now), - # but fork a child to remove $tmp a little later, so that the - # browser has time to load it first. - delete $main::tempnames{$tmp}; - if (fork() == 0) { - sleep 5; - unlink($tmp); - exit(0); - } - } - } else { - exit(1); - } - } - } else { - InteractiveMode($profile, $symbols, $libs, $total); - } - - cleanup(); - exit(0); -} - -##### Entry Point ##### - -Main(); - -# Temporary code to detect if we're running on a Goobuntu system. -# These systems don't have the right stuff installed for the special -# Readline libraries to work, so as a temporary workaround, we default -# to using the normal stdio code, rather than the fancier readline-based -# code -sub ReadlineMightFail { - if (-e '/lib/libtermcap.so.2') { - return 0; # libtermcap exists, so readline should be okay - } else { - return 1; - } -} - -sub RunGV { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >$DEVNULL 2>&1")) { - # Options using double dash are supported by this gv version. - # Also, turn on noantialias to better handle bug in gv for - # postscript files with large dimensions. - # TODO: Maybe we should not pass the --noantialias flag - # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); - } else { - # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); - } -} - -sub RunWeb { - my $fname = shift; - print STDERR "Loading web page file:///$fname\n"; - - my $uname = `uname`; - if ($uname =~ /Darwin/) { - # OS X: open will use standard preference for SVG files. - system("/usr/bin/open", $fname); - return; - } - - if ($uname =~ /CYGWIN/) { - # Windows(cygwin): open will use standard preference for SVG files. - my $winname = `cygpath -wa $fname`; - system("explorer.exe", $winname); - return; - } - if ($uname =~ /MINGW/) { - # Windows(MinGW): open will use standard preference for SVG files. - system("cmd", "/c", "start", $fname); - return; - } - - # Some kind of Unix; try generic symlinks, then specific browsers. - # (Stop once we find one.) - # Works best if the browser is already running. - my @alt = ( - "/etc/alternatives/gnome-www-browser", - "/etc/alternatives/x-www-browser", - "google-chrome", - "firefox", - ); - foreach my $b (@alt) { - if (system($b, $fname) == 0) { - return; - } - } - - print STDERR "Could not load web browser.\n"; -} - -sub RunKcachegrind { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); -} - - -##### Interactive helper routines ##### - -sub InteractiveMode { - $| = 1; # Make output unbuffered for interactive mode - my ($orig_profile, $symbols, $libs, $total) = @_; - - print STDERR "Welcome to pprof! For help, type 'help'.\n"; - - # Use ReadLine if it's installed and input comes from a console. - if ( -t STDIN && - !ReadlineMightFail() && - defined(eval {require Term::ReadLine}) ) { - my $term = new Term::ReadLine 'pprof'; - while ( defined ($_ = $term->readline('(pprof) '))) { - $term->addhistory($_) if /\S/; - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - } - } else { # don't have readline - while (1) { - print STDERR "(pprof) "; - $_ = ; - last if ! defined $_ ; - s/\r//g; # turn windows-looking lines into unix-looking lines - - # Save some flags that might be reset by InteractiveCommand() - my $save_opt_lines = $main::opt_lines; - - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - - # Restore flags - $main::opt_lines = $save_opt_lines; - } - } -} - -# Takes two args: orig profile, and command to run. -# Returns 1 if we should keep going, or 0 if we were asked to quit -sub InteractiveCommand { - my($orig_profile, $symbols, $libs, $total, $command) = @_; - $_ = $command; # just to make future m//'s easier - if (!defined($_)) { - print STDERR "\n"; - return 0; - } - if (m/^\s*quit/) { - return 0; - } - if (m/^\s*help/) { - InteractiveHelpMessage(); - return 1; - } - # Clear all the mode options -- mode is controlled by "$command" - $main::opt_text = 0; - $main::opt_callgrind = 0; - $main::opt_disasm = 0; - $main::opt_list = 0; - $main::opt_gv = 0; - $main::opt_cum = 0; - - if (m/^\s*(text|top)(\d*)\s*(.*)/) { - $main::opt_text = 1; - - my $line_limit = ($2 ne "") ? int($2) : 10; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($3); - - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintText($symbols, $flat, $cumulative, $total, $line_limit); - return 1; - } - if (m/^\s*callgrind\s*([^ \n]*)/) { - $main::opt_callgrind = 1; - - # Get derived profiles - my $calls = ExtractCalls($symbols, $orig_profile); - my $filename = $1; - if ( $1 eq '' ) { - $filename = TempName($main::next_tmpfile, "callgrind"); - } - PrintCallgrind($calls, $filename); - if ( $1 eq '' ) { - RunKcachegrind($filename, " & "); - $main::next_tmpfile++; - } - - return 1; - } - if (m/^\s*(web)?list\s*(.+)/) { - my $html = (defined($1) && ($1 eq "web")); - $main::opt_list = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($2); - - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintListing($total, $libs, $flat, $cumulative, $routine, $html); - return 1; - } - if (m/^\s*disasm\s*(.+)/) { - $main::opt_disasm = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); - - # Process current profile to account for various settings - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintDisassembly($libs, $flat, $cumulative, $routine, $total); - return 1; - } - if (m/^\s*(gv|web)\s*(.*)/) { - $main::opt_gv = 0; - $main::opt_web = 0; - if ($1 eq "gv") { - $main::opt_gv = 1; - } elsif ($1 eq "web") { - $main::opt_web = 1; - } - - my $focus; - my $ignore; - ($focus, $ignore) = ParseInteractiveArgs($2); - - # Process current profile to account for various settings - my $profile = ProcessProfile($total, $orig_profile, $symbols, $focus, $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), " &"); - } elsif ($main::opt_web) { - RunWeb(TempName($main::next_tmpfile, "svg")); - } - $main::next_tmpfile++; - } - return 1; - } - if (m/^\s*$/) { - return 1; - } - print STDERR "Unknown command: try 'help'.\n"; - return 1; -} - - -sub ProcessProfile { - my $total_count = shift; - my $orig_profile = shift; - my $symbols = shift; - my $focus = shift; - my $ignore = shift; - - # Process current profile to account for various settings - my $profile = $orig_profile; - printf("Total: %s %s\n", Unparse($total_count), Units()); - if ($focus ne '') { - $profile = FocusProfile($symbols, $profile, $focus); - my $focus_count = TotalProfile($profile); - Infof("After focusing on '%s': %s %s of %s (%0.1f%%)\n", - $focus, - Unparse($focus_count), Units(), - Unparse($total_count), ($focus_count*100.0) / $total_count); - } - if ($ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $ignore); - my $ignore_count = TotalProfile($profile); - Infof("After ignoring '%s': %s %s of %s (%0.1f%%)\n", - $ignore, - Unparse($ignore_count), Units(), - Unparse($total_count), - ($ignore_count*100.0) / $total_count); - } - - return $profile; -} - -sub InteractiveHelpMessage { - print STDERR <{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - my $depth = $#addrs + 1; - # int(foo / 2**32) is the only reliable way to get rid of bottom - # 32 bits on both 32- and 64-bit systems. - print pack('L*', $count & 0xFFFFFFFF, int($count / 2**32)); - print pack('L*', $depth & 0xFFFFFFFF, int($depth / 2**32)); - - foreach my $full_addr (@addrs) { - my $addr = $full_addr; - $addr =~ s/0x0*//; # strip off leading 0x, zeroes - if (length($addr) > 16) { - print STDERR "Invalid address in profile: $full_addr\n"; - next; - } - my $low_addr = substr($addr, -8); # get last 8 hex chars - my $high_addr = substr($addr, -16, 8); # get up to 8 more hex chars - print pack('L*', hex('0x' . $low_addr), hex('0x' . $high_addr)); - } - } - } -} - -# Print symbols and profile data -sub PrintSymbolizedProfile { - my $symbols = shift; - my $profile = shift; - my $prog = shift; - - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - - print '--- ', $symbol_marker, "\n"; - if (defined($prog)) { - print 'binary=', $prog, "\n"; - } - while (my ($pc, $name) = each(%{$symbols})) { - my $sep = ' '; - print '0x', $pc; - # We have a list of function names, which include the inlined - # calls. They are separated (and terminated) by --, which is - # illegal in function names. - for (my $j = 2; $j <= $#{$name}; $j += 3) { - print $sep, $name->[$j]; - $sep = '--'; - } - print "\n"; - } - print '---', "\n"; - - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - print '--- ', $profile_marker, "\n"; - if (defined($main::collected_profile)) { - # if used with remote fetch, simply dump the collected profile to output. - open(SRC, "<$main::collected_profile"); - while () { - print $_; - } - close(SRC); - } else { - # dump a cpu-format profile to standard out - PrintProfileData($profile); - } -} - -# Print information conditionally filtered out depending on the output -# format. -sub Infof { - my $format = shift; - my @args = @_; - return if $main::opt_svg; - printf($format, @args); -} - -# Print text output -sub PrintText { - my $symbols = shift; - my $flat = shift; - my $cumulative = shift; - my $total = shift; - my $line_limit = shift; - - # Which profile to sort by? - my $s = $main::opt_cum ? $cumulative : $flat; - - my $running_sum = 0; - my $lines = 0; - foreach my $k (sort { GetEntry($s, $b) <=> GetEntry($s, $a) || $a cmp $b } - keys(%{$cumulative})) { - my $f = GetEntry($flat, $k); - my $c = GetEntry($cumulative, $k); - $running_sum += $f; - - my $sym = $k; - if (exists($symbols->{$k})) { - $sym = $symbols->{$k}->[0] . " " . $symbols->{$k}->[1]; - if ($main::opt_addresses) { - $sym = $k . " " . $sym; - } - } - - if ($f != 0 || $c != 0) { - printf("%8s %6s %6s %8s %6s %s\n", - Unparse($f), - Percent($f, $total), - Percent($running_sum, $total), - Unparse($c), - Percent($c, $total), - $sym); - } - $lines++; - last if ($line_limit >= 0 && $lines >= $line_limit); - } -} - -# Print the call graph in a way that's suiteable for callgrind. -sub PrintCallgrind { - my $calls = shift; - my $filename; - if ($main::opt_interactive) { - $filename = shift; - print STDERR "Writing callgrind file to '$filename'.\n" - } else { - $filename = "&STDOUT"; - } - open(CG, ">".$filename ); - printf CG ("events: Hits\n\n"); - foreach my $call ( map { $_->[0] } - sort { $a->[1] cmp $b ->[1] || - $a->[2] <=> $b->[2] } - map { /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - [$_, $1, $2] } - keys %$calls ) { - my $count = int($calls->{$call}); - $call =~ /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - my ( $caller_file, $caller_line, $caller_function, - $callee_file, $callee_line, $callee_function ) = - ( $1, $2, $3, $5, $6, $7 ); - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); - if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); - printf CG ("calls=$count $callee_line\n"); - } - printf CG ("$caller_line $count\n\n"); - } -} - -# Print disassembly for all all routines that match $main::opt_disasm -sub PrintDisassembly { - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $disasm_opts = shift; - my $total = shift; - - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - # See if there are any samples in this routine - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - PrintDisassembledFunction($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr, $total); - last; - } - $addr = AddressInc($addr); - } - } - } -} - -# Return reference to array of tuples of the form: -# [start_address, filename, linenumber, instruction, limit_address] -# E.g., -# ["0x806c43d", "/foo/bar.cc", 131, "ret", "0x806c440"] -sub Disassemble { - my $prog = shift; - my $offset = shift; - my $start_addr = shift; - my $end_addr = shift; - - my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - - if (system("$objdump --help >$DEVNULL 2>&1") != 0) { - # objdump must not exist. Fall back to go tool objdump. - $objdump = "go tool objdump"; - $cmd = "$objdump $prog 0x$start_addr 0x$end_addr"; - } - - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); - my @result = (); - my $filename = ""; - my $linenumber = -1; - my $last = ["", "", "", ""]; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - chop; - if (m|\s*(.+):(\d+)\s*$|) { - # Location line of the form: - # : - $filename = $1; - $linenumber = $2; - } elsif (m/^ +([0-9a-f]+):\s*(.*)/) { - # Disassembly line -- zero-extend address to full length - my $addr = HexExtend($1); - my $k = AddressAdd($addr, $offset); - $last->[4] = $k; # Store ending address for previous instruction - $last = [$k, $filename, $linenumber, $2, $end_addr]; - push(@result, $last); - } - } - close(OBJDUMP); - return @result; -} - -# The input file should contain lines of the form /proc/maps-like -# output (same format as expected from the profiles) or that looks -# like hex addresses (like "0xDEADBEEF"). We will parse all -# /proc/maps output, and for all the hex addresses, we will output -# "short" symbol names, one per line, in the same order as the input. -sub PrintSymbols { - my $maps_and_symbols_file = shift; - - # ParseLibraries expects pcs to be in a set. Fine by us... - my @pclist = (); # pcs in sorted order - my $pcs = {}; - my $map = ""; - foreach my $line (<$maps_and_symbols_file>) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ($line =~ /\b(0x[0-9a-f]+)\b/i) { - push(@pclist, HexExtend($1)); - $pcs->{$pclist[-1]} = 1; - } else { - $map .= $line; - } - } - - my $libs = ParseLibraries($main::prog, $map, $pcs); - my $symbols = ExtractSymbols($libs, $pcs); - - foreach my $pc (@pclist) { - # ->[0] is the shortname, ->[2] is the full name - print(($symbols->{$pc}->[0] || "??") . "\n"); - } -} - - -# For sorting functions by name -sub ByName { - return ShortFunctionName($a) cmp ShortFunctionName($b); -} - -# Print source-listing for all all routines that match $main::opt_list -sub PrintListing { - my $total = shift; - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $list_opts = shift; - my $html = shift; - - my $output = \*STDOUT; - my $fname = ""; - - - if ($html) { - # Arrange to write the output to a temporary file - $fname = TempName($main::next_tmpfile, "html"); - $main::next_tmpfile++; - if (!open(TEMP, ">$fname")) { - print STDERR "$fname: $!\n"; - return; - } - $output = \*TEMP; - print $output HtmlListingHeader(); - printf $output ("
    %s
    Total: %s %s
    \n", - $main::prog, Unparse($total), Units()); - } - - my $listed = 0; - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - # Print if there are any samples in this routine - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - $listed += PrintSource( - $lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr, - $html, - $output); - last; - } - $addr = AddressInc($addr); - } - } - } - - if ($html) { - if ($listed > 0) { - print $output HtmlListingFooter(); - close($output); - RunWeb($fname); - } else { - close($output); - unlink($fname); - } - } -} - -sub HtmlListingHeader { - return <<'EOF'; - - - -Pprof listing - - - - -EOF -} - -sub HtmlListingFooter { - return <<'EOF'; - - -EOF -} - -sub HtmlEscape { - my $text = shift; - $text =~ s/&/&/g; - $text =~ s//>/g; - return $text; -} - -# Returns the indentation of the line, if it has any non-whitespace -# characters. Otherwise, returns -1. -sub Indentation { - my $line = shift; - if (m/^(\s*)\S/) { - return length($1); - } else { - return -1; - } -} - -# Print source-listing for one routine -sub PrintSource { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - my $html = shift; - my $output = shift; - - # Disassemble all instructions (just to get line numbers) - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Hack 1: assume that the first source file encountered in the - # disassembly contains the routine - my $filename = undef; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[2] >= 0) { - $filename = $instructions[$i]->[1]; - last; - } - } - if (!defined($filename)) { - print STDERR "no filename found in $routine\n"; - return 0; - } - - # Hack 2: assume that the largest line number from $filename is the - # end of the procedure. This is typically safe since if P1 contains - # an inlined call to P2, then P2 usually occurs earlier in the - # source file. If this does not work, we might have to compute a - # density profile or just print all regions we find. - my $lastline = 0; - for (my $i = 0; $i <= $#instructions; $i++) { - my $f = $instructions[$i]->[1]; - my $l = $instructions[$i]->[2]; - if (($f eq $filename) && ($l > $lastline)) { - $lastline = $l; - } - } - - # Hack 3: assume the first source location from "filename" is the start of - # the source code. - my $firstline = 1; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[1] eq $filename) { - $firstline = $instructions[$i]->[2]; - last; - } - } - - # Hack 4: Extend last line forward until its indentation is less than - # the indentation we saw on $firstline - my $oldlastline = $lastline; - { - if (!open(FILE, "<$filename")) { - print STDERR "$filename: $!\n"; - return 0; - } - my $l = 0; - my $first_indentation = -1; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $l++; - my $indent = Indentation($_); - if ($l >= $firstline) { - if ($first_indentation < 0 && $indent >= 0) { - $first_indentation = $indent; - last if ($first_indentation == 0); - } - } - if ($l >= $lastline && $indent >= 0) { - if ($indent >= $first_indentation) { - $lastline = $l+1; - } else { - last; - } - } - } - close(FILE); - } - - # Assign all samples to the range $firstline,$lastline, - # Hack 4: If an instruction does not occur in the range, its samples - # are moved to the next instruction that occurs in the range. - my $samples1 = {}; # Map from line number to flat count - my $samples2 = {}; # Map from line number to cumulative count - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts - my %disasm = (); # Map from line number to disassembly - my $running_disasm = ""; # Unassigned disassembly - my $skip_marker = "---\n"; - if ($html) { - $skip_marker = ""; - for (my $l = $firstline; $l <= $lastline; $l++) { - $disasm{$l} = ""; - } - } - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - - if ($html) { - $running_disasm .= sprintf(" %6s %6s \t\t%8s: %s\n", - HtmlPrintNumber($c1), - HtmlPrintNumber($c2), - $e->[0], - CleanDisassembly($e->[3])); - } - - $running1 += $c1; - $running2 += $c2; - $total1 += $c1; - $total2 += $c2; - my $file = $e->[1]; - my $line = $e->[2]; - if (($file eq $filename) && - ($line >= $firstline) && - ($line <= $lastline)) { - # Assign all accumulated samples to this line - AddEntry($samples1, $line, $running1); - AddEntry($samples2, $line, $running2); - $running1 = 0; - $running2 = 0; - if ($html) { - $disasm{$line} .= $running_disasm; - $running_disasm = ''; - } - } - } - - # Assign any leftover samples to $lastline - AddEntry($samples1, $lastline, $running1); - AddEntry($samples2, $lastline, $running2); - - if ($html) { - printf $output ( - "

    %s

    %s\n
    \n" .
-      "Total:%6s %6s (flat / cumulative %s)\n",
-      HtmlEscape(ShortFunctionName($routine)),
-      HtmlEscape($filename),
-      Unparse($total1),
-      Unparse($total2),
-      Units());
-  } else {
-    printf $output (
-      "ROUTINE ====================== %s in %s\n" .
-      "%6s %6s Total %s (flat / cumulative)\n",
-      ShortFunctionName($routine),
-      $filename,
-      Unparse($total1),
-      Unparse($total2),
-      Units());
-  }
-  if (!open(FILE, "<$filename")) {
-    print STDERR "$filename: $!\n";
-    return 0;
-  }
-  my $l = 0;
-  while () {
-    s/\r//g;         # turn windows-looking lines into unix-looking lines
-    $l++;
-    if ($l >= $firstline - 5 &&
-        (($l <= $oldlastline + 5) || ($l <= $lastline))) {
-      chop;
-      my $text = $_;
-      if ($l == $firstline) { print $output $skip_marker; }
-      my $n1 = GetEntry($samples1, $l);
-      my $n2 = GetEntry($samples2, $l);
-      if ($html) {
-        my $dis = $disasm{$l};
-        if (!defined($dis) || $n1 + $n2 == 0) {
-          # No samples/disassembly for this source line
-          printf $output (
-            "%5d " .
-            "%6s %6s %s\n",
-            $l,
-            HtmlPrintNumber($n1),
-            HtmlPrintNumber($n2),
-            HtmlEscape($text));
-        } else {
-          printf $output (
-            "%5d " .
-            "%6s %6s %s" .
-            "%s\n",
-            $l,
-            HtmlPrintNumber($n1),
-            HtmlPrintNumber($n2),
-            HtmlEscape($text),
-            HtmlEscape($dis));
-        }
-      } else {
-        printf $output(
-          "%6s %6s %4d: %s\n",
-          UnparseAlt($n1),
-          UnparseAlt($n2),
-          $l,
-          $text);
-      }
-      if ($l == $lastline)  { print $output $skip_marker; }
-    };
-  }
-  close(FILE);
-  if ($html) {
-    print $output "
    \n"; - } - return 1; -} - -# Return the source line for the specified file/linenumber. -# Returns undef if not found. -sub SourceLine { - my $file = shift; - my $line = shift; - - # Look in cache - if (!defined($main::source_cache{$file})) { - if (100 < scalar keys(%main::source_cache)) { - # Clear the cache when it gets too big - $main::source_cache = (); - } - - # Read all lines from the file - if (!open(FILE, "<$file")) { - print STDERR "$file: $!\n"; - $main::source_cache{$file} = []; # Cache the negative result - return undef; - } - my $lines = []; - push(@{$lines}, ""); # So we can use 1-based line numbers as indices - while () { - push(@{$lines}, $_); - } - close(FILE); - - # Save the lines in the cache - $main::source_cache{$file} = $lines; - } - - my $lines = $main::source_cache{$file}; - if (($line < 0) || ($line > $#{$lines})) { - return undef; - } else { - return $lines->[$line]; - } -} - -# Print disassembly for one routine with interspersed source if available -sub PrintDisassembledFunction { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - my $total = shift; - - # Disassemble all instructions - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Make array of counts per instruction - my @flat_count = (); - my @cum_count = (); - my $flat_total = 0; - my $cum_total = 0; - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - push(@flat_count, $c1); - push(@cum_count, $c2); - $flat_total += $c1; - $cum_total += $c2; - } - - # Print header with total counts - printf("ROUTINE ====================== %s\n" . - "%6s %6s %s (flat, cumulative) %.1f%% of total\n", - ShortFunctionName($routine), - Unparse($flat_total), - Unparse($cum_total), - Units(), - ($cum_total * 100.0) / $total); - - # Process instructions in order - my $current_file = ""; - for (my $i = 0; $i <= $#instructions; ) { - my $e = $instructions[$i]; - - # Print the new file name whenever we switch files - if ($e->[1] ne $current_file) { - $current_file = $e->[1]; - my $fname = $current_file; - $fname =~ s|^\./||; # Trim leading "./" - - # Shorten long file names - if (length($fname) >= 58) { - $fname = "..." . substr($fname, -55); - } - printf("-------------------- %s\n", $fname); - } - - # TODO: Compute range of lines to print together to deal with - # small reorderings. - my $first_line = $e->[2]; - my $last_line = $first_line; - my %flat_sum = (); - my %cum_sum = (); - for (my $l = $first_line; $l <= $last_line; $l++) { - $flat_sum{$l} = 0; - $cum_sum{$l} = 0; - } - - # Find run of instructions for this range of source lines - my $first_inst = $i; - while (($i <= $#instructions) && - ($instructions[$i]->[2] >= $first_line) && - ($instructions[$i]->[2] <= $last_line)) { - $e = $instructions[$i]; - $flat_sum{$e->[2]} += $flat_count[$i]; - $cum_sum{$e->[2]} += $cum_count[$i]; - $i++; - } - my $last_inst = $i - 1; - - # Print source lines - for (my $l = $first_line; $l <= $last_line; $l++) { - my $line = SourceLine($current_file, $l); - if (!defined($line)) { - $line = "?\n"; - next; - } else { - $line =~ s/^\s+//; - } - printf("%6s %6s %5d: %s", - UnparseAlt($flat_sum{$l}), - UnparseAlt($cum_sum{$l}), - $l, - $line); - } - - # Print disassembly - for (my $x = $first_inst; $x <= $last_inst; $x++) { - my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - printf("%6s %6s %8s: %6s\n", - UnparseAlt($flat_count[$x]), - UnparseAlt($cum_count[$x]), - $address, - CleanDisassembly($e->[3])); - } - } -} - -# Print DOT graph -sub PrintDot { - my $prog = shift; - my $symbols = shift; - my $raw = shift; - my $flat = shift; - my $cumulative = shift; - my $overall_total = shift; - - # Get total - my $local_total = TotalProfile($flat); - my $nodelimit = int($main::opt_nodefraction * $local_total); - my $edgelimit = int($main::opt_edgefraction * $local_total); - my $nodecount = $main::opt_nodecount; - - # Find nodes to include - my @list = (sort { abs(GetEntry($cumulative, $b)) <=> - abs(GetEntry($cumulative, $a)) - || $a cmp $b } - keys(%{$cumulative})); - my $last = $nodecount - 1; - if ($last > $#list) { - $last = $#list; - } - while (($last >= 0) && - (abs(GetEntry($cumulative, $list[$last])) <= $nodelimit)) { - $last--; - } - if ($last < 0) { - print STDERR "No nodes to print\n"; - cleanup(); - return 0; - } - - if ($nodelimit > 0 || $edgelimit > 0) { - printf STDERR ("Dropping nodes with <= %s %s; edges with <= %s abs(%s)\n", - Unparse($nodelimit), Units(), - Unparse($edgelimit), Units()); - } - - # Open DOT output file - my $output; - if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); - } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; - } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; - } elsif ($main::opt_web || $main::opt_svg) { - # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); - } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; - } else { - $output = ">&STDOUT"; - } - open(DOT, $output) || error("$output: $!\n"); - - # Title - printf DOT ("digraph \"%s; %s %s\" {\n", - $prog, - Unparse($overall_total), - Units()); - if ($main::opt_pdf) { - # The output is more printable if we set the page size for dot. - printf DOT ("size=\"8,11\"\n"); - } - printf DOT ("node [width=0.375,height=0.25];\n"); - - # Print legend - printf DOT ("Legend [shape=box,fontsize=24,shape=plaintext," . - "label=\"%s\\l%s\\l%s\\l%s\\l%s\\l\"];\n", - $prog, - sprintf("Total %s: %s", Units(), Unparse($overall_total)), - sprintf("Focusing on: %s", Unparse($local_total)), - sprintf("Dropped nodes with <= %s abs(%s)", - Unparse($nodelimit), Units()), - sprintf("Dropped edges with <= %s %s", - Unparse($edgelimit), Units()) - ); - - # Print nodes - my %node = (); - my $nextnode = 1; - foreach my $a (@list[0..$last]) { - # Pick font size - my $f = GetEntry($flat, $a); - my $c = GetEntry($cumulative, $a); - - my $fs = 8; - if ($local_total > 0) { - $fs = 8 + (50.0 * sqrt(abs($f * 1.0 / $local_total))); - } - - $node{$a} = $nextnode++; - my $sym = $a; - $sym =~ s/\s+/\\n/g; - $sym =~ s/::/\\n/g; - - # Extra cumulative info to print for non-leaves - my $extra = ""; - if ($f != $c) { - $extra = sprintf("\\rof %s (%s)", - Unparse($c), - Percent($c, $overall_total)); - } - my $style = ""; - if ($main::opt_heapcheck) { - if ($f > 0) { - # make leak-causing nodes more visible (add a background) - $style = ",style=filled,fillcolor=gray" - } elsif ($f < 0) { - # make anti-leak-causing nodes (which almost never occur) - # stand out as well (triple border) - $style = ",peripheries=3" - } - } - - printf DOT ("N%d [label=\"%s\\n%s (%s)%s\\r" . - "\",shape=box,fontsize=%.1f%s];\n", - $node{$a}, - $sym, - Unparse($f), - Percent($f, $overall_total), - $extra, - $fs, - $style, - ); - } - - # Get edges and counts per edge - my %edge = (); - my $n; - foreach my $k (keys(%{$raw})) { - # TODO: omit low %age edges - $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); - for (my $i = 1; $i <= $#translated; $i++) { - my $src = $translated[$i]; - my $dst = $translated[$i-1]; - #next if ($src eq $dst); # Avoid self-edges? - if (exists($node{$src}) && exists($node{$dst})) { - my $edge_label = "$src\001$dst"; - if (!exists($edge{$edge_label})) { - $edge{$edge_label} = 0; - } - $edge{$edge_label} += $n; - } - } - } - - # Print edges - foreach my $e (keys(%edge)) { - my @x = split(/\001/, $e); - $n = $edge{$e}; - - if (abs($n) > $edgelimit) { - # Compute line width based on edge count - my $fraction = abs($local_total ? (3 * ($n / $local_total)) : 0); - if ($fraction > 1) { $fraction = 1; } - my $w = $fraction * 2; - if ($w < 1 && ($main::opt_web || $main::opt_svg)) { - # SVG output treats line widths < 1 poorly. - $w = 1; - } - - # Dot sometimes segfaults if given edge weights that are too large, so - # we cap the weights at a large value - my $edgeweight = abs($n) ** 0.7; - if ($edgeweight > 100000) { $edgeweight = 100000; } - $edgeweight = int($edgeweight); - - my $style = sprintf("setlinewidth(%f)", $w); - if ($x[1] =~ m/\(inline\)/) { - $style .= ",dashed"; - } - - # Use a slightly squashed function of the edge count as the weight - printf DOT ("N%s -> N%s [label=%s, weight=%d, style=\"%s\"];\n", - $node{$x[0]}, - $node{$x[1]}, - Unparse($n), - $edgeweight, - $style); - } - } - - print DOT ("}\n"); - close(DOT); - - if ($main::opt_web || $main::opt_svg) { - # Rewrite SVG to be more usable inside web browser. - RewriteSvg(TempName($main::next_tmpfile, "svg")); - } - - return 1; -} - -sub RewriteSvg { - my $svgfile = shift; - - open(SVG, $svgfile) || die "open temp svg: $!"; - my @svg = ; - close(SVG); - unlink $svgfile; - my $svg = join('', @svg); - - # Dot's SVG output is - # - # - # - # ... - # - # - # - # Change it to - # - # - # $svg_javascript - # - # - # ... - # - # - # - - # Fix width, height; drop viewBox. - $svg =~ s/(?s) above first - my $svg_javascript = SvgJavascript(); - my $viewport = "\n"; - $svg =~ s/ above . - $svg =~ s/(.*)(<\/svg>)/$1<\/g>$2/; - $svg =~ s/$svgfile") || die "open $svgfile: $!"; - print SVG $svg; - close(SVG); - } -} - -sub SvgJavascript { - return <<'EOF'; - -EOF -} - -# Translate a stack of addresses into a stack of symbols -sub TranslateStack { - my $symbols = shift; - my $k = shift; - - my @addrs = split(/\n/, $k); - my @result = (); - for (my $i = 0; $i <= $#addrs; $i++) { - my $a = $addrs[$i]; - - # Skip large addresses since they sometimes show up as fake entries on RH9 - if (length($a) > 8 && $a gt "7fffffffffffffff") { - next; - } - - if ($main::opt_disasm || $main::opt_list) { - # We want just the address for the key - push(@result, $a); - next; - } - - my $symlist = $symbols->{$a}; - if (!defined($symlist)) { - $symlist = [$a, "", $a]; - } - - # We can have a sequence of symbols for a particular entry - # (more than one symbol in the case of inlining). Callers - # come before callees in symlist, so walk backwards since - # the translated stack should contain callees before callers. - for (my $j = $#{$symlist}; $j >= 2; $j -= 3) { - my $func = $symlist->[$j-2]; - my $fileline = $symlist->[$j-1]; - my $fullfunc = $symlist->[$j]; - if ($j > 2) { - $func = "$func (inline)"; - } - if ($main::opt_addresses) { - push(@result, "$a $func $fileline"); - } elsif ($main::opt_lines) { - if ($func eq '??' && $fileline eq '??:0') { - push(@result, "$a"); - } else { - push(@result, "$func $fileline"); - } - } elsif ($main::opt_functions) { - if ($func eq '??') { - push(@result, "$a"); - } else { - push(@result, $func); - } - } elsif ($main::opt_files) { - if ($fileline eq '??:0' || $fileline eq '') { - push(@result, "$a"); - } else { - my $f = $fileline; - $f =~ s/:\d+$//; - push(@result, $f); - } - } else { - push(@result, $a); - last; # Do not print inlined info - } - } - } - - # print join(",", @addrs), " => ", join(",", @result), "\n"; - return @result; -} - -# Generate percent string for a number and a total -sub Percent { - my $num = shift; - my $tot = shift; - if ($tot != 0) { - return sprintf("%.1f%%", $num * 100.0 / $tot); - } else { - return ($num == 0) ? "nan" : (($num > 0) ? "+inf" : "-inf"); - } -} - -# Generate pretty-printed form of number -sub Unparse { - my $num = shift; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return sprintf("%d", $num); - } else { - if ($main::opt_show_bytes) { - return sprintf("%d", $num); - } else { - return sprintf("%.1f", $num / 1048576.0); - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return sprintf("%.3f", $num / 1e9); # Convert nanoseconds to seconds - } else { - return sprintf("%d", $num); - } -} - -# Alternate pretty-printed form: 0 maps to "." -sub UnparseAlt { - my $num = shift; - if ($num == 0) { - return "."; - } else { - return Unparse($num); - } -} - -# Alternate pretty-printed form: 0 maps to "" -sub HtmlPrintNumber { - my $num = shift; - if ($num == 0) { - return ""; - } else { - return Unparse($num); - } -} - -# Return output units -sub Units { - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return "objects"; - } else { - if ($main::opt_show_bytes) { - return "B"; - } else { - return "MB"; - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return "seconds"; - } elsif ($main::profile_type eq 'thread') { - return "threads"; - } else { - return "samples"; - } -} - -##### Profile manipulation code ##### - -# Generate flattened profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a] -sub FlatProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - AddEntry($result, $addrs[0], $count); - } - } - return $result; -} - -# Generate cumulative profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a], [b], [c], [d] -sub CumulativeProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - AddEntry($result, $a, $count); - } - } - return $result; -} - -# If the second-youngest PC on the stack is always the same, returns -# that pc. Otherwise, returns undef. -sub IsSecondPcAlwaysTheSame { - my $profile = shift; - - my $second_pc = undef; - foreach my $k (keys(%{$profile})) { - my @addrs = split(/\n/, $k); - if ($#addrs < 1) { - return undef; - } - if (not defined $second_pc) { - $second_pc = $addrs[1]; - } else { - if ($second_pc ne $addrs[1]) { - return undef; - } - } - } - return $second_pc; -} - -sub ExtractSymbolLocation { - my $symbols = shift; - my $address = shift; - # 'addr2line' outputs "??:0" for unknown locations; we do the - # same to be consistent. - my $location = "??:0:unknown"; - if (exists $symbols->{$address}) { - my $file = $symbols->{$address}->[1]; - if ($file eq "?") { - $file = "??:0" - } - $location = $file . ":" . $symbols->{$address}->[0]; - } - return $location; -} - -# Extracts a graph of calls. -sub ExtractCalls { - my $symbols = shift; - my $profile = shift; - - my $calls = {}; - while( my ($stack_trace, $count) = each %$profile ) { - my @address = split(/\n/, $stack_trace); - my $destination = ExtractSymbolLocation($symbols, $address[0]); - AddEntry($calls, $destination, $count); - for (my $i = 1; $i <= $#address; $i++) { - my $source = ExtractSymbolLocation($symbols, $address[$i]); - my $call = "$source -> $destination"; - AddEntry($calls, $call, $count); - $destination = $source; - } - } - - return $calls; -} - -sub RemoveUninterestingFrames { - my $symbols = shift; - my $profile = shift; - - # List of function names to skip - my %skip = (); - my $skip_regexp = 'NOMATCH'; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - foreach my $name ('calloc', - 'cfree', - 'malloc', - 'free', - 'memalign', - 'posix_memalign', - 'pvalloc', - 'valloc', - 'realloc', - 'tc_calloc', - 'tc_cfree', - 'tc_malloc', - 'tc_free', - 'tc_memalign', - 'tc_posix_memalign', - 'tc_pvalloc', - 'tc_valloc', - 'tc_realloc', - 'tc_new', - 'tc_delete', - 'tc_newarray', - 'tc_deletearray', - 'tc_new_nothrow', - 'tc_newarray_nothrow', - 'do_malloc', - '::do_malloc', # new name -- got moved to an unnamed ns - '::do_malloc_or_cpp_alloc', - 'DoSampledAllocation', - 'simple_alloc::allocate', - '__malloc_alloc_template::allocate', - '__builtin_delete', - '__builtin_new', - '__builtin_vec_delete', - '__builtin_vec_new', - 'operator new', - 'operator new[]', - # Go - 'catstring', - 'cnew', - 'copyin', - 'gostring', - 'gostringsize', - 'growslice1', - 'appendslice1', - 'hash_init', - 'hash_subtable_new', - 'hash_conv', - 'hash_grow', - 'hash_insert_internal', - 'hash_insert', - 'mapassign', - 'runtime.mapassign', - 'runtime.appendslice', - 'runtime.mapassign1', - 'makechan', - 'makemap', - 'mal', - 'profilealloc', - 'runtime.new', - 'makeslice1', - 'runtime.malloc', - 'unsafe.New', - 'runtime.mallocgc', - 'runtime.catstring', - 'runtime.cnew', - 'runtime.cnewarray', - 'runtime.growslice', - 'runtime.ifaceT2E', - 'runtime.ifaceT2I', - 'runtime.makechan', - 'runtime.makechan_c', - 'runtime.makemap', - 'runtime.makemap_c', - 'runtime.makeslice', - 'runtime.mal', - 'runtime.settype', - 'runtime.settype_flush', - 'runtime.slicebytetostring', - 'runtime.sliceinttostring', - 'runtime.stringtoslicebyte', - 'runtime.stringtosliceint', - # These mark the beginning/end of our custom sections - '__start_google_malloc', - '__stop_google_malloc', - '__start_malloc_hook', - '__stop_malloc_hook') { - $skip{$name} = 1; - $skip{"_" . $name} = 1; # Mach (OS X) adds a _ prefix to everything - } - # TODO: Remove TCMalloc once everything has been - # moved into the tcmalloc:: namespace and we have flushed - # old code out of the system. - $skip_regexp = "TCMalloc|^tcmalloc::"; - } elsif ($main::profile_type eq 'contention') { - foreach my $vname ('Mutex::Unlock', 'Mutex::UnlockSlow') { - $skip{$vname} = 1; - } - } elsif ($main::profile_type eq 'cpu') { - # Drop signal handlers used for CPU profile collection - # TODO(dpeng): this should not be necessary; it's taken - # care of by the general 2nd-pc mechanism below. - foreach my $name ('ProfileData::Add', # historical - 'ProfileData::prof_handler', # historical - 'CpuProfiler::prof_handler', - '__FRAME_END__', - '__pthread_sighandler', - '__restore') { - $skip{$name} = 1; - } - } else { - # Nothing skipped for unknown types - } - - # Go doesn't have the problem that this heuristic tries to fix. Disable. - if (0 && $main::profile_type eq 'cpu') { - # If all the second-youngest program counters are the same, - # this STRONGLY suggests that it is an artifact of measurement, - # i.e., stack frames pushed by the CPU profiler signal handler. - # Hence, we delete them. - # (The topmost PC is read from the signal structure, not from - # the stack, so it does not get involved.) - while (my $second_pc = IsSecondPcAlwaysTheSame($profile)) { - my $result = {}; - my $func = ''; - if (exists($symbols->{$second_pc})) { - $second_pc = $symbols->{$second_pc}->[0]; - } - print STDERR "Removing $second_pc from all stack traces.\n"; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - splice @addrs, 1, 1; - my $reduced_path = join("\n", @addrs); - AddEntry($result, $reduced_path, $count); - } - $profile = $result; - } - } - - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my @path = (); - foreach my $a (@addrs) { - if (exists($symbols->{$a})) { - my $func = $symbols->{$a}->[0]; - if ($skip{$func} || ($func =~ m/$skip_regexp/)) { - next; - } - } - push(@path, $a); - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Reduce profile to granularity given by user -sub ReduceProfile { - my $symbols = shift; - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); - my @path = (); - my %seen = (); - $seen{''} = 1; # So that empty keys are skipped - foreach my $e (@translated) { - # To avoid double-counting due to recursion, skip a stack-trace - # entry if it has already been seen - if (!$seen{$e}) { - $seen{$e} = 1; - push(@path, $e); - } - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Does the specified symbol array match the regexp? -sub SymbolMatches { - my $sym = shift; - my $re = shift; - if (defined($sym)) { - for (my $i = 0; $i < $#{$sym}; $i += 3) { - if ($sym->[$i] =~ m/$re/ || $sym->[$i+1] =~ m/$re/) { - return 1; - } - } - } - return 0; -} - -# Focus only on paths involving specified regexps -sub FocusProfile { - my $symbols = shift; - my $profile = shift; - my $focus = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$focus/) || SymbolMatches($symbols->{$a}, $focus)) { - AddEntry($result, $k, $count); - last; - } - } - } - return $result; -} - -# Focus only on paths not involving specified regexps -sub IgnoreProfile { - my $symbols = shift; - my $profile = shift; - my $ignore = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my $matched = 0; - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$ignore/) || SymbolMatches($symbols->{$a}, $ignore)) { - $matched = 1; - last; - } - } - if (!$matched) { - AddEntry($result, $k, $count); - } - } - return $result; -} - -# Get total count in profile -sub TotalProfile { - my $profile = shift; - my $result = 0; - foreach my $k (keys(%{$profile})) { - $result += $profile->{$k}; - } - return $result; -} - -# Add A to B -sub AddProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - my $v = $A->{$k}; - AddEntry($R, $k, $v); - } - # add all keys in B - foreach my $k (keys(%{$B})) { - my $v = $B->{$k}; - AddEntry($R, $k, $v); - } - return $R; -} - -# Merges symbol maps -sub MergeSymbols { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - $R->{$k} = $A->{$k}; - } - if (defined($B)) { - foreach my $k (keys(%{$B})) { - $R->{$k} = $B->{$k}; - } - } - return $R; -} - - -# Add A to B -sub AddPcs { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - $R->{$k} = 1 - } - # add all keys in B - foreach my $k (keys(%{$B})) { - $R->{$k} = 1 - } - return $R; -} - -# Subtract B from A -sub SubtractProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - my $v = $A->{$k} - GetEntry($B, $k); - if ($v < 0 && $main::opt_drop_negative) { - $v = 0; - } - AddEntry($R, $k, $v); - } - if (!$main::opt_drop_negative) { - # Take care of when subtracted profile has more entries - foreach my $k (keys(%{$B})) { - if (!exists($A->{$k})) { - AddEntry($R, $k, 0 - $B->{$k}); - } - } - } - return $R; -} - -# Get entry from profile; zero if not present -sub GetEntry { - my $profile = shift; - my $k = shift; - if (exists($profile->{$k})) { - return $profile->{$k}; - } else { - return 0; - } -} - -# Add entry to specified profile -sub AddEntry { - my $profile = shift; - my $k = shift; - my $n = shift; - if (!exists($profile->{$k})) { - $profile->{$k} = 0; - } - $profile->{$k} += $n; -} - -# Add a stack of entries to specified profile, and add them to the $pcs -# list. -sub AddEntries { - my $profile = shift; - my $pcs = shift; - my $stack = shift; - my $count = shift; - my @k = (); - - foreach my $e (split(/\s+/, $stack)) { - my $pc = HexExtend($e); - $pcs->{$pc} = 1; - push @k, $pc; - } - AddEntry($profile, (join "\n", @k), $count); -} - -sub IsSymbolizedProfileFile { - my $file_name = shift; - - if (!(-e $file_name) || !(-r $file_name)) { - return 0; - } - - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - # Check if the file contains a symbol-section marker. - open(TFILE, "<$file_name"); - my @lines = ; - my $result = grep(/^--- *$symbol_marker/, @lines); - close(TFILE); - return $result > 0; -} - -##### Code to profile a server dynamically ##### - -sub CheckSymbolPage { - my $url = SymbolPageURL(); -print STDERR "Read $url\n"; - - my $line = FetchHTTP($url); - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - unless (defined($line)) { - error("$url doesn't exist\n"); - } - - if ($line =~ /^num_symbols:\s+(\d+)$/) { - if ($1 == 0) { - error("Stripped binary. No symbols available.\n"); - } - } else { - error("Failed to get the number of symbols from $url\n"); - } -} - -sub IsProfileURL { - my $profile_name = shift; - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - return defined($host) and defined($port) and defined($path); -} - -sub ParseProfileURL { - my $profile_name = shift; - if (defined($profile_name) && - $profile_name =~ m,^(?:(https?)://|)([^/:]+):(\d+)(|\@\d+)(|/|(.*?)($PROFILE_PAGE|$PMUPROFILE_PAGE|$HEAP_PAGE|$GROWTH_PAGE|$THREAD_PAGE|$BLOCK_PAGE|$CONTENTION_PAGE|$WALL_PAGE|$FILTEREDPROFILE_PAGE))$,o) { - # $7 is $PROFILE_PAGE/$HEAP_PAGE/etc. $5 is *everything* after - # the hostname, as long as that everything is the empty string, - # a slash, or something ending in $PROFILE_PAGE/$HEAP_PAGE/etc. - # So "$7 || $5" is $PROFILE_PAGE/etc if there, or else it's "/" or "". - return ($1 || "http", $2, $3, $6, $7 || $5); - } - return (); -} - -# We fetch symbols from the first profile argument. -sub SymbolPageURL { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); - return "$scheme://$host:$port$prefix$SYMBOL_PAGE"; -} - -sub FetchProgramName() { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); - my $url = "$scheme://$host:$port$prefix$PROGRAM_NAME_PAGE"; - - my $cmdline = FetchHTTP($url); - $cmdline =~ s/\n.*//s; # first line only - $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines - error("Failed to get program name from $url\n") unless defined($cmdline); - $cmdline =~ s/\x00.+//; # Remove argv[1] and latters. - $cmdline =~ s!\n!!g; # Remove LFs. - return $cmdline; -} - -# Reads a symbol map from the file handle name given as $1, returning -# the resulting symbol map. Also processes variables relating to symbols. -# Currently, the only variable processed is 'binary=' which updates -# $main::prog to have the correct program name. -sub ReadSymbols { - my $in = shift; - my $map = shift; - while (<$in>) { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Removes all the leading zeroes from the symbols, see comment below. - if (m/^0x0*([0-9a-f]+)\s+(.+)/) { - $map->{$1} = $2; - } elsif (m/^---/) { - last; - } elsif (m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1, $2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "binary") { - if ($main::prog ne $UNKNOWN_BINARY && $main::prog ne $value) { - printf STDERR ("Warning: Mismatched binary name '%s', using '%s'.\n", - $main::prog, $value); - } - $main::prog = $value; - } else { - printf STDERR ("Ignoring unknown variable in symbols list: " . - "'%s' = '%s'\n", $variable, $value); - } - } - } - return $map; -} - -# Fetches and processes symbols to prepare them for use in the profile output -# code. If the optional 'symbol_map' arg is not given, fetches symbols from -# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols -# are assumed to have already been fetched into 'symbol_map' and are simply -# extracted and processed. -sub FetchSymbols { - my $pcset = shift; - my $symbol_map = shift; - - my %seen = (); - my @pcs = grep { !$seen{$_}++ } keys(%$pcset); # uniq - - if (!defined($symbol_map)) { - $symbol_map = {}; - - my $post_data = join("+", sort((map {"0x" . "$_"} @pcs))); - my $url = SymbolPageURL(); - my $content = PostHTTP($url, $post_data); - - my $tmp_symbol = File::Temp->new()->filename; - open(SYMBOL, ">$tmp_symbol"); - print SYMBOL $content; - close(SYMBOL); - - open(SYMBOL, "<$tmp_symbol") || error("$tmp_symbol"); - ReadSymbols(*SYMBOL{IO}, $symbol_map); - close(SYMBOL); - } - - my $symbols = {}; - foreach my $pc (@pcs) { - my $fullname; - # For 64 bits binaries, symbols are extracted with 8 leading zeroes. - # Then /symbol reads the long symbols in as uint64, and outputs - # the result with a "0x%08llx" format which get rid of the zeroes. - # By removing all the leading zeroes in both $pc and the symbols from - # /symbol, the symbols match and are retrievable from the map. - my $shortpc = $pc; - $shortpc =~ s/^0*//; - # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. - my $fullnames; - if (defined($symbol_map->{$shortpc})) { - $fullnames = $symbol_map->{$shortpc}; - } else { - $fullnames = "0x" . $pc; # Just use addresses - } - my $sym = []; - $symbols->{$pc} = $sym; - foreach my $fullname (split("--", $fullnames)) { - my $name = ShortFunctionName($fullname); - push(@{$sym}, $name, "?", $fullname); - } - } - return $symbols; -} - -sub BaseName { - my $file_name = shift; - $file_name =~ s!^.*/!!; # Remove directory name - return $file_name; -} - -sub MakeProfileBaseName { - my ($binary_name, $profile_name) = @_; - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - my $binary_shortname = BaseName($binary_name); - return sprintf("%s.%s.%s-port%s", - $binary_shortname, $main::op_time, $host, $port); -} - -sub FetchDynamicProfile { - my $binary_name = shift; - my $profile_name = shift; - my $fetch_name_only = shift; - my $encourage_patience = shift; - - if (!IsProfileURL($profile_name)) { - return $profile_name; - } else { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - if ($path eq "" || $path eq "/") { - # Missing type specifier defaults to cpu-profile - $path = $PROFILE_PAGE; - } - - my $profile_file = MakeProfileBaseName($binary_name, $profile_name); - - my $url; - my $timeout; - if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)) { - if ($path =~ m/$PROFILE_PAGE/) { - $url = sprintf("$scheme://$host:$port$prefix$path?seconds=%d", - $main::opt_seconds); - } else { - if ($profile_name =~ m/[?]/) { - $profile_name .= "&" - } else { - $profile_name .= "?" - } - $url = sprintf("$scheme://$profile_name" . "seconds=%d", - $main::opt_seconds); - } - $timeout = int($main::opt_seconds * 1.01 + 60); - } else { - # For non-CPU profiles, we add a type-extension to - # the target profile file name. - my $suffix = $path; - $suffix =~ s,/,.,g; - $profile_file .= "$suffix"; - $url = "$scheme://$host:$port$prefix$path"; - } - - my $tmp_profile = File::Temp->new()->filename; - my $real_profile = File::Temp->new()->filename; - - if ($fetch_name_only > 0) { - return $real_profile; - } - - if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)){ - print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; - if ($encourage_patience) { - print STDERR "Be patient...\n"; - } - } else { - print STDERR "Fetching $path profile from $host:$port to\n ${real_profile}\n"; - } - - my $content = FetchHTTP($url, $timeout); - - open(OUTFILE, ">$tmp_profile"); - binmode(OUTFILE); - print OUTFILE $content; - close(OUTFILE); - - my $line = $content; - $line !~ /^Could not enable CPU profiling/ || error($line); - - copy($tmp_profile, $real_profile) || error("Unable to copy profile\n"); - print STDERR "Wrote profile to $real_profile\n"; - $main::collected_profile = $real_profile; - return $main::collected_profile; - } -} - -# Collect profiles in parallel -sub FetchDynamicProfiles { - my $items = scalar(@main::pfile_args); - my $levels = log($items) / log(2); - - if ($items == 1) { - $main::profile_files[0] = FetchDynamicProfile($main::prog, $main::pfile_args[0], 0, 1); - } else { - # math rounding issues - if ((2 ** $levels) < $items) { - $levels++; - } - my $count = scalar(@main::pfile_args); - for (my $i = 0; $i < $count; $i++) { - $main::profile_files[$i] = FetchDynamicProfile($main::prog, $main::pfile_args[$i], 1, 0); - } - print STDERR "Fetching $count profiles, Be patient...\n"; - FetchDynamicProfilesRecurse($levels, 0, 0); - $main::collected_profile = join(" \\\n ", @main::profile_files); - } -} - -# Recursively fork a process to get enough processes -# collecting profiles -sub FetchDynamicProfilesRecurse { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if (my $pid = fork()) { - $position = 0 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - wait; - } else { - $position = 1 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - exit(0); - } -} - -# Collect a single profile -sub TryCollectProfile { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if ($level >= ($maxlevel - 1)) { - if ($position < scalar(@main::pfile_args)) { - FetchDynamicProfile($main::prog, $main::pfile_args[$position], 0, 0); - } - } else { - FetchDynamicProfilesRecurse($maxlevel, $level+1, $position); - } -} - -##### Parsing code ##### - -# Provide a small streaming-read module to handle very large -# cpu-profile files. Stream in chunks along a sliding window. -# Provides an interface to get one 'slot', correctly handling -# endian-ness differences. A slot is one 32-bit or 64-bit word -# (depending on the input profile). We tell endianness and bit-size -# for the profile by looking at the first 8 bytes: in cpu profiles, -# the second slot is always 3 (we'll accept anything that's not 0). -BEGIN { - package CpuProfileStream; - - sub new { - my ($class, $file, $fname) = @_; - my $self = { file => $file, - base => 0, - stride => 512 * 1024, # must be a multiple of bitsize/8 - slots => [], - unpack_code => "", # N for big-endian, V for little - }; - bless $self, $class; - # Let unittests adjust the stride - if ($main::opt_test_stride > 0) { - $self->{stride} = $main::opt_test_stride; - } - # Read the first two slots to figure out bitsize and endianness. - my $slots = $self->{slots}; - my $str; - read($self->{file}, $str, 8); - # Set the global $address_length based on what we see here. - # 8 is 32-bit (8 hexadecimal chars); 16 is 64-bit (16 hexadecimal chars). - $address_length = ($str eq (chr(0)x8)) ? 16 : 8; - if ($address_length == 8) { - if (substr($str, 6, 2) eq chr(0)x2) { - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 4, 2) eq chr(0)x2) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**16\n"); - } - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # If we're a 64-bit profile, make sure we're a 64-bit-capable - # perl. Otherwise, each slot will be represented as a float - # instead of an int64, losing precision and making all the - # 64-bit addresses right. We *could* try to handle this with - # software emulation of 64-bit ints, but that's added complexity - # for no clear benefit (yet). We use 'Q' to test for 64-bit-ness; - # perl docs say it's only available on 64-bit perl systems. - my $has_q = 0; - eval { $has_q = pack("Q", "1") ? 1 : 1; }; - if (!$has_q) { - ::error("$fname: need a 64-bit perl to process this 64-bit profile.\n"); - } - read($self->{file}, $str, 8); - if (substr($str, 4, 4) eq chr(0)x4) { - # We'd love to use 'Q', but it's a) not universal, b) not endian-proof. - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 0, 4) eq chr(0)x4) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**32\n"); - } - my @pair = unpack($self->{unpack_code} . "*", $str); - # Since we know one of the pair is 0, it's fine to just add them. - @$slots = (0, $pair[0] + $pair[1]); - } - return $self; - } - - # Load more data when we access slots->get(X) which is not yet in memory. - sub overflow { - my ($self) = @_; - my $slots = $self->{slots}; - $self->{base} += $#$slots + 1; # skip over data we're replacing - my $str; - read($self->{file}, $str, $self->{stride}); - if ($address_length == 8) { # the 32-bit case - # This is the easy case: unpack provides 32-bit unpacking primitives. - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # We need to unpack 32 bits at a time and combine. - my @b32_values = unpack($self->{unpack_code} . "*", $str); - my @b64_values = (); - for (my $i = 0; $i < $#b32_values; $i += 2) { - # TODO(csilvers): if this is a 32-bit perl, the math below - # could end up in a too-large int, which perl will promote - # to a double, losing necessary precision. Deal with that. - if ($self->{unpack_code} eq 'V') { # little-endian - push(@b64_values, $b32_values[$i] + $b32_values[$i+1] * (2**32)); - } else { - push(@b64_values, $b32_values[$i] * (2**32) + $b32_values[$i+1]); - } - } - @$slots = @b64_values; - } - } - - # Access the i-th long in the file (logically), or -1 at EOF. - sub get { - my ($self, $idx) = @_; - my $slots = $self->{slots}; - while ($#$slots >= 0) { - if ($idx < $self->{base}) { - # The only time we expect a reference to $slots[$i - something] - # after referencing $slots[$i] is reading the very first header. - # Since $stride > |header|, that shouldn't cause any lookback - # errors. And everything after the header is sequential. - print STDERR "Unexpected look-back reading CPU profile"; - return -1; # shrug, don't know what better to return - } elsif ($idx > $self->{base} + $#$slots) { - $self->overflow(); - } else { - return $slots->[$idx - $self->{base}]; - } - } - # If we get here, $slots is [], which means we've reached EOF - return -1; # unique since slots is supposed to hold unsigned numbers - } -} - -# Parse profile generated by common/profiler.cc and return a reference -# to a map: -# $result->{version} Version number of profile file -# $result->{period} Sampling period (in microseconds) -# $result->{profile} Profile object -# $result->{map} Memory map info from profile -# $result->{pcs} Hash of all PC values seen, key is hex address -sub ReadProfile { - my $prog = shift; - my $fname = shift; - - if (IsSymbolizedProfileFile($fname) && !$main::use_symbolized_profile) { - # we have both a binary and symbolized profiles, abort - usage("Symbolized profile '$fname' cannot be used with a binary arg. " . - "Try again without passing '$prog'."); - } - - $main::profile_type = ''; - - $CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $contention_marker = $&; - $GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $growth_marker = $&; - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - - # Look at first line to see if it is a heap or a CPU profile. - # CPU profile may start with no header at all, and just binary data - # (starting with \0\0\0\0) -- in that case, don't try to read the - # whole firstline, since it may be gigabytes(!) of data. - open(PROFILE, "<$fname") || error("$fname: $!\n"); - binmode PROFILE; # New perls do UTF-8 processing - my $firstchar = ""; - my $header = ""; - read(PROFILE, $firstchar, 1); - seek(PROFILE, -1, 1); # unread the firstchar - if ($firstchar ne "\0") { - $header = ; - if (!defined($header)) { - error("Profile is empty.\n"); - } - $header =~ s/\r//g; # turn windows-looking lines into unix-looking lines - } - - my $symbols; - if ($header =~ m/^--- *$symbol_marker/o) { - # read the symbol section of the symbolized profile file - $symbols = ReadSymbols(*PROFILE{IO}); - - # read the next line to get the header for the remaining profile - $header = ""; - read(PROFILE, $firstchar, 1); - seek(PROFILE, -1, 1); # unread the firstchar - if ($firstchar ne "\0") { - $header = ; - $header =~ s/\r//g; - } - } - - my $result; - - if ($header =~ m/^heap profile:.*$growth_marker/o) { - $main::profile_type = 'growth'; - $result = ReadHeapProfile($prog, $fname, $header); - } elsif ($header =~ m/^heap profile:/) { - $main::profile_type = 'heap'; - $result = ReadHeapProfile($prog, $fname, $header); - } elsif ($header =~ m/^--- *$contention_marker/o) { - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, $fname); - } elsif ($header =~ m/^--- *Stacks:/) { - print STDERR - "Old format contention profile: mistakenly reports " . - "condition variable signals as lock contentions.\n"; - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, $fname); - } elsif ($header =~ m/^thread creation profile:/) { - $main::profile_type = 'thread'; - $result = ReadThreadProfile($prog, $fname); - } elsif ($header =~ m/^--- *$profile_marker/) { - # the binary cpu profile data starts immediately after this line - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname); - } else { - if (defined($symbols)) { - # a symbolized profile contains a format we don't recognize, bail out - error("$fname: Cannot recognize profile section after symbols.\n"); - } - # no ascii header present -- must be a CPU profile - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname); - } - - # if we got symbols along with the profile, return those as well - if (defined($symbols)) { - $result->{symbols} = $symbols; - } - - return $result; -} - -# Subtract one from caller pc so we map back to call instr. -# However, don't do this if we're reading a symbolized profile -# file, in which case the subtract-one was done when the file -# was written. -# -# We apply the same logic to all readers, though ReadCPUProfile uses an -# independent implementation. -sub FixCallerAddresses { - my $stack = shift; - if ($main::use_symbolized_profile) { - return $stack; - } else { - $stack =~ /(\s)/; - my $delimiter = $1; - my @addrs = split(' ', $stack); - my @fixedaddrs; - $#fixedaddrs = $#addrs; - if ($#addrs >= 0) { - $fixedaddrs[0] = $addrs[0]; - } - for (my $i = 1; $i <= $#addrs; $i++) { - $fixedaddrs[$i] = AddressSub($addrs[$i], "0x1"); - } - return join $delimiter, @fixedaddrs; - } -} - -# CPU profile reader -sub ReadCPUProfile { - my $prog = shift; - my $fname = shift; - my $version; - my $period; - my $i; - my $profile = {}; - my $pcs = {}; - - # Parse string into array of slots. - my $slots = CpuProfileStream->new(*PROFILE, $fname); - - # Read header. The current header version is a 5-element structure - # containing: - # 0: header count (always 0) - # 1: header "words" (after this one: 3) - # 2: format version (0) - # 3: sampling period (usec) - # 4: unused padding (always 0) - if ($slots->get(0) != 0 ) { - error("$fname: not a profile file, or old format profile file\n"); - } - $i = 2 + $slots->get(1); - $version = $slots->get(2); - $period = $slots->get(3); - # Do some sanity checking on these header values. - if ($version > (2**32) || $period > (2**32) || $i > (2**32) || $i < 5) { - error("$fname: not a profile file, or corrupted profile file\n"); - } - - # Parse profile - while ($slots->get($i) != -1) { - my $n = $slots->get($i++); - my $d = $slots->get($i++); - if ($d > (2**16)) { # TODO(csilvers): what's a reasonable max-stack-depth? - my $addr = sprintf("0%o", $i * ($address_length == 8 ? 4 : 8)); - print STDERR "At index $i (address $addr):\n"; - error("$fname: stack trace depth >= 2**32\n"); - } - if ($slots->get($i) == 0) { - # End of profile data marker - $i += $d; - last; - } - - # Make key out of the stack entries - my @k = (); - for (my $j = 0; $j < $d; $j++) { - my $pc = $slots->get($i+$j); - # Subtract one from caller pc so we map back to call instr. - # However, don't do this if we're reading a symbolized profile - # file, in which case the subtract-one was done when the file - # was written. - if ($j > 0 && !$main::use_symbolized_profile) { - $pc--; - } - $pc = sprintf("%0*x", $address_length, $pc); - $pcs->{$pc} = 1; - push @k, $pc; - } - - AddEntry($profile, (join "\n", @k), $n); - $i += $d; - } - - # Parse map - my $map = ''; - seek(PROFILE, $i * 4, 0); - read(PROFILE, $map, (stat PROFILE)[7]); - close(PROFILE); - - my $r = {}; - $r->{version} = $version; - $r->{period} = $period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - - return $r; -} - -sub ReadHeapProfile { - my $prog = shift; - my $fname = shift; - my $header = shift; - - my $index = 1; - if ($main::opt_inuse_space) { - $index = 1; - } elsif ($main::opt_inuse_objects) { - $index = 0; - } elsif ($main::opt_alloc_space) { - $index = 3; - } elsif ($main::opt_alloc_objects) { - $index = 2; - } - - # Find the type of this profile. The header line looks like: - # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 - # There are two pairs , the first inuse objects/space, and the - # second allocated objects/space. This is followed optionally by a profile - # type, and if that is present, optionally by a sampling frequency. - # For remote heap profiles (v1): - # The interpretation of the sampling frequency is that the profiler, for - # each sample, calculates a uniformly distributed random integer less than - # the given value, and records the next sample after that many bytes have - # been allocated. Therefore, the expected sample interval is half of the - # given frequency. By default, if not specified, the expected sample - # interval is 128KB. Only remote-heap-page profiles are adjusted for - # sample size. - # For remote heap profiles (v2): - # The sampling frequency is the rate of a Poisson process. This means that - # the probability of sampling an allocation of size X with sampling rate Y - # is 1 - exp(-X/Y) - # For version 2, a typical header line might look like this: - # heap profile: 1922: 127792360 [ 1922: 127792360] @ _v2/524288 - # the trailing number (524288) is the sampling rate. (Version 1 showed - # double the 'rate' here) - my $sampling_algorithm = 0; - my $sample_adjustment = 0; - chomp($header); - my $type = "unknown"; - if ($header =~ m"^heap profile:\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\](\s*@\s*([^/]*)(/(\d+))?)?") { - if (defined($6) && ($6 ne '')) { - $type = $6; - my $sample_period = $8; - # $type is "heapprofile" for profiles generated by the - # heap-profiler, and either "heap" or "heap_v2" for profiles - # generated by sampling directly within tcmalloc. It can also - # be "growth" for heap-growth profiles. The first is typically - # found for profiles generated locally, and the others for - # remote profiles. - if (($type eq "heapprofile") || ($type !~ /heap/) ) { - # No need to adjust for the sampling rate with heap-profiler-derived data - $sampling_algorithm = 0; - } elsif ($type =~ /_v2/) { - $sampling_algorithm = 2; # version 2 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period); - } - } else { - $sampling_algorithm = 1; # version 1 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period)/2; - } - } - } else { - # We detect whether or not this is a remote-heap profile by checking - # that the total-allocated stats ($n2,$s2) are exactly the - # same as the in-use stats ($n1,$s1). It is remotely conceivable - # that a non-remote-heap profile may pass this check, but it is hard - # to imagine how that could happen. - # In this case it's so old it's guaranteed to be remote-heap version 1. - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - if (($n1 == $n2) && ($s1 == $s2)) { - # This is likely to be a remote-heap based sample profile - $sampling_algorithm = 1; - } - } - } - - if ($sampling_algorithm > 0) { - # For remote-heap generated profiles, adjust the counts and sizes to - # account for the sample rate (we sample once every 128KB by default). - if ($sample_adjustment == 0) { - # Turn on profile adjustment. - $sample_adjustment = 128*1024; - print STDERR "Adjusting heap profiles for 1-in-128KB sampling rate\n"; - } else { - printf STDERR ("Adjusting heap profiles for 1-in-%d sampling rate\n", - $sample_adjustment); - } - if ($sampling_algorithm > 1) { - # We don't bother printing anything for the original version (version 1) - printf STDERR "Heap version $sampling_algorithm\n"; - } - } - - my $profile = {}; - my $pcs = {}; - my $map = ""; - - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } - last; - } - - if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } - last; - } - - # Read entry of the form: - # : [: ] @ a1 a2 a3 ... an - s/^\s*//; - s/\s*$//; - if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { - my $stack = $5; - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - - if ($sample_adjustment) { - if ($sampling_algorithm == 2) { - # Remote-heap version 2 - # The sampling frequency is the rate of a Poisson process. - # This means that the probability of sampling an allocation of - # size X with sampling rate Y is 1 - exp(-X/Y) - my $ratio; - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor; - $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } else { - # Remote-heap version 1 - my $ratio; - if ($n1 > 0) { - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - if ($ratio < 1) { - $n1 /= $ratio; - $s1 /= $ratio; - } - } - if ($n2 > 0) { - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - if ($ratio < 1) { - $n2 /= $ratio; - $s2 /= $ratio; - } - } - } - } - - my @counts = ($n1, $s1, $n2, $s2); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); - } - } - - my $r = {}; - $r->{version} = "heap"; - $r->{period} = 1; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -sub ReadThreadProfile { - my $prog = shift; - my $fname = shift; - - my $profile = {}; - my $pcs = {}; - my $map = ""; - - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } - last; - } - - if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } - last; - } - - # Read entry of the form: - # @ a1 a2 a3 ... an - s/^\s*//; - s/\s*$//; - if (m/^@\s+(.*)$/) { - AddEntries($profile, $pcs, FixCallerAddresses($1), 1); - } - } - - my $r = {}; - $r->{version} = "thread"; - $r->{period} = 1; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -sub ReadSynchProfile { - my ($prog, $fname, $header) = @_; - - my $map = ''; - my $profile = {}; - my $pcs = {}; - my $sampling_period = 1; - my $cyclespernanosec = 2.8; # Default assumption for old binaries - my $seen_clockrate = 0; - my $line; - - my $index = 0; - if ($main::opt_total_delay) { - $index = 0; - } elsif ($main::opt_contentions) { - $index = 1; - } elsif ($main::opt_mean_delay) { - $index = 2; - } - - while ( $line = ) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ( $line =~ /^\s*(\d+)\s+(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $count, $stack) = ($1, $2, $3); - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - $count *= $sampling_period; - - my @values = ($cycles, $count, $cycles / $count); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $values[$index]); - - } elsif ( $line =~ /^(slow release).*thread \d+ \@\s*(.*?)\s*$/ || - $line =~ /^\s*(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $stack) = ($1, $2); - if ($cycles !~ /^\d+$/) { - next; - } - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - - AddEntries($profile, $pcs, FixCallerAddresses($stack), $cycles); - - } elsif ( $line =~ m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1,$2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "cycles/second") { - $cyclespernanosec = $value / 1e9; - $seen_clockrate = 1; - } elsif ($variable eq "sampling period") { - $sampling_period = $value; - } elsif ($variable eq "ms since reset") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } elsif ($variable eq "discarded samples") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } else { - printf STDERR ("Ignoring unnknown variable in /contention output: " . - "'%s' = '%s'\n",$variable,$value); - } - } else { - # Memory map entry - $map .= $line; - } - } - close PROFILE; - - if (!$seen_clockrate) { - printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n", - $cyclespernanosec); - } - - my $r = {}; - $r->{version} = 0; - $r->{period} = $sampling_period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... -sub HexExtend { - my $addr = shift; - - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; - } - - return substr("000000000000000".$addr, -$address_length); -} - -##### Symbol extraction ##### - -# Aggressively search the lib_prefix values for the given library -# If all else fails, just return the name of the library unmodified. -# If the lib_prefix is "/my/path,/other/path" and $file is "/lib/dir/mylib.so" -# it will search the following locations in this order, until it finds a file: -# /my/path/lib/dir/mylib.so -# /other/path/lib/dir/mylib.so -# /my/path/dir/mylib.so -# /other/path/dir/mylib.so -# /my/path/mylib.so -# /other/path/mylib.so -# /lib/dir/mylib.so (returned as last resort) -sub FindLibrary { - my $file = shift; - my $suffix = $file; - - # Search for the library as described above - do { - foreach my $prefix (@prefix_list) { - my $fullpath = $prefix . $suffix; - if (-e $fullpath) { - return $fullpath; - } - } - } while ($suffix =~ s|^/[^/]+/|/|); - return $file; -} - -# Return path to library with debugging symbols. -# For libc libraries, the copy in /usr/lib/debug contains debugging symbols -sub DebuggingLibrary { - my $file = shift; - if ($file =~ m|^/| && -f "/usr/lib/debug$file") { - return "/usr/lib/debug$file"; - } - return undef; -} - -# Parse text section header of a library using objdump -sub ParseTextSectionHeaderFromObjdump { - my $lib = shift; - - my $size = undef; - my $vma; - my $file_offset; - # Get objdump output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Idx Name Size VMA LMA File off Algn - # 10 .text 00104b2c 420156f0 420156f0 000156f0 2**4 - # For 64-bit objects, VMA and LMA will be 16 hex digits, size and file - # offset may still be 8. But AddressSub below will still handle that. - my @x = split; - if (($#x >= 6) && ($x[1] eq '.text')) { - $size = $x[2]; - $vma = $x[3]; - $file_offset = $x[5]; - last; - } - } - close(OBJDUMP); - - if (!defined($size)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -# Parse text section header of a library using otool (on OS X) -sub ParseTextSectionHeaderFromOtool { - my $lib = shift; - - my $size = undef; - my $vma = undef; - my $file_offset = undef; - # Get otool output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); - my $cmd = ""; - my $sectname = ""; - my $segname = ""; - foreach my $line () { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - # Load command <#> - # cmd LC_SEGMENT - # [...] - # Section - # sectname __text - # segname __TEXT - # addr 0x000009f8 - # size 0x00018b9e - # offset 2552 - # align 2^2 (4) - # We will need to strip off the leading 0x from the hex addresses, - # and convert the offset into hex. - if ($line =~ /Load command/) { - $cmd = ""; - $sectname = ""; - $segname = ""; - } elsif ($line =~ /Section/) { - $sectname = ""; - $segname = ""; - } elsif ($line =~ /cmd (\w+)/) { - $cmd = $1; - } elsif ($line =~ /sectname (\w+)/) { - $sectname = $1; - } elsif ($line =~ /segname (\w+)/) { - $segname = $1; - } elsif (!(($cmd eq "LC_SEGMENT" || $cmd eq "LC_SEGMENT_64") && - $sectname eq "__text" && - $segname eq "__TEXT")) { - next; - } elsif ($line =~ /\baddr 0x([0-9a-fA-F]+)/) { - $vma = $1; - } elsif ($line =~ /\bsize 0x([0-9a-fA-F]+)/) { - $size = $1; - } elsif ($line =~ /\boffset ([0-9]+)/) { - $file_offset = sprintf("%016x", $1); - } - if (defined($vma) && defined($size) && defined($file_offset)) { - last; - } - } - close(OTOOL); - - if (!defined($vma) || !defined($size) || !defined($file_offset)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -sub ParseTextSectionHeader { - # obj_tool_map("otool") is only defined if we're in a Mach-O environment - if (defined($obj_tool_map{"otool"})) { - my $r = ParseTextSectionHeaderFromOtool(@_); - if (defined($r)){ - return $r; - } - } - # If otool doesn't work, or we don't have it, fall back to objdump - return ParseTextSectionHeaderFromObjdump(@_); -} - -# Split /proc/pid/maps dump into a list of libraries -sub ParseLibraries { - return if $main::use_symbol_page; # We don't need libraries info. - my $prog = shift; - my $map = shift; - my $pcs = shift; - - my $result = []; - my $h = "[a-f0-9]+"; - my $zero_offset = HexExtend("0"); - - my $buildvar = ""; - foreach my $l (split("\n", $map)) { - if ($l =~ m/^\s*build=(.*)$/) { - $buildvar = $1; - } - - my $start; - my $finish; - my $offset; - my $lib; - if ($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?)$/i) { - # Full line from /proc/self/maps. Example: - # 40000000-40015000 r-xp 00000000 03:01 12845071 /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = HexExtend($3); - $lib = $4; - $lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths - } elsif ($l =~ /^\s*($h)-($h):\s*(\S+\.so(\.\d+)*)/) { - # Cooked line from DumpAddressMap. Example: - # 40000000-40015000: /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = $zero_offset; - $lib = $3; - } else { - next; - } - - # Expand "$build" variable if available - $lib =~ s/\$build\b/$buildvar/g; - - $lib = FindLibrary($lib); - - # Check for pre-relocated libraries, which use pre-relocated symbol tables - # and thus require adjusting the offset that we'll use to translate - # VM addresses into symbol table addresses. - # Only do this if we're not going to fetch the symbol table from a - # debugging copy of the library. - if (!DebuggingLibrary($lib)) { - my $text = ParseTextSectionHeader($lib); - if (defined($text)) { - my $vma_offset = AddressSub($text->{vma}, $text->{file_offset}); - $offset = AddressAdd($offset, $vma_offset); - } - } - - push(@{$result}, [$lib, $start, $finish, $offset]); - } - - # Append special entry for additional library (not relocated) - if ($main::opt_lib ne "") { - my $text = ParseTextSectionHeader($main::opt_lib); - if (defined($text)) { - my $start = $text->{vma}; - my $finish = AddressAdd($start, $text->{size}); - - push(@{$result}, [$main::opt_lib, $start, $finish, $start]); - } - } - - # Append special entry for the main program. This covers - # 0..max_pc_value_seen, so that we assume pc values not found in one - # of the library ranges will be treated as coming from the main - # program binary. - my $min_pc = HexExtend("0"); - my $max_pc = $min_pc; # find the maximal PC value in any sample - foreach my $pc (keys(%{$pcs})) { - if (HexExtend($pc) gt $max_pc) { $max_pc = HexExtend($pc); } - } - push(@{$result}, [$prog, $min_pc, $max_pc, $zero_offset]); - - return $result; -} - -# Add two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressAdd { - my $addr1 = shift; - my $addr2 = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr1)+hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - - if ($main::opt_debug and $main::opt_test) { - print STDERR "AddressAdd $addr1 + $addr2 = "; - } - - my $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - my $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2); - my $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - my $r = sprintf("%07x", $sum); - - $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2) + $c; - $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - $r = sprintf("%07x", $sum) . $r; - - $sum = hex($addr1) + hex($addr2) + $c; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - if ($main::opt_debug and $main::opt_test) { print STDERR "$r\n"; } - - return $r; - } -} - - -# Subtract two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressSub { - my $addr1 = shift; - my $addr2 = shift; - my $diff; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $diff = (hex($addr1)-hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $diff); - - } else { - # Do the addition in 7-nibble chunks to trivialize borrow handling. - # if ($main::opt_debug) { print STDERR "AddressSub $addr1 - $addr2 = "; } - - my $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - my $a2 = hex(substr($addr2,-7)); - $addr2 = substr($addr2,0,-7); - my $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - my $r = sprintf("%07x", $diff); - - $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - $a2 = hex(substr($addr2,-7)) + $b; - $addr2 = substr($addr2,0,-7); - $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - $r = sprintf("%07x", $diff) . $r; - - $a1 = hex($addr1); - $a2 = hex($addr2) + $b; - if ($a2 > $a1) { $a1 += 0x100; } - $diff = $a1 - $a2; - $r = sprintf("%02x", $diff) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - - return $r; - } -} - -# Increment a hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressInc { - my $addr = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr)+1) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - # We are always doing this to step through the addresses in a function, - # and will almost never overflow the first chunk, so we check for this - # case and exit early. - - # if ($main::opt_debug) { print STDERR "AddressInc $addr1 = "; } - - my $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - my $r = sprintf("%07x", $sum); - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "0000000"; - } - - $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - $r = sprintf("%07x", $sum) . $r; - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "00000000000000"; - } - - $sum = hex($addr) + 1; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - return $r; - } -} - -# Extract symbols for all PC values found in profile -sub ExtractSymbols { - my $libs = shift; - my $pcset = shift; - - my $symbols = {}; - - # Map each PC value to the containing library - my %seen = (); - foreach my $lib (@{$libs}) { - my $libname = $lib->[0]; - my $start = $lib->[1]; - my $finish = $lib->[2]; - my $offset = $lib->[3]; - - # Get list of pcs that belong in this library. - my $contained = []; - foreach my $pc (keys(%{$pcset})) { - if (!$seen{$pc} && ($pc ge $start) && ($pc le $finish)) { - $seen{$pc} = 1; - push(@{$contained}, $pc); - } - } - # Map to symbols - MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); - } - - return $symbols; -} - -# Map list of PC values to symbols for a given image -sub MapToSymbols { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - my $debug = 0; - - # Ignore empty binaries - if ($#{$pclist} < 0) { return; } - - # Figure out the addr2line command to use - my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; - if (exists $obj_tool_map{"addr2line_pdb"}) { - $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; - } - - # Use the go version because we know it works on all platforms - $addr2line = "go tool addr2line"; - $cmd = "$addr2line $image"; - - # If "addr2line" isn't installed on the system at all, just use - # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >$DEVNULL 2>&1") != 0) { - MapSymbolsWithNM($image, $offset, $pclist, $symbols); - return; - } - - # "addr2line -i" can produce a variable number of lines per input - # address, with no separator that allows us to tell when data for - # the next address starts. So we find the address for a special - # symbol (_fini) and interleave this address between all real - # addresses passed to addr2line. The name of this special symbol - # can then be used as a separator. - $sep_address = undef; # May be filled in by MapSymbolsWithNM() - my $nm_symbols = {}; - MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. - if (defined($sep_address)) { - # Only add " -i" to addr2line if the binary supports it. - # addr2line --help returns 0, but not if it sees an unknown flag first. - if (system("$cmd -i --help >$DEVNULL 2>&1") == 0) { - $cmd .= " -i"; - } else { - $sep_address = undef; # no need for sep_address if we don't support -i - } - } - - # Make file with all PC values with intervening 'sep_address' so - # that we can reliably detect the end of inlined function list - open(ADDRESSES, ">$main::tmpfile_sym") || error("$main::tmpfile_sym: $!\n"); - if ($debug) { print("---- $image ---\n"); } - for (my $i = 0; $i <= $#{$pclist}; $i++) { - # addr2line always reads hex addresses, and does not need '0x' prefix. - if ($debug) { printf STDERR ("%s\n", $pclist->[$i]); } - printf ADDRESSES ("%s\n", AddressSub($pclist->[$i], $offset)); - if (defined($sep_address)) { - printf ADDRESSES ("%s\n", $sep_address); - } - } - close(ADDRESSES); - if ($debug) { - print("----\n"); - system("cat $main::tmpfile_sym"); - print("---- $cmd\n"); - system("$cmd <$main::tmpfile_sym"); - print("----\n"); - } - - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); - my $count = 0; # Index in pclist - while () { - # Read fullfunction and filelineinfo from next pair of lines - s/\r?\n$//g; - my $fullfunction = $_; - $_ = ; - s/\r?\n$//g; - my $filelinenum = $_; - - if (defined($sep_address) && $fullfunction eq $sep_symbol) { - # Terminating marker for data for this address - $count++; - next; - } - - $filelinenum =~ s|\\|/|g; # turn windows-style paths into unix-style paths - - my $pcstr = $pclist->[$count]; - my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { - $function = $nms->[0]; - $fullfunction = $nms->[2]; - } - } - - # Prepend to accumulated symbols for pcstr - # (so that caller comes before callee) - my $sym = $symbols->{$pcstr}; - if (!defined($sym)) { - $sym = []; - $symbols->{$pcstr} = $sym; - } - unshift(@{$sym}, $function, $filelinenum, $fullfunction); - if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } - if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately - $count++; - } - } - close(SYMBOLS); -} - -# Use nm to map the list of referenced PCs to symbols. Return true iff we -# are able to read procedure information via nm. -sub MapSymbolsWithNM { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - # Get nm output sorted by increasing address - my $symbol_table = GetProcedureBoundaries($image, "."); - if (!%{$symbol_table}) { - return 0; - } - # Start addresses are already the right length (8 or 16 hex digits). - my @names = sort { $symbol_table->{$a}->[0] cmp $symbol_table->{$b}->[0] } - keys(%{$symbol_table}); - - if ($#names < 0) { - # No symbols: just use addresses - foreach my $pc (@{$pclist}) { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - return 0; - } - - # Sort addresses so we can do a join against nm output - my $index = 0; - my $fullname = $names[0]; - my $name = ShortFunctionName($fullname); - foreach my $pc (sort { $a cmp $b } @{$pclist}) { - # Adjust for mapped offset - my $mpc = AddressSub($pc, $offset); - while (($index < $#names) && ($mpc ge $symbol_table->{$fullname}->[1])){ - $index++; - $fullname = $names[$index]; - $name = ShortFunctionName($fullname); - } - if ($mpc lt $symbol_table->{$fullname}->[1]) { - $symbols->{$pc} = [$name, "?", $fullname]; - } else { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - } - return 1; -} - -sub ShortFunctionName { - my $function = shift; - while ($function =~ s/(?]*>//g) { } # Remove template arguments - $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type - return $function; -} - -# Trim overly long symbols found in disassembler output -sub CleanDisassembly { - my $d = shift; - while ($d =~ s/(?]*>/$1/g) { } # Remove template arguments - return $d; -} - -##### Miscellaneous ##### - -# Find the right versions of the above object tools to use. The -# argument is the program file being analyzed, and should be an ELF -# 32-bit or ELF 64-bit executable file. The location of the tools -# is determined by considering the following options in this order: -# 1) --tools option, if set -# 2) PPROF_TOOLS environment variable, if set -# 3) the environment -sub ConfigureObjTools { - my $prog_file = shift; - - # Check for the existence of $prog_file because /usr/bin/file does not - # predictably return error status in prod. - (-e $prog_file) || error("$prog_file does not exist.\n"); - - # Follow symlinks (at least for systems where "file" supports that) - my $file_cmd = "/usr/bin/file -L $prog_file 2>$DEVNULL || /usr/bin/file $prog_file 2>$DEVNULL"; - if ($^O eq "MSWin32") { - $file_cmd = "file -L $prog_file 2>NUL || file $prog_file 2>NUL"; - } - my $file_type = `$file_cmd`; - - if ($file_type =~ /64-bit/) { - # Change $address_length to 16 if the program file is ELF 64-bit. - # We can't detect this from many (most?) heap or lock contention - # profiles, since the actual addresses referenced are generally in low - # memory even for 64-bit programs. - $address_length = 16; - } - - if (($file_type =~ /MS Windows/) || ($OS eq "windows")) { - # For windows, we provide a version of nm and addr2line as part of - # the opensource release, which is capable of parsing - # Windows-style PDB executables. It should live in the path, or - # in the same directory as pprof. - $obj_tool_map{"nm_pdb"} = "nm-pdb"; - $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; - $obj_tool_map{"objdump"} = "false"; # no objdump - } - - if ($file_type =~ /Mach-O/) { - # OS X uses otool to examine Mach-O files, rather than objdump. - $obj_tool_map{"otool"} = "otool"; - $obj_tool_map{"addr2line"} = "false"; # no addr2line - $obj_tool_map{"objdump"} = "false"; # no objdump - } - - # Go fill in %obj_tool_map with the pathnames to use: - foreach my $tool (keys %obj_tool_map) { - $obj_tool_map{$tool} = ConfigureTool($obj_tool_map{$tool}); - } -} - -# Returns the path of a caller-specified object tool. If --tools or -# PPROF_TOOLS are specified, then returns the full path to the tool -# with that prefix. Otherwise, returns the path unmodified (which -# means we will look for it on PATH). -sub ConfigureTool { - my $tool = shift; - my $path; - - if ($main::opt_tools ne "") { - # Use a prefix specified by the --tools option... - $path = $main::opt_tools . $tool; - if (!-x $path) { - error("No '$tool' found with prefix specified by --tools $main::opt_tools\n"); - } - } elsif (exists $ENV{"PPROF_TOOLS"} && - $ENV{"PPROF_TOOLS"} ne "") { - #... or specified with the PPROF_TOOLS environment variable... - $path = $ENV{"PPROF_TOOLS"} . $tool; - if (!-x $path) { - error("No '$tool' found with prefix specified by PPROF_TOOLS=$ENV{PPROF_TOOLS}\n"); - } - } else { - # ... otherwise use the version that exists in the same directory as - # pprof. If there's nothing there, use $PATH. - $0 =~ m,[^/]*$,; # this is everything after the last slash - my $dirname = $`; # this is everything up to and including the last slash - if (-x "$dirname$tool") { - $path = "$dirname$tool"; - } else { - $path = $tool; - } - } - if ($main::opt_debug) { print STDERR "Using '$path' for '$tool'.\n"; } - return $path; -} - -# FetchHTTP retrieves a URL using either curl or LWP::UserAgent. -# It returns the entire body of the page on success, or exits the program -# with an error message on any failure. -sub FetchHTTP { - my $url = shift; - my $timeout = shift; # optional, in seconds - eval "use LWP::UserAgent ();"; - if ($@) { - my @max; - push @max, "--max-time", $timeout if $timeout; - open(my $fh, "-|", "curl", @max, "-s", $url) or error("Neither LWP::UserAgent nor curl is installed: $!\n"); - my $slurp = do { local $/; <$fh> }; - close($fh); - if ($? != 0) { - error("Error fetching $url with curl: exit $?") - } - return $slurp; - } - my $ua = LWP::UserAgent->new; - $ua->timeout($timeout) if $timeout; - my $res = $ua->get($url); - error("Failed to fetch $url\n") unless $res->is_success(); - return $res->content(); -} - -sub PostHTTP { - my ($url, $post_data) = @_; - eval "use LWP::UserAgent ();"; - if ($@) { - open(POSTFILE, ">$main::tmpfile_sym"); - print POSTFILE $post_data; - close(POSTFILE); - - open(my $fh, "-|", "curl", "-s", "-d", "\@$main::tmpfile_sym", $url) or error("Neither LWP::UserAgent nor curl is installed: $!\n"); - my $slurp = do { local $/; <$fh> }; - close($fh); - if ($? != 0) { - error("Error fetching $url with curl: exit $?") - } - return $slurp; - } - my $req = HTTP::Request->new(POST => $url); - $req->content($post_data); - my $ua = LWP::UserAgent->new; - my $res = $ua->request($req); - error("Failed to POST to $url\n") unless $res->is_success(); - return $res->content(); -} - -sub cleanup { - unlink($main::tmpfile_sym) if defined $main::tmpfile_sym; - unlink(keys %main::tempnames) if %main::tempnames; - unlink($main::collected_profile) if defined $main::collected_profile; - - # We leave any collected profiles in $HOME/pprof in case the user wants - # to look at them later. We print a message informing them of this. - if ((scalar(@main::profile_files) > 0) && - defined($main::collected_profile)) { - if (scalar(@main::profile_files) == 1) { - print STDERR "Dynamically gathered profile is in $main::collected_profile\n"; - } - print STDERR "If you want to investigate this profile further, you can do:\n"; - print STDERR "\n"; - print STDERR " pprof \\\n"; - print STDERR " $main::prog \\\n"; - print STDERR " $main::collected_profile\n"; - print STDERR "\n"; - } -} - -sub sighandler { - cleanup(); - exit(1); -} - -sub error { - my $msg = shift; - print STDERR $msg; - cleanup(); - exit(1); -} - - -# Run $nm_command and get all the resulting procedure boundaries whose -# names match "$regexp" and returns them in a hashtable mapping from -# procedure name to a two-element vector of [start address, end address] -sub GetProcedureBoundariesViaNm { - my $nm_command = shift; - my $regexp = shift; - - my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); - my $last_start = "0"; - my $routine = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (m/^\s*([0-9a-f]+) (.) (..*)/) { - my $start_val = $1; - my $type = $2; - my $this_routine = $3; - - # It's possible for two symbols to share the same address, if - # one is a zero-length variable (like __start_google_malloc) or - # one symbol is a weak alias to another (like __libc_malloc). - # In such cases, we want to ignore all values except for the - # actual symbol, which in nm-speak has type "T". The logic - # below does this, though it's a bit tricky: what happens when - # we have a series of lines with the same address, is the first - # one gets queued up to be processed. However, it won't - # *actually* be processed until later, when we read a line with - # a different address. That means that as long as we're reading - # lines with the same address, we have a chance to replace that - # item in the queue, which we do whenever we see a 'T' entry -- - # that is, a line with type 'T'. If we never see a 'T' entry, - # we'll just go ahead and process the first entry (which never - # got touched in the queue), and ignore the others. - if ($start_val eq $last_start && $type =~ /t/i) { - # We are the 'T' symbol at this address, replace previous symbol. - $routine = $this_routine; - next; - } elsif ($start_val eq $last_start) { - # We're not the 'T' symbol at this address, so ignore us. - next; - } - - if ($this_routine eq $sep_symbol) { - $sep_address = HexExtend($start_val); - } - - # Tag this routine with the starting address in case the image - # has multiple occurrences of this routine. We use a syntax - # that resembles template paramters that are automatically - # stripped out by ShortFunctionName() - $this_routine .= "<$start_val>"; - - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($start_val)]; - } - $last_start = $start_val; - $routine = $this_routine; - } elsif (m/^Loaded image name: (.+)/) { - # The win32 nm workalike emits information about the binary it is using. - if ($main::opt_debug) { print STDERR "Using Image $1\n"; } - } elsif (m/^PDB file name: (.+)/) { - # The win32 nm workalike emits information about the pdb it is using. - if ($main::opt_debug) { print STDERR "Using PDB $1\n"; } - } - } - close(NM); - # Handle the last line in the nm output. Unfortunately, we don't know - # how big this last symbol is, because we don't know how big the file - # is. For now, we just give it a size of 0. - # TODO(csilvers): do better here. - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($last_start)]; - } - return $symbol_table; -} - -# Gets the procedure boundaries for all routines in "$image" whose names -# match "$regexp" and returns them in a hashtable mapping from procedure -# name to a two-element vector of [start address, end address]. -# Will return an empty map if nm is not installed or not working properly. -sub GetProcedureBoundaries { - my $image = shift; - my $regexp = shift; - - # For libc libraries, the copy in /usr/lib/debug contains debugging symbols - my $debugging = DebuggingLibrary($image); - if ($debugging) { - $image = $debugging; - } - - my $nm = $obj_tool_map{"nm"}; - - # nm can fail for two reasons: 1) $image isn't a debug library; 2) nm - # binary doesn't support --demangle. In addition, for OS X we need - # to use the -f flag to get 'flat' nm output (otherwise we don't sort - # properly and get incorrect results). Unfortunately, GNU nm uses -f - # in an incompatible way. So first we test whether our nm supports - # --demangle and -f. - my $demangle_flag = ""; - if (system("$nm --demangle $image >$DEVNULL 2>&1") == 0) { - # In this mode, we do "nm --demangle " - $demangle_flag = "--demangle"; - } - my $flatten_flag = ""; - if (system("$nm -f $image >$DEVNULL 2>&1") == 0) { - $flatten_flag = "-f"; - } - - # Finally, in the case $image isn't a debug library, we try again with - # -D to at least get *exported* symbols. If we can't use --demangle, too bad. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>$DEVNULL", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>$DEVNULL", - # go tool nm is for Go binaries - "go tool nm $image 2>$DEVNULL | sort"); - - foreach my $nm_command (@nm_commands) { - my $symbol_table = GetProcedureBoundariesViaNm($nm_command, $regexp); - return $symbol_table if (%{$symbol_table}); - } - my $symbol_table = {}; - return $symbol_table; -} - - -# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. -# To make them more readable, we add underscores at interesting places. -# This routine removes the underscores, producing the canonical representation -# used by pprof to represent addresses, particularly in the tested routines. -sub CanonicalHex { - my $arg = shift; - return join '', (split '_',$arg); -} - - -# Unit test for AddressAdd: -sub AddressAddUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressAddUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd ($row->[0], $row->[1]); - if ($sum ne $row->[2]) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - my $expected = join '', (split '_',$row->[2]); - if ($sum ne CanonicalHex($row->[2])) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressSub: -sub AddressSubUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressSubUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub ($row->[0], $row->[1]); - if ($sum ne $row->[3]) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - if ($sum ne CanonicalHex($row->[3])) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressInc: -sub AddressIncUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressIncUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc ($row->[0]); - if ($sum ne $row->[4]) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc (CanonicalHex($row->[0])); - if ($sum ne CanonicalHex($row->[4])) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Driver for unit tests. -# Currently just the address add/subtract/increment routines for 64-bit. -sub RunUnitTests { - my $error_count = 0; - - # This is a list of tuples [a, b, a+b, a-b, a+1] - my $unit_test_data_8 = [ - [qw(aaaaaaaa 50505050 fafafafa 5a5a5a5a aaaaaaab)], - [qw(50505050 aaaaaaaa fafafafa a5a5a5a6 50505051)], - [qw(ffffffff aaaaaaaa aaaaaaa9 55555555 00000000)], - [qw(00000001 ffffffff 00000000 00000002 00000002)], - [qw(00000001 fffffff0 fffffff1 00000011 00000002)], - ]; - my $unit_test_data_16 = [ - # The implementation handles data in 7-nibble chunks, so those are the - # interesting boundaries. - [qw(aaaaaaaa 50505050 - 00_000000f_afafafa 00_0000005_a5a5a5a 00_000000a_aaaaaab)], - [qw(50505050 aaaaaaaa - 00_000000f_afafafa ff_ffffffa_5a5a5a6 00_0000005_0505051)], - [qw(ffffffff aaaaaaaa - 00_000001a_aaaaaa9 00_0000005_5555555 00_0000010_0000000)], - [qw(00000001 ffffffff - 00_0000010_0000000 ff_ffffff0_0000002 00_0000000_0000002)], - [qw(00000001 fffffff0 - 00_000000f_ffffff1 ff_ffffff0_0000011 00_0000000_0000002)], - - [qw(00_a00000a_aaaaaaa 50505050 - 00_a00000f_afafafa 00_a000005_a5a5a5a 00_a00000a_aaaaaab)], - [qw(0f_fff0005_0505050 aaaaaaaa - 0f_fff000f_afafafa 0f_ffefffa_5a5a5a6 0f_fff0005_0505051)], - [qw(00_000000f_fffffff 01_800000a_aaaaaaa - 01_800001a_aaaaaa9 fe_8000005_5555555 00_0000010_0000000)], - [qw(00_0000000_0000001 ff_fffffff_fffffff - 00_0000000_0000000 00_0000000_0000002 00_0000000_0000002)], - [qw(00_0000000_0000001 ff_fffffff_ffffff0 - ff_fffffff_ffffff1 00_0000000_0000011 00_0000000_0000002)], - ]; - - $error_count += AddressAddUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressSubUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressIncUnitTest($unit_test_data_8, $unit_test_data_16); - if ($error_count > 0) { - print STDERR $error_count, " errors: FAILED\n"; - } else { - print STDERR "PASS\n"; - } - exit ($error_count); -} diff --git a/src/cmd/dist/build.c b/src/cmd/dist/build.c index 31c4da3817..d638ae4ebe 100644 --- a/src/cmd/dist/build.c +++ b/src/cmd/dist/build.c @@ -670,13 +670,6 @@ install(char *dir) bpathf(&final_path, "%s/src/%s", goroot_final, dir); name = lastelem(dir); - // For misc/prof, copy into the tool directory and we're done. - if(hasprefix(dir, "misc/")) { - copyfile(bpathf(&b, "%s/%s", tooldir, name), - bpathf(&b1, "%s/misc/%s", goroot, name), 1); - goto out; - } - // set up gcc command line on first run. if(gccargs.len == 0) { bprintf(&b, "%s %s", defaultcc, defaultcflags); @@ -1293,8 +1286,6 @@ static char *buildorder[] = { "libbio", "liblink", - "misc/pprof", - "cmd/cc", // must be before c "cmd/gc", // must be before g "cmd/%sl", // must be before a, c, g From 2cd05c34046c4802fb2494c718d651045c2a3eb0 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 20:20:45 -0500 Subject: [PATCH 56/59] runtime/cgo: add +build tags to files named for $GOOS These are being built into the runtime/cgo for every operating system. It doesn't seem to matter, but restore the Go 1.3 behavior anyway. LGTM=r R=r, dave CC=golang-codereviews https://golang.org/cl/171290043 --- src/runtime/cgo/dragonfly.c | 2 ++ src/runtime/cgo/freebsd.c | 2 ++ src/runtime/cgo/netbsd.c | 2 ++ src/runtime/cgo/openbsd.c | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/runtime/cgo/dragonfly.c b/src/runtime/cgo/dragonfly.c index 3c95ff354e..c233c8ba9a 100644 --- a/src/runtime/cgo/dragonfly.c +++ b/src/runtime/cgo/dragonfly.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build dragonfly + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/freebsd.c b/src/runtime/cgo/freebsd.c index aefc481e64..4876b2abe4 100644 --- a/src/runtime/cgo/freebsd.c +++ b/src/runtime/cgo/freebsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build freebsd + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/netbsd.c b/src/runtime/cgo/netbsd.c index de38bb7707..076cc87f12 100644 --- a/src/runtime/cgo/netbsd.c +++ b/src/runtime/cgo/netbsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build netbsd + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/openbsd.c b/src/runtime/cgo/openbsd.c index 7c2b6c1737..476649544d 100644 --- a/src/runtime/cgo/openbsd.c +++ b/src/runtime/cgo/openbsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build openbsd + #include "textflag.h" // Supply environ, __progname and __guard_local, because From 2ad99f09609de1e709aeacaeb277d637729e8081 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 20:21:03 -0500 Subject: [PATCH 57/59] runtime: fix sudog leak in syncsemrelease Manifested as increased memory usage in a Google production system. Not an unbounded leak, but can significantly increase the number of sudogs allocated between garbage collections. I checked all the other calls to acquireSudog. This is the only one that was missing a releaseSudog. LGTM=r, dneil R=dneil, r CC=golang-codereviews https://golang.org/cl/169260043 --- src/runtime/sema.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/sema.go b/src/runtime/sema.go index a42a29988a..d2a028c01b 100644 --- a/src/runtime/sema.go +++ b/src/runtime/sema.go @@ -259,6 +259,7 @@ func syncsemrelease(s *syncSema, n uint32) { } s.tail = w goparkunlock(&s.lock, "semarelease") + releaseSudog(w) } else { unlock(&s.lock) } From a697c4b439852821aa7bb0e179b64b66e9f9f812 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 20:21:37 -0500 Subject: [PATCH 58/59] cmd/internal/objfile: minor edits Follow-up in response to comments on TBR'ed CL 171260043. LGTM=r R=r CC=golang-codereviews https://golang.org/cl/172080043 --- src/cmd/internal/objfile/disasm.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/cmd/internal/objfile/disasm.go b/src/cmd/internal/objfile/disasm.go index 7982faa47c..1a339c3214 100644 --- a/src/cmd/internal/objfile/disasm.go +++ b/src/cmd/internal/objfile/disasm.go @@ -1,3 +1,7 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package objfile import ( @@ -17,14 +21,14 @@ import ( // Disasm is a disassembler for a given File. type Disasm struct { - syms []Sym - pcln *gosym.Table - text []byte - textStart uint64 - textEnd uint64 - goarch string - disasm disasmFunc - byteOrder binary.ByteOrder + syms []Sym //symbols in file, sorted by address + pcln *gosym.Table // pcln table + text []byte // bytes of text segment (actual instructions) + textStart uint64 // start PC of text + textEnd uint64 // end PC of text + goarch string // GOARCH string + disasm disasmFunc // disassembler function for goarch + byteOrder binary.ByteOrder // byte order for goarch } // Disasm returns a disassembler for the file f. @@ -89,7 +93,8 @@ func (d *Disasm) lookup(addr uint64) (name string, base uint64) { } // base returns the final element in the path. -// It works on both Windows and Unix paths. +// It works on both Windows and Unix paths, +// regardless of host operating system. func base(path string) string { path = path[strings.LastIndex(path, "/")+1:] path = path[strings.LastIndex(path, `\`)+1:] From c99616fc6795123ec1a6b4d742099789865bd939 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 9 Nov 2014 21:10:49 -0500 Subject: [PATCH 59/59] test: fix nacl build Disable linkx_run.go and sinit_run.go, because they exec subprocesses, which NaCl cannot. TBR=r CC=golang-codereviews https://golang.org/cl/171350043 --- test/linkx_run.go | 1 + test/sinit_run.go | 1 + 2 files changed, 2 insertions(+) diff --git a/test/linkx_run.go b/test/linkx_run.go index abfc342a6c..5b67ce7d3d 100644 --- a/test/linkx_run.go +++ b/test/linkx_run.go @@ -1,3 +1,4 @@ +// +build !nacl // run // Copyright 2014 The Go Authors. All rights reserved. diff --git a/test/sinit_run.go b/test/sinit_run.go index a21bd1077d..b0a91ce5b1 100644 --- a/test/sinit_run.go +++ b/test/sinit_run.go @@ -1,3 +1,4 @@ +// +build !nacl // run // Copyright 2014 The Go Authors. All rights reserved.