diff --git a/.hgtags b/.hgtags index 5a5c4aed44..edd2163f70 100644 --- a/.hgtags +++ b/.hgtags @@ -135,3 +135,4 @@ f8b50ad4cac4d4c4ecf48324b4f512f65e82cc1c go1.3beta1 85518b1d6f8d6e16133b9ed2c9db6807522d37de go1.3.2 f44017549ff9c3cc5eef74ebe7276cd0dfc066b6 go1.3.3 f44017549ff9c3cc5eef74ebe7276cd0dfc066b6 release +1fdfd7dfaedb1b7702141617e621ab7328a236a1 go1.4beta1 diff --git a/AUTHORS b/AUTHORS index 03e686e751..48a262bbd7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -65,6 +65,7 @@ Aulus Egnatius Varialus Ben Olive Benjamin Black Benny Siegert +Benoit Sigoure Berengar Lehr Billie Harold Cleek Bjorn Tillenius @@ -156,6 +157,7 @@ Evan Shaw Ewan Chou Fabrizio Milo Fan Hongjian +Fastly, Inc. Fatih Arslan Fazlul Shahriar Felix Geisendörfer @@ -166,6 +168,7 @@ Francisco Souza Frederick Kelly Mayle III Fredrik Enestad Frithjof Schulze +Gabriel Aszalos Gary Burd Gautham Thambidorai Georg Reinke @@ -315,6 +318,7 @@ Moriyoshi Koizumi Môshe van der Sterre Nan Deng Nathan John Youngman +Nathan P Finch ngmoco, LLC Nicholas Katsaros Nicholas Presta diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 7679f78742..ec69858b60 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -104,6 +104,7 @@ Ben Lynn Ben Olive Benjamin Black Benny Siegert +Benoit Sigoure Berengar Lehr Bill Neubauer Bill Thiede @@ -241,6 +242,7 @@ Fredrik Enestad Frithjof Schulze Fumitoshi Ukai Gaal Yahas +Gabriel Aszalos Gary Burd Gautham Thambidorai Georg Reinke @@ -299,6 +301,7 @@ Jason Del Ponte Jason Travis Jay Weisskopf Jean-Marc Eurin +Jed Denlea Jeff Hodges Jeff R. Allen Jeff Sickel @@ -444,6 +447,7 @@ Môshe van der Sterre Mrunal Patel Nan Deng Nathan John Youngman +Nathan P Finch Nicholas Katsaros Nicholas Presta Nicholas Sullivan diff --git a/doc/articles/go_command.html b/doc/articles/go_command.html index 246b8c956d..2978628cd2 100644 --- a/doc/articles/go_command.html +++ b/doc/articles/go_command.html @@ -78,17 +78,18 @@ well-established conventions.

source code. For Bitbucket, GitHub, Google Code, and Launchpad, the root directory of the repository is identified by the repository's main URL, without the http:// prefix. Subdirectories are named by -adding to that path. For example, the supplemental networking -libraries for Go are obtained by running

+adding to that path. +For example, the Go example programs are obtained by running

-hg clone http://code.google.com/p/go.net
+git clone https://github.com/golang/example

and thus the import path for the root directory of that repository is -"code.google.com/p/go.net". The websocket package is stored in a -subdirectory, so its import path is -"code.google.com/p/go.net/websocket".

+"github.com/golang/example". +The stringutil +package is stored in a subdirectory, so its import path is +"github.com/golang/example/stringutil".

These paths are on the long side, but in exchange we get an automatically managed name space for import paths and the ability for diff --git a/doc/cmd.html b/doc/cmd.html index 132ea275fa..5d20d3887a 100644 --- a/doc/cmd.html +++ b/doc/cmd.html @@ -62,7 +62,7 @@ details. -cover +cover      Cover is a program for creating and analyzing the coverage profiles generated by "go test -coverprofile". @@ -83,13 +83,13 @@ gofmt command with more general options. -godoc +godoc      Godoc extracts and generates documentation for Go packages. -vet +vet      Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string. diff --git a/doc/code.html b/doc/code.html index f019306fa2..ce9f8636fa 100644 --- a/doc/code.html +++ b/doc/code.html @@ -60,37 +60,35 @@ To give you an idea of how a workspace looks in practice, here's an example: 

 bin/
-    streak                         # command executable
-    todo                           # command executable
+    hello                          # command executable
+    outyet                         # command executable
 pkg/
     linux_amd64/
-        code.google.com/p/goauth2/
-            oauth.a                # package object
-        github.com/nf/todo/
-            task.a                 # package object
+        github.com/golang/example/
+            stringutil.a           # package object
 src/
-    code.google.com/p/goauth2/
-        .hg/                       # mercurial repository metadata
-        oauth/
-            oauth.go               # package source
-            oauth_test.go          # test source
-    github.com/nf/
-        streak/
-            .git/                  # git repository metadata
-            oauth.go               # command source
-            streak.go              # command source
-        todo/
-            .git/                  # git repository metadata
-            task/
-                task.go            # package source
-            todo.go                # command source
+    github.com/golang/example/
+        .git/                      # Git repository metadata
+	hello/
+	    hello.go               # command source
+	outyet/
+	    main.go                # command source
+	    main_test.go           # test source
+	stringutil/
+	    reverse.go             # package source
+	    reverse_test.go        # test source

-This workspace contains three repositories (goauth2, -streak, and todo) comprising two commands -(streak and todo) and two libraries -(oauth and task). +This workspace contains one repository (example) +comprising two commands (hello and outyet) +and one library (stringutil). +

+ +

+A typical workspace would contain many source repositories containing many +packages and commands. Most Go programmers keep all their Go source code +and dependencies in a single workspace.

@@ -277,29 +275,29 @@ Let's write a library and use it from the hello program.

Again, the first step is to choose a package path (we'll use -github.com/user/newmath) and create the package directory: +github.com/user/stringutil) and create the package directory: 

-$ mkdir $GOPATH/src/github.com/user/newmath
+$ mkdir $GOPATH/src/github.com/user/stringutil

-Next, create a file named sqrt.go in that directory with the +Next, create a file named reverse.go in that directory with the following contents. 

-// Package newmath is a trivial example package.
-package newmath
+// Package stringutil contains utility functions for working with strings.
+package stringutil
 
-// Sqrt returns an approximation to the square root of x.
-func Sqrt(x float64) float64 {
-	z := 1.0
-	for i := 0; i < 1000; i++ {
-		z -= (z*z - x) / (2 * z)
+// Reverse returns its argument string reversed rune-wise left to right.
+func Reverse(s string) string {
+	r := []rune(s)
+	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
+		r[i], r[j] = r[j], r[i]
 	}
-	return z
+	return string(r)
 }
@@ -308,7 +306,7 @@ Now, test that the package compiles with go build: 

-$ go build github.com/user/newmath
+$ go build github.com/user/stringutil

@@ -326,7 +324,7 @@ directory of the workspace.

-After confirming that the newmath package builds, +After confirming that the stringutil package builds, modify your original hello.go (which is in $GOPATH/src/github.com/user/hello) to use it:

@@ -337,18 +335,18 @@ package main import ( "fmt" - "github.com/user/newmath" + "github.com/user/stringutil" ) func main() { - fmt.Printf("Hello, world. Sqrt(2) = %v\n", newmath.Sqrt(2)) + fmt.Printf(stringutil.Reverse("!oG ,olleH")) }

Whenever the go tool installs a package or binary, it also -installs whatever dependencies it has. So when you install the hello -program +installs whatever dependencies it has. +So when you install the hello program 

@@ -356,16 +354,16 @@ $ go install github.com/user/hello

-the newmath package will be installed as well, automatically. +the stringutil package will be installed as well, automatically.

-Running the new version of the program, you should see some numerical output: +Running the new version of the program, you should see a new, reversed message: 

 $ hello
-Hello, world.  Sqrt(2) = 1.414213562373095
+Hello, Go!

@@ -374,22 +372,22 @@ After the steps above, your workspace should look like this: 

 bin/
-    hello              # command executable
+    hello                 # command executable
 pkg/
-    linux_amd64/       # this will reflect your OS and architecture
+    linux_amd64/          # this will reflect your OS and architecture
         github.com/user/
-            newmath.a  # package object
+            stringutil.a  # package object
 src/
     github.com/user/
         hello/
-            hello.go   # command source
-        newmath/
-            sqrt.go    # package source
+            hello.go      # command source
+        stringutil/
+            reverse.go    # package source

-Note that go install placed the newmath.a object in a -directory inside pkg/linux_amd64 that mirrors its source +Note that go install placed the stringutil.a object +in a directory inside pkg/linux_amd64 that mirrors its source directory. This is so that future invocations of the go tool can find the package object and avoid recompiling the package unnecessarily. @@ -457,20 +455,29 @@ if the function calls a failure function such as t.Error or

-Add a test to the newmath package by creating the file -$GOPATH/src/github.com/user/newmath/sqrt_test.go containing the -following Go code. +Add a test to the stringutil package by creating the file +$GOPATH/src/github.com/user/stringutil/reverse_test.go containing +the following Go code. 

-package newmath
+package stringutil
 
 import "testing"
 
-func TestSqrt(t *testing.T) {
-	const in, out = 4, 2
-	if x := Sqrt(in); x != out {
-		t.Errorf("Sqrt(%v) = %v, want %v", in, x, out)
+func TestReverse(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"Hello, world", "dlrow ,olleH"},
+		{"Hello, 世界", "界世 ,olleH"},
+		{"", ""},
+	}
+	for _, c := range cases {
+		got := Reverse(c.in)
+		if got != c.want {
+			t.Errorf("Reverse(%q) == %q, want %q", c.in, got, c.want)
+		}
 	}
 }
@@ -480,8 +487,8 @@ Then run the test with go test: 

-$ go test github.com/user/newmath
-ok  	github.com/user/newmath 0.165s
+$ go test github.com/user/stringutil
+ok  	github.com/user/stringutil 0.165s

@@ -491,7 +498,7 @@ directory, you can omit the package path: 

 $ go test
-ok  	github.com/user/newmath 0.165s
+ok  	github.com/user/stringutil 0.165s

@@ -507,16 +514,16 @@ An import path can describe how to obtain the package source code using a revision control system such as Git or Mercurial. The go tool uses this property to automatically fetch packages from remote repositories. For instance, the examples described in this document are also kept in a -Mercurial repository hosted at Google Code, -code.google.com/p/go.example. +Git repository hosted at GitHub +github.com/golang/example. If you include the repository URL in the package's import path, go get will fetch, build, and install it automatically: 

-$ go get code.google.com/p/go.example/hello
+$ go get github.com/golang/example/hello
 $ $GOPATH/bin/hello
-Hello, world.  Sqrt(2) = 1.414213562373095
+Hello, Go examples!

@@ -533,37 +540,39 @@ tree should now look like this: 

 bin/
-    hello                 # command executable
+    hello                           # command executable
 pkg/
     linux_amd64/
-        code.google.com/p/go.example/
-            newmath.a     # package object
+        github.com/golang/example/
+            stringutil.a            # package object
         github.com/user/
-            newmath.a     # package object
+            stringutil.a            # package object
 src/
-    code.google.com/p/go.example/
+    github.com/golang/example/
+	.git/                       # Git repository metadata
         hello/
-            hello.go      # command source
-        newmath/
-            sqrt.go       # package source
-            sqrt_test.go  # test source
+            hello.go                # command source
+        stringutil/
+            reverse.go              # package source
+            reverse_test.go         # test source
     github.com/user/
         hello/
-            hello.go      # command source
-        newmath/
-            sqrt.go       # package source
-            sqrt_test.go  # test source
+            hello.go                # command source
+        stringutil/
+            reverse.go              # package source
+            reverse_test.go         # test source

-The hello command hosted at Google Code depends on the -newmath package within the same repository. The imports in -hello.go file use the same import path convention, so the go -get command is able to locate and install the dependent package, too. +The hello command hosted at GitHub depends on the +stringutil package within the same repository. The imports in +hello.go file use the same import path convention, so the +go get command is able to locate and install the dependent +package, too. 

-import "code.google.com/p/go.example/newmath"
+import "github.com/golang/example/stringutil"

diff --git a/doc/contribute.html b/doc/contribute.html index 90c3f10a1d..92fd88b485 100644 --- a/doc/contribute.html +++ b/doc/contribute.html @@ -121,7 +121,7 @@ are inside the go directory when issuing commands.

To contribute to subrepositories, edit the .hg/hgrc for each subrepository in the same way. For example, add the codereview extension to -code.google.com/p/go.tools/.hg/hgrc. +golang.org/x/tools/.hg/hgrc.

Understanding the extension

diff --git a/doc/go1.4.html b/doc/go1.4.html index 7e670c47cb..ac63ade60a 100644 --- a/doc/go1.4.html +++ b/doc/go1.4.html @@ -18,8 +18,7 @@ Stacks are now contiguous, reallocated when necessary rather than linking on new "segments"; this release therefore eliminates the notorious "hot stack split" problem. There are some new tools available including support in the go command -for build-time source code generation -and TODO. +for build-time source code generation. The release also adds support for ARM processors on Android and Native Client (NaCl) and AMD64 on Plan 9. As always, Go 1.4 keeps the promise @@ -121,9 +120,9 @@ compile but is easy to fix by adding an explicit dereference.

Go 1.4 can build binaries for ARM processors running the Android operating system. It can also build a .so library that can be loaded by an Android application -using the supporting packages in the go.mobile repository. +using the supporting packages in the mobile subrepository. A brief description of the plans for this experimental port are available -here. +here.

NaCl on ARM

@@ -194,13 +193,12 @@ A consequence is that stacks are no longer segmented, eliminating the "hot split When a stack limit is reached, a new, larger stack is allocated, all active frames for the goroutine are copied there, and any pointers into the stack are updated. Performance can be noticeably better in some cases and is always more predictable. -Details are available in the design document. +Details are available in the design document.

The use of contiguous stacks means that stacks can start smaller without triggering performance issues, so the default starting size for a goroutine's stack in 1.4 has been reduced to 2048 bytes from 8192 bytes. -TODO: It may be bumped to 4096 for the release.

@@ -281,7 +279,9 @@ More information about these changes is in the assembly docum

Status of gccgo

-TODO gccgo news +The release schedules for the GCC and Go projects do not coincide. +GCC release 4.9 contains the Go 1.2 version of gccgo. +The next release, GCC 5, will likely have the Go 1.4 version of gccgo.

Internal packages

@@ -319,7 +319,7 @@ from 1.5 and onward it will be enforced for any repository.

Full details of the mechanism are in -the design document. +the design document.

Canonical import paths

@@ -370,9 +370,36 @@ fails because of this check, the mis-imported package has been copied to the loc and should be removed manually.

+

+To complement this new feature, a check has been added at update time to verify +that the local package's remote repository matches that of its custom import. +The go get -u command will fail to +update a package if its remote repository has changed since it was first +downloaded. +The new -f flag overrides this check. +

+

Further information is in -the design document. +the design document. +

+ +

Import paths for the subrepositories

+ +

+The Go project subrepositories (code.google.com/p/go.tools and so on) +are now available under custom import paths replacing code.google.com/p/go. with golang.org/x/, +as in golang.org/x/tools. +We will add canonical import comments to the code around June 1, 2015, +at which point Go 1.4 and later will stop accepting the old code.google.com paths. +

+ +

+Updating: All code that imports from subrepositories should change +to use the new golang.org paths. +Go 1.0 and later can resolve and import the new paths, so updating will not break +compatibility with older releases. +Code that has not updated will stop compiling with Go 1.4 around June 1, 2015.

The go generate subcommand

@@ -384,13 +411,13 @@ to automate the running of tools to generate source code before compilation. For example, it can be used to run the yacc compiler-compiler on a .y file to produce the Go source file implementing the grammar, or to automate the generation of String methods for typed constants using the new -stringer -tool in the go.tools repository. +stringer +tool in the golang.org/x/tools subrepository.

For more information, see the -design document. +design document.

Change to file name handling

@@ -465,17 +492,12 @@ rebuild the standard library and commands, to avoid overwriting the installation -

Changes to godoc

-

-TODO godoc news -

-

Changes to package source layout

In the main Go source repository, the source code for the packages was kept in the directory src/pkg, which made sense but differed from -other repositories, including the Go sub-repositories such as go.tools. +other repositories, including the Go subrepositories. In Go 1.4, the pkg level of the source tree is now gone, so for example the fmt package's source, once kept in directory src/pkg/fmt, now lives one level higher in src/fmt. @@ -487,6 +509,16 @@ need to know about the new location. All tools and services maintained by the Go have been updated.

+ +

SWIG

+ +

+Due to the runtime changes in this release, Go 1.4 will require SWIG 3.0.3. +At time of writing that has not yet been released, but we expect it to be by +Go 1.4's release date. +TODO +

+

Miscellany

@@ -544,14 +576,57 @@ There are no new packages in this release.

Major changes to the library

+

bufio.Scanner

+

-TODO major changes +The Scanner type in the +bufio package +has had a bug fixed that may require changes to custom +split functions. +The bug made it impossible to generate an empty token at EOF; the fix +changes the end conditions seen by the split function. +Previously, scanning stopped at EOF if there was no more data. +As of 1.4, the split function will be called once at EOF after input is exhausted, +so the split function can generate a final empty token +as the documentation already promised.

-
-encoding/gob: remove unsafe (CL 102680045)
-syscall: now frozen (CL 129820043); go.sys subrepo created: http://golang.org/s/go1.4-syscall
-
+

+Updating: Custom split functions may need to be modified to +handle empty tokens at EOF as desired. +

+ +

syscall

+ +

+The syscall package is now frozen except +for changes needed to maintain the core repository. +In particular, it will no longer be extended to support new or different system calls +that are not used by the core. +The reasons are described at length in a +separate document. +

+ +

+A new subrepository, golang.org/x/sys, +has been created to serve as the location for new developments to support system +calls on all kernels. +It has a nicer structure, with three packages that each hold the implementation of +system calls for one of +Unix, +Windows and +Plan 9. +These packages will be curated more generously, accepting all reasonable changes +that reflect kernel interfaces in those operating systems. +See the documentation and the article mentioned above for more information. +

+ +

+Updating: Existing programs are not affected as the syscall +package is largely unchanged from the 1.3 release. +Future development that requires system calls not in the syscall package +should build on golang.org/x/sys instead. +

Minor changes to the library

@@ -562,37 +637,199 @@ See the relevant package documentation for more information about each change. +
  • +The crypto/tls package +now supports ALPN as defined in RFC 7301. +
    • + +
  • +The crypto/tls package +now supports programmatic selection of server certificates +through the new CertificateForName function +of the Config struct. +
    • + +
  • +Also in the crypto/tls package, the server now supports +TLS_FALLBACK_SCSV +to help clients detect fallback attacks. +(The Go client does not support fallback at all, so it is not vulnerable to +those attacks.) +
    • + +
  • +In the encoding/asn1 package, +optional elements with a default value will now only be omitted if they have that value. +
    • + +
  • +The encoding/csv package no longer +quotes empty strings but does quote the end-of-data marker \. (backslash dot). +This is permitted by the definition of CSV and allows it to work better with Postgres. +
    • + +
  • +The encoding/gob package has been rewritten to eliminate +the use of unsafe operations, allowing it to be used in environments that do not permit use of the +unsafe package. +For typical uses it will be 10-30% slower, but the delta is dependent on the type of the data and +in some cases, especially involving arrays, it can be faster. +There is no functional change. +
    • + +
  • +In the fmt package, +formatting of pointers to maps has changed to be consistent with that of pointers +to structs, arrays, and so on. +For instance, &map[string]int{"one": 1} now prints by default as +&map[one: 1] rather than as a hexadecimal pointer value. +
    • + +
  • +The net/http package's +Request type +has a new BasicAuth method +that returns the username and password from authenticated requests using the +HTTP Basic Authentication +Scheme. +
    • + +
  • The net/http package's +Transport type +has a new DialTLS hook +that allows customizing the behavior of outbound TLS connections. +
    • + +
  • +The net/http/httputil package's +ReverseProxy type +has a new field, +ErrorLog, that +provides user control of logging. +
    • + +
  • +The os package +now implements symbolic links on the Windows operating system +through the Symlink function. +Other operating systems already have this functionality. +
    • + +
  • +The reflect package's +Type interface +has a new method, Comparable, +that reports whether the type implements general comparisons. +
    • + +
  • +Also in the reflect package, the +Value interface is now three instead of four words +because of changes to the implementation of interfaces in the runtime. +This saves memory but has no semantic effect. +
    • + +
  • +The runtime package +now implements monotonic clocks on Windows, +as it already did for the other systems. +
    • + +
  • +The runtime package's +Mallocs counter +now counts very small allocations that were missed in Go 1.3. +This may break tests using ReadMemStats +or AllocsPerRun +due to the more accurate answer. +
    • + +
  • +In the runtime package, +an array PauseEnd +has been added to the +MemStats +and GCStats structs. +This array is a circular buffer of times when garbage collection pauses ended. +The corresponding pause durations are already recorded in +PauseNs +
    • + +
  • +The runtime/race package +now supports FreeBSD, which means the +go command's -race +flag now works on FreeBSD. +
    • + +
  • +The sync/atomic package +has a new type, Value. +Value provides an efficient mechanism for atomic loads and +stores of values of arbitrary type. +
    • + +
  • +In the syscall package's +implementation on Linux, the +Setuid +and Setgid have been disabled +because those system calls operate on the calling thread, not the whole process, which is +different from other platforms and not the expected result. +
    • + +
  • +The testing package +has a new facility to provide more control over running a set of tests. +If the test code contains a function 
    -
-cmd/6l, liblink: use pc-relative addressing for all memory references, so that linking Go binaries at high addresses works (CL 125140043). This cuts the maximum size of a Go binary's text+data+bss from 4GB to 2GB.
-
-bufio: handling of empty tokens at EOF changed, may require scanner change (CL 145390043)
-compress/flate, compress/gzip, compress/zlib: Reset support (https://codereview.appspot.com/97140043)
-crypto/tls: add support for ALPN (RFC 7301) (CL 108710046)
-crypto/tls: support programmatic selection of server certificates (CL 107400043)
-encoding/asn1: optional elements with a default value will now only be omitted if they have that value (CL 86960045)
-fmt: print type *map[T]T as &map[k:v] (CL 154870043)
-encoding/csv: do not quote empty strings, quote \. (CL 164760043)
-net/http: add Request.BasicAuth method (CL 76540043)
-net/http: add Transport.DialTLS hook (CL 137940043)
-net/http/httputil: add ReverseProxy.ErrorLog (CL 132750043)
-os: implement symlink support for windows (CL 86160044)
-reflect: add type.Comparable (CL 144020043)
-reflect: Value is one word smaller
-runtime: implement monotonic clocks on windows (CL 108700045)
-runtime: MemStats.Mallocs now counts very small allocations missed in Go 1.3. This may break tests using runtime.ReadMemStats or testing.AllocsPerRun by giving a more accurate answer than Go 1.3 did (CL 143150043).
-runtime/race: freebsd is supported (CL 107270043)
-runtime: add PauseEnd array to MemStats and GCStats (CL 153670043)
-swig: Due to runtime changes Go 1.4 will require SWIG 3.0.3 (not yet released)
-sync/atomic: add Value (CL 136710045)
-syscall: Setuid, Setgid are disabled on linux platforms. On linux those syscalls operate on the calling thread, not the whole process. This does not match the semantics of other platforms, nor the expectations of the caller, so the operations have been disabled until issue 1435 is resolved (CL 106170043)
-testing: add Coverage (CL 98150043)
-testing: add TestMain support (CL 148770043)
-text/scanner: add IsIdentRune field of Scanner. (CL 108030044)
-text/template: allow comparison of signed and unsigned integers (CL 149780043)
-time: use the micro symbol (µ (U+00B5)) to print microsecond duration (CL 105030046)
+func TestMain(m *testing.M)
    + +that function will be called instead of running the tests directly. +The M struct contains methods to access and run the tests. +
    • + +
  • +Also in the testing package, +a new Coverage +function reports the current test coverage fraction, +enabling individual tests to report how much they are contributing to the +overall coverage. +
    • + +
  • +The text/scanner package's +Scanner type +has a new function, +IsIdentRune, +allowing one to control the definition of an identifier when scanning. +
    • + +
  • +The text/template package's boolean +functions eq, lt, and so on have been generalized to allow comparison +of signed and unsigned integers, simplifying their use in practice. +(Previously one could only compare values of the same signedness.) +All negative values compare less than all unsigned values. +
    • + +
  • +The time package now uses the standard symbol for the micro prefix, +the micro symbol (U+00B5 'µ'), to print microsecond durations. +ParseDuration still accepts us +but the package no longer prints microseconds as us. +
    +Updating: Code that depends on the output format of durations +but does not use ParseDuration will need to be updated. +
    • + + diff --git a/doc/go1compat.html b/doc/go1compat.html index 94c48d2ce3..d800dec0c0 100644 --- a/doc/go1compat.html +++ b/doc/go1compat.html @@ -153,7 +153,7 @@ developed software based on Go 1.

    Code in sub-repositories of the main go tree, such as -code.google.com/p/go.net, +golang.org/x/net, may be developed under looser compatibility requirements. However, the sub-repositories will be tagged as appropriate to identify versions that are compatible @@ -170,9 +170,9 @@ is therefore outside the purview of the guarantees made here. As of Go version 1.4, the syscall package is frozen. Any evolution of the system call interface must be supported elsewhere, such as in the -go.sys subrepository. +go.sys subrepository. For details and background, see -this document. +this document.

    Tools

    diff --git a/doc/go_faq.html b/doc/go_faq.html index 9aac058388..7597997798 100644 --- a/doc/go_faq.html +++ b/doc/go_faq.html @@ -1616,7 +1616,7 @@ Go is a fine language in which to implement a self-hosting compiler: a native lexer and parser are already available in the go package and a separate type checking -package +package has also been written.

    @@ -1715,7 +1715,7 @@ func main() {

    Nowadays, most Go programmers use a tool, -goimports, +goimports, which automatically rewrites a Go source file to have the correct imports, eliminating the unused imports issue in practice. This program is easily connected to most editors to run automatically when a Go source file is written. diff --git a/doc/install-source.html b/doc/install-source.html index 82859b50fb..f53deb404c 100644 --- a/doc/install-source.html +++ b/doc/install-source.html @@ -241,12 +241,12 @@ provides essential setup instructions for using the Go tools.

    The source code for several Go tools (including godoc) -is kept in the go.tools repository. +is kept in the go.tools repository. To install all of them, run the go get command: 

    -$ go get code.google.com/p/go.tools/cmd/...
+$ go get golang.org/x/tools/cmd/...

    @@ -254,7 +254,7 @@ Or if you just want to install a specific command (godoc in this ca 

    -$ go get code.google.com/p/go.tools/cmd/godoc
+$ go get golang.org/x/tools/cmd/godoc

    diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index 3aac8f43c9..0c9b27a318 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -1631,7 +1631,7 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): try: cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32") except: - return "hgapplydiff: " + ExceptionDetail() + "\nInstall hgapplydiff with:\n$ go get code.google.com/p/go.codereview/cmd/hgapplydiff\n" + return "hgapplydiff: " + ExceptionDetail() + "\nInstall hgapplydiff with:\n$ go get golang.org/x/codereview/cmd/hgapplydiff\n" out, err = cmd.communicate(patch) if cmd.returncode != 0 and not opts["ignore_hgapplydiff_failure"]: @@ -3451,6 +3451,7 @@ class FakeMercurialUI(object): def __init__(self): self.quiet = True self.output = '' + self.debugflag = False def write(self, *args, **opts): self.output += ' '.join(args) diff --git a/misc/benchcmp b/misc/benchcmp index 28a37392d8..84d92eefd4 100755 --- a/misc/benchcmp +++ b/misc/benchcmp @@ -1,5 +1,5 @@ #!/bin/bash echo 'misc/benchcmp has moved:' >&2 -echo ' go get -u code.google.com/p/go.tools/cmd/benchcmp' >&2 +echo ' go get -u golang.org/x/tools/cmd/benchcmp' >&2 exit 2 diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go index 3b289ba7b5..fbdfac87ac 100644 --- a/misc/cgo/test/cgo_test.go +++ b/misc/cgo/test/cgo_test.go @@ -62,5 +62,6 @@ func Test8517(t *testing.T) { test8517(t) } func Test8811(t *testing.T) { test8811(t) } func TestReturnAfterGrow(t *testing.T) { testReturnAfterGrow(t) } func TestReturnAfterGrowFromGo(t *testing.T) { testReturnAfterGrowFromGo(t) } +func Test9026(t *testing.T) { test9026(t) } func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) } diff --git a/misc/cgo/test/issue6997_linux.go b/misc/cgo/test/issue6997_linux.go index 871bd517a7..5455f0c536 100644 --- a/misc/cgo/test/issue6997_linux.go +++ b/misc/cgo/test/issue6997_linux.go @@ -34,7 +34,7 @@ func test6997(t *testing.T) { if r == 0 { t.Error("pthread finished but wasn't cancelled??") } - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): t.Error("hung in pthread_cancel/pthread_join") } } diff --git a/misc/cgo/test/issue9026.go b/misc/cgo/test/issue9026.go new file mode 100644 index 0000000000..8848d0e811 --- /dev/null +++ b/misc/cgo/test/issue9026.go @@ -0,0 +1,9 @@ +package cgotest + +import ( + "testing" + + "./issue9026" +) + +func test9026(t *testing.T) { issue9026.Test(t) } diff --git a/misc/cgo/test/issue9026/issue9026.go b/misc/cgo/test/issue9026/issue9026.go new file mode 100644 index 0000000000..0af86e64da --- /dev/null +++ b/misc/cgo/test/issue9026/issue9026.go @@ -0,0 +1,36 @@ +package issue9026 + +// This file appears in its own package since the assertion tests the +// per-package counter used to create fresh identifiers. + +/* +typedef struct {} git_merge_file_input; + +typedef struct {} git_merge_file_options; + +void git_merge_file( + git_merge_file_input *in, + git_merge_file_options *opts) {} +*/ +import "C" +import ( + "fmt" + "testing" +) + +func Test(t *testing.T) { + var in C.git_merge_file_input + var opts *C.git_merge_file_options + C.git_merge_file(&in, opts) + + // Test that the generated type names are deterministic. + // (Previously this would fail about 10% of the time.) + // + // Brittle: the assertion may fail spuriously when the algorithm + // changes, but should remain stable otherwise. + got := fmt.Sprintf("%T %T", in, opts) + want := "issue9026._Ctype_struct___0 *issue9026._Ctype_struct___1" + if got != want { + t.Errorf("Non-deterministic type names: got %s, want %s", got, want) + } +} diff --git a/misc/makerelease/makerelease.go b/misc/makerelease/makerelease.go index 9b2373307f..e94efdbcee 100644 --- a/misc/makerelease/makerelease.go +++ b/misc/makerelease/makerelease.go @@ -53,8 +53,8 @@ var ( ) const ( - blogPath = "code.google.com/p/go.blog" - toolPath = "code.google.com/p/go.tools" + blogPath = "golang.org/x/blog" + toolPath = "golang.org/x/tools" tourPath = "code.google.com/p/go-tour" defaultToolTag = "release-branch.go1.3" defaultTourTag = "release-branch.go1.3" @@ -64,9 +64,9 @@ const ( // These must be the command that cmd/go knows to install to $GOROOT/bin // or $GOROOT/pkg/tool. var toolPaths = []string{ - "code.google.com/p/go.tools/cmd/cover", - "code.google.com/p/go.tools/cmd/godoc", - "code.google.com/p/go.tools/cmd/vet", + "golang.org/x/tools/cmd/cover", + "golang.org/x/tools/cmd/godoc", + "golang.org/x/tools/cmd/vet", } var preBuildCleanFiles = []string{ diff --git a/misc/pprof b/misc/pprof deleted file mode 100755 index f83e6fb659..0000000000 --- a/misc/pprof +++ /dev/null @@ -1,5100 +0,0 @@ -#! /usr/bin/env perl - -# This is a copy of http://google-perftools.googlecode.com/svn/trunk/src/pprof -# with local modifications to handle generation of SVG images and -# the Go-style pprof paths. These modifications will probably filter -# back into the official source before long. -# It's convenient to have a copy here because we need just the one -# Perl script, not all the C++ libraries that surround it. - -# Copyright (c) 1998-2007, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# --- -# Program for printing the profile generated by common/profiler.cc, -# or by the heap profiler (common/debugallocation.cc) -# -# The profile contains a sequence of entries of the form: -# -# This program parses the profile, and generates user-readable -# output. -# -# Examples: -# -# % tools/pprof "program" "profile" -# Enters "interactive" mode -# -# % tools/pprof --text "program" "profile" -# Generates one line per procedure -# -# % tools/pprof --gv "program" "profile" -# Generates annotated call-graph and displays via "gv" -# -# % tools/pprof --gv --focus=Mutex "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# -# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# and does not match "string" -# -# % tools/pprof --list=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --list= pattern. The listing is -# annotated with the flat and cumulative sample counts at each line. -# -# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --disasm= pattern. The listing is -# annotated with the flat and cumulative sample counts at each PC value. -# -# TODO: Use color to indicate files? - -use strict; -use warnings; -use Getopt::Long; -use File::Temp; -use File::Copy; - -my $PPROF_VERSION = "1.5"; - -# NOTE: All mentions of c++filt have been expunged from this script -# because (1) we don't use C++, and (2) the copy of c++filt that ships -# on OS X is from 2007 and destroys nm output by "demangling" the -# first two columns (address and symbol type). - -# These are the object tools we use which can come from a -# user-specified location using --tools, from the PPROF_TOOLS -# environment variable, or from the environment. -my %obj_tool_map = ( - "objdump" => "objdump", - "nm" => "nm", - "addr2line" => "addr2line", - ## ConfigureObjTools may add architecture-specific entries: - #"nm_pdb" => "nm-pdb", # for reading windows (PDB-format) executables - #"addr2line_pdb" => "addr2line-pdb", # ditto - #"otool" => "otool", # equivalent of objdump on OS X -); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; -# These are used for dynamic profiles - -# These are the web pages that servers need to support for dynamic profiles -my $HEAP_PAGE = "/pprof/heap"; -my $THREAD_PAGE = "/pprof/thread"; -my $PROFILE_PAGE = "/pprof/profile"; # must support cgi-param "?seconds=#" -my $BLOCK_PAGE = "/pprof/block"; -my $PMUPROFILE_PAGE = "/pprof/pmuprofile(?:\\?.*)?"; # must support cgi-param - # ?seconds=#&event=x&period=n -my $GROWTH_PAGE = "/pprof/growth"; -my $CONTENTION_PAGE = "/pprof/contention"; -my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter -my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST -my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; - -# default binary name -my $UNKNOWN_BINARY = "(unknown)"; - -# There is a pervasive dependency on the length (in hex characters, -# i.e., nibbles) of an address, distinguishing between 32-bit and -# 64-bit profiles. To err on the safe size, default to 64-bit here: -my $address_length = 16; - -# A list of paths to search for shared object files -my @prefix_list = (); - -# Special routine name that should not have any symbols. -# Used as separator to parse "addr2line -i" output. -my $sep_symbol = '_fini'; -my $sep_address = undef; - -my $OS = $^O; -my $DEVNULL = "/dev/null"; -if ($^O =~ /MSWin32|cygwin|msys/) { - $OS = "windows"; - $DEVNULL = "NUL"; -} - -##### Argument parsing ##### - -sub usage_string { - return < - is a space separated list of profile names. -pprof [options] - is a list of profile files where each file contains - the necessary symbol mappings as well as profile data (likely generated - with --raw). -pprof [options] - is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE - - Each name can be: - /path/to/profile - a path to a profile file - host:port[/] - a location of a service to get profile from - - The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, - $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, - $THREAD_PAGE, $BLOCK_PAGE or /pprof/filteredprofile. - For instance: - pprof http://myserver.com:80$HEAP_PAGE - If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). -pprof --symbols - Maps addresses to symbol names. In this mode, stdin should be a - list of library mappings, in the same format as is found in the heap- - and cpu-profile files (this loosely matches that of /proc/self/maps - on linux), followed by a list of hex addresses to map, one per line. - - For more help with querying remote servers, including how to add the - necessary server-side support code, see this filename (or one like it): - - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html - -Options: - --cum Sort by cumulative data - --base= Subtract from before display - --interactive Run in interactive mode (interactive "help" gives help) [default] - --seconds= Length of time for dynamic profiles [default=30 secs] - --add_lib= Read additional symbols and line info from the given library - --lib_prefix=

    Comma separated list of library path prefixes - -Reporting Granularity: - --addresses Report at address level - --lines Report at source line level - --functions Report at function level [default] - --files Report at source file level - -Output type: - --text Generate text report - --callgrind Generate callgrind format to stdout - --gv Generate Postscript and display - --web Generate SVG and display - --list= Generate source listing of matching routines - --disasm= Generate disassembly of matching routines - --symbols Print demangled symbol names found at given addresses - --dot Generate DOT file to stdout - --ps Generate Postcript to stdout - --pdf Generate PDF to stdout - --svg Generate SVG to stdout - --gif Generate GIF to stdout - --raw Generate symbolized pprof data (useful with remote fetch) - -Heap-Profile Options: - --inuse_space Display in-use (mega)bytes [default] - --inuse_objects Display in-use objects - --alloc_space Display allocated (mega)bytes - --alloc_objects Display allocated objects - --show_bytes Display space in bytes - --drop_negative Ignore negative differences - -Contention-profile options: - --total_delay Display total delay at each region [default] - --contentions Display number of delays at each region - --mean_delay Display mean delay at each region - -Call-graph Options: - --nodecount= Show at most so many nodes [default=80] - --nodefraction= Hide nodes below *total [default=.005] - --edgefraction= Hide edges below *total [default=.001] - --focus= Focus on nodes matching - --ignore= Ignore nodes matching - --scale= Set GV scaling [default=0] - --heapcheck Make nodes with non-0 object counts - (i.e. direct leak generators) more visible - -Miscellaneous: - --tools= Prefix for object tool pathnames - --test Run unit tests - --help This message - --version Version information - -Environment Variables: - PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof - PPROF_TOOLS Prefix for object tools pathnames - -Examples: - -pprof /bin/ls ls.prof - Enters "interactive" mode -pprof --text /bin/ls ls.prof - Outputs one line per procedure -pprof --web /bin/ls ls.prof - Displays annotated call-graph in web browser -pprof --gv /bin/ls ls.prof - Displays annotated call-graph via 'gv' -pprof --gv --focus=Mutex /bin/ls ls.prof - Restricts to code paths including a .*Mutex.* entry -pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof - Code paths including Mutex but not string -pprof --list=getdir /bin/ls ls.prof - (Per-line) annotated source listing for getdir() -pprof --disasm=getdir /bin/ls ls.prof - (Per-PC) annotated disassembly for getdir() - -pprof http://localhost:1234/ - Enters "interactive" mode -pprof --text localhost:1234 - Outputs one line per procedure for localhost:1234 -pprof --raw localhost:1234 > ./local.raw -pprof --text ./local.raw - Fetches a remote profile for later analysis and then - analyzes it in text mode. -EOF -} - -sub version_string { - return <new()->filename; - $main::tmpfile_ps = File::Temp->new()->filename; - - $main::next_tmpfile = 0; - $SIG{'INT'} = \&sighandler; - - # Cache from filename/linenumber to source code - $main::source_cache = (); - - $main::opt_help = 0; - $main::opt_version = 0; - - $main::opt_cum = 0; - $main::opt_base = ''; - $main::opt_addresses = 0; - $main::opt_lines = 0; - $main::opt_functions = 0; - $main::opt_files = 0; - $main::opt_lib_prefix = ""; - - $main::opt_text = 0; - $main::opt_callgrind = 0; - $main::opt_list = ""; - $main::opt_disasm = ""; - $main::opt_symbols = 0; - $main::opt_gv = 0; - $main::opt_web = 0; - $main::opt_dot = 0; - $main::opt_ps = 0; - $main::opt_pdf = 0; - $main::opt_gif = 0; - $main::opt_svg = 0; - $main::opt_raw = 0; - - $main::opt_nodecount = 80; - $main::opt_nodefraction = 0.005; - $main::opt_edgefraction = 0.001; - $main::opt_focus = ''; - $main::opt_ignore = ''; - $main::opt_scale = 0; - $main::opt_heapcheck = 0; - $main::opt_seconds = 30; - $main::opt_lib = ""; - - $main::opt_inuse_space = 0; - $main::opt_inuse_objects = 0; - $main::opt_alloc_space = 0; - $main::opt_alloc_objects = 0; - $main::opt_show_bytes = 0; - $main::opt_drop_negative = 0; - $main::opt_interactive = 0; - - $main::opt_total_delay = 0; - $main::opt_contentions = 0; - $main::opt_mean_delay = 0; - - $main::opt_tools = ""; - $main::opt_debug = 0; - $main::opt_test = 0; - - # These are undocumented flags used only by unittests. - $main::opt_test_stride = 0; - - # Are we using $SYMBOL_PAGE? - $main::use_symbol_page = 0; - - # Files returned by TempName. - %main::tempnames = (); - - # Type of profile we are dealing with - # Supported types: - # cpu - # heap - # growth - # contention - $main::profile_type = ''; # Empty type means "unknown" - - GetOptions("help!" => \$main::opt_help, - "version!" => \$main::opt_version, - "cum!" => \$main::opt_cum, - "base=s" => \$main::opt_base, - "seconds=i" => \$main::opt_seconds, - "add_lib=s" => \$main::opt_lib, - "lib_prefix=s" => \$main::opt_lib_prefix, - "functions!" => \$main::opt_functions, - "lines!" => \$main::opt_lines, - "addresses!" => \$main::opt_addresses, - "files!" => \$main::opt_files, - "text!" => \$main::opt_text, - "callgrind!" => \$main::opt_callgrind, - "list=s" => \$main::opt_list, - "disasm=s" => \$main::opt_disasm, - "symbols!" => \$main::opt_symbols, - "gv!" => \$main::opt_gv, - "web!" => \$main::opt_web, - "dot!" => \$main::opt_dot, - "ps!" => \$main::opt_ps, - "pdf!" => \$main::opt_pdf, - "svg!" => \$main::opt_svg, - "gif!" => \$main::opt_gif, - "raw!" => \$main::opt_raw, - "interactive!" => \$main::opt_interactive, - "nodecount=i" => \$main::opt_nodecount, - "nodefraction=f" => \$main::opt_nodefraction, - "edgefraction=f" => \$main::opt_edgefraction, - "focus=s" => \$main::opt_focus, - "ignore=s" => \$main::opt_ignore, - "scale=i" => \$main::opt_scale, - "heapcheck" => \$main::opt_heapcheck, - "inuse_space!" => \$main::opt_inuse_space, - "inuse_objects!" => \$main::opt_inuse_objects, - "alloc_space!" => \$main::opt_alloc_space, - "alloc_objects!" => \$main::opt_alloc_objects, - "show_bytes!" => \$main::opt_show_bytes, - "drop_negative!" => \$main::opt_drop_negative, - "total_delay!" => \$main::opt_total_delay, - "contentions!" => \$main::opt_contentions, - "mean_delay!" => \$main::opt_mean_delay, - "tools=s" => \$main::opt_tools, - "test!" => \$main::opt_test, - "debug!" => \$main::opt_debug, - # Undocumented flags used only by unittests: - "test_stride=i" => \$main::opt_test_stride, - ) || usage("Invalid option(s)"); - - # Deal with the standard --help and --version - if ($main::opt_help) { - print usage_string(); - exit(0); - } - - if ($main::opt_version) { - print version_string(); - exit(0); - } - - # Disassembly/listing/symbols mode requires address-level info - if ($main::opt_disasm || $main::opt_list || $main::opt_symbols) { - $main::opt_functions = 0; - $main::opt_lines = 0; - $main::opt_addresses = 1; - $main::opt_files = 0; - } - - # Check heap-profiling flags - if ($main::opt_inuse_space + - $main::opt_inuse_objects + - $main::opt_alloc_space + - $main::opt_alloc_objects > 1) { - usage("Specify at most on of --inuse/--alloc options"); - } - - # Check output granularities - my $grains = - $main::opt_functions + - $main::opt_lines + - $main::opt_addresses + - $main::opt_files + - 0; - if ($grains > 1) { - usage("Only specify one output granularity option"); - } - if ($grains == 0) { - $main::opt_functions = 1; - } - - # Check output modes - my $modes = - $main::opt_text + - $main::opt_callgrind + - ($main::opt_list eq '' ? 0 : 1) + - ($main::opt_disasm eq '' ? 0 : 1) + - ($main::opt_symbols == 0 ? 0 : 1) + - $main::opt_gv + - $main::opt_web + - $main::opt_dot + - $main::opt_ps + - $main::opt_pdf + - $main::opt_svg + - $main::opt_gif + - $main::opt_raw + - $main::opt_interactive + - 0; - if ($modes > 1) { - usage("Only specify one output mode"); - } - if ($modes == 0) { - if (-t STDOUT) { # If STDOUT is a tty, activate interactive mode - $main::opt_interactive = 1; - } else { - $main::opt_text = 1; - } - } - - if ($main::opt_test) { - RunUnitTests(); - # Should not return - exit(1); - } - - # Binary name and profile arguments list - $main::prog = ""; - @main::pfile_args = (); - - # Remote profiling without a binary (using $SYMBOL_PAGE instead) - if (IsProfileURL($ARGV[0])) { - $main::use_symbol_page = 1; - } elsif ($ARGV[0] && IsSymbolizedProfileFile($ARGV[0])) { - $main::use_symbolized_profile = 1; - $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file - } - - if ($main::use_symbol_page || $main::use_symbolized_profile) { - # We don't need a binary! - my %disabled = ('--lines' => $main::opt_lines, - '--disasm' => $main::opt_disasm); - for my $option (keys %disabled) { - usage("$option cannot be used without a binary") if $disabled{$option}; - } - # Set $main::prog later... - scalar(@ARGV) || usage("Did not specify profile file"); - } elsif ($main::opt_symbols) { - # --symbols needs a binary-name (to run nm on, etc) but not profiles - $main::prog = shift(@ARGV) || usage("Did not specify program"); - } else { - $main::prog = shift(@ARGV) || usage("Did not specify program"); - scalar(@ARGV) || usage("Did not specify profile file"); - } - - # Parse profile file/location arguments - foreach my $farg (@ARGV) { - if ($farg =~ m/(.*)\@([0-9]+)(|\/.*)$/ ) { - my $machine = $1; - my $num_machines = $2; - my $path = $3; - for (my $i = 0; $i < $num_machines; $i++) { - unshift(@main::pfile_args, "$i.$machine$path"); - } - } else { - unshift(@main::pfile_args, $farg); - } - } - - if ($main::use_symbol_page) { - unless (IsProfileURL($main::pfile_args[0])) { - error("The first profile should be a remote form to use $SYMBOL_PAGE\n"); - } - CheckSymbolPage(); - $main::prog = FetchProgramName(); - } elsif (!$main::use_symbolized_profile) { # may not need objtools! - ConfigureObjTools($main::prog) - } - - # Break the opt_lib_prefix into the prefix_list array - @prefix_list = split (',', $main::opt_lib_prefix); - - # Remove trailing / from the prefixes, in the list to prevent - # searching things like /my/path//lib/mylib.so - foreach (@prefix_list) { - s|/+$||; - } -} - -sub Main() { - Init(); - $main::collected_profile = undef; - @main::profile_files = (); - $main::op_time = time(); - - # Printing symbols is special and requires a lot less info that most. - if ($main::opt_symbols) { - PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin - return; - } - - # Fetch all profile data - FetchDynamicProfiles(); - - # this will hold symbols that we read from the profile files - my $symbol_map = {}; - - # Read one profile, pick the last item on the list - my $data = ReadProfile($main::prog, pop(@main::profile_files)); - my $profile = $data->{profile}; - my $pcs = $data->{pcs}; - my $libs = $data->{libs}; # Info about main program and shared libraries - $symbol_map = MergeSymbols($symbol_map, $data->{symbols}); - - # Add additional profiles, if available. - if (scalar(@main::profile_files) > 0) { - foreach my $pname (@main::profile_files) { - my $data2 = ReadProfile($main::prog, $pname); - $profile = AddProfile($profile, $data2->{profile}); - $pcs = AddPcs($pcs, $data2->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $data2->{symbols}); - } - } - - # Subtract base from profile, if specified - if ($main::opt_base ne '') { - my $base = ReadProfile($main::prog, $main::opt_base); - $profile = SubtractProfile($profile, $base->{profile}); - $pcs = AddPcs($pcs, $base->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); - } - - # Get total data in profile - my $total = TotalProfile($profile); - - # Collect symbols - my $symbols; - if ($main::use_symbolized_profile) { - $symbols = FetchSymbols($pcs, $symbol_map); - } elsif ($main::use_symbol_page) { - $symbols = FetchSymbols($pcs); - } else { - $symbols = ExtractSymbols($libs, $pcs); - } - - # Remove uniniteresting stack items - $profile = RemoveUninterestingFrames($symbols, $profile); - - # Focus? - if ($main::opt_focus ne '') { - $profile = FocusProfile($symbols, $profile, $main::opt_focus); - } - - # Ignore? - if ($main::opt_ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); - } - - my $calls = ExtractCalls($symbols, $profile); - - # Reduce profiles to required output granularity, and also clean - # each stack trace so a given entry exists at most once. - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - # Print - if (!$main::opt_interactive) { - if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); - } elsif ($main::opt_list) { - PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); - } elsif ($main::opt_text) { - # Make sure the output is empty when have nothing to report - # (only matters when --heapcheck is given but we must be - # compatible with old branches that did not pass --heapcheck always): - if ($total != 0) { - Infof("Total: %s %s\n", Unparse($total), Units()); - } - PrintText($symbols, $flat, $cumulative, $total, -1); - } elsif ($main::opt_raw) { - PrintSymbolizedProfile($symbols, $profile, $main::prog); - } elsif ($main::opt_callgrind) { - PrintCallgrind($calls); - } else { - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), ""); - } elsif ($main::opt_web) { - my $tmp = TempName($main::next_tmpfile, "svg"); - RunWeb($tmp); - # The command we run might hand the file name off - # to an already running browser instance and then exit. - # Normally, we'd remove $tmp on exit (right now), - # but fork a child to remove $tmp a little later, so that the - # browser has time to load it first. - delete $main::tempnames{$tmp}; - if (fork() == 0) { - sleep 5; - unlink($tmp); - exit(0); - } - } - } else { - exit(1); - } - } - } else { - InteractiveMode($profile, $symbols, $libs, $total); - } - - cleanup(); - exit(0); -} - -##### Entry Point ##### - -Main(); - -# Temporary code to detect if we're running on a Goobuntu system. -# These systems don't have the right stuff installed for the special -# Readline libraries to work, so as a temporary workaround, we default -# to using the normal stdio code, rather than the fancier readline-based -# code -sub ReadlineMightFail { - if (-e '/lib/libtermcap.so.2') { - return 0; # libtermcap exists, so readline should be okay - } else { - return 1; - } -} - -sub RunGV { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >$DEVNULL 2>&1")) { - # Options using double dash are supported by this gv version. - # Also, turn on noantialias to better handle bug in gv for - # postscript files with large dimensions. - # TODO: Maybe we should not pass the --noantialias flag - # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); - } else { - # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); - } -} - -sub RunWeb { - my $fname = shift; - print STDERR "Loading web page file:///$fname\n"; - - my $uname = `uname`; - if ($uname =~ /Darwin/) { - # OS X: open will use standard preference for SVG files. - system("/usr/bin/open", $fname); - return; - } - - if ($uname =~ /CYGWIN/) { - # Windows(cygwin): open will use standard preference for SVG files. - my $winname = `cygpath -wa $fname`; - system("explorer.exe", $winname); - return; - } - if ($uname =~ /MINGW/) { - # Windows(MinGW): open will use standard preference for SVG files. - system("cmd", "/c", "start", $fname); - return; - } - - # Some kind of Unix; try generic symlinks, then specific browsers. - # (Stop once we find one.) - # Works best if the browser is already running. - my @alt = ( - "/etc/alternatives/gnome-www-browser", - "/etc/alternatives/x-www-browser", - "google-chrome", - "firefox", - ); - foreach my $b (@alt) { - if (system($b, $fname) == 0) { - return; - } - } - - print STDERR "Could not load web browser.\n"; -} - -sub RunKcachegrind { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); -} - - -##### Interactive helper routines ##### - -sub InteractiveMode { - $| = 1; # Make output unbuffered for interactive mode - my ($orig_profile, $symbols, $libs, $total) = @_; - - print STDERR "Welcome to pprof! For help, type 'help'.\n"; - - # Use ReadLine if it's installed and input comes from a console. - if ( -t STDIN && - !ReadlineMightFail() && - defined(eval {require Term::ReadLine}) ) { - my $term = new Term::ReadLine 'pprof'; - while ( defined ($_ = $term->readline('(pprof) '))) { - $term->addhistory($_) if /\S/; - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - } - } else { # don't have readline - while (1) { - print STDERR "(pprof) "; - $_ = ; - last if ! defined $_ ; - s/\r//g; # turn windows-looking lines into unix-looking lines - - # Save some flags that might be reset by InteractiveCommand() - my $save_opt_lines = $main::opt_lines; - - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - - # Restore flags - $main::opt_lines = $save_opt_lines; - } - } -} - -# Takes two args: orig profile, and command to run. -# Returns 1 if we should keep going, or 0 if we were asked to quit -sub InteractiveCommand { - my($orig_profile, $symbols, $libs, $total, $command) = @_; - $_ = $command; # just to make future m//'s easier - if (!defined($_)) { - print STDERR "\n"; - return 0; - } - if (m/^\s*quit/) { - return 0; - } - if (m/^\s*help/) { - InteractiveHelpMessage(); - return 1; - } - # Clear all the mode options -- mode is controlled by "$command" - $main::opt_text = 0; - $main::opt_callgrind = 0; - $main::opt_disasm = 0; - $main::opt_list = 0; - $main::opt_gv = 0; - $main::opt_cum = 0; - - if (m/^\s*(text|top)(\d*)\s*(.*)/) { - $main::opt_text = 1; - - my $line_limit = ($2 ne "") ? int($2) : 10; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($3); - - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintText($symbols, $flat, $cumulative, $total, $line_limit); - return 1; - } - if (m/^\s*callgrind\s*([^ \n]*)/) { - $main::opt_callgrind = 1; - - # Get derived profiles - my $calls = ExtractCalls($symbols, $orig_profile); - my $filename = $1; - if ( $1 eq '' ) { - $filename = TempName($main::next_tmpfile, "callgrind"); - } - PrintCallgrind($calls, $filename); - if ( $1 eq '' ) { - RunKcachegrind($filename, " & "); - $main::next_tmpfile++; - } - - return 1; - } - if (m/^\s*(web)?list\s*(.+)/) { - my $html = (defined($1) && ($1 eq "web")); - $main::opt_list = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($2); - - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintListing($total, $libs, $flat, $cumulative, $routine, $html); - return 1; - } - if (m/^\s*disasm\s*(.+)/) { - $main::opt_disasm = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); - - # Process current profile to account for various settings - my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintDisassembly($libs, $flat, $cumulative, $routine, $total); - return 1; - } - if (m/^\s*(gv|web)\s*(.*)/) { - $main::opt_gv = 0; - $main::opt_web = 0; - if ($1 eq "gv") { - $main::opt_gv = 1; - } elsif ($1 eq "web") { - $main::opt_web = 1; - } - - my $focus; - my $ignore; - ($focus, $ignore) = ParseInteractiveArgs($2); - - # Process current profile to account for various settings - my $profile = ProcessProfile($total, $orig_profile, $symbols, $focus, $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), " &"); - } elsif ($main::opt_web) { - RunWeb(TempName($main::next_tmpfile, "svg")); - } - $main::next_tmpfile++; - } - return 1; - } - if (m/^\s*$/) { - return 1; - } - print STDERR "Unknown command: try 'help'.\n"; - return 1; -} - - -sub ProcessProfile { - my $total_count = shift; - my $orig_profile = shift; - my $symbols = shift; - my $focus = shift; - my $ignore = shift; - - # Process current profile to account for various settings - my $profile = $orig_profile; - printf("Total: %s %s\n", Unparse($total_count), Units()); - if ($focus ne '') { - $profile = FocusProfile($symbols, $profile, $focus); - my $focus_count = TotalProfile($profile); - Infof("After focusing on '%s': %s %s of %s (%0.1f%%)\n", - $focus, - Unparse($focus_count), Units(), - Unparse($total_count), ($focus_count*100.0) / $total_count); - } - if ($ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $ignore); - my $ignore_count = TotalProfile($profile); - Infof("After ignoring '%s': %s %s of %s (%0.1f%%)\n", - $ignore, - Unparse($ignore_count), Units(), - Unparse($total_count), - ($ignore_count*100.0) / $total_count); - } - - return $profile; -} - -sub InteractiveHelpMessage { - print STDERR <{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - my $depth = $#addrs + 1; - # int(foo / 2**32) is the only reliable way to get rid of bottom - # 32 bits on both 32- and 64-bit systems. - print pack('L*', $count & 0xFFFFFFFF, int($count / 2**32)); - print pack('L*', $depth & 0xFFFFFFFF, int($depth / 2**32)); - - foreach my $full_addr (@addrs) { - my $addr = $full_addr; - $addr =~ s/0x0*//; # strip off leading 0x, zeroes - if (length($addr) > 16) { - print STDERR "Invalid address in profile: $full_addr\n"; - next; - } - my $low_addr = substr($addr, -8); # get last 8 hex chars - my $high_addr = substr($addr, -16, 8); # get up to 8 more hex chars - print pack('L*', hex('0x' . $low_addr), hex('0x' . $high_addr)); - } - } - } -} - -# Print symbols and profile data -sub PrintSymbolizedProfile { - my $symbols = shift; - my $profile = shift; - my $prog = shift; - - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - - print '--- ', $symbol_marker, "\n"; - if (defined($prog)) { - print 'binary=', $prog, "\n"; - } - while (my ($pc, $name) = each(%{$symbols})) { - my $sep = ' '; - print '0x', $pc; - # We have a list of function names, which include the inlined - # calls. They are separated (and terminated) by --, which is - # illegal in function names. - for (my $j = 2; $j <= $#{$name}; $j += 3) { - print $sep, $name->[$j]; - $sep = '--'; - } - print "\n"; - } - print '---', "\n"; - - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - print '--- ', $profile_marker, "\n"; - if (defined($main::collected_profile)) { - # if used with remote fetch, simply dump the collected profile to output. - open(SRC, "<$main::collected_profile"); - while () { - print $_; - } - close(SRC); - } else { - # dump a cpu-format profile to standard out - PrintProfileData($profile); - } -} - -# Print information conditionally filtered out depending on the output -# format. -sub Infof { - my $format = shift; - my @args = @_; - return if $main::opt_svg; - printf($format, @args); -} - -# Print text output -sub PrintText { - my $symbols = shift; - my $flat = shift; - my $cumulative = shift; - my $total = shift; - my $line_limit = shift; - - # Which profile to sort by? - my $s = $main::opt_cum ? $cumulative : $flat; - - my $running_sum = 0; - my $lines = 0; - foreach my $k (sort { GetEntry($s, $b) <=> GetEntry($s, $a) || $a cmp $b } - keys(%{$cumulative})) { - my $f = GetEntry($flat, $k); - my $c = GetEntry($cumulative, $k); - $running_sum += $f; - - my $sym = $k; - if (exists($symbols->{$k})) { - $sym = $symbols->{$k}->[0] . " " . $symbols->{$k}->[1]; - if ($main::opt_addresses) { - $sym = $k . " " . $sym; - } - } - - if ($f != 0 || $c != 0) { - printf("%8s %6s %6s %8s %6s %s\n", - Unparse($f), - Percent($f, $total), - Percent($running_sum, $total), - Unparse($c), - Percent($c, $total), - $sym); - } - $lines++; - last if ($line_limit >= 0 && $lines >= $line_limit); - } -} - -# Print the call graph in a way that's suiteable for callgrind. -sub PrintCallgrind { - my $calls = shift; - my $filename; - if ($main::opt_interactive) { - $filename = shift; - print STDERR "Writing callgrind file to '$filename'.\n" - } else { - $filename = "&STDOUT"; - } - open(CG, ">".$filename ); - printf CG ("events: Hits\n\n"); - foreach my $call ( map { $_->[0] } - sort { $a->[1] cmp $b ->[1] || - $a->[2] <=> $b->[2] } - map { /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - [$_, $1, $2] } - keys %$calls ) { - my $count = int($calls->{$call}); - $call =~ /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - my ( $caller_file, $caller_line, $caller_function, - $callee_file, $callee_line, $callee_function ) = - ( $1, $2, $3, $5, $6, $7 ); - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); - if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); - printf CG ("calls=$count $callee_line\n"); - } - printf CG ("$caller_line $count\n\n"); - } -} - -# Print disassembly for all all routines that match $main::opt_disasm -sub PrintDisassembly { - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $disasm_opts = shift; - my $total = shift; - - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - # See if there are any samples in this routine - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - PrintDisassembledFunction($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr, $total); - last; - } - $addr = AddressInc($addr); - } - } - } -} - -# Return reference to array of tuples of the form: -# [start_address, filename, linenumber, instruction, limit_address] -# E.g., -# ["0x806c43d", "/foo/bar.cc", 131, "ret", "0x806c440"] -sub Disassemble { - my $prog = shift; - my $offset = shift; - my $start_addr = shift; - my $end_addr = shift; - - my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - - if (system("$objdump --help >$DEVNULL 2>&1") != 0) { - # objdump must not exist. Fall back to go tool objdump. - $objdump = "go tool objdump"; - $cmd = "$objdump $prog 0x$start_addr 0x$end_addr"; - } - - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); - my @result = (); - my $filename = ""; - my $linenumber = -1; - my $last = ["", "", "", ""]; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - chop; - if (m|\s*(.+):(\d+)\s*$|) { - # Location line of the form: - # : - $filename = $1; - $linenumber = $2; - } elsif (m/^ +([0-9a-f]+):\s*(.*)/) { - # Disassembly line -- zero-extend address to full length - my $addr = HexExtend($1); - my $k = AddressAdd($addr, $offset); - $last->[4] = $k; # Store ending address for previous instruction - $last = [$k, $filename, $linenumber, $2, $end_addr]; - push(@result, $last); - } - } - close(OBJDUMP); - return @result; -} - -# The input file should contain lines of the form /proc/maps-like -# output (same format as expected from the profiles) or that looks -# like hex addresses (like "0xDEADBEEF"). We will parse all -# /proc/maps output, and for all the hex addresses, we will output -# "short" symbol names, one per line, in the same order as the input. -sub PrintSymbols { - my $maps_and_symbols_file = shift; - - # ParseLibraries expects pcs to be in a set. Fine by us... - my @pclist = (); # pcs in sorted order - my $pcs = {}; - my $map = ""; - foreach my $line (<$maps_and_symbols_file>) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ($line =~ /\b(0x[0-9a-f]+)\b/i) { - push(@pclist, HexExtend($1)); - $pcs->{$pclist[-1]} = 1; - } else { - $map .= $line; - } - } - - my $libs = ParseLibraries($main::prog, $map, $pcs); - my $symbols = ExtractSymbols($libs, $pcs); - - foreach my $pc (@pclist) { - # ->[0] is the shortname, ->[2] is the full name - print(($symbols->{$pc}->[0] || "??") . "\n"); - } -} - - -# For sorting functions by name -sub ByName { - return ShortFunctionName($a) cmp ShortFunctionName($b); -} - -# Print source-listing for all all routines that match $main::opt_list -sub PrintListing { - my $total = shift; - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $list_opts = shift; - my $html = shift; - - my $output = \*STDOUT; - my $fname = ""; - - - if ($html) { - # Arrange to write the output to a temporary file - $fname = TempName($main::next_tmpfile, "html"); - $main::next_tmpfile++; - if (!open(TEMP, ">$fname")) { - print STDERR "$fname: $!\n"; - return; - } - $output = \*TEMP; - print $output HtmlListingHeader(); - printf $output ("
    %s
    Total: %s %s
    \n", - $main::prog, Unparse($total), Units()); - } - - my $listed = 0; - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - # Print if there are any samples in this routine - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - $listed += PrintSource( - $lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr, - $html, - $output); - last; - } - $addr = AddressInc($addr); - } - } - } - - if ($html) { - if ($listed > 0) { - print $output HtmlListingFooter(); - close($output); - RunWeb($fname); - } else { - close($output); - unlink($fname); - } - } -} - -sub HtmlListingHeader { - return <<'EOF'; - - - -Pprof listing - - - - -EOF -} - -sub HtmlListingFooter { - return <<'EOF'; - - -EOF -} - -sub HtmlEscape { - my $text = shift; - $text =~ s/&/&/g; - $text =~ s//>/g; - return $text; -} - -# Returns the indentation of the line, if it has any non-whitespace -# characters. Otherwise, returns -1. -sub Indentation { - my $line = shift; - if (m/^(\s*)\S/) { - return length($1); - } else { - return -1; - } -} - -# Print source-listing for one routine -sub PrintSource { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - my $html = shift; - my $output = shift; - - # Disassemble all instructions (just to get line numbers) - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Hack 1: assume that the first source file encountered in the - # disassembly contains the routine - my $filename = undef; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[2] >= 0) { - $filename = $instructions[$i]->[1]; - last; - } - } - if (!defined($filename)) { - print STDERR "no filename found in $routine\n"; - return 0; - } - - # Hack 2: assume that the largest line number from $filename is the - # end of the procedure. This is typically safe since if P1 contains - # an inlined call to P2, then P2 usually occurs earlier in the - # source file. If this does not work, we might have to compute a - # density profile or just print all regions we find. - my $lastline = 0; - for (my $i = 0; $i <= $#instructions; $i++) { - my $f = $instructions[$i]->[1]; - my $l = $instructions[$i]->[2]; - if (($f eq $filename) && ($l > $lastline)) { - $lastline = $l; - } - } - - # Hack 3: assume the first source location from "filename" is the start of - # the source code. - my $firstline = 1; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[1] eq $filename) { - $firstline = $instructions[$i]->[2]; - last; - } - } - - # Hack 4: Extend last line forward until its indentation is less than - # the indentation we saw on $firstline - my $oldlastline = $lastline; - { - if (!open(FILE, "<$filename")) { - print STDERR "$filename: $!\n"; - return 0; - } - my $l = 0; - my $first_indentation = -1; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $l++; - my $indent = Indentation($_); - if ($l >= $firstline) { - if ($first_indentation < 0 && $indent >= 0) { - $first_indentation = $indent; - last if ($first_indentation == 0); - } - } - if ($l >= $lastline && $indent >= 0) { - if ($indent >= $first_indentation) { - $lastline = $l+1; - } else { - last; - } - } - } - close(FILE); - } - - # Assign all samples to the range $firstline,$lastline, - # Hack 4: If an instruction does not occur in the range, its samples - # are moved to the next instruction that occurs in the range. - my $samples1 = {}; # Map from line number to flat count - my $samples2 = {}; # Map from line number to cumulative count - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts - my %disasm = (); # Map from line number to disassembly - my $running_disasm = ""; # Unassigned disassembly - my $skip_marker = "---\n"; - if ($html) { - $skip_marker = ""; - for (my $l = $firstline; $l <= $lastline; $l++) { - $disasm{$l} = ""; - } - } - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - - if ($html) { - $running_disasm .= sprintf(" %6s %6s \t\t%8s: %s\n", - HtmlPrintNumber($c1), - HtmlPrintNumber($c2), - $e->[0], - CleanDisassembly($e->[3])); - } - - $running1 += $c1; - $running2 += $c2; - $total1 += $c1; - $total2 += $c2; - my $file = $e->[1]; - my $line = $e->[2]; - if (($file eq $filename) && - ($line >= $firstline) && - ($line <= $lastline)) { - # Assign all accumulated samples to this line - AddEntry($samples1, $line, $running1); - AddEntry($samples2, $line, $running2); - $running1 = 0; - $running2 = 0; - if ($html) { - $disasm{$line} .= $running_disasm; - $running_disasm = ''; - } - } - } - - # Assign any leftover samples to $lastline - AddEntry($samples1, $lastline, $running1); - AddEntry($samples2, $lastline, $running2); - - if ($html) { - printf $output ( - "

    %s

    %s\n
    \n" .
-      "Total:%6s %6s (flat / cumulative %s)\n",
-      HtmlEscape(ShortFunctionName($routine)),
-      HtmlEscape($filename),
-      Unparse($total1),
-      Unparse($total2),
-      Units());
-  } else {
-    printf $output (
-      "ROUTINE ====================== %s in %s\n" .
-      "%6s %6s Total %s (flat / cumulative)\n",
-      ShortFunctionName($routine),
-      $filename,
-      Unparse($total1),
-      Unparse($total2),
-      Units());
-  }
-  if (!open(FILE, "<$filename")) {
-    print STDERR "$filename: $!\n";
-    return 0;
-  }
-  my $l = 0;
-  while () {
-    s/\r//g;         # turn windows-looking lines into unix-looking lines
-    $l++;
-    if ($l >= $firstline - 5 &&
-        (($l <= $oldlastline + 5) || ($l <= $lastline))) {
-      chop;
-      my $text = $_;
-      if ($l == $firstline) { print $output $skip_marker; }
-      my $n1 = GetEntry($samples1, $l);
-      my $n2 = GetEntry($samples2, $l);
-      if ($html) {
-        my $dis = $disasm{$l};
-        if (!defined($dis) || $n1 + $n2 == 0) {
-          # No samples/disassembly for this source line
-          printf $output (
-            "%5d " .
-            "%6s %6s %s\n",
-            $l,
-            HtmlPrintNumber($n1),
-            HtmlPrintNumber($n2),
-            HtmlEscape($text));
-        } else {
-          printf $output (
-            "%5d " .
-            "%6s %6s %s" .
-            "%s\n",
-            $l,
-            HtmlPrintNumber($n1),
-            HtmlPrintNumber($n2),
-            HtmlEscape($text),
-            HtmlEscape($dis));
-        }
-      } else {
-        printf $output(
-          "%6s %6s %4d: %s\n",
-          UnparseAlt($n1),
-          UnparseAlt($n2),
-          $l,
-          $text);
-      }
-      if ($l == $lastline)  { print $output $skip_marker; }
-    };
-  }
-  close(FILE);
-  if ($html) {
-    print $output "
    \n"; - } - return 1; -} - -# Return the source line for the specified file/linenumber. -# Returns undef if not found. -sub SourceLine { - my $file = shift; - my $line = shift; - - # Look in cache - if (!defined($main::source_cache{$file})) { - if (100 < scalar keys(%main::source_cache)) { - # Clear the cache when it gets too big - $main::source_cache = (); - } - - # Read all lines from the file - if (!open(FILE, "<$file")) { - print STDERR "$file: $!\n"; - $main::source_cache{$file} = []; # Cache the negative result - return undef; - } - my $lines = []; - push(@{$lines}, ""); # So we can use 1-based line numbers as indices - while () { - push(@{$lines}, $_); - } - close(FILE); - - # Save the lines in the cache - $main::source_cache{$file} = $lines; - } - - my $lines = $main::source_cache{$file}; - if (($line < 0) || ($line > $#{$lines})) { - return undef; - } else { - return $lines->[$line]; - } -} - -# Print disassembly for one routine with interspersed source if available -sub PrintDisassembledFunction { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - my $total = shift; - - # Disassemble all instructions - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Make array of counts per instruction - my @flat_count = (); - my @cum_count = (); - my $flat_total = 0; - my $cum_total = 0; - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - push(@flat_count, $c1); - push(@cum_count, $c2); - $flat_total += $c1; - $cum_total += $c2; - } - - # Print header with total counts - printf("ROUTINE ====================== %s\n" . - "%6s %6s %s (flat, cumulative) %.1f%% of total\n", - ShortFunctionName($routine), - Unparse($flat_total), - Unparse($cum_total), - Units(), - ($cum_total * 100.0) / $total); - - # Process instructions in order - my $current_file = ""; - for (my $i = 0; $i <= $#instructions; ) { - my $e = $instructions[$i]; - - # Print the new file name whenever we switch files - if ($e->[1] ne $current_file) { - $current_file = $e->[1]; - my $fname = $current_file; - $fname =~ s|^\./||; # Trim leading "./" - - # Shorten long file names - if (length($fname) >= 58) { - $fname = "..." . substr($fname, -55); - } - printf("-------------------- %s\n", $fname); - } - - # TODO: Compute range of lines to print together to deal with - # small reorderings. - my $first_line = $e->[2]; - my $last_line = $first_line; - my %flat_sum = (); - my %cum_sum = (); - for (my $l = $first_line; $l <= $last_line; $l++) { - $flat_sum{$l} = 0; - $cum_sum{$l} = 0; - } - - # Find run of instructions for this range of source lines - my $first_inst = $i; - while (($i <= $#instructions) && - ($instructions[$i]->[2] >= $first_line) && - ($instructions[$i]->[2] <= $last_line)) { - $e = $instructions[$i]; - $flat_sum{$e->[2]} += $flat_count[$i]; - $cum_sum{$e->[2]} += $cum_count[$i]; - $i++; - } - my $last_inst = $i - 1; - - # Print source lines - for (my $l = $first_line; $l <= $last_line; $l++) { - my $line = SourceLine($current_file, $l); - if (!defined($line)) { - $line = "?\n"; - next; - } else { - $line =~ s/^\s+//; - } - printf("%6s %6s %5d: %s", - UnparseAlt($flat_sum{$l}), - UnparseAlt($cum_sum{$l}), - $l, - $line); - } - - # Print disassembly - for (my $x = $first_inst; $x <= $last_inst; $x++) { - my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - printf("%6s %6s %8s: %6s\n", - UnparseAlt($flat_count[$x]), - UnparseAlt($cum_count[$x]), - $address, - CleanDisassembly($e->[3])); - } - } -} - -# Print DOT graph -sub PrintDot { - my $prog = shift; - my $symbols = shift; - my $raw = shift; - my $flat = shift; - my $cumulative = shift; - my $overall_total = shift; - - # Get total - my $local_total = TotalProfile($flat); - my $nodelimit = int($main::opt_nodefraction * $local_total); - my $edgelimit = int($main::opt_edgefraction * $local_total); - my $nodecount = $main::opt_nodecount; - - # Find nodes to include - my @list = (sort { abs(GetEntry($cumulative, $b)) <=> - abs(GetEntry($cumulative, $a)) - || $a cmp $b } - keys(%{$cumulative})); - my $last = $nodecount - 1; - if ($last > $#list) { - $last = $#list; - } - while (($last >= 0) && - (abs(GetEntry($cumulative, $list[$last])) <= $nodelimit)) { - $last--; - } - if ($last < 0) { - print STDERR "No nodes to print\n"; - cleanup(); - return 0; - } - - if ($nodelimit > 0 || $edgelimit > 0) { - printf STDERR ("Dropping nodes with <= %s %s; edges with <= %s abs(%s)\n", - Unparse($nodelimit), Units(), - Unparse($edgelimit), Units()); - } - - # Open DOT output file - my $output; - if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); - } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; - } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; - } elsif ($main::opt_web || $main::opt_svg) { - # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); - } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; - } else { - $output = ">&STDOUT"; - } - open(DOT, $output) || error("$output: $!\n"); - - # Title - printf DOT ("digraph \"%s; %s %s\" {\n", - $prog, - Unparse($overall_total), - Units()); - if ($main::opt_pdf) { - # The output is more printable if we set the page size for dot. - printf DOT ("size=\"8,11\"\n"); - } - printf DOT ("node [width=0.375,height=0.25];\n"); - - # Print legend - printf DOT ("Legend [shape=box,fontsize=24,shape=plaintext," . - "label=\"%s\\l%s\\l%s\\l%s\\l%s\\l\"];\n", - $prog, - sprintf("Total %s: %s", Units(), Unparse($overall_total)), - sprintf("Focusing on: %s", Unparse($local_total)), - sprintf("Dropped nodes with <= %s abs(%s)", - Unparse($nodelimit), Units()), - sprintf("Dropped edges with <= %s %s", - Unparse($edgelimit), Units()) - ); - - # Print nodes - my %node = (); - my $nextnode = 1; - foreach my $a (@list[0..$last]) { - # Pick font size - my $f = GetEntry($flat, $a); - my $c = GetEntry($cumulative, $a); - - my $fs = 8; - if ($local_total > 0) { - $fs = 8 + (50.0 * sqrt(abs($f * 1.0 / $local_total))); - } - - $node{$a} = $nextnode++; - my $sym = $a; - $sym =~ s/\s+/\\n/g; - $sym =~ s/::/\\n/g; - - # Extra cumulative info to print for non-leaves - my $extra = ""; - if ($f != $c) { - $extra = sprintf("\\rof %s (%s)", - Unparse($c), - Percent($c, $overall_total)); - } - my $style = ""; - if ($main::opt_heapcheck) { - if ($f > 0) { - # make leak-causing nodes more visible (add a background) - $style = ",style=filled,fillcolor=gray" - } elsif ($f < 0) { - # make anti-leak-causing nodes (which almost never occur) - # stand out as well (triple border) - $style = ",peripheries=3" - } - } - - printf DOT ("N%d [label=\"%s\\n%s (%s)%s\\r" . - "\",shape=box,fontsize=%.1f%s];\n", - $node{$a}, - $sym, - Unparse($f), - Percent($f, $overall_total), - $extra, - $fs, - $style, - ); - } - - # Get edges and counts per edge - my %edge = (); - my $n; - foreach my $k (keys(%{$raw})) { - # TODO: omit low %age edges - $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); - for (my $i = 1; $i <= $#translated; $i++) { - my $src = $translated[$i]; - my $dst = $translated[$i-1]; - #next if ($src eq $dst); # Avoid self-edges? - if (exists($node{$src}) && exists($node{$dst})) { - my $edge_label = "$src\001$dst"; - if (!exists($edge{$edge_label})) { - $edge{$edge_label} = 0; - } - $edge{$edge_label} += $n; - } - } - } - - # Print edges - foreach my $e (keys(%edge)) { - my @x = split(/\001/, $e); - $n = $edge{$e}; - - if (abs($n) > $edgelimit) { - # Compute line width based on edge count - my $fraction = abs($local_total ? (3 * ($n / $local_total)) : 0); - if ($fraction > 1) { $fraction = 1; } - my $w = $fraction * 2; - if ($w < 1 && ($main::opt_web || $main::opt_svg)) { - # SVG output treats line widths < 1 poorly. - $w = 1; - } - - # Dot sometimes segfaults if given edge weights that are too large, so - # we cap the weights at a large value - my $edgeweight = abs($n) ** 0.7; - if ($edgeweight > 100000) { $edgeweight = 100000; } - $edgeweight = int($edgeweight); - - my $style = sprintf("setlinewidth(%f)", $w); - if ($x[1] =~ m/\(inline\)/) { - $style .= ",dashed"; - } - - # Use a slightly squashed function of the edge count as the weight - printf DOT ("N%s -> N%s [label=%s, weight=%d, style=\"%s\"];\n", - $node{$x[0]}, - $node{$x[1]}, - Unparse($n), - $edgeweight, - $style); - } - } - - print DOT ("}\n"); - close(DOT); - - if ($main::opt_web || $main::opt_svg) { - # Rewrite SVG to be more usable inside web browser. - RewriteSvg(TempName($main::next_tmpfile, "svg")); - } - - return 1; -} - -sub RewriteSvg { - my $svgfile = shift; - - open(SVG, $svgfile) || die "open temp svg: $!"; - my @svg = ; - close(SVG); - unlink $svgfile; - my $svg = join('', @svg); - - # Dot's SVG output is - # - # - # - # ... - # - # - # - # Change it to - # - # - # $svg_javascript - # - # - # ... - # - # - # - - # Fix width, height; drop viewBox. - $svg =~ s/(?s) above first - my $svg_javascript = SvgJavascript(); - my $viewport = "\n"; - $svg =~ s/ above . - $svg =~ s/(.*)(<\/svg>)/$1<\/g>$2/; - $svg =~ s/$svgfile") || die "open $svgfile: $!"; - print SVG $svg; - close(SVG); - } -} - -sub SvgJavascript { - return <<'EOF'; - -EOF -} - -# Translate a stack of addresses into a stack of symbols -sub TranslateStack { - my $symbols = shift; - my $k = shift; - - my @addrs = split(/\n/, $k); - my @result = (); - for (my $i = 0; $i <= $#addrs; $i++) { - my $a = $addrs[$i]; - - # Skip large addresses since they sometimes show up as fake entries on RH9 - if (length($a) > 8 && $a gt "7fffffffffffffff") { - next; - } - - if ($main::opt_disasm || $main::opt_list) { - # We want just the address for the key - push(@result, $a); - next; - } - - my $symlist = $symbols->{$a}; - if (!defined($symlist)) { - $symlist = [$a, "", $a]; - } - - # We can have a sequence of symbols for a particular entry - # (more than one symbol in the case of inlining). Callers - # come before callees in symlist, so walk backwards since - # the translated stack should contain callees before callers. - for (my $j = $#{$symlist}; $j >= 2; $j -= 3) { - my $func = $symlist->[$j-2]; - my $fileline = $symlist->[$j-1]; - my $fullfunc = $symlist->[$j]; - if ($j > 2) { - $func = "$func (inline)"; - } - if ($main::opt_addresses) { - push(@result, "$a $func $fileline"); - } elsif ($main::opt_lines) { - if ($func eq '??' && $fileline eq '??:0') { - push(@result, "$a"); - } else { - push(@result, "$func $fileline"); - } - } elsif ($main::opt_functions) { - if ($func eq '??') { - push(@result, "$a"); - } else { - push(@result, $func); - } - } elsif ($main::opt_files) { - if ($fileline eq '??:0' || $fileline eq '') { - push(@result, "$a"); - } else { - my $f = $fileline; - $f =~ s/:\d+$//; - push(@result, $f); - } - } else { - push(@result, $a); - last; # Do not print inlined info - } - } - } - - # print join(",", @addrs), " => ", join(",", @result), "\n"; - return @result; -} - -# Generate percent string for a number and a total -sub Percent { - my $num = shift; - my $tot = shift; - if ($tot != 0) { - return sprintf("%.1f%%", $num * 100.0 / $tot); - } else { - return ($num == 0) ? "nan" : (($num > 0) ? "+inf" : "-inf"); - } -} - -# Generate pretty-printed form of number -sub Unparse { - my $num = shift; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return sprintf("%d", $num); - } else { - if ($main::opt_show_bytes) { - return sprintf("%d", $num); - } else { - return sprintf("%.1f", $num / 1048576.0); - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return sprintf("%.3f", $num / 1e9); # Convert nanoseconds to seconds - } else { - return sprintf("%d", $num); - } -} - -# Alternate pretty-printed form: 0 maps to "." -sub UnparseAlt { - my $num = shift; - if ($num == 0) { - return "."; - } else { - return Unparse($num); - } -} - -# Alternate pretty-printed form: 0 maps to "" -sub HtmlPrintNumber { - my $num = shift; - if ($num == 0) { - return ""; - } else { - return Unparse($num); - } -} - -# Return output units -sub Units { - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return "objects"; - } else { - if ($main::opt_show_bytes) { - return "B"; - } else { - return "MB"; - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return "seconds"; - } elsif ($main::profile_type eq 'thread') { - return "threads"; - } else { - return "samples"; - } -} - -##### Profile manipulation code ##### - -# Generate flattened profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a] -sub FlatProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - AddEntry($result, $addrs[0], $count); - } - } - return $result; -} - -# Generate cumulative profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a], [b], [c], [d] -sub CumulativeProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - AddEntry($result, $a, $count); - } - } - return $result; -} - -# If the second-youngest PC on the stack is always the same, returns -# that pc. Otherwise, returns undef. -sub IsSecondPcAlwaysTheSame { - my $profile = shift; - - my $second_pc = undef; - foreach my $k (keys(%{$profile})) { - my @addrs = split(/\n/, $k); - if ($#addrs < 1) { - return undef; - } - if (not defined $second_pc) { - $second_pc = $addrs[1]; - } else { - if ($second_pc ne $addrs[1]) { - return undef; - } - } - } - return $second_pc; -} - -sub ExtractSymbolLocation { - my $symbols = shift; - my $address = shift; - # 'addr2line' outputs "??:0" for unknown locations; we do the - # same to be consistent. - my $location = "??:0:unknown"; - if (exists $symbols->{$address}) { - my $file = $symbols->{$address}->[1]; - if ($file eq "?") { - $file = "??:0" - } - $location = $file . ":" . $symbols->{$address}->[0]; - } - return $location; -} - -# Extracts a graph of calls. -sub ExtractCalls { - my $symbols = shift; - my $profile = shift; - - my $calls = {}; - while( my ($stack_trace, $count) = each %$profile ) { - my @address = split(/\n/, $stack_trace); - my $destination = ExtractSymbolLocation($symbols, $address[0]); - AddEntry($calls, $destination, $count); - for (my $i = 1; $i <= $#address; $i++) { - my $source = ExtractSymbolLocation($symbols, $address[$i]); - my $call = "$source -> $destination"; - AddEntry($calls, $call, $count); - $destination = $source; - } - } - - return $calls; -} - -sub RemoveUninterestingFrames { - my $symbols = shift; - my $profile = shift; - - # List of function names to skip - my %skip = (); - my $skip_regexp = 'NOMATCH'; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - foreach my $name ('calloc', - 'cfree', - 'malloc', - 'free', - 'memalign', - 'posix_memalign', - 'pvalloc', - 'valloc', - 'realloc', - 'tc_calloc', - 'tc_cfree', - 'tc_malloc', - 'tc_free', - 'tc_memalign', - 'tc_posix_memalign', - 'tc_pvalloc', - 'tc_valloc', - 'tc_realloc', - 'tc_new', - 'tc_delete', - 'tc_newarray', - 'tc_deletearray', - 'tc_new_nothrow', - 'tc_newarray_nothrow', - 'do_malloc', - '::do_malloc', # new name -- got moved to an unnamed ns - '::do_malloc_or_cpp_alloc', - 'DoSampledAllocation', - 'simple_alloc::allocate', - '__malloc_alloc_template::allocate', - '__builtin_delete', - '__builtin_new', - '__builtin_vec_delete', - '__builtin_vec_new', - 'operator new', - 'operator new[]', - # Go - 'catstring', - 'cnew', - 'copyin', - 'gostring', - 'gostringsize', - 'growslice1', - 'appendslice1', - 'hash_init', - 'hash_subtable_new', - 'hash_conv', - 'hash_grow', - 'hash_insert_internal', - 'hash_insert', - 'mapassign', - 'runtime.mapassign', - 'runtime.appendslice', - 'runtime.mapassign1', - 'makechan', - 'makemap', - 'mal', - 'profilealloc', - 'runtime.new', - 'makeslice1', - 'runtime.malloc', - 'unsafe.New', - 'runtime.mallocgc', - 'runtime.catstring', - 'runtime.cnew', - 'runtime.cnewarray', - 'runtime.growslice', - 'runtime.ifaceT2E', - 'runtime.ifaceT2I', - 'runtime.makechan', - 'runtime.makechan_c', - 'runtime.makemap', - 'runtime.makemap_c', - 'runtime.makeslice', - 'runtime.mal', - 'runtime.settype', - 'runtime.settype_flush', - 'runtime.slicebytetostring', - 'runtime.sliceinttostring', - 'runtime.stringtoslicebyte', - 'runtime.stringtosliceint', - # These mark the beginning/end of our custom sections - '__start_google_malloc', - '__stop_google_malloc', - '__start_malloc_hook', - '__stop_malloc_hook') { - $skip{$name} = 1; - $skip{"_" . $name} = 1; # Mach (OS X) adds a _ prefix to everything - } - # TODO: Remove TCMalloc once everything has been - # moved into the tcmalloc:: namespace and we have flushed - # old code out of the system. - $skip_regexp = "TCMalloc|^tcmalloc::"; - } elsif ($main::profile_type eq 'contention') { - foreach my $vname ('Mutex::Unlock', 'Mutex::UnlockSlow') { - $skip{$vname} = 1; - } - } elsif ($main::profile_type eq 'cpu') { - # Drop signal handlers used for CPU profile collection - # TODO(dpeng): this should not be necessary; it's taken - # care of by the general 2nd-pc mechanism below. - foreach my $name ('ProfileData::Add', # historical - 'ProfileData::prof_handler', # historical - 'CpuProfiler::prof_handler', - '__FRAME_END__', - '__pthread_sighandler', - '__restore') { - $skip{$name} = 1; - } - } else { - # Nothing skipped for unknown types - } - - # Go doesn't have the problem that this heuristic tries to fix. Disable. - if (0 && $main::profile_type eq 'cpu') { - # If all the second-youngest program counters are the same, - # this STRONGLY suggests that it is an artifact of measurement, - # i.e., stack frames pushed by the CPU profiler signal handler. - # Hence, we delete them. - # (The topmost PC is read from the signal structure, not from - # the stack, so it does not get involved.) - while (my $second_pc = IsSecondPcAlwaysTheSame($profile)) { - my $result = {}; - my $func = ''; - if (exists($symbols->{$second_pc})) { - $second_pc = $symbols->{$second_pc}->[0]; - } - print STDERR "Removing $second_pc from all stack traces.\n"; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - splice @addrs, 1, 1; - my $reduced_path = join("\n", @addrs); - AddEntry($result, $reduced_path, $count); - } - $profile = $result; - } - } - - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my @path = (); - foreach my $a (@addrs) { - if (exists($symbols->{$a})) { - my $func = $symbols->{$a}->[0]; - if ($skip{$func} || ($func =~ m/$skip_regexp/)) { - next; - } - } - push(@path, $a); - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Reduce profile to granularity given by user -sub ReduceProfile { - my $symbols = shift; - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); - my @path = (); - my %seen = (); - $seen{''} = 1; # So that empty keys are skipped - foreach my $e (@translated) { - # To avoid double-counting due to recursion, skip a stack-trace - # entry if it has already been seen - if (!$seen{$e}) { - $seen{$e} = 1; - push(@path, $e); - } - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Does the specified symbol array match the regexp? -sub SymbolMatches { - my $sym = shift; - my $re = shift; - if (defined($sym)) { - for (my $i = 0; $i < $#{$sym}; $i += 3) { - if ($sym->[$i] =~ m/$re/ || $sym->[$i+1] =~ m/$re/) { - return 1; - } - } - } - return 0; -} - -# Focus only on paths involving specified regexps -sub FocusProfile { - my $symbols = shift; - my $profile = shift; - my $focus = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$focus/) || SymbolMatches($symbols->{$a}, $focus)) { - AddEntry($result, $k, $count); - last; - } - } - } - return $result; -} - -# Focus only on paths not involving specified regexps -sub IgnoreProfile { - my $symbols = shift; - my $profile = shift; - my $ignore = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my $matched = 0; - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$ignore/) || SymbolMatches($symbols->{$a}, $ignore)) { - $matched = 1; - last; - } - } - if (!$matched) { - AddEntry($result, $k, $count); - } - } - return $result; -} - -# Get total count in profile -sub TotalProfile { - my $profile = shift; - my $result = 0; - foreach my $k (keys(%{$profile})) { - $result += $profile->{$k}; - } - return $result; -} - -# Add A to B -sub AddProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - my $v = $A->{$k}; - AddEntry($R, $k, $v); - } - # add all keys in B - foreach my $k (keys(%{$B})) { - my $v = $B->{$k}; - AddEntry($R, $k, $v); - } - return $R; -} - -# Merges symbol maps -sub MergeSymbols { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - $R->{$k} = $A->{$k}; - } - if (defined($B)) { - foreach my $k (keys(%{$B})) { - $R->{$k} = $B->{$k}; - } - } - return $R; -} - - -# Add A to B -sub AddPcs { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - $R->{$k} = 1 - } - # add all keys in B - foreach my $k (keys(%{$B})) { - $R->{$k} = 1 - } - return $R; -} - -# Subtract B from A -sub SubtractProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - my $v = $A->{$k} - GetEntry($B, $k); - if ($v < 0 && $main::opt_drop_negative) { - $v = 0; - } - AddEntry($R, $k, $v); - } - if (!$main::opt_drop_negative) { - # Take care of when subtracted profile has more entries - foreach my $k (keys(%{$B})) { - if (!exists($A->{$k})) { - AddEntry($R, $k, 0 - $B->{$k}); - } - } - } - return $R; -} - -# Get entry from profile; zero if not present -sub GetEntry { - my $profile = shift; - my $k = shift; - if (exists($profile->{$k})) { - return $profile->{$k}; - } else { - return 0; - } -} - -# Add entry to specified profile -sub AddEntry { - my $profile = shift; - my $k = shift; - my $n = shift; - if (!exists($profile->{$k})) { - $profile->{$k} = 0; - } - $profile->{$k} += $n; -} - -# Add a stack of entries to specified profile, and add them to the $pcs -# list. -sub AddEntries { - my $profile = shift; - my $pcs = shift; - my $stack = shift; - my $count = shift; - my @k = (); - - foreach my $e (split(/\s+/, $stack)) { - my $pc = HexExtend($e); - $pcs->{$pc} = 1; - push @k, $pc; - } - AddEntry($profile, (join "\n", @k), $count); -} - -sub IsSymbolizedProfileFile { - my $file_name = shift; - - if (!(-e $file_name) || !(-r $file_name)) { - return 0; - } - - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - # Check if the file contains a symbol-section marker. - open(TFILE, "<$file_name"); - my @lines = ; - my $result = grep(/^--- *$symbol_marker/, @lines); - close(TFILE); - return $result > 0; -} - -##### Code to profile a server dynamically ##### - -sub CheckSymbolPage { - my $url = SymbolPageURL(); -print STDERR "Read $url\n"; - - my $line = FetchHTTP($url); - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - unless (defined($line)) { - error("$url doesn't exist\n"); - } - - if ($line =~ /^num_symbols:\s+(\d+)$/) { - if ($1 == 0) { - error("Stripped binary. No symbols available.\n"); - } - } else { - error("Failed to get the number of symbols from $url\n"); - } -} - -sub IsProfileURL { - my $profile_name = shift; - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - return defined($host) and defined($port) and defined($path); -} - -sub ParseProfileURL { - my $profile_name = shift; - if (defined($profile_name) && - $profile_name =~ m,^(?:(https?)://|)([^/:]+):(\d+)(|\@\d+)(|/|(.*?)($PROFILE_PAGE|$PMUPROFILE_PAGE|$HEAP_PAGE|$GROWTH_PAGE|$THREAD_PAGE|$BLOCK_PAGE|$CONTENTION_PAGE|$WALL_PAGE|$FILTEREDPROFILE_PAGE))$,o) { - # $7 is $PROFILE_PAGE/$HEAP_PAGE/etc. $5 is *everything* after - # the hostname, as long as that everything is the empty string, - # a slash, or something ending in $PROFILE_PAGE/$HEAP_PAGE/etc. - # So "$7 || $5" is $PROFILE_PAGE/etc if there, or else it's "/" or "". - return ($1 || "http", $2, $3, $6, $7 || $5); - } - return (); -} - -# We fetch symbols from the first profile argument. -sub SymbolPageURL { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); - return "$scheme://$host:$port$prefix$SYMBOL_PAGE"; -} - -sub FetchProgramName() { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); - my $url = "$scheme://$host:$port$prefix$PROGRAM_NAME_PAGE"; - - my $cmdline = FetchHTTP($url); - $cmdline =~ s/\n.*//s; # first line only - $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines - error("Failed to get program name from $url\n") unless defined($cmdline); - $cmdline =~ s/\x00.+//; # Remove argv[1] and latters. - $cmdline =~ s!\n!!g; # Remove LFs. - return $cmdline; -} - -# Reads a symbol map from the file handle name given as $1, returning -# the resulting symbol map. Also processes variables relating to symbols. -# Currently, the only variable processed is 'binary=' which updates -# $main::prog to have the correct program name. -sub ReadSymbols { - my $in = shift; - my $map = shift; - while (<$in>) { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Removes all the leading zeroes from the symbols, see comment below. - if (m/^0x0*([0-9a-f]+)\s+(.+)/) { - $map->{$1} = $2; - } elsif (m/^---/) { - last; - } elsif (m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1, $2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "binary") { - if ($main::prog ne $UNKNOWN_BINARY && $main::prog ne $value) { - printf STDERR ("Warning: Mismatched binary name '%s', using '%s'.\n", - $main::prog, $value); - } - $main::prog = $value; - } else { - printf STDERR ("Ignoring unknown variable in symbols list: " . - "'%s' = '%s'\n", $variable, $value); - } - } - } - return $map; -} - -# Fetches and processes symbols to prepare them for use in the profile output -# code. If the optional 'symbol_map' arg is not given, fetches symbols from -# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols -# are assumed to have already been fetched into 'symbol_map' and are simply -# extracted and processed. -sub FetchSymbols { - my $pcset = shift; - my $symbol_map = shift; - - my %seen = (); - my @pcs = grep { !$seen{$_}++ } keys(%$pcset); # uniq - - if (!defined($symbol_map)) { - $symbol_map = {}; - - my $post_data = join("+", sort((map {"0x" . "$_"} @pcs))); - my $url = SymbolPageURL(); - my $content = PostHTTP($url, $post_data); - - my $tmp_symbol = File::Temp->new()->filename; - open(SYMBOL, ">$tmp_symbol"); - print SYMBOL $content; - close(SYMBOL); - - open(SYMBOL, "<$tmp_symbol") || error("$tmp_symbol"); - ReadSymbols(*SYMBOL{IO}, $symbol_map); - close(SYMBOL); - } - - my $symbols = {}; - foreach my $pc (@pcs) { - my $fullname; - # For 64 bits binaries, symbols are extracted with 8 leading zeroes. - # Then /symbol reads the long symbols in as uint64, and outputs - # the result with a "0x%08llx" format which get rid of the zeroes. - # By removing all the leading zeroes in both $pc and the symbols from - # /symbol, the symbols match and are retrievable from the map. - my $shortpc = $pc; - $shortpc =~ s/^0*//; - # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. - my $fullnames; - if (defined($symbol_map->{$shortpc})) { - $fullnames = $symbol_map->{$shortpc}; - } else { - $fullnames = "0x" . $pc; # Just use addresses - } - my $sym = []; - $symbols->{$pc} = $sym; - foreach my $fullname (split("--", $fullnames)) { - my $name = ShortFunctionName($fullname); - push(@{$sym}, $name, "?", $fullname); - } - } - return $symbols; -} - -sub BaseName { - my $file_name = shift; - $file_name =~ s!^.*/!!; # Remove directory name - return $file_name; -} - -sub MakeProfileBaseName { - my ($binary_name, $profile_name) = @_; - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - my $binary_shortname = BaseName($binary_name); - return sprintf("%s.%s.%s-port%s", - $binary_shortname, $main::op_time, $host, $port); -} - -sub FetchDynamicProfile { - my $binary_name = shift; - my $profile_name = shift; - my $fetch_name_only = shift; - my $encourage_patience = shift; - - if (!IsProfileURL($profile_name)) { - return $profile_name; - } else { - my ($scheme, $host, $port, $prefix, $path) = ParseProfileURL($profile_name); - if ($path eq "" || $path eq "/") { - # Missing type specifier defaults to cpu-profile - $path = $PROFILE_PAGE; - } - - my $profile_file = MakeProfileBaseName($binary_name, $profile_name); - - my $url; - my $timeout; - if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)) { - if ($path =~ m/$PROFILE_PAGE/) { - $url = sprintf("$scheme://$host:$port$prefix$path?seconds=%d", - $main::opt_seconds); - } else { - if ($profile_name =~ m/[?]/) { - $profile_name .= "&" - } else { - $profile_name .= "?" - } - $url = sprintf("$scheme://$profile_name" . "seconds=%d", - $main::opt_seconds); - } - $timeout = int($main::opt_seconds * 1.01 + 60); - } else { - # For non-CPU profiles, we add a type-extension to - # the target profile file name. - my $suffix = $path; - $suffix =~ s,/,.,g; - $profile_file .= "$suffix"; - $url = "$scheme://$host:$port$prefix$path"; - } - - my $tmp_profile = File::Temp->new()->filename; - my $real_profile = File::Temp->new()->filename; - - if ($fetch_name_only > 0) { - return $real_profile; - } - - if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)){ - print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; - if ($encourage_patience) { - print STDERR "Be patient...\n"; - } - } else { - print STDERR "Fetching $path profile from $host:$port to\n ${real_profile}\n"; - } - - my $content = FetchHTTP($url, $timeout); - - open(OUTFILE, ">$tmp_profile"); - binmode(OUTFILE); - print OUTFILE $content; - close(OUTFILE); - - my $line = $content; - $line !~ /^Could not enable CPU profiling/ || error($line); - - copy($tmp_profile, $real_profile) || error("Unable to copy profile\n"); - print STDERR "Wrote profile to $real_profile\n"; - $main::collected_profile = $real_profile; - return $main::collected_profile; - } -} - -# Collect profiles in parallel -sub FetchDynamicProfiles { - my $items = scalar(@main::pfile_args); - my $levels = log($items) / log(2); - - if ($items == 1) { - $main::profile_files[0] = FetchDynamicProfile($main::prog, $main::pfile_args[0], 0, 1); - } else { - # math rounding issues - if ((2 ** $levels) < $items) { - $levels++; - } - my $count = scalar(@main::pfile_args); - for (my $i = 0; $i < $count; $i++) { - $main::profile_files[$i] = FetchDynamicProfile($main::prog, $main::pfile_args[$i], 1, 0); - } - print STDERR "Fetching $count profiles, Be patient...\n"; - FetchDynamicProfilesRecurse($levels, 0, 0); - $main::collected_profile = join(" \\\n ", @main::profile_files); - } -} - -# Recursively fork a process to get enough processes -# collecting profiles -sub FetchDynamicProfilesRecurse { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if (my $pid = fork()) { - $position = 0 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - wait; - } else { - $position = 1 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - exit(0); - } -} - -# Collect a single profile -sub TryCollectProfile { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if ($level >= ($maxlevel - 1)) { - if ($position < scalar(@main::pfile_args)) { - FetchDynamicProfile($main::prog, $main::pfile_args[$position], 0, 0); - } - } else { - FetchDynamicProfilesRecurse($maxlevel, $level+1, $position); - } -} - -##### Parsing code ##### - -# Provide a small streaming-read module to handle very large -# cpu-profile files. Stream in chunks along a sliding window. -# Provides an interface to get one 'slot', correctly handling -# endian-ness differences. A slot is one 32-bit or 64-bit word -# (depending on the input profile). We tell endianness and bit-size -# for the profile by looking at the first 8 bytes: in cpu profiles, -# the second slot is always 3 (we'll accept anything that's not 0). -BEGIN { - package CpuProfileStream; - - sub new { - my ($class, $file, $fname) = @_; - my $self = { file => $file, - base => 0, - stride => 512 * 1024, # must be a multiple of bitsize/8 - slots => [], - unpack_code => "", # N for big-endian, V for little - }; - bless $self, $class; - # Let unittests adjust the stride - if ($main::opt_test_stride > 0) { - $self->{stride} = $main::opt_test_stride; - } - # Read the first two slots to figure out bitsize and endianness. - my $slots = $self->{slots}; - my $str; - read($self->{file}, $str, 8); - # Set the global $address_length based on what we see here. - # 8 is 32-bit (8 hexadecimal chars); 16 is 64-bit (16 hexadecimal chars). - $address_length = ($str eq (chr(0)x8)) ? 16 : 8; - if ($address_length == 8) { - if (substr($str, 6, 2) eq chr(0)x2) { - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 4, 2) eq chr(0)x2) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**16\n"); - } - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # If we're a 64-bit profile, make sure we're a 64-bit-capable - # perl. Otherwise, each slot will be represented as a float - # instead of an int64, losing precision and making all the - # 64-bit addresses right. We *could* try to handle this with - # software emulation of 64-bit ints, but that's added complexity - # for no clear benefit (yet). We use 'Q' to test for 64-bit-ness; - # perl docs say it's only available on 64-bit perl systems. - my $has_q = 0; - eval { $has_q = pack("Q", "1") ? 1 : 1; }; - if (!$has_q) { - ::error("$fname: need a 64-bit perl to process this 64-bit profile.\n"); - } - read($self->{file}, $str, 8); - if (substr($str, 4, 4) eq chr(0)x4) { - # We'd love to use 'Q', but it's a) not universal, b) not endian-proof. - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 0, 4) eq chr(0)x4) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**32\n"); - } - my @pair = unpack($self->{unpack_code} . "*", $str); - # Since we know one of the pair is 0, it's fine to just add them. - @$slots = (0, $pair[0] + $pair[1]); - } - return $self; - } - - # Load more data when we access slots->get(X) which is not yet in memory. - sub overflow { - my ($self) = @_; - my $slots = $self->{slots}; - $self->{base} += $#$slots + 1; # skip over data we're replacing - my $str; - read($self->{file}, $str, $self->{stride}); - if ($address_length == 8) { # the 32-bit case - # This is the easy case: unpack provides 32-bit unpacking primitives. - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # We need to unpack 32 bits at a time and combine. - my @b32_values = unpack($self->{unpack_code} . "*", $str); - my @b64_values = (); - for (my $i = 0; $i < $#b32_values; $i += 2) { - # TODO(csilvers): if this is a 32-bit perl, the math below - # could end up in a too-large int, which perl will promote - # to a double, losing necessary precision. Deal with that. - if ($self->{unpack_code} eq 'V') { # little-endian - push(@b64_values, $b32_values[$i] + $b32_values[$i+1] * (2**32)); - } else { - push(@b64_values, $b32_values[$i] * (2**32) + $b32_values[$i+1]); - } - } - @$slots = @b64_values; - } - } - - # Access the i-th long in the file (logically), or -1 at EOF. - sub get { - my ($self, $idx) = @_; - my $slots = $self->{slots}; - while ($#$slots >= 0) { - if ($idx < $self->{base}) { - # The only time we expect a reference to $slots[$i - something] - # after referencing $slots[$i] is reading the very first header. - # Since $stride > |header|, that shouldn't cause any lookback - # errors. And everything after the header is sequential. - print STDERR "Unexpected look-back reading CPU profile"; - return -1; # shrug, don't know what better to return - } elsif ($idx > $self->{base} + $#$slots) { - $self->overflow(); - } else { - return $slots->[$idx - $self->{base}]; - } - } - # If we get here, $slots is [], which means we've reached EOF - return -1; # unique since slots is supposed to hold unsigned numbers - } -} - -# Parse profile generated by common/profiler.cc and return a reference -# to a map: -# $result->{version} Version number of profile file -# $result->{period} Sampling period (in microseconds) -# $result->{profile} Profile object -# $result->{map} Memory map info from profile -# $result->{pcs} Hash of all PC values seen, key is hex address -sub ReadProfile { - my $prog = shift; - my $fname = shift; - - if (IsSymbolizedProfileFile($fname) && !$main::use_symbolized_profile) { - # we have both a binary and symbolized profiles, abort - usage("Symbolized profile '$fname' cannot be used with a binary arg. " . - "Try again without passing '$prog'."); - } - - $main::profile_type = ''; - - $CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $contention_marker = $&; - $GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $growth_marker = $&; - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - - # Look at first line to see if it is a heap or a CPU profile. - # CPU profile may start with no header at all, and just binary data - # (starting with \0\0\0\0) -- in that case, don't try to read the - # whole firstline, since it may be gigabytes(!) of data. - open(PROFILE, "<$fname") || error("$fname: $!\n"); - binmode PROFILE; # New perls do UTF-8 processing - my $firstchar = ""; - my $header = ""; - read(PROFILE, $firstchar, 1); - seek(PROFILE, -1, 1); # unread the firstchar - if ($firstchar ne "\0") { - $header = ; - if (!defined($header)) { - error("Profile is empty.\n"); - } - $header =~ s/\r//g; # turn windows-looking lines into unix-looking lines - } - - my $symbols; - if ($header =~ m/^--- *$symbol_marker/o) { - # read the symbol section of the symbolized profile file - $symbols = ReadSymbols(*PROFILE{IO}); - - # read the next line to get the header for the remaining profile - $header = ""; - read(PROFILE, $firstchar, 1); - seek(PROFILE, -1, 1); # unread the firstchar - if ($firstchar ne "\0") { - $header = ; - $header =~ s/\r//g; - } - } - - my $result; - - if ($header =~ m/^heap profile:.*$growth_marker/o) { - $main::profile_type = 'growth'; - $result = ReadHeapProfile($prog, $fname, $header); - } elsif ($header =~ m/^heap profile:/) { - $main::profile_type = 'heap'; - $result = ReadHeapProfile($prog, $fname, $header); - } elsif ($header =~ m/^--- *$contention_marker/o) { - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, $fname); - } elsif ($header =~ m/^--- *Stacks:/) { - print STDERR - "Old format contention profile: mistakenly reports " . - "condition variable signals as lock contentions.\n"; - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, $fname); - } elsif ($header =~ m/^thread creation profile:/) { - $main::profile_type = 'thread'; - $result = ReadThreadProfile($prog, $fname); - } elsif ($header =~ m/^--- *$profile_marker/) { - # the binary cpu profile data starts immediately after this line - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname); - } else { - if (defined($symbols)) { - # a symbolized profile contains a format we don't recognize, bail out - error("$fname: Cannot recognize profile section after symbols.\n"); - } - # no ascii header present -- must be a CPU profile - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname); - } - - # if we got symbols along with the profile, return those as well - if (defined($symbols)) { - $result->{symbols} = $symbols; - } - - return $result; -} - -# Subtract one from caller pc so we map back to call instr. -# However, don't do this if we're reading a symbolized profile -# file, in which case the subtract-one was done when the file -# was written. -# -# We apply the same logic to all readers, though ReadCPUProfile uses an -# independent implementation. -sub FixCallerAddresses { - my $stack = shift; - if ($main::use_symbolized_profile) { - return $stack; - } else { - $stack =~ /(\s)/; - my $delimiter = $1; - my @addrs = split(' ', $stack); - my @fixedaddrs; - $#fixedaddrs = $#addrs; - if ($#addrs >= 0) { - $fixedaddrs[0] = $addrs[0]; - } - for (my $i = 1; $i <= $#addrs; $i++) { - $fixedaddrs[$i] = AddressSub($addrs[$i], "0x1"); - } - return join $delimiter, @fixedaddrs; - } -} - -# CPU profile reader -sub ReadCPUProfile { - my $prog = shift; - my $fname = shift; - my $version; - my $period; - my $i; - my $profile = {}; - my $pcs = {}; - - # Parse string into array of slots. - my $slots = CpuProfileStream->new(*PROFILE, $fname); - - # Read header. The current header version is a 5-element structure - # containing: - # 0: header count (always 0) - # 1: header "words" (after this one: 3) - # 2: format version (0) - # 3: sampling period (usec) - # 4: unused padding (always 0) - if ($slots->get(0) != 0 ) { - error("$fname: not a profile file, or old format profile file\n"); - } - $i = 2 + $slots->get(1); - $version = $slots->get(2); - $period = $slots->get(3); - # Do some sanity checking on these header values. - if ($version > (2**32) || $period > (2**32) || $i > (2**32) || $i < 5) { - error("$fname: not a profile file, or corrupted profile file\n"); - } - - # Parse profile - while ($slots->get($i) != -1) { - my $n = $slots->get($i++); - my $d = $slots->get($i++); - if ($d > (2**16)) { # TODO(csilvers): what's a reasonable max-stack-depth? - my $addr = sprintf("0%o", $i * ($address_length == 8 ? 4 : 8)); - print STDERR "At index $i (address $addr):\n"; - error("$fname: stack trace depth >= 2**32\n"); - } - if ($slots->get($i) == 0) { - # End of profile data marker - $i += $d; - last; - } - - # Make key out of the stack entries - my @k = (); - for (my $j = 0; $j < $d; $j++) { - my $pc = $slots->get($i+$j); - # Subtract one from caller pc so we map back to call instr. - # However, don't do this if we're reading a symbolized profile - # file, in which case the subtract-one was done when the file - # was written. - if ($j > 0 && !$main::use_symbolized_profile) { - $pc--; - } - $pc = sprintf("%0*x", $address_length, $pc); - $pcs->{$pc} = 1; - push @k, $pc; - } - - AddEntry($profile, (join "\n", @k), $n); - $i += $d; - } - - # Parse map - my $map = ''; - seek(PROFILE, $i * 4, 0); - read(PROFILE, $map, (stat PROFILE)[7]); - close(PROFILE); - - my $r = {}; - $r->{version} = $version; - $r->{period} = $period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - - return $r; -} - -sub ReadHeapProfile { - my $prog = shift; - my $fname = shift; - my $header = shift; - - my $index = 1; - if ($main::opt_inuse_space) { - $index = 1; - } elsif ($main::opt_inuse_objects) { - $index = 0; - } elsif ($main::opt_alloc_space) { - $index = 3; - } elsif ($main::opt_alloc_objects) { - $index = 2; - } - - # Find the type of this profile. The header line looks like: - # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 - # There are two pairs , the first inuse objects/space, and the - # second allocated objects/space. This is followed optionally by a profile - # type, and if that is present, optionally by a sampling frequency. - # For remote heap profiles (v1): - # The interpretation of the sampling frequency is that the profiler, for - # each sample, calculates a uniformly distributed random integer less than - # the given value, and records the next sample after that many bytes have - # been allocated. Therefore, the expected sample interval is half of the - # given frequency. By default, if not specified, the expected sample - # interval is 128KB. Only remote-heap-page profiles are adjusted for - # sample size. - # For remote heap profiles (v2): - # The sampling frequency is the rate of a Poisson process. This means that - # the probability of sampling an allocation of size X with sampling rate Y - # is 1 - exp(-X/Y) - # For version 2, a typical header line might look like this: - # heap profile: 1922: 127792360 [ 1922: 127792360] @ _v2/524288 - # the trailing number (524288) is the sampling rate. (Version 1 showed - # double the 'rate' here) - my $sampling_algorithm = 0; - my $sample_adjustment = 0; - chomp($header); - my $type = "unknown"; - if ($header =~ m"^heap profile:\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\](\s*@\s*([^/]*)(/(\d+))?)?") { - if (defined($6) && ($6 ne '')) { - $type = $6; - my $sample_period = $8; - # $type is "heapprofile" for profiles generated by the - # heap-profiler, and either "heap" or "heap_v2" for profiles - # generated by sampling directly within tcmalloc. It can also - # be "growth" for heap-growth profiles. The first is typically - # found for profiles generated locally, and the others for - # remote profiles. - if (($type eq "heapprofile") || ($type !~ /heap/) ) { - # No need to adjust for the sampling rate with heap-profiler-derived data - $sampling_algorithm = 0; - } elsif ($type =~ /_v2/) { - $sampling_algorithm = 2; # version 2 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period); - } - } else { - $sampling_algorithm = 1; # version 1 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period)/2; - } - } - } else { - # We detect whether or not this is a remote-heap profile by checking - # that the total-allocated stats ($n2,$s2) are exactly the - # same as the in-use stats ($n1,$s1). It is remotely conceivable - # that a non-remote-heap profile may pass this check, but it is hard - # to imagine how that could happen. - # In this case it's so old it's guaranteed to be remote-heap version 1. - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - if (($n1 == $n2) && ($s1 == $s2)) { - # This is likely to be a remote-heap based sample profile - $sampling_algorithm = 1; - } - } - } - - if ($sampling_algorithm > 0) { - # For remote-heap generated profiles, adjust the counts and sizes to - # account for the sample rate (we sample once every 128KB by default). - if ($sample_adjustment == 0) { - # Turn on profile adjustment. - $sample_adjustment = 128*1024; - print STDERR "Adjusting heap profiles for 1-in-128KB sampling rate\n"; - } else { - printf STDERR ("Adjusting heap profiles for 1-in-%d sampling rate\n", - $sample_adjustment); - } - if ($sampling_algorithm > 1) { - # We don't bother printing anything for the original version (version 1) - printf STDERR "Heap version $sampling_algorithm\n"; - } - } - - my $profile = {}; - my $pcs = {}; - my $map = ""; - - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } - last; - } - - if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } - last; - } - - # Read entry of the form: - # : [: ] @ a1 a2 a3 ... an - s/^\s*//; - s/\s*$//; - if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { - my $stack = $5; - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - - if ($sample_adjustment) { - if ($sampling_algorithm == 2) { - # Remote-heap version 2 - # The sampling frequency is the rate of a Poisson process. - # This means that the probability of sampling an allocation of - # size X with sampling rate Y is 1 - exp(-X/Y) - my $ratio; - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor; - $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } else { - # Remote-heap version 1 - my $ratio; - if ($n1 > 0) { - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - if ($ratio < 1) { - $n1 /= $ratio; - $s1 /= $ratio; - } - } - if ($n2 > 0) { - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - if ($ratio < 1) { - $n2 /= $ratio; - $s2 /= $ratio; - } - } - } - } - - my @counts = ($n1, $s1, $n2, $s2); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); - } - } - - my $r = {}; - $r->{version} = "heap"; - $r->{period} = 1; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -sub ReadThreadProfile { - my $prog = shift; - my $fname = shift; - - my $profile = {}; - my $pcs = {}; - my $map = ""; - - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } - last; - } - - if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } - last; - } - - # Read entry of the form: - # @ a1 a2 a3 ... an - s/^\s*//; - s/\s*$//; - if (m/^@\s+(.*)$/) { - AddEntries($profile, $pcs, FixCallerAddresses($1), 1); - } - } - - my $r = {}; - $r->{version} = "thread"; - $r->{period} = 1; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -sub ReadSynchProfile { - my ($prog, $fname, $header) = @_; - - my $map = ''; - my $profile = {}; - my $pcs = {}; - my $sampling_period = 1; - my $cyclespernanosec = 2.8; # Default assumption for old binaries - my $seen_clockrate = 0; - my $line; - - my $index = 0; - if ($main::opt_total_delay) { - $index = 0; - } elsif ($main::opt_contentions) { - $index = 1; - } elsif ($main::opt_mean_delay) { - $index = 2; - } - - while ( $line = ) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ( $line =~ /^\s*(\d+)\s+(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $count, $stack) = ($1, $2, $3); - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - $count *= $sampling_period; - - my @values = ($cycles, $count, $cycles / $count); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $values[$index]); - - } elsif ( $line =~ /^(slow release).*thread \d+ \@\s*(.*?)\s*$/ || - $line =~ /^\s*(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $stack) = ($1, $2); - if ($cycles !~ /^\d+$/) { - next; - } - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - - AddEntries($profile, $pcs, FixCallerAddresses($stack), $cycles); - - } elsif ( $line =~ m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1,$2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "cycles/second") { - $cyclespernanosec = $value / 1e9; - $seen_clockrate = 1; - } elsif ($variable eq "sampling period") { - $sampling_period = $value; - } elsif ($variable eq "ms since reset") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } elsif ($variable eq "discarded samples") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } else { - printf STDERR ("Ignoring unnknown variable in /contention output: " . - "'%s' = '%s'\n",$variable,$value); - } - } else { - # Memory map entry - $map .= $line; - } - } - close PROFILE; - - if (!$seen_clockrate) { - printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n", - $cyclespernanosec); - } - - my $r = {}; - $r->{version} = 0; - $r->{period} = $sampling_period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... -sub HexExtend { - my $addr = shift; - - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; - } - - return substr("000000000000000".$addr, -$address_length); -} - -##### Symbol extraction ##### - -# Aggressively search the lib_prefix values for the given library -# If all else fails, just return the name of the library unmodified. -# If the lib_prefix is "/my/path,/other/path" and $file is "/lib/dir/mylib.so" -# it will search the following locations in this order, until it finds a file: -# /my/path/lib/dir/mylib.so -# /other/path/lib/dir/mylib.so -# /my/path/dir/mylib.so -# /other/path/dir/mylib.so -# /my/path/mylib.so -# /other/path/mylib.so -# /lib/dir/mylib.so (returned as last resort) -sub FindLibrary { - my $file = shift; - my $suffix = $file; - - # Search for the library as described above - do { - foreach my $prefix (@prefix_list) { - my $fullpath = $prefix . $suffix; - if (-e $fullpath) { - return $fullpath; - } - } - } while ($suffix =~ s|^/[^/]+/|/|); - return $file; -} - -# Return path to library with debugging symbols. -# For libc libraries, the copy in /usr/lib/debug contains debugging symbols -sub DebuggingLibrary { - my $file = shift; - if ($file =~ m|^/| && -f "/usr/lib/debug$file") { - return "/usr/lib/debug$file"; - } - return undef; -} - -# Parse text section header of a library using objdump -sub ParseTextSectionHeaderFromObjdump { - my $lib = shift; - - my $size = undef; - my $vma; - my $file_offset; - # Get objdump output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Idx Name Size VMA LMA File off Algn - # 10 .text 00104b2c 420156f0 420156f0 000156f0 2**4 - # For 64-bit objects, VMA and LMA will be 16 hex digits, size and file - # offset may still be 8. But AddressSub below will still handle that. - my @x = split; - if (($#x >= 6) && ($x[1] eq '.text')) { - $size = $x[2]; - $vma = $x[3]; - $file_offset = $x[5]; - last; - } - } - close(OBJDUMP); - - if (!defined($size)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -# Parse text section header of a library using otool (on OS X) -sub ParseTextSectionHeaderFromOtool { - my $lib = shift; - - my $size = undef; - my $vma = undef; - my $file_offset = undef; - # Get otool output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); - my $cmd = ""; - my $sectname = ""; - my $segname = ""; - foreach my $line () { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - # Load command <#> - # cmd LC_SEGMENT - # [...] - # Section - # sectname __text - # segname __TEXT - # addr 0x000009f8 - # size 0x00018b9e - # offset 2552 - # align 2^2 (4) - # We will need to strip off the leading 0x from the hex addresses, - # and convert the offset into hex. - if ($line =~ /Load command/) { - $cmd = ""; - $sectname = ""; - $segname = ""; - } elsif ($line =~ /Section/) { - $sectname = ""; - $segname = ""; - } elsif ($line =~ /cmd (\w+)/) { - $cmd = $1; - } elsif ($line =~ /sectname (\w+)/) { - $sectname = $1; - } elsif ($line =~ /segname (\w+)/) { - $segname = $1; - } elsif (!(($cmd eq "LC_SEGMENT" || $cmd eq "LC_SEGMENT_64") && - $sectname eq "__text" && - $segname eq "__TEXT")) { - next; - } elsif ($line =~ /\baddr 0x([0-9a-fA-F]+)/) { - $vma = $1; - } elsif ($line =~ /\bsize 0x([0-9a-fA-F]+)/) { - $size = $1; - } elsif ($line =~ /\boffset ([0-9]+)/) { - $file_offset = sprintf("%016x", $1); - } - if (defined($vma) && defined($size) && defined($file_offset)) { - last; - } - } - close(OTOOL); - - if (!defined($vma) || !defined($size) || !defined($file_offset)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -sub ParseTextSectionHeader { - # obj_tool_map("otool") is only defined if we're in a Mach-O environment - if (defined($obj_tool_map{"otool"})) { - my $r = ParseTextSectionHeaderFromOtool(@_); - if (defined($r)){ - return $r; - } - } - # If otool doesn't work, or we don't have it, fall back to objdump - return ParseTextSectionHeaderFromObjdump(@_); -} - -# Split /proc/pid/maps dump into a list of libraries -sub ParseLibraries { - return if $main::use_symbol_page; # We don't need libraries info. - my $prog = shift; - my $map = shift; - my $pcs = shift; - - my $result = []; - my $h = "[a-f0-9]+"; - my $zero_offset = HexExtend("0"); - - my $buildvar = ""; - foreach my $l (split("\n", $map)) { - if ($l =~ m/^\s*build=(.*)$/) { - $buildvar = $1; - } - - my $start; - my $finish; - my $offset; - my $lib; - if ($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?)$/i) { - # Full line from /proc/self/maps. Example: - # 40000000-40015000 r-xp 00000000 03:01 12845071 /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = HexExtend($3); - $lib = $4; - $lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths - } elsif ($l =~ /^\s*($h)-($h):\s*(\S+\.so(\.\d+)*)/) { - # Cooked line from DumpAddressMap. Example: - # 40000000-40015000: /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = $zero_offset; - $lib = $3; - } else { - next; - } - - # Expand "$build" variable if available - $lib =~ s/\$build\b/$buildvar/g; - - $lib = FindLibrary($lib); - - # Check for pre-relocated libraries, which use pre-relocated symbol tables - # and thus require adjusting the offset that we'll use to translate - # VM addresses into symbol table addresses. - # Only do this if we're not going to fetch the symbol table from a - # debugging copy of the library. - if (!DebuggingLibrary($lib)) { - my $text = ParseTextSectionHeader($lib); - if (defined($text)) { - my $vma_offset = AddressSub($text->{vma}, $text->{file_offset}); - $offset = AddressAdd($offset, $vma_offset); - } - } - - push(@{$result}, [$lib, $start, $finish, $offset]); - } - - # Append special entry for additional library (not relocated) - if ($main::opt_lib ne "") { - my $text = ParseTextSectionHeader($main::opt_lib); - if (defined($text)) { - my $start = $text->{vma}; - my $finish = AddressAdd($start, $text->{size}); - - push(@{$result}, [$main::opt_lib, $start, $finish, $start]); - } - } - - # Append special entry for the main program. This covers - # 0..max_pc_value_seen, so that we assume pc values not found in one - # of the library ranges will be treated as coming from the main - # program binary. - my $min_pc = HexExtend("0"); - my $max_pc = $min_pc; # find the maximal PC value in any sample - foreach my $pc (keys(%{$pcs})) { - if (HexExtend($pc) gt $max_pc) { $max_pc = HexExtend($pc); } - } - push(@{$result}, [$prog, $min_pc, $max_pc, $zero_offset]); - - return $result; -} - -# Add two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressAdd { - my $addr1 = shift; - my $addr2 = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr1)+hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - - if ($main::opt_debug and $main::opt_test) { - print STDERR "AddressAdd $addr1 + $addr2 = "; - } - - my $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - my $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2); - my $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - my $r = sprintf("%07x", $sum); - - $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2) + $c; - $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - $r = sprintf("%07x", $sum) . $r; - - $sum = hex($addr1) + hex($addr2) + $c; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - if ($main::opt_debug and $main::opt_test) { print STDERR "$r\n"; } - - return $r; - } -} - - -# Subtract two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressSub { - my $addr1 = shift; - my $addr2 = shift; - my $diff; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $diff = (hex($addr1)-hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $diff); - - } else { - # Do the addition in 7-nibble chunks to trivialize borrow handling. - # if ($main::opt_debug) { print STDERR "AddressSub $addr1 - $addr2 = "; } - - my $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - my $a2 = hex(substr($addr2,-7)); - $addr2 = substr($addr2,0,-7); - my $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - my $r = sprintf("%07x", $diff); - - $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - $a2 = hex(substr($addr2,-7)) + $b; - $addr2 = substr($addr2,0,-7); - $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - $r = sprintf("%07x", $diff) . $r; - - $a1 = hex($addr1); - $a2 = hex($addr2) + $b; - if ($a2 > $a1) { $a1 += 0x100; } - $diff = $a1 - $a2; - $r = sprintf("%02x", $diff) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - - return $r; - } -} - -# Increment a hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressInc { - my $addr = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr)+1) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - # We are always doing this to step through the addresses in a function, - # and will almost never overflow the first chunk, so we check for this - # case and exit early. - - # if ($main::opt_debug) { print STDERR "AddressInc $addr1 = "; } - - my $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - my $r = sprintf("%07x", $sum); - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "0000000"; - } - - $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - $r = sprintf("%07x", $sum) . $r; - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "00000000000000"; - } - - $sum = hex($addr) + 1; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - return $r; - } -} - -# Extract symbols for all PC values found in profile -sub ExtractSymbols { - my $libs = shift; - my $pcset = shift; - - my $symbols = {}; - - # Map each PC value to the containing library - my %seen = (); - foreach my $lib (@{$libs}) { - my $libname = $lib->[0]; - my $start = $lib->[1]; - my $finish = $lib->[2]; - my $offset = $lib->[3]; - - # Get list of pcs that belong in this library. - my $contained = []; - foreach my $pc (keys(%{$pcset})) { - if (!$seen{$pc} && ($pc ge $start) && ($pc le $finish)) { - $seen{$pc} = 1; - push(@{$contained}, $pc); - } - } - # Map to symbols - MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); - } - - return $symbols; -} - -# Map list of PC values to symbols for a given image -sub MapToSymbols { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - my $debug = 0; - - # Ignore empty binaries - if ($#{$pclist} < 0) { return; } - - # Figure out the addr2line command to use - my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; - if (exists $obj_tool_map{"addr2line_pdb"}) { - $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; - } - - # Use the go version because we know it works on all platforms - $addr2line = "go tool addr2line"; - $cmd = "$addr2line $image"; - - # If "addr2line" isn't installed on the system at all, just use - # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >$DEVNULL 2>&1") != 0) { - MapSymbolsWithNM($image, $offset, $pclist, $symbols); - return; - } - - # "addr2line -i" can produce a variable number of lines per input - # address, with no separator that allows us to tell when data for - # the next address starts. So we find the address for a special - # symbol (_fini) and interleave this address between all real - # addresses passed to addr2line. The name of this special symbol - # can then be used as a separator. - $sep_address = undef; # May be filled in by MapSymbolsWithNM() - my $nm_symbols = {}; - MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. - if (defined($sep_address)) { - # Only add " -i" to addr2line if the binary supports it. - # addr2line --help returns 0, but not if it sees an unknown flag first. - if (system("$cmd -i --help >$DEVNULL 2>&1") == 0) { - $cmd .= " -i"; - } else { - $sep_address = undef; # no need for sep_address if we don't support -i - } - } - - # Make file with all PC values with intervening 'sep_address' so - # that we can reliably detect the end of inlined function list - open(ADDRESSES, ">$main::tmpfile_sym") || error("$main::tmpfile_sym: $!\n"); - if ($debug) { print("---- $image ---\n"); } - for (my $i = 0; $i <= $#{$pclist}; $i++) { - # addr2line always reads hex addresses, and does not need '0x' prefix. - if ($debug) { printf STDERR ("%s\n", $pclist->[$i]); } - printf ADDRESSES ("%s\n", AddressSub($pclist->[$i], $offset)); - if (defined($sep_address)) { - printf ADDRESSES ("%s\n", $sep_address); - } - } - close(ADDRESSES); - if ($debug) { - print("----\n"); - system("cat $main::tmpfile_sym"); - print("---- $cmd\n"); - system("$cmd <$main::tmpfile_sym"); - print("----\n"); - } - - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); - my $count = 0; # Index in pclist - while () { - # Read fullfunction and filelineinfo from next pair of lines - s/\r?\n$//g; - my $fullfunction = $_; - $_ = ; - s/\r?\n$//g; - my $filelinenum = $_; - - if (defined($sep_address) && $fullfunction eq $sep_symbol) { - # Terminating marker for data for this address - $count++; - next; - } - - $filelinenum =~ s|\\|/|g; # turn windows-style paths into unix-style paths - - my $pcstr = $pclist->[$count]; - my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { - $function = $nms->[0]; - $fullfunction = $nms->[2]; - } - } - - # Prepend to accumulated symbols for pcstr - # (so that caller comes before callee) - my $sym = $symbols->{$pcstr}; - if (!defined($sym)) { - $sym = []; - $symbols->{$pcstr} = $sym; - } - unshift(@{$sym}, $function, $filelinenum, $fullfunction); - if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } - if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately - $count++; - } - } - close(SYMBOLS); -} - -# Use nm to map the list of referenced PCs to symbols. Return true iff we -# are able to read procedure information via nm. -sub MapSymbolsWithNM { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - # Get nm output sorted by increasing address - my $symbol_table = GetProcedureBoundaries($image, "."); - if (!%{$symbol_table}) { - return 0; - } - # Start addresses are already the right length (8 or 16 hex digits). - my @names = sort { $symbol_table->{$a}->[0] cmp $symbol_table->{$b}->[0] } - keys(%{$symbol_table}); - - if ($#names < 0) { - # No symbols: just use addresses - foreach my $pc (@{$pclist}) { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - return 0; - } - - # Sort addresses so we can do a join against nm output - my $index = 0; - my $fullname = $names[0]; - my $name = ShortFunctionName($fullname); - foreach my $pc (sort { $a cmp $b } @{$pclist}) { - # Adjust for mapped offset - my $mpc = AddressSub($pc, $offset); - while (($index < $#names) && ($mpc ge $symbol_table->{$fullname}->[1])){ - $index++; - $fullname = $names[$index]; - $name = ShortFunctionName($fullname); - } - if ($mpc lt $symbol_table->{$fullname}->[1]) { - $symbols->{$pc} = [$name, "?", $fullname]; - } else { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - } - return 1; -} - -sub ShortFunctionName { - my $function = shift; - while ($function =~ s/(?]*>//g) { } # Remove template arguments - $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type - return $function; -} - -# Trim overly long symbols found in disassembler output -sub CleanDisassembly { - my $d = shift; - while ($d =~ s/(?]*>/$1/g) { } # Remove template arguments - return $d; -} - -##### Miscellaneous ##### - -# Find the right versions of the above object tools to use. The -# argument is the program file being analyzed, and should be an ELF -# 32-bit or ELF 64-bit executable file. The location of the tools -# is determined by considering the following options in this order: -# 1) --tools option, if set -# 2) PPROF_TOOLS environment variable, if set -# 3) the environment -sub ConfigureObjTools { - my $prog_file = shift; - - # Check for the existence of $prog_file because /usr/bin/file does not - # predictably return error status in prod. - (-e $prog_file) || error("$prog_file does not exist.\n"); - - # Follow symlinks (at least for systems where "file" supports that) - my $file_cmd = "/usr/bin/file -L $prog_file 2>$DEVNULL || /usr/bin/file $prog_file 2>$DEVNULL"; - if ($^O eq "MSWin32") { - $file_cmd = "file -L $prog_file 2>NUL || file $prog_file 2>NUL"; - } - my $file_type = `$file_cmd`; - - if ($file_type =~ /64-bit/) { - # Change $address_length to 16 if the program file is ELF 64-bit. - # We can't detect this from many (most?) heap or lock contention - # profiles, since the actual addresses referenced are generally in low - # memory even for 64-bit programs. - $address_length = 16; - } - - if (($file_type =~ /MS Windows/) || ($OS eq "windows")) { - # For windows, we provide a version of nm and addr2line as part of - # the opensource release, which is capable of parsing - # Windows-style PDB executables. It should live in the path, or - # in the same directory as pprof. - $obj_tool_map{"nm_pdb"} = "nm-pdb"; - $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; - $obj_tool_map{"objdump"} = "false"; # no objdump - } - - if ($file_type =~ /Mach-O/) { - # OS X uses otool to examine Mach-O files, rather than objdump. - $obj_tool_map{"otool"} = "otool"; - $obj_tool_map{"addr2line"} = "false"; # no addr2line - $obj_tool_map{"objdump"} = "false"; # no objdump - } - - # Go fill in %obj_tool_map with the pathnames to use: - foreach my $tool (keys %obj_tool_map) { - $obj_tool_map{$tool} = ConfigureTool($obj_tool_map{$tool}); - } -} - -# Returns the path of a caller-specified object tool. If --tools or -# PPROF_TOOLS are specified, then returns the full path to the tool -# with that prefix. Otherwise, returns the path unmodified (which -# means we will look for it on PATH). -sub ConfigureTool { - my $tool = shift; - my $path; - - if ($main::opt_tools ne "") { - # Use a prefix specified by the --tools option... - $path = $main::opt_tools . $tool; - if (!-x $path) { - error("No '$tool' found with prefix specified by --tools $main::opt_tools\n"); - } - } elsif (exists $ENV{"PPROF_TOOLS"} && - $ENV{"PPROF_TOOLS"} ne "") { - #... or specified with the PPROF_TOOLS environment variable... - $path = $ENV{"PPROF_TOOLS"} . $tool; - if (!-x $path) { - error("No '$tool' found with prefix specified by PPROF_TOOLS=$ENV{PPROF_TOOLS}\n"); - } - } else { - # ... otherwise use the version that exists in the same directory as - # pprof. If there's nothing there, use $PATH. - $0 =~ m,[^/]*$,; # this is everything after the last slash - my $dirname = $`; # this is everything up to and including the last slash - if (-x "$dirname$tool") { - $path = "$dirname$tool"; - } else { - $path = $tool; - } - } - if ($main::opt_debug) { print STDERR "Using '$path' for '$tool'.\n"; } - return $path; -} - -# FetchHTTP retrieves a URL using either curl or LWP::UserAgent. -# It returns the entire body of the page on success, or exits the program -# with an error message on any failure. -sub FetchHTTP { - my $url = shift; - my $timeout = shift; # optional, in seconds - eval "use LWP::UserAgent ();"; - if ($@) { - my @max; - push @max, "--max-time", $timeout if $timeout; - open(my $fh, "-|", "curl", @max, "-s", $url) or error("Neither LWP::UserAgent nor curl is installed: $!\n"); - my $slurp = do { local $/; <$fh> }; - close($fh); - if ($? != 0) { - error("Error fetching $url with curl: exit $?") - } - return $slurp; - } - my $ua = LWP::UserAgent->new; - $ua->timeout($timeout) if $timeout; - my $res = $ua->get($url); - error("Failed to fetch $url\n") unless $res->is_success(); - return $res->content(); -} - -sub PostHTTP { - my ($url, $post_data) = @_; - eval "use LWP::UserAgent ();"; - if ($@) { - open(POSTFILE, ">$main::tmpfile_sym"); - print POSTFILE $post_data; - close(POSTFILE); - - open(my $fh, "-|", "curl", "-s", "-d", "\@$main::tmpfile_sym", $url) or error("Neither LWP::UserAgent nor curl is installed: $!\n"); - my $slurp = do { local $/; <$fh> }; - close($fh); - if ($? != 0) { - error("Error fetching $url with curl: exit $?") - } - return $slurp; - } - my $req = HTTP::Request->new(POST => $url); - $req->content($post_data); - my $ua = LWP::UserAgent->new; - my $res = $ua->request($req); - error("Failed to POST to $url\n") unless $res->is_success(); - return $res->content(); -} - -sub cleanup { - unlink($main::tmpfile_sym) if defined $main::tmpfile_sym; - unlink(keys %main::tempnames) if %main::tempnames; - unlink($main::collected_profile) if defined $main::collected_profile; - - # We leave any collected profiles in $HOME/pprof in case the user wants - # to look at them later. We print a message informing them of this. - if ((scalar(@main::profile_files) > 0) && - defined($main::collected_profile)) { - if (scalar(@main::profile_files) == 1) { - print STDERR "Dynamically gathered profile is in $main::collected_profile\n"; - } - print STDERR "If you want to investigate this profile further, you can do:\n"; - print STDERR "\n"; - print STDERR " pprof \\\n"; - print STDERR " $main::prog \\\n"; - print STDERR " $main::collected_profile\n"; - print STDERR "\n"; - } -} - -sub sighandler { - cleanup(); - exit(1); -} - -sub error { - my $msg = shift; - print STDERR $msg; - cleanup(); - exit(1); -} - - -# Run $nm_command and get all the resulting procedure boundaries whose -# names match "$regexp" and returns them in a hashtable mapping from -# procedure name to a two-element vector of [start address, end address] -sub GetProcedureBoundariesViaNm { - my $nm_command = shift; - my $regexp = shift; - - my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); - my $last_start = "0"; - my $routine = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (m/^\s*([0-9a-f]+) (.) (..*)/) { - my $start_val = $1; - my $type = $2; - my $this_routine = $3; - - # It's possible for two symbols to share the same address, if - # one is a zero-length variable (like __start_google_malloc) or - # one symbol is a weak alias to another (like __libc_malloc). - # In such cases, we want to ignore all values except for the - # actual symbol, which in nm-speak has type "T". The logic - # below does this, though it's a bit tricky: what happens when - # we have a series of lines with the same address, is the first - # one gets queued up to be processed. However, it won't - # *actually* be processed until later, when we read a line with - # a different address. That means that as long as we're reading - # lines with the same address, we have a chance to replace that - # item in the queue, which we do whenever we see a 'T' entry -- - # that is, a line with type 'T'. If we never see a 'T' entry, - # we'll just go ahead and process the first entry (which never - # got touched in the queue), and ignore the others. - if ($start_val eq $last_start && $type =~ /t/i) { - # We are the 'T' symbol at this address, replace previous symbol. - $routine = $this_routine; - next; - } elsif ($start_val eq $last_start) { - # We're not the 'T' symbol at this address, so ignore us. - next; - } - - if ($this_routine eq $sep_symbol) { - $sep_address = HexExtend($start_val); - } - - # Tag this routine with the starting address in case the image - # has multiple occurrences of this routine. We use a syntax - # that resembles template paramters that are automatically - # stripped out by ShortFunctionName() - $this_routine .= "<$start_val>"; - - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($start_val)]; - } - $last_start = $start_val; - $routine = $this_routine; - } elsif (m/^Loaded image name: (.+)/) { - # The win32 nm workalike emits information about the binary it is using. - if ($main::opt_debug) { print STDERR "Using Image $1\n"; } - } elsif (m/^PDB file name: (.+)/) { - # The win32 nm workalike emits information about the pdb it is using. - if ($main::opt_debug) { print STDERR "Using PDB $1\n"; } - } - } - close(NM); - # Handle the last line in the nm output. Unfortunately, we don't know - # how big this last symbol is, because we don't know how big the file - # is. For now, we just give it a size of 0. - # TODO(csilvers): do better here. - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($last_start)]; - } - return $symbol_table; -} - -# Gets the procedure boundaries for all routines in "$image" whose names -# match "$regexp" and returns them in a hashtable mapping from procedure -# name to a two-element vector of [start address, end address]. -# Will return an empty map if nm is not installed or not working properly. -sub GetProcedureBoundaries { - my $image = shift; - my $regexp = shift; - - # For libc libraries, the copy in /usr/lib/debug contains debugging symbols - my $debugging = DebuggingLibrary($image); - if ($debugging) { - $image = $debugging; - } - - my $nm = $obj_tool_map{"nm"}; - - # nm can fail for two reasons: 1) $image isn't a debug library; 2) nm - # binary doesn't support --demangle. In addition, for OS X we need - # to use the -f flag to get 'flat' nm output (otherwise we don't sort - # properly and get incorrect results). Unfortunately, GNU nm uses -f - # in an incompatible way. So first we test whether our nm supports - # --demangle and -f. - my $demangle_flag = ""; - if (system("$nm --demangle $image >$DEVNULL 2>&1") == 0) { - # In this mode, we do "nm --demangle " - $demangle_flag = "--demangle"; - } - my $flatten_flag = ""; - if (system("$nm -f $image >$DEVNULL 2>&1") == 0) { - $flatten_flag = "-f"; - } - - # Finally, in the case $image isn't a debug library, we try again with - # -D to at least get *exported* symbols. If we can't use --demangle, too bad. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>$DEVNULL", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>$DEVNULL", - # go tool nm is for Go binaries - "go tool nm $image 2>$DEVNULL | sort"); - - foreach my $nm_command (@nm_commands) { - my $symbol_table = GetProcedureBoundariesViaNm($nm_command, $regexp); - return $symbol_table if (%{$symbol_table}); - } - my $symbol_table = {}; - return $symbol_table; -} - - -# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. -# To make them more readable, we add underscores at interesting places. -# This routine removes the underscores, producing the canonical representation -# used by pprof to represent addresses, particularly in the tested routines. -sub CanonicalHex { - my $arg = shift; - return join '', (split '_',$arg); -} - - -# Unit test for AddressAdd: -sub AddressAddUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressAddUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd ($row->[0], $row->[1]); - if ($sum ne $row->[2]) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - my $expected = join '', (split '_',$row->[2]); - if ($sum ne CanonicalHex($row->[2])) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressSub: -sub AddressSubUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressSubUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub ($row->[0], $row->[1]); - if ($sum ne $row->[3]) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - if ($sum ne CanonicalHex($row->[3])) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressInc: -sub AddressIncUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressIncUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc ($row->[0]); - if ($sum ne $row->[4]) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc (CanonicalHex($row->[0])); - if ($sum ne CanonicalHex($row->[4])) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Driver for unit tests. -# Currently just the address add/subtract/increment routines for 64-bit. -sub RunUnitTests { - my $error_count = 0; - - # This is a list of tuples [a, b, a+b, a-b, a+1] - my $unit_test_data_8 = [ - [qw(aaaaaaaa 50505050 fafafafa 5a5a5a5a aaaaaaab)], - [qw(50505050 aaaaaaaa fafafafa a5a5a5a6 50505051)], - [qw(ffffffff aaaaaaaa aaaaaaa9 55555555 00000000)], - [qw(00000001 ffffffff 00000000 00000002 00000002)], - [qw(00000001 fffffff0 fffffff1 00000011 00000002)], - ]; - my $unit_test_data_16 = [ - # The implementation handles data in 7-nibble chunks, so those are the - # interesting boundaries. - [qw(aaaaaaaa 50505050 - 00_000000f_afafafa 00_0000005_a5a5a5a 00_000000a_aaaaaab)], - [qw(50505050 aaaaaaaa - 00_000000f_afafafa ff_ffffffa_5a5a5a6 00_0000005_0505051)], - [qw(ffffffff aaaaaaaa - 00_000001a_aaaaaa9 00_0000005_5555555 00_0000010_0000000)], - [qw(00000001 ffffffff - 00_0000010_0000000 ff_ffffff0_0000002 00_0000000_0000002)], - [qw(00000001 fffffff0 - 00_000000f_ffffff1 ff_ffffff0_0000011 00_0000000_0000002)], - - [qw(00_a00000a_aaaaaaa 50505050 - 00_a00000f_afafafa 00_a000005_a5a5a5a 00_a00000a_aaaaaab)], - [qw(0f_fff0005_0505050 aaaaaaaa - 0f_fff000f_afafafa 0f_ffefffa_5a5a5a6 0f_fff0005_0505051)], - [qw(00_000000f_fffffff 01_800000a_aaaaaaa - 01_800001a_aaaaaa9 fe_8000005_5555555 00_0000010_0000000)], - [qw(00_0000000_0000001 ff_fffffff_fffffff - 00_0000000_0000000 00_0000000_0000002 00_0000000_0000002)], - [qw(00_0000000_0000001 ff_fffffff_ffffff0 - ff_fffffff_ffffff1 00_0000000_0000011 00_0000000_0000002)], - ]; - - $error_count += AddressAddUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressSubUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressIncUnitTest($unit_test_data_8, $unit_test_data_16); - if ($error_count > 0) { - print STDERR $error_count, " errors: FAILED\n"; - } else { - print STDERR "PASS\n"; - } - exit ($error_count); -} diff --git a/src/bufio/scan.go b/src/bufio/scan.go index a41451524d..364d159613 100644 --- a/src/bufio/scan.go +++ b/src/bufio/scan.go @@ -36,6 +36,7 @@ type Scanner struct { start int // First non-processed byte in buf. end int // End of data in buf. err error // Sticky error. + empties int // Count of successive empty tokens. } // SplitFunc is the signature of the split function used to tokenize the @@ -108,6 +109,8 @@ func (s *Scanner) Text() string { // After Scan returns false, the Err method will return any error that // occurred during scanning, except that if it was io.EOF, Err // will return nil. +// Split panics if the split function returns 100 empty tokens without +// advancing the input. This is a common error mode for scanners. func (s *Scanner) Scan() bool { // Loop until we have a token. for { @@ -125,6 +128,15 @@ func (s *Scanner) Scan() bool { } s.token = token if token != nil { + if s.err == nil || advance > 0 { + s.empties = 0 + } else { + // Returning tokens not advancing input at EOF. + s.empties++ + if s.empties > 100 { + panic("bufio.Scan: 100 empty tokens without progressing") + } + } return true } } @@ -172,6 +184,7 @@ func (s *Scanner) Scan() bool { break } if n > 0 { + s.empties = 0 break } loop++ diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go index 1454a8113c..eea87cbf7b 100644 --- a/src/bufio/scan_test.go +++ b/src/bufio/scan_test.go @@ -455,3 +455,70 @@ func TestEmptyTokens(t *testing.T) { t.Fatal(err) } } + +func loopAtEOFSplit(data []byte, atEOF bool) (advance int, token []byte, err error) { + if len(data) > 0 { + return 1, data[:1], nil + } + return 0, data, nil +} + +func TestDontLoopForever(t *testing.T) { + s := NewScanner(strings.NewReader("abc")) + s.Split(loopAtEOFSplit) + // Expect a panic + defer func() { + err := recover() + if err == nil { + t.Fatal("should have panicked") + } + if msg, ok := err.(string); !ok || !strings.Contains(msg, "empty tokens") { + panic(err) + } + }() + for count := 0; s.Scan(); count++ { + if count > 1000 { + t.Fatal("looping") + } + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } +} + +func TestBlankLines(t *testing.T) { + s := NewScanner(strings.NewReader(strings.Repeat("\n", 1000))) + for count := 0; s.Scan(); count++ { + if count > 2000 { + t.Fatal("looping") + } + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } +} + +type countdown int + +func (c *countdown) split(data []byte, atEOF bool) (advance int, token []byte, err error) { + if *c > 0 { + *c-- + return 1, data[:1], nil + } + return 0, nil, nil +} + +// Check that the looping-at-EOF check doesn't trigger for merely empty tokens. +func TestEmptyLinesOK(t *testing.T) { + c := countdown(10000) + s := NewScanner(strings.NewReader(strings.Repeat("\n", 10000))) + s.Split(c.split) + for s.Scan() { + } + if s.Err() != nil { + t.Fatal("after scan:", s.Err()) + } + if c != 0 { + t.Fatalf("stopped with %d left to process", c) + } +} diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c index 8e49a2d9c8..27d9d3e8be 100644 --- a/src/cmd/5g/reg.c +++ b/src/cmd/5g/reg.c @@ -199,7 +199,7 @@ regopt(Prog *firstp) proginfo(&info, p); // Avoid making variables for direct-called functions. - if(p->as == ABL && p->to.type == D_EXTERN) + if(p->as == ABL && p->to.name == D_EXTERN) continue; bit = mkvar(r, &p->from); diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index d77d56c22a..abdd369d71 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -944,6 +944,8 @@ type typeConv struct { // Map from types to incomplete pointers to those types. ptrs map[dwarf.Type][]*Type + // Keys of ptrs in insertion order (deterministic worklist) + ptrKeys []dwarf.Type // Predeclared types. bool ast.Expr @@ -1061,16 +1063,17 @@ func (tr *TypeRepr) Set(repr string, fargs ...interface{}) { func (c *typeConv) FinishType(pos token.Pos) { // Completing one pointer type might produce more to complete. // Keep looping until they're all done. - for len(c.ptrs) > 0 { - for dtype := range c.ptrs { - // Note Type might invalidate c.ptrs[dtype]. - t := c.Type(dtype, pos) - for _, ptr := range c.ptrs[dtype] { - ptr.Go.(*ast.StarExpr).X = t.Go - ptr.C.Set("%s*", t.C) - } - delete(c.ptrs, dtype) + for len(c.ptrKeys) > 0 { + dtype := c.ptrKeys[0] + c.ptrKeys = c.ptrKeys[1:] + + // Note Type might invalidate c.ptrs[dtype]. + t := c.Type(dtype, pos) + for _, ptr := range c.ptrs[dtype] { + ptr.Go.(*ast.StarExpr).X = t.Go + ptr.C.Set("%s*", t.C) } + c.ptrs[dtype] = nil // retain the map key } } @@ -1237,6 +1240,9 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { // Placeholder initialization; completed in FinishType. t.Go = &ast.StarExpr{} t.C.Set("*") + if _, ok := c.ptrs[dt.Type]; !ok { + c.ptrKeys = append(c.ptrKeys, dt.Type) + } c.ptrs[dt.Type] = append(c.ptrs[dt.Type], t) case *dwarf.QualType: diff --git a/src/cmd/dist/build.c b/src/cmd/dist/build.c index 8fd2e998a4..9c81dd8b22 100644 --- a/src/cmd/dist/build.c +++ b/src/cmd/dist/build.c @@ -691,13 +691,6 @@ install(char *dir) bpathf(&final_path, "%s/src/%s", goroot_final, dir); name = lastelem(dir); - // For misc/prof, copy into the tool directory and we're done. - if(hasprefix(dir, "misc/")) { - copyfile(bpathf(&b, "%s/%s", tooldir, name), - bpathf(&b1, "%s/misc/%s", goroot, name), 1); - goto out; - } - // set up gcc command line on first run. if(gccargs.len == 0) { bprintf(&b, "%s %s", defaultcc, defaultcflags); @@ -1328,8 +1321,6 @@ static char *buildorder[] = { "libbio", "liblink", - "misc/pprof", - "cmd/cc", // must be before c "cmd/gc", // must be before g "cmd/%sl", // must be before a, c, g diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c index 2303b442cd..523ba37aa7 100644 --- a/src/cmd/gc/lex.c +++ b/src/cmd/gc/lex.c @@ -344,8 +344,8 @@ main(int argc, char *argv[]) break; } } - if(j == nelem(debugtab)) - fatal("unknown debug information -d '%s'\n", f[i]); + if(debugtab[j].name == nil) + sysfatal("unknown debug information -d '%s'\n", f[i]); } } diff --git a/src/cmd/go/build.go b/src/cmd/go/build.go index 79a27116a1..1dd4314da6 100644 --- a/src/cmd/go/build.go +++ b/src/cmd/go/build.go @@ -1826,7 +1826,15 @@ func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) cfile = mkAbs(p.Dir, cfile) - args := stringList(tool(archChar+"c"), "-F", "-V", "-w", "-trimpath", b.work, "-I", objdir, "-I", inc, "-o", ofile, buildCcflags, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, cfile) + warn := []string{"-w"} + if p.usesSwig() { + // When using SWIG, this compiler is only used to + // compile the C files generated by SWIG. + // We don't want warnings. + // See issue 9065 for details. + warn = nil + } + args := stringList(tool(archChar+"c"), "-F", "-V", warn, "-trimpath", b.work, "-I", objdir, "-I", inc, "-o", ofile, buildCcflags, "-D", "GOOS_"+goos, "-D", "GOARCH_"+goarch, cfile) return b.run(p.Dir, p.ImportPath, nil, args) } diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 314c69bd8c..43a3159440 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -247,7 +247,7 @@ The arguments are space-separated tokens or double-quoted strings passed to the generator as individual arguments when it is run. Quoted strings use Go syntax and are evaluated before execution; a -quoted string appears a single argument to the generator. +quoted string appears as a single argument to the generator. Go generate sets several variables when it runs the generator: @@ -260,7 +260,7 @@ Go generate sets several variables when it runs the generator: $GOPACKAGE The name of the package of the file containing the directive. -Other than variable substition and quoted-string evaluation, no +Other than variable substitution and quoted-string evaluation, no special processing such as "globbing" is performed on the command line. @@ -590,7 +590,7 @@ Usage: Vet runs the Go vet command on the packages named by the import paths. -For more about vet, see 'godoc code.google.com/p/go.tools/cmd/vet'. +For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. For more about specifying packages, see 'go help packages'. To run the vet tool with specific options, run 'go tool vet'. diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go index 4227abbe7c..a83cce8f7a 100644 --- a/src/cmd/go/generate.go +++ b/src/cmd/go/generate.go @@ -58,7 +58,7 @@ Go generate sets several variables when it runs the generator: $GOPACKAGE The name of the package of the file containing the directive. -Other than variable substition and quoted-string evaluation, no +Other than variable substitution and quoted-string evaluation, no special processing such as "globbing" is performed on the command line. diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index e17326442c..b71feb7a67 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -383,9 +383,10 @@ func findInternal(path string) (index int, ok bool) { type targetDir int const ( - toRoot targetDir = iota // to bin dir inside package root (default) - toTool // GOROOT/pkg/tool - toBin // GOROOT/bin + toRoot targetDir = iota // to bin dir inside package root (default) + toTool // GOROOT/pkg/tool + toBin // GOROOT/bin + stalePath // the old import path; fail to build ) // goTools is a map of Go program import path to install target directory. @@ -398,10 +399,14 @@ var goTools = map[string]targetDir{ "cmd/nm": toTool, "cmd/objdump": toTool, "cmd/pack": toTool, + "cmd/pprof": toTool, "cmd/yacc": toTool, - "code.google.com/p/go.tools/cmd/cover": toTool, - "code.google.com/p/go.tools/cmd/godoc": toBin, - "code.google.com/p/go.tools/cmd/vet": toTool, + "golang.org/x/tools/cmd/cover": toTool, + "golang.org/x/tools/cmd/godoc": toBin, + "golang.org/x/tools/cmd/vet": toTool, + "code.google.com/p/go.tools/cmd/cover": stalePath, + "code.google.com/p/go.tools/cmd/godoc": stalePath, + "code.google.com/p/go.tools/cmd/vet": stalePath, } // expandScanner expands a scanner.List error into all the errors in the list. @@ -462,6 +467,13 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } if p.Name == "main" { + // Report an error when the old code.google.com/p/go.tools paths are used. + if goTools[p.ImportPath] == stalePath { + newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) + e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) + p.Error = &PackageError{Err: e} + return p + } _, elem := filepath.Split(p.Dir) full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { diff --git a/src/cmd/go/test.bash b/src/cmd/go/test.bash index 2b5230b1aa..e0f066f186 100755 --- a/src/cmd/go/test.bash +++ b/src/cmd/go/test.bash @@ -433,20 +433,20 @@ TEST godoc installs into GOBIN d=$(mktemp -d -t testgoXXX) export GOPATH=$d mkdir $d/gobin -GOBIN=$d/gobin ./testgo get code.google.com/p/go.tools/cmd/godoc || ok=false +GOBIN=$d/gobin ./testgo get golang.org/x/tools/cmd/godoc || ok=false if [ ! -x $d/gobin/godoc ]; then echo did not install godoc to '$GOBIN' - GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc || true + GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' golang.org/x/tools/cmd/godoc || true ok=false fi TEST godoc installs into GOROOT GOROOT=$(./testgo env GOROOT) rm -f $GOROOT/bin/godoc -./testgo install code.google.com/p/go.tools/cmd/godoc || ok=false +./testgo install golang.org/x/tools/cmd/godoc || ok=false if [ ! -x $GOROOT/bin/godoc ]; then echo did not install godoc to '$GOROOT/bin' - ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc || true + ./testgo list -f 'Target: {{.Target}}' golang.org/x/tools/cmd/godoc || true ok=false fi @@ -561,8 +561,8 @@ fi TEST without GOPATH, go get fails d=$(mktemp -d -t testgoXXX) mkdir -p $d/src -if GOPATH= GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with $GOPATH unset' +if GOPATH= GOROOT=$d ./testgo get -d golang.org/x/codereview/cmd/hgpatch ; then + echo 'go get golang.org/x/codereview/cmd/hgpatch should not succeed with $GOPATH unset' ok=false fi rm -rf $d @@ -571,8 +571,8 @@ rm -rf $d TEST with GOPATH=GOROOT, go get fails d=$(mktemp -d -t testgoXXX) mkdir -p $d/src -if GOPATH=$d GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with GOPATH=$GOROOT' +if GOPATH=$d GOROOT=$d ./testgo get -d golang.org/x/codereview/cmd/hgpatch ; then + echo 'go get golang.org/x/codereview/cmd/hgpatch should not succeed with GOPATH=$GOROOT' ok=false fi rm -rf $d @@ -728,7 +728,7 @@ elif ! grep "case-insensitive file name collision" $d/out >/dev/null; then fi TEST go get cover -./testgo get code.google.com/p/go.tools/cmd/cover || ok=false +./testgo get golang.org/x/tools/cmd/cover || ok=false unset GOPATH rm -rf $d diff --git a/src/cmd/go/tool.go b/src/cmd/go/tool.go index 6d26f7a4b4..c96161e0f9 100644 --- a/src/cmd/go/tool.go +++ b/src/cmd/go/tool.go @@ -53,7 +53,7 @@ func tool(toolName string) string { // Give a nice message if there is no tool with that name. if _, err := os.Stat(toolPath); err != nil { if isInGoToolsRepo(toolName) { - fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get code.google.com/p/go.tools/cmd/%s\n", toolName, toolName) + fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName) } else { fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName) } diff --git a/src/cmd/go/vet.go b/src/cmd/go/vet.go index de7befc611..02ff54b2ac 100644 --- a/src/cmd/go/vet.go +++ b/src/cmd/go/vet.go @@ -17,7 +17,7 @@ var cmdVet = &Command{ Long: ` Vet runs the Go vet command on the packages named by the import paths. -For more about vet, see 'godoc code.google.com/p/go.tools/cmd/vet'. +For more about vet, see 'godoc golang.org/x/tools/cmd/vet'. For more about specifying packages, see 'go help packages'. To run the vet tool with specific options, run 'go tool vet'. diff --git a/src/cmd/internal/objfile/disasm.go b/src/cmd/internal/objfile/disasm.go new file mode 100644 index 0000000000..1a339c3214 --- /dev/null +++ b/src/cmd/internal/objfile/disasm.go @@ -0,0 +1,248 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objfile + +import ( + "bufio" + "debug/gosym" + "encoding/binary" + "fmt" + "io" + "regexp" + "sort" + "strings" + "text/tabwriter" + + "cmd/internal/rsc.io/arm/armasm" + "cmd/internal/rsc.io/x86/x86asm" +) + +// Disasm is a disassembler for a given File. +type Disasm struct { + syms []Sym //symbols in file, sorted by address + pcln *gosym.Table // pcln table + text []byte // bytes of text segment (actual instructions) + textStart uint64 // start PC of text + textEnd uint64 // end PC of text + goarch string // GOARCH string + disasm disasmFunc // disassembler function for goarch + byteOrder binary.ByteOrder // byte order for goarch +} + +// Disasm returns a disassembler for the file f. +func (f *File) Disasm() (*Disasm, error) { + syms, err := f.Symbols() + if err != nil { + return nil, err + } + + pcln, err := f.PCLineTable() + if err != nil { + return nil, err + } + + textStart, textBytes, err := f.Text() + if err != nil { + return nil, err + } + + goarch := f.GOARCH() + disasm := disasms[goarch] + byteOrder := byteOrders[goarch] + if disasm == nil || byteOrder == nil { + return nil, fmt.Errorf("unsupported architecture") + } + + // Filter out section symbols, overwriting syms in place. + keep := syms[:0] + for _, sym := range syms { + switch sym.Name { + case "runtime.text", "text", "_text", "runtime.etext", "etext", "_etext": + // drop + default: + keep = append(keep, sym) + } + } + syms = keep + d := &Disasm{ + syms: syms, + pcln: pcln, + text: textBytes, + textStart: textStart, + textEnd: textStart + uint64(len(textBytes)), + goarch: goarch, + disasm: disasm, + byteOrder: byteOrder, + } + + return d, nil +} + +// lookup finds the symbol name containing addr. +func (d *Disasm) lookup(addr uint64) (name string, base uint64) { + i := sort.Search(len(d.syms), func(i int) bool { return addr < d.syms[i].Addr }) + if i > 0 { + s := d.syms[i-1] + if s.Addr != 0 && s.Addr <= addr && addr < s.Addr+uint64(s.Size) { + return s.Name, s.Addr + } + } + return "", 0 +} + +// base returns the final element in the path. +// It works on both Windows and Unix paths, +// regardless of host operating system. +func base(path string) string { + path = path[strings.LastIndex(path, "/")+1:] + path = path[strings.LastIndex(path, `\`)+1:] + return path +} + +// Print prints a disassembly of the file to w. +// If filter is non-nil, the disassembly only includes functions with names matching filter. +// The disassembly only includes functions that overlap the range [start, end). +func (d *Disasm) Print(w io.Writer, filter *regexp.Regexp, start, end uint64) { + if start < d.textStart { + start = d.textStart + } + if end > d.textEnd { + end = d.textEnd + } + printed := false + bw := bufio.NewWriter(w) + for _, sym := range d.syms { + symStart := sym.Addr + symEnd := sym.Addr + uint64(sym.Size) + if sym.Code != 'T' && sym.Code != 't' || + symStart < d.textStart || + symEnd <= start || end <= symStart || + filter != nil && !filter.MatchString(sym.Name) { + continue + } + if printed { + fmt.Fprintf(bw, "\n") + } + printed = true + + file, _, _ := d.pcln.PCToLine(sym.Addr) + fmt.Fprintf(bw, "TEXT %s(SB) %s\n", sym.Name, file) + + tw := tabwriter.NewWriter(bw, 1, 8, 1, '\t', 0) + if symEnd > end { + symEnd = end + } + code := d.text[:end-d.textStart] + d.Decode(symStart, symEnd, func(pc, size uint64, file string, line int, text string) { + i := pc - d.textStart + fmt.Fprintf(tw, "\t%s:%d\t%#x\t", base(file), line, pc) + if size%4 != 0 || d.goarch == "386" || d.goarch == "amd64" { + // Print instruction as bytes. + fmt.Fprintf(tw, "%x", code[i:i+size]) + } else { + // Print instruction as 32-bit words. + for j := uint64(0); j < size; j += 4 { + if j > 0 { + fmt.Fprintf(tw, " ") + } + fmt.Fprintf(tw, "%08x", d.byteOrder.Uint32(code[i+j:])) + } + } + fmt.Fprintf(tw, "\t%s\n", text) + }) + tw.Flush() + } + bw.Flush() +} + +// Decode disassembles the text segment range [start, end), calling f for each instruction. +func (d *Disasm) Decode(start, end uint64, f func(pc, size uint64, file string, line int, text string)) { + if start < d.textStart { + start = d.textStart + } + if end > d.textEnd { + end = d.textEnd + } + code := d.text[:end-d.textStart] + lookup := d.lookup + for pc := start; pc < end; { + i := pc - d.textStart + text, size := d.disasm(code[i:], pc, lookup) + file, line, _ := d.pcln.PCToLine(pc) + f(pc, uint64(size), file, line, text) + pc += uint64(size) + } +} + +type lookupFunc func(addr uint64) (sym string, base uint64) +type disasmFunc func(code []byte, pc uint64, lookup lookupFunc) (text string, size int) + +func disasm_386(code []byte, pc uint64, lookup lookupFunc) (string, int) { + return disasm_x86(code, pc, lookup, 32) +} + +func disasm_amd64(code []byte, pc uint64, lookup lookupFunc) (string, int) { + return disasm_x86(code, pc, lookup, 64) +} + +func disasm_x86(code []byte, pc uint64, lookup lookupFunc, arch int) (string, int) { + inst, err := x86asm.Decode(code, 64) + var text string + size := inst.Len + if err != nil || size == 0 || inst.Op == 0 { + size = 1 + text = "?" + } else { + text = x86asm.Plan9Syntax(inst, pc, lookup) + } + return text, size +} + +type textReader struct { + code []byte + pc uint64 +} + +func (r textReader) ReadAt(data []byte, off int64) (n int, err error) { + if off < 0 || uint64(off) < r.pc { + return 0, io.EOF + } + d := uint64(off) - r.pc + if d >= uint64(len(r.code)) { + return 0, io.EOF + } + n = copy(data, r.code[d:]) + if n < len(data) { + err = io.ErrUnexpectedEOF + } + return +} + +func disasm_arm(code []byte, pc uint64, lookup lookupFunc) (string, int) { + inst, err := armasm.Decode(code, armasm.ModeARM) + var text string + size := inst.Len + if err != nil || size == 0 || inst.Op == 0 { + size = 4 + text = "?" + } else { + text = armasm.Plan9Syntax(inst, pc, lookup, textReader{code, pc}) + } + return text, size +} + +var disasms = map[string]disasmFunc{ + "386": disasm_386, + "amd64": disasm_amd64, + "arm": disasm_arm, +} + +var byteOrders = map[string]binary.ByteOrder{ + "386": binary.LittleEndian, + "amd64": binary.LittleEndian, + "arm": binary.LittleEndian, + "power64": binary.BigEndian, + "power64le": binary.LittleEndian, +} diff --git a/src/cmd/internal/objfile/elf.go b/src/cmd/internal/objfile/elf.go index 8495fa7532..17755b84d2 100644 --- a/src/cmd/internal/objfile/elf.go +++ b/src/cmd/internal/objfile/elf.go @@ -8,6 +8,7 @@ package objfile import ( "debug/elf" + "fmt" "os" ) @@ -77,3 +78,27 @@ func (f *elfFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { } return textStart, symtab, pclntab, nil } + +func (f *elfFile) text() (textStart uint64, text []byte, err error) { + sect := f.elf.Section(".text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = sect.Addr + text, err = sect.Data() + return +} + +func (f *elfFile) goarch() string { + switch f.elf.Machine { + case elf.EM_386: + return "386" + case elf.EM_X86_64: + return "amd64" + case elf.EM_ARM: + return "arm" + case elf.EM_PPC64: + return "power64" + } + return "" +} diff --git a/src/cmd/internal/objfile/goobj.go b/src/cmd/internal/objfile/goobj.go index a4f49ebe44..a1d773023d 100644 --- a/src/cmd/internal/objfile/goobj.go +++ b/src/cmd/internal/objfile/goobj.go @@ -79,3 +79,15 @@ func (f *goobjFile) symbols() ([]Sym, error) { func (f *goobjFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { return 0, nil, nil, fmt.Errorf("pcln not available in go object file") } + +// text does not make sense for Go object files, because +// each function has a separate section. +func (f *goobjFile) text() (textStart uint64, text []byte, err error) { + return 0, nil, fmt.Errorf("text not available in go object file") +} + +// goarch makes sense but is not exposed in debug/goobj's API, +// and we don't need it yet for any users of internal/objfile. +func (f *goobjFile) goarch() string { + return "GOARCH unimplemented for debug/goobj files" +} diff --git a/src/cmd/internal/objfile/macho.go b/src/cmd/internal/objfile/macho.go index f845792ffa..7dd84a339d 100644 --- a/src/cmd/internal/objfile/macho.go +++ b/src/cmd/internal/objfile/macho.go @@ -85,6 +85,30 @@ func (f *machoFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) return textStart, symtab, pclntab, nil } +func (f *machoFile) text() (textStart uint64, text []byte, err error) { + sect := f.macho.Section("__text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = sect.Addr + text, err = sect.Data() + return +} + +func (f *machoFile) goarch() string { + switch f.macho.Cpu { + case macho.Cpu386: + return "386" + case macho.CpuAmd64: + return "amd64" + case macho.CpuArm: + return "arm" + case macho.CpuPpc64: + return "power64" + } + return "" +} + type uint64s []uint64 func (x uint64s) Len() int { return len(x) } diff --git a/src/cmd/internal/objfile/objfile.go b/src/cmd/internal/objfile/objfile.go index 09fa63e60b..9227ef387f 100644 --- a/src/cmd/internal/objfile/objfile.go +++ b/src/cmd/internal/objfile/objfile.go @@ -9,11 +9,14 @@ import ( "debug/gosym" "fmt" "os" + "sort" ) type rawFile interface { symbols() (syms []Sym, err error) pcln() (textStart uint64, symtab, pclntab []byte, err error) + text() (textStart uint64, text []byte, err error) + goarch() string } // A File is an opened executable file. @@ -60,9 +63,20 @@ func (f *File) Close() error { } func (f *File) Symbols() ([]Sym, error) { - return f.raw.symbols() + syms, err := f.raw.symbols() + if err != nil { + return nil, err + } + sort.Sort(byAddr(syms)) + return syms, nil } +type byAddr []Sym + +func (x byAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } +func (x byAddr) Len() int { return len(x) } +func (x byAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + func (f *File) PCLineTable() (*gosym.Table, error) { textStart, symtab, pclntab, err := f.raw.pcln() if err != nil { @@ -70,3 +84,11 @@ func (f *File) PCLineTable() (*gosym.Table, error) { } return gosym.NewTable(symtab, gosym.NewLineTable(pclntab, textStart)) } + +func (f *File) Text() (uint64, []byte, error) { + return f.raw.text() +} + +func (f *File) GOARCH() string { + return f.raw.goarch() +} diff --git a/src/cmd/internal/objfile/pe.go b/src/cmd/internal/objfile/pe.go index 868709eaf9..67e59c226b 100644 --- a/src/cmd/internal/objfile/pe.go +++ b/src/cmd/internal/objfile/pe.go @@ -133,6 +133,25 @@ func (f *peFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) { return textStart, symtab, pclntab, nil } +func (f *peFile) text() (textStart uint64, text []byte, err error) { + var imageBase uint64 + switch oh := f.pe.OptionalHeader.(type) { + case *pe.OptionalHeader32: + imageBase = uint64(oh.ImageBase) + case *pe.OptionalHeader64: + imageBase = oh.ImageBase + default: + return 0, nil, fmt.Errorf("pe file format not recognized") + } + sect := f.pe.Section(".text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = imageBase + uint64(sect.VirtualAddress) + text, err = sect.Data() + return +} + func findPESymbol(f *pe.File, name string) (*pe.Symbol, error) { for _, s := range f.Symbols { if s.Name != name { @@ -168,3 +187,15 @@ func loadPETable(f *pe.File, sname, ename string) ([]byte, error) { } return data[ssym.Value:esym.Value], nil } + +func (f *peFile) goarch() string { + // Not sure how to get the info we want from PE header. + // Look in symbol table for telltale rt0 symbol. + if _, err := findPESymbol(f.pe, "_rt0_386_windows"); err == nil { + return "386" + } + if _, err := findPESymbol(f.pe, "_rt0_amd64_windows"); err == nil { + return "amd64" + } + return "" +} diff --git a/src/cmd/internal/objfile/plan9obj.go b/src/cmd/internal/objfile/plan9obj.go index 80744f82a8..eb6cba5eb1 100644 --- a/src/cmd/internal/objfile/plan9obj.go +++ b/src/cmd/internal/objfile/plan9obj.go @@ -88,6 +88,16 @@ func (f *plan9File) pcln() (textStart uint64, symtab, pclntab []byte, err error) return textStart, symtab, pclntab, nil } +func (f *plan9File) text() (textStart uint64, text []byte, err error) { + sect := f.plan9.Section("text") + if sect == nil { + return 0, nil, fmt.Errorf("text section not found") + } + textStart = f.plan9.LoadAddress + f.plan9.HdrSize + text, err = sect.Data() + return +} + func findPlan9Symbol(f *plan9obj.File, name string) (*plan9obj.Sym, error) { syms, err := f.Symbols() if err != nil { @@ -122,3 +132,15 @@ func loadPlan9Table(f *plan9obj.File, sname, ename string) ([]byte, error) { textStart := f.LoadAddress + f.HdrSize return data[ssym.Value-textStart : esym.Value-textStart], nil } + +func (f *plan9File) goarch() string { + switch f.plan9.Magic { + case plan9obj.Magic386: + return "386" + case plan9obj.MagicAMD64: + return "amd64" + case plan9obj.MagicARM: + return "arm" + } + return "" +} diff --git a/src/cmd/objdump/Makefile b/src/cmd/objdump/Makefile deleted file mode 100644 index 1b66c26bab..0000000000 --- a/src/cmd/objdump/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -all: x86.go armasm.go - -x86.go: bundle - ./bundle -p main -x x86_ rsc.io/x86/x86asm | gofmt >x86.go - -armasm.go: bundle - ./bundle -p main -x arm_ rsc.io/arm/armasm | gofmt >armasm.go - -bundle: - go build -o bundle code.google.com/p/rsc/cmd/bundle diff --git a/src/cmd/objdump/elf.go b/src/cmd/objdump/elf.go deleted file mode 100644 index 906e903532..0000000000 --- a/src/cmd/objdump/elf.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of ELF executables (Linux, FreeBSD, and so on). - -package main - -import ( - "debug/elf" - "os" -) - -func elfSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := elf.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - elfSyms, err := p.Symbols() - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - switch p.Machine { - case elf.EM_X86_64: - goarch = "amd64" - case elf.EM_386: - goarch = "386" - case elf.EM_ARM: - goarch = "arm" - } - - for _, s := range elfSyms { - sym := Sym{Addr: s.Value, Name: s.Name, Size: int64(s.Size), Code: '?'} - switch s.Section { - case elf.SHN_UNDEF: - sym.Code = 'U' - case elf.SHN_COMMON: - sym.Code = 'B' - default: - i := int(s.Section) - if i < 0 || i >= len(p.Sections) { - break - } - sect := p.Sections[i] - switch sect.Flags & (elf.SHF_WRITE | elf.SHF_ALLOC | elf.SHF_EXECINSTR) { - case elf.SHF_ALLOC | elf.SHF_EXECINSTR: - sym.Code = 'T' - case elf.SHF_ALLOC: - sym.Code = 'R' - case elf.SHF_ALLOC | elf.SHF_WRITE: - sym.Code = 'D' - } - } - if elf.ST_BIND(s.Info) == elf.STB_LOCAL { - sym.Code += 'a' - 'A' - } - syms = append(syms, sym) - } - - return -} diff --git a/src/cmd/objdump/macho.go b/src/cmd/objdump/macho.go deleted file mode 100644 index 6e0ad223d4..0000000000 --- a/src/cmd/objdump/macho.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of Mach-O executables (OS X). - -package main - -import ( - "debug/macho" - "os" - "sort" -) - -func machoSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := macho.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - if p.Symtab == nil { - errorf("%s: no symbol table", f.Name()) - return - } - - switch p.Cpu { - case macho.Cpu386: - goarch = "386" - case macho.CpuAmd64: - goarch = "amd64" - case macho.CpuArm: - goarch = "arm" - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - for _, s := range p.Symtab.Syms { - addrs = append(addrs, s.Value) - } - sort.Sort(uint64s(addrs)) - - for _, s := range p.Symtab.Syms { - sym := Sym{Name: s.Name, Addr: s.Value, Code: '?'} - i := sort.Search(len(addrs), func(x int) bool { return addrs[x] > s.Value }) - if i < len(addrs) { - sym.Size = int64(addrs[i] - s.Value) - } - if s.Sect == 0 { - sym.Code = 'U' - } else if int(s.Sect) <= len(p.Sections) { - sect := p.Sections[s.Sect-1] - switch sect.Seg { - case "__TEXT": - sym.Code = 'R' - case "__DATA": - sym.Code = 'D' - } - switch sect.Seg + " " + sect.Name { - case "__TEXT __text": - sym.Code = 'T' - case "__DATA __bss", "__DATA __noptrbss": - sym.Code = 'B' - } - } - syms = append(syms, sym) - } - - return -} - -type uint64s []uint64 - -func (x uint64s) Len() int { return len(x) } -func (x uint64s) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x uint64s) Less(i, j int) bool { return x[i] < x[j] } diff --git a/src/cmd/objdump/main.go b/src/cmd/objdump/main.go index 0f66f20a40..708a853702 100644 --- a/src/cmd/objdump/main.go +++ b/src/cmd/objdump/main.go @@ -32,27 +32,15 @@ package main import ( - "bufio" - "bytes" - "debug/elf" - "debug/gosym" - "debug/macho" - "debug/pe" - "debug/plan9obj" - "encoding/binary" "flag" "fmt" - "io" "log" "os" "regexp" - "sort" "strconv" "strings" - "text/tabwriter" - "cmd/internal/rsc.io/arm/armasm" - "cmd/internal/rsc.io/x86/x86asm" + "cmd/internal/objfile" ) var symregexp = flag.String("s", "", "only dump symbols matching this regexp") @@ -85,446 +73,35 @@ func main() { symRE = re } - f, err := os.Open(flag.Arg(0)) + f, err := objfile.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } - textStart, textData, symtab, pclntab, err := loadTables(f) + dis, err := f.Disasm() if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) + log.Fatal("disassemble %s: %v", flag.Arg(0), err) } - syms, goarch, err := loadSymbols(f) - if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) - } + switch flag.NArg() { + default: + usage() + case 1: + // disassembly of entire object + dis.Print(os.Stdout, symRE, 0, ^uint64(0)) + os.Exit(0) - // Filter out section symbols, overwriting syms in place. - keep := syms[:0] - for _, sym := range syms { - switch sym.Name { - case "runtime.text", "text", "_text", "runtime.etext", "etext", "_etext": - // drop - default: - keep = append(keep, sym) + case 3: + // disassembly of PC range + start, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(1), "0x"), 16, 64) + if err != nil { + log.Fatalf("invalid start PC: %v", err) } - } - syms = keep - - disasm := disasms[goarch] - if disasm == nil { - log.Fatalf("reading %s: unknown architecture", flag.Arg(0)) - } - - lookup := func(addr uint64) (string, uint64) { - i := sort.Search(len(syms), func(i int) bool { return syms[i].Addr > addr }) - if i > 0 { - s := syms[i-1] - if s.Addr <= addr && addr < s.Addr+uint64(s.Size) && s.Name != "runtime.etext" && s.Name != "etext" && s.Name != "_etext" { - return s.Name, s.Addr - } + end, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(2), "0x"), 16, 64) + if err != nil { + log.Fatalf("invalid end PC: %v", err) } - return "", 0 - } - - pcln := gosym.NewLineTable(pclntab, textStart) - tab, err := gosym.NewTable(symtab, pcln) - if err != nil { - log.Fatalf("reading %s: %v", flag.Arg(0), err) - } - - if flag.NArg() == 1 { - // disassembly of entire object - our format - dump(tab, lookup, disasm, goarch, syms, textData, textStart) - os.Exit(exitCode) - } - - // disassembly of specific piece of object - gnu objdump format for pprof - gnuDump(tab, lookup, disasm, textData, textStart) - os.Exit(exitCode) -} - -// base returns the final element in the path. -// It works on both Windows and Unix paths. -func base(path string) string { - path = path[strings.LastIndex(path, "/")+1:] - path = path[strings.LastIndex(path, `\`)+1:] - return path -} - -func dump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, goarch string, syms []Sym, textData []byte, textStart uint64) { - stdout := bufio.NewWriter(os.Stdout) - defer stdout.Flush() - - printed := false - for _, sym := range syms { - if (sym.Code != 'T' && sym.Code != 't') || sym.Size == 0 || sym.Name == "_text" || sym.Name == "text" || sym.Addr < textStart || symRE != nil && !symRE.MatchString(sym.Name) { - continue - } - if sym.Addr >= textStart+uint64(len(textData)) || sym.Addr+uint64(sym.Size) > textStart+uint64(len(textData)) { - break - } - if printed { - fmt.Fprintf(stdout, "\n") - } else { - printed = true - } - file, _, _ := tab.PCToLine(sym.Addr) - fmt.Fprintf(stdout, "TEXT %s(SB) %s\n", sym.Name, file) - tw := tabwriter.NewWriter(stdout, 1, 8, 1, '\t', 0) - start := sym.Addr - end := sym.Addr + uint64(sym.Size) - for pc := start; pc < end; { - i := pc - textStart - text, size := disasm(textData[i:end-textStart], pc, lookup) - file, line, _ := tab.PCToLine(pc) - - // ARM is word-based, so show actual word hex, not byte hex. - // Since ARM is little endian, they're different. - if goarch == "arm" && size == 4 { - fmt.Fprintf(tw, "\t%s:%d\t%#x\t%08x\t%s\n", base(file), line, pc, binary.LittleEndian.Uint32(textData[i:i+uint64(size)]), text) - } else { - fmt.Fprintf(tw, "\t%s:%d\t%#x\t%x\t%s\n", base(file), line, pc, textData[i:i+uint64(size)], text) - } - pc += uint64(size) - } - tw.Flush() + dis.Print(os.Stdout, symRE, start, end) + os.Exit(0) } } - -func disasm_386(code []byte, pc uint64, lookup lookupFunc) (string, int) { - return disasm_x86(code, pc, lookup, 32) -} - -func disasm_amd64(code []byte, pc uint64, lookup lookupFunc) (string, int) { - return disasm_x86(code, pc, lookup, 64) -} - -func disasm_x86(code []byte, pc uint64, lookup lookupFunc, arch int) (string, int) { - inst, err := x86asm.Decode(code, 64) - var text string - size := inst.Len - if err != nil || size == 0 || inst.Op == 0 { - size = 1 - text = "?" - } else { - text = x86asm.Plan9Syntax(inst, pc, lookup) - } - return text, size -} - -type textReader struct { - code []byte - pc uint64 -} - -func (r textReader) ReadAt(data []byte, off int64) (n int, err error) { - if off < 0 || uint64(off) < r.pc { - return 0, io.EOF - } - d := uint64(off) - r.pc - if d >= uint64(len(r.code)) { - return 0, io.EOF - } - n = copy(data, r.code[d:]) - if n < len(data) { - err = io.ErrUnexpectedEOF - } - return -} - -func disasm_arm(code []byte, pc uint64, lookup lookupFunc) (string, int) { - inst, err := armasm.Decode(code, armasm.ModeARM) - var text string - size := inst.Len - if err != nil || size == 0 || inst.Op == 0 { - size = 4 - text = "?" - } else { - text = armasm.Plan9Syntax(inst, pc, lookup, textReader{code, pc}) - } - return text, size -} - -var disasms = map[string]disasmFunc{ - "386": disasm_386, - "amd64": disasm_amd64, - "arm": disasm_arm, -} - -func gnuDump(tab *gosym.Table, lookup lookupFunc, disasm disasmFunc, textData []byte, textStart uint64) { - start, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(1), "0x"), 16, 64) - if err != nil { - log.Fatalf("invalid start PC: %v", err) - } - end, err := strconv.ParseUint(strings.TrimPrefix(flag.Arg(2), "0x"), 16, 64) - if err != nil { - log.Fatalf("invalid end PC: %v", err) - } - if start < textStart { - start = textStart - } - if end < start { - end = start - } - if end > textStart+uint64(len(textData)) { - end = textStart + uint64(len(textData)) - } - - stdout := bufio.NewWriter(os.Stdout) - defer stdout.Flush() - - // For now, find spans of same PC/line/fn and - // emit them as having dummy instructions. - var ( - spanPC uint64 - spanFile string - spanLine int - spanFn *gosym.Func - ) - - flush := func(endPC uint64) { - if spanPC == 0 { - return - } - fmt.Fprintf(stdout, "%s:%d\n", spanFile, spanLine) - for pc := spanPC; pc < endPC; { - text, size := disasm(textData[pc-textStart:], pc, lookup) - fmt.Fprintf(stdout, " %x: %s\n", pc, text) - pc += uint64(size) - } - spanPC = 0 - } - - for pc := start; pc < end; pc++ { - file, line, fn := tab.PCToLine(pc) - if file != spanFile || line != spanLine || fn != spanFn { - flush(pc) - spanPC, spanFile, spanLine, spanFn = pc, file, line, fn - } - } - flush(end) -} - -func loadTables(f *os.File) (textStart uint64, textData, symtab, pclntab []byte, err error) { - if obj, err := elf.NewFile(f); err == nil { - if sect := obj.Section(".text"); sect != nil { - textStart = sect.Addr - textData, _ = sect.Data() - } - if sect := obj.Section(".gosymtab"); sect != nil { - if symtab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - if sect := obj.Section(".gopclntab"); sect != nil { - if pclntab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - if obj, err := macho.NewFile(f); err == nil { - if sect := obj.Section("__text"); sect != nil { - textStart = sect.Addr - textData, _ = sect.Data() - } - if sect := obj.Section("__gosymtab"); sect != nil { - if symtab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - if sect := obj.Section("__gopclntab"); sect != nil { - if pclntab, err = sect.Data(); err != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - if obj, err := pe.NewFile(f); err == nil { - var imageBase uint64 - switch oh := obj.OptionalHeader.(type) { - case *pe.OptionalHeader32: - imageBase = uint64(oh.ImageBase) - case *pe.OptionalHeader64: - imageBase = oh.ImageBase - default: - return 0, nil, nil, nil, fmt.Errorf("pe file format not recognized") - } - if sect := obj.Section(".text"); sect != nil { - textStart = imageBase + uint64(sect.VirtualAddress) - textData, _ = sect.Data() - } - if pclntab, err = loadPETable(obj, "runtime.pclntab", "runtime.epclntab"); err != nil { - // We didn't find the symbols, so look for the names used in 1.3 and earlier. - // TODO: Remove code looking for the old symbols when we no longer care about 1.3. - var err2 error - if pclntab, err2 = loadPETable(obj, "pclntab", "epclntab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - if symtab, err = loadPETable(obj, "runtime.symtab", "runtime.esymtab"); err != nil { - // Same as above. - var err2 error - if symtab, err2 = loadPETable(obj, "symtab", "esymtab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - if obj, err := plan9obj.NewFile(f); err == nil { - textStart = obj.LoadAddress + obj.HdrSize - if sect := obj.Section("text"); sect != nil { - textData, _ = sect.Data() - } - if pclntab, err = loadPlan9Table(obj, "runtime.pclntab", "runtime.epclntab"); err != nil { - // We didn't find the symbols, so look for the names used in 1.3 and earlier. - // TODO: Remove code looking for the old symbols when we no longer care about 1.3. - var err2 error - if pclntab, err2 = loadPlan9Table(obj, "pclntab", "epclntab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - if symtab, err = loadPlan9Table(obj, "runtime.symtab", "runtime.esymtab"); err != nil { - // Same as above. - var err2 error - if symtab, err2 = loadPlan9Table(obj, "symtab", "esymtab"); err2 != nil { - return 0, nil, nil, nil, err - } - } - return textStart, textData, symtab, pclntab, nil - } - - return 0, nil, nil, nil, fmt.Errorf("unrecognized binary format") -} - -func findPESymbol(f *pe.File, name string) (*pe.Symbol, error) { - for _, s := range f.Symbols { - if s.Name != name { - continue - } - if s.SectionNumber <= 0 { - return nil, fmt.Errorf("symbol %s: invalid section number %d", name, s.SectionNumber) - } - if len(f.Sections) < int(s.SectionNumber) { - return nil, fmt.Errorf("symbol %s: section number %d is larger than max %d", name, s.SectionNumber, len(f.Sections)) - } - return s, nil - } - return nil, fmt.Errorf("no %s symbol found", name) -} - -func loadPETable(f *pe.File, sname, ename string) ([]byte, error) { - ssym, err := findPESymbol(f, sname) - if err != nil { - return nil, err - } - esym, err := findPESymbol(f, ename) - if err != nil { - return nil, err - } - if ssym.SectionNumber != esym.SectionNumber { - return nil, fmt.Errorf("%s and %s symbols must be in the same section", sname, ename) - } - sect := f.Sections[ssym.SectionNumber-1] - data, err := sect.Data() - if err != nil { - return nil, err - } - return data[ssym.Value:esym.Value], nil -} - -func findPlan9Symbol(f *plan9obj.File, name string) (*plan9obj.Sym, error) { - syms, err := f.Symbols() - if err != nil { - return nil, err - } - for _, s := range syms { - if s.Name != name { - continue - } - return &s, nil - } - return nil, fmt.Errorf("no %s symbol found", name) -} - -func loadPlan9Table(f *plan9obj.File, sname, ename string) ([]byte, error) { - ssym, err := findPlan9Symbol(f, sname) - if err != nil { - return nil, err - } - esym, err := findPlan9Symbol(f, ename) - if err != nil { - return nil, err - } - sect := f.Section("text") - if sect == nil { - return nil, err - } - data, err := sect.Data() - if err != nil { - return nil, err - } - textStart := f.LoadAddress + f.HdrSize - return data[ssym.Value-textStart : esym.Value-textStart], nil -} - -// TODO(rsc): This code is taken from cmd/nm. Arrange some way to share the code. - -var exitCode = 0 - -func errorf(format string, args ...interface{}) { - log.Printf(format, args...) - exitCode = 1 -} - -func loadSymbols(f *os.File) (syms []Sym, goarch string, err error) { - f.Seek(0, 0) - buf := make([]byte, 16) - io.ReadFull(f, buf) - f.Seek(0, 0) - - for _, p := range parsers { - if bytes.HasPrefix(buf, p.prefix) { - syms, goarch = p.parse(f) - sort.Sort(byAddr(syms)) - return - } - } - err = fmt.Errorf("unknown file format") - return -} - -type Sym struct { - Addr uint64 - Size int64 - Code rune - Name string - Type string -} - -var parsers = []struct { - prefix []byte - parse func(*os.File) ([]Sym, string) -}{ - {[]byte("\x7FELF"), elfSymbols}, - {[]byte("\xFE\xED\xFA\xCE"), machoSymbols}, - {[]byte("\xFE\xED\xFA\xCF"), machoSymbols}, - {[]byte("\xCE\xFA\xED\xFE"), machoSymbols}, - {[]byte("\xCF\xFA\xED\xFE"), machoSymbols}, - {[]byte("MZ"), peSymbols}, - {[]byte("\x00\x00\x01\xEB"), plan9Symbols}, // 386 - {[]byte("\x00\x00\x04\x07"), plan9Symbols}, // mips - {[]byte("\x00\x00\x06\x47"), plan9Symbols}, // arm - {[]byte("\x00\x00\x8A\x97"), plan9Symbols}, // amd64 -} - -type byAddr []Sym - -func (x byAddr) Len() int { return len(x) } -func (x byAddr) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x byAddr) Less(i, j int) bool { return x[i].Addr < x[j].Addr } diff --git a/src/cmd/objdump/objdump_test.go b/src/cmd/objdump/objdump_test.go index 0a2d2565a7..2bb74663c3 100644 --- a/src/cmd/objdump/objdump_test.go +++ b/src/cmd/objdump/objdump_test.go @@ -5,113 +5,15 @@ package main import ( - "bufio" - "bytes" - "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "runtime" - "strconv" "strings" "testing" ) -func loadSyms(t *testing.T) map[string]string { - switch runtime.GOOS { - case "android", "nacl": - t.Skipf("skipping on %s", runtime.GOOS) - } - - cmd := exec.Command("go", "tool", "nm", os.Args[0]) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out)) - } - syms := make(map[string]string) - scanner := bufio.NewScanner(bytes.NewReader(out)) - for scanner.Scan() { - f := strings.Fields(scanner.Text()) - if len(f) < 3 { - continue - } - syms[f[2]] = f[0] - } - if err := scanner.Err(); err != nil { - t.Fatalf("error reading symbols: %v", err) - } - return syms -} - -func runObjDump(t *testing.T, exe, startaddr, endaddr string) (path, lineno string) { - switch runtime.GOOS { - case "android", "nacl": - t.Skipf("skipping on %s", runtime.GOOS) - } - - cmd := exec.Command(exe, os.Args[0], startaddr, endaddr) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("go tool objdump %v: %v\n%s", os.Args[0], err, string(out)) - } - f := strings.Split(string(out), "\n") - if len(f) < 1 { - t.Fatal("objdump output must have at least one line") - } - pathAndLineNo := f[0] - f = strings.Split(pathAndLineNo, ":") - if runtime.GOOS == "windows" { - switch len(f) { - case 2: - return f[0], f[1] - case 3: - return f[0] + ":" + f[1], f[2] - default: - t.Fatalf("no line number found in %q", pathAndLineNo) - } - } - if len(f) != 2 { - t.Fatalf("no line number found in %q", pathAndLineNo) - } - return f[0], f[1] -} - -func testObjDump(t *testing.T, exe, startaddr, endaddr string, line int) { - srcPath, srcLineNo := runObjDump(t, exe, startaddr, endaddr) - fi1, err := os.Stat("objdump_test.go") - if err != nil { - t.Fatalf("Stat failed: %v", err) - } - fi2, err := os.Stat(srcPath) - if err != nil { - t.Fatalf("Stat failed: %v", err) - } - if !os.SameFile(fi1, fi2) { - t.Fatalf("objdump_test.go and %s are not same file", srcPath) - } - if srcLineNo != fmt.Sprint(line) { - t.Fatalf("line number = %v; want %d", srcLineNo, line) - } -} - -func TestObjDump(t *testing.T) { - _, _, line, _ := runtime.Caller(0) - syms := loadSyms(t) - - tmp, exe := buildObjdump(t) - defer os.RemoveAll(tmp) - - startaddr := syms["cmd/objdump.TestObjDump"] - addr, err := strconv.ParseUint(startaddr, 16, 64) - if err != nil { - t.Fatalf("invalid start address %v: %v", startaddr, err) - } - endaddr := fmt.Sprintf("%x", addr+10) - testObjDump(t, exe, startaddr, endaddr, line-1) - testObjDump(t, exe, "0x"+startaddr, "0x"+endaddr, line-1) -} - func buildObjdump(t *testing.T) (tmp, exe string) { switch runtime.GOOS { case "android", "nacl": @@ -143,7 +45,7 @@ var x86Need = []string{ var armNeed = []string{ "fmthello.go:6", "TEXT main.main(SB)", - "B.LS main.main(SB)", + //"B.LS main.main(SB)", // TODO(rsc): restore; golang.org/issue/9021 "BL fmt.Println(SB)", "RET", } diff --git a/src/cmd/objdump/pe.go b/src/cmd/objdump/pe.go deleted file mode 100644 index 38190095a3..0000000000 --- a/src/cmd/objdump/pe.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of PE executables (Microsoft Windows). - -package main - -import ( - "debug/pe" - "os" - "sort" -) - -func peSymbols(f *os.File) (syms []Sym, goarch string) { - p, err := pe.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - - var imageBase uint64 - switch oh := p.OptionalHeader.(type) { - case *pe.OptionalHeader32: - imageBase = uint64(oh.ImageBase) - goarch = "386" - case *pe.OptionalHeader64: - imageBase = oh.ImageBase - goarch = "amd64" - default: - errorf("parsing %s: file format not recognized", f.Name()) - return - } - - for _, s := range p.Symbols { - const ( - N_UNDEF = 0 // An undefined (extern) symbol - N_ABS = -1 // An absolute symbol (e_value is a constant, not an address) - N_DEBUG = -2 // A debugging symbol - ) - sym := Sym{Name: s.Name, Addr: uint64(s.Value), Code: '?'} - switch s.SectionNumber { - case N_UNDEF: - sym.Code = 'U' - case N_ABS: - sym.Code = 'C' - case N_DEBUG: - sym.Code = '?' - default: - if s.SectionNumber < 0 { - errorf("parsing %s: invalid section number %d", f.Name(), s.SectionNumber) - return - } - if len(p.Sections) < int(s.SectionNumber) { - errorf("parsing %s: section number %d is large then max %d", f.Name(), s.SectionNumber, len(p.Sections)) - return - } - sect := p.Sections[s.SectionNumber-1] - const ( - text = 0x20 - data = 0x40 - bss = 0x80 - permX = 0x20000000 - permR = 0x40000000 - permW = 0x80000000 - ) - ch := sect.Characteristics - switch { - case ch&text != 0: - sym.Code = 'T' - case ch&data != 0: - if ch&permW == 0 { - sym.Code = 'R' - } else { - sym.Code = 'D' - } - case ch&bss != 0: - sym.Code = 'B' - } - sym.Addr += imageBase + uint64(sect.VirtualAddress) - } - syms = append(syms, sym) - addrs = append(addrs, sym.Addr) - } - - sort.Sort(uint64s(addrs)) - for i := range syms { - j := sort.Search(len(addrs), func(x int) bool { return addrs[x] > syms[i].Addr }) - if j < len(addrs) { - syms[i].Size = int64(addrs[j] - syms[i].Addr) - } - } - - return -} diff --git a/src/cmd/objdump/plan9obj.go b/src/cmd/objdump/plan9obj.go deleted file mode 100644 index f851d4158c..0000000000 --- a/src/cmd/objdump/plan9obj.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Parsing of Plan 9 a.out executables. - -package main - -import ( - "debug/plan9obj" - "os" - "sort" -) - -var validSymType = map[rune]bool{ - 'T': true, - 't': true, - 'D': true, - 'd': true, - 'B': true, - 'b': true, -} - -func plan9Symbols(f *os.File) (syms []Sym, goarch string) { - p, err := plan9obj.NewFile(f) - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - plan9Syms, err := p.Symbols() - if err != nil { - errorf("parsing %s: %v", f.Name(), err) - return - } - - switch p.Magic { - case plan9obj.MagicAMD64: - goarch = "amd64" - case plan9obj.Magic386: - goarch = "386" - case plan9obj.MagicARM: - goarch = "arm" - } - - // Build sorted list of addresses of all symbols. - // We infer the size of a symbol by looking at where the next symbol begins. - var addrs []uint64 - for _, s := range plan9Syms { - if !validSymType[s.Type] { - continue - } - addrs = append(addrs, s.Value) - } - sort.Sort(uint64s(addrs)) - - for _, s := range plan9Syms { - if !validSymType[s.Type] { - continue - } - sym := Sym{Addr: s.Value, Name: s.Name, Code: rune(s.Type)} - i := sort.Search(len(addrs), func(x int) bool { return addrs[x] > s.Value }) - if i < len(addrs) { - sym.Size = int64(addrs[i] - s.Value) - } - syms = append(syms, sym) - } - - return -} diff --git a/src/cmd/pprof/README b/src/cmd/pprof/README new file mode 100644 index 0000000000..a728ef2353 --- /dev/null +++ b/src/cmd/pprof/README @@ -0,0 +1,8 @@ +The pprof in this directory is adapted from the pprof used inside Google +for C++, Java, and Go programs. Because it was developed for that broader +context, it is overgeneralized when used here for the specific use case +of profiling standard Go programs. However, we've left the abstractions +intact in order to share updates between this copy and Google's internal one. + +Please do not take the level of abstraction in this program as an example +to follow in your own. diff --git a/src/cmd/pprof/doc.go b/src/cmd/pprof/doc.go new file mode 100644 index 0000000000..c6ff11d102 --- /dev/null +++ b/src/cmd/pprof/doc.go @@ -0,0 +1,12 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Pprof interprets and displays profiles of Go programs. +// +// Usage: +// +// go tool pprof binary profile +// +// For more information, see http://blog.golang.org/profiling-go-programs. +package main diff --git a/src/cmd/pprof/pprof.go b/src/cmd/pprof/pprof.go index 89a5bb7d22..44f4f6cb72 100644 --- a/src/cmd/pprof/pprof.go +++ b/src/cmd/pprof/pprof.go @@ -11,6 +11,7 @@ import ( "os" "regexp" "strings" + "sync" "cmd/internal/objfile" "cmd/pprof/internal/commands" @@ -100,7 +101,10 @@ func (flags) ExtraUsage() string { // objTool implements plugin.ObjTool using Go libraries // (instead of invoking GNU binutils). -type objTool struct{} +type objTool struct { + mu sync.Mutex + disasmCache map[string]*objfile.Disasm +} func (*objTool) Open(name string, start uint64) (plugin.ObjFile, error) { of, err := objfile.Open(name) @@ -119,8 +123,39 @@ func (*objTool) Demangle(names []string) (map[string]string, error) { return make(map[string]string), nil } -func (*objTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) { - return nil, fmt.Errorf("disassembly not supported") +func (t *objTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) { + d, err := t.cachedDisasm(file) + if err != nil { + return nil, err + } + var asm []plugin.Inst + d.Decode(start, end, func(pc, size uint64, file string, line int, text string) { + asm = append(asm, plugin.Inst{Addr: pc, File: file, Line: line, Text: text}) + }) + return asm, nil +} + +func (t *objTool) cachedDisasm(file string) (*objfile.Disasm, error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.disasmCache == nil { + t.disasmCache = make(map[string]*objfile.Disasm) + } + d := t.disasmCache[file] + if d != nil { + return d, nil + } + f, err := objfile.Open(file) + if err != nil { + return nil, err + } + d, err = f.Disasm() + f.Close() + if err != nil { + return nil, err + } + t.disasmCache[file] = d + return d, nil } func (*objTool) SetConfig(config string) { diff --git a/src/compress/lzw/reader.go b/src/compress/lzw/reader.go index 0835bd8b90..526620c827 100644 --- a/src/compress/lzw/reader.go +++ b/src/compress/lzw/reader.go @@ -11,7 +11,7 @@ // two non-literal codes are a clear code and an EOF code. // // The TIFF file format uses a similar but incompatible version of the LZW -// algorithm. See the code.google.com/p/go.image/tiff/lzw package for an +// algorithm. See the golang.org/x/image/tiff/lzw package for an // implementation. package lzw diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index 5a91baca0e..59b23e93f5 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -21,7 +21,7 @@ func (h Hash) HashFunc() Hash { } const ( - MD4 Hash = 1 + iota // import code.google.com/p/go.crypto/md4 + MD4 Hash = 1 + iota // import golang.org/x/crypto/md4 MD5 // import crypto/md5 SHA1 // import crypto/sha1 SHA224 // import crypto/sha256 @@ -29,11 +29,11 @@ const ( SHA384 // import crypto/sha512 SHA512 // import crypto/sha512 MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA - RIPEMD160 // import code.google.com/p/go.crypto/ripemd160 - SHA3_224 // import code.google.com/p/go.crypto/sha3 - SHA3_256 // import code.google.com/p/go.crypto/sha3 - SHA3_384 // import code.google.com/p/go.crypto/sha3 - SHA3_512 // import code.google.com/p/go.crypto/sha3 + RIPEMD160 // import golang.org/x/crypto/ripemd160 + SHA3_224 // import golang.org/x/crypto/sha3 + SHA3_256 // import golang.org/x/crypto/sha3 + SHA3_384 // import golang.org/x/crypto/sha3 + SHA3_512 // import golang.org/x/crypto/sha3 maxHash ) diff --git a/src/database/sql/fakedb_test.go b/src/database/sql/fakedb_test.go index 171c322d49..a993fd46ed 100644 --- a/src/database/sql/fakedb_test.go +++ b/src/database/sql/fakedb_test.go @@ -141,6 +141,8 @@ type Dummy struct { } func TestDrivers(t *testing.T) { + unregisterAllDrivers() + Register("test", fdriver) Register("invalid", Dummy{}) all := Drivers() if len(all) < 2 || !sort.StringsAreSorted(all) || !contains(all, "test") || !contains(all, "invalid") { diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index ad9179cf7d..6e6f246aee 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -37,6 +37,11 @@ func Register(name string, driver driver.Driver) { drivers[name] = driver } +func unregisterAllDrivers() { + // For tests. + drivers = make(map[string]driver.Driver) +} + // Drivers returns a sorted list of the names of the registered drivers. func Drivers() []string { var list []string diff --git a/src/debug/goobj/read_test.go b/src/debug/goobj/read_test.go index dee140533c..cc991e5d91 100644 --- a/src/debug/goobj/read_test.go +++ b/src/debug/goobj/read_test.go @@ -12,7 +12,7 @@ var importPathToPrefixTests = []struct { }{ {"runtime", "runtime"}, {"sync/atomic", "sync/atomic"}, - {"code.google.com/p/go.tools/godoc", "code.google.com/p/go.tools/godoc"}, + {"golang.org/x/tools/godoc", "golang.org/x/tools/godoc"}, {"foo.bar/baz.quux", "foo.bar/baz%2equux"}, {"", ""}, {"%foo%bar", "%25foo%25bar"}, diff --git a/src/net/http/cookiejar/jar.go b/src/net/http/cookiejar/jar.go index 389ab58e41..0e0fac9286 100644 --- a/src/net/http/cookiejar/jar.go +++ b/src/net/http/cookiejar/jar.go @@ -30,7 +30,7 @@ import ( // set a cookie for bar.com. // // A public suffix list implementation is in the package -// code.google.com/p/go.net/publicsuffix. +// golang.org/x/net/publicsuffix. type PublicSuffixList interface { // PublicSuffix returns the public suffix of domain. // diff --git a/src/net/http/main_test.go b/src/net/http/main_test.go index 9f1dfc3727..b8c71fd19f 100644 --- a/src/net/http/main_test.go +++ b/src/net/http/main_test.go @@ -70,7 +70,7 @@ func goroutineLeaked() bool { } fmt.Fprintf(os.Stderr, "Too many goroutines running after net/http test(s).\n") for stack, count := range stackCount { - fmt.Fprintf(os.Stderr, "%d instances of:\n%s", count, stack) + fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack) } return true } diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index bb44ac8537..5e0a0053c0 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -2819,6 +2819,7 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, useTLS bool) { InsecureSkipVerify: true, }, } + defer noVerifyTransport.CloseIdleConnections() client := &Client{Transport: noVerifyTransport} for pb.Next() { res, err := client.Get(ts.URL) diff --git a/src/os/exec/exec_test.go b/src/os/exec/exec_test.go index bc9c00effe..197d3e8b40 100644 --- a/src/os/exec/exec_test.go +++ b/src/os/exec/exec_test.go @@ -246,7 +246,7 @@ func TestPipeLookPathLeak(t *testing.T) { } func numOpenFDS(t *testing.T) (n int, lsof []byte) { - lsof, err := exec.Command("lsof", "-n", "-p", strconv.Itoa(os.Getpid())).Output() + lsof, err := exec.Command("lsof", "-b", "-n", "-p", strconv.Itoa(os.Getpid())).Output() if err != nil { t.Skip("skipping test; error finding or running lsof") } diff --git a/src/os/file_plan9.go b/src/os/file_plan9.go index 5efc2a4f1d..132594eede 100644 --- a/src/os/file_plan9.go +++ b/src/os/file_plan9.go @@ -25,7 +25,8 @@ type file struct { dirinfo *dirInfo // nil unless directory being read } -// Fd returns the integer Unix file descriptor referencing the open file. +// Fd returns the integer Plan 9 file descriptor referencing the open file. +// The file descriptor is valid only until f.Close is called or f is garbage collected. func (f *File) Fd() uintptr { if f == nil { return ^(uintptr(0)) diff --git a/src/os/file_unix.go b/src/os/file_unix.go index f59d563e69..ff4fc7d12e 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -29,6 +29,7 @@ type file struct { } // Fd returns the integer Unix file descriptor referencing the open file. +// The file descriptor is valid only until f.Close is called or f is garbage collected. func (f *File) Fd() uintptr { if f == nil { return ^(uintptr(0)) diff --git a/src/os/file_windows.go b/src/os/file_windows.go index 3b5519390b..2a90a50559 100644 --- a/src/os/file_windows.go +++ b/src/os/file_windows.go @@ -36,6 +36,7 @@ type file struct { } // Fd returns the Windows handle referencing the open file. +// The handle is valid only until f.Close is called or f is garbage collected. func (file *File) Fd() uintptr { if file == nil { return uintptr(syscall.InvalidHandle) diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index d456e6bca4..501e64b094 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -2284,3 +2284,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 MOVL m_curg(AX), AX MOVL (g_stack+stack_hi)(AX), AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 5d176575c3..1aa2d71a80 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -2229,3 +2229,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 MOVQ m_curg(AX), AX MOVQ (g_stack+stack_hi)(AX), AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index 2b2155753e..153564b14e 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -1079,3 +1079,9 @@ TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 TEXT runtime·return0(SB), NOSPLIT, $0 MOVL $0, AX RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 + BYTE $0x90 // NOP + CALL runtime·goexit1(SB) // does not return diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 9a58fdc51e..58aebf3884 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -1320,3 +1320,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$8 MOVW saveG-8(SP), g MOVW saveR11-4(SP), R11 RET + +// The top-most function running on a goroutine +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$-4-0 + MOVW R0, R0 // NOP + BL runtime·goexit1(SB) // does not return diff --git a/src/runtime/cgo/dragonfly.c b/src/runtime/cgo/dragonfly.c index 3c95ff354e..c233c8ba9a 100644 --- a/src/runtime/cgo/dragonfly.c +++ b/src/runtime/cgo/dragonfly.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build dragonfly + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/freebsd.c b/src/runtime/cgo/freebsd.c index aefc481e64..4876b2abe4 100644 --- a/src/runtime/cgo/freebsd.c +++ b/src/runtime/cgo/freebsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build freebsd + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/netbsd.c b/src/runtime/cgo/netbsd.c index de38bb7707..076cc87f12 100644 --- a/src/runtime/cgo/netbsd.c +++ b/src/runtime/cgo/netbsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build netbsd + #include "textflag.h" // Supply environ and __progname, because we don't diff --git a/src/runtime/cgo/openbsd.c b/src/runtime/cgo/openbsd.c index 7c2b6c1737..476649544d 100644 --- a/src/runtime/cgo/openbsd.c +++ b/src/runtime/cgo/openbsd.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build openbsd + #include "textflag.h" // Supply environ, __progname and __guard_local, because diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index 5958ad8914..972eedc624 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -36,10 +36,14 @@ func TestCgoTraceback(t *testing.T) { } func TestCgoExternalThreadPanic(t *testing.T) { - if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + if runtime.GOOS == "plan9" { t.Skipf("no pthreads on %s", runtime.GOOS) } - got := executeTest(t, cgoExternalThreadPanicSource, nil, "main.c", cgoExternalThreadPanicC) + csrc := cgoExternalThreadPanicC + if runtime.GOOS == "windows" { + csrc = cgoExternalThreadPanicC_windows + } + got := executeTest(t, cgoExternalThreadPanicSource, nil, "main.c", csrc) want := "panic: BOOM" if !strings.Contains(got, want) { t.Fatalf("want failure containing %q. output:\n%s\n", want, got) @@ -169,3 +173,24 @@ start(void) printf("pthread_create failed\n"); } ` + +const cgoExternalThreadPanicC_windows = ` +#include +#include + +void gopanic(void); + +static void* +die(void* x) +{ + gopanic(); + return 0; +} + +void +start(void) +{ + if(_beginthreadex(0, 0, die, 0, 0, 0) != 0) + printf("_beginthreadex failed\n"); +} +` diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 1b8052bb56..6cc5df810c 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -117,11 +117,20 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) { return } -// Callers fills the slice pc with the program counters of function invocations +// Callers fills the slice pc with the return program counters of function invocations // on the calling goroutine's stack. The argument skip is the number of stack frames // to skip before recording in pc, with 0 identifying the frame for Callers itself and // 1 identifying the caller of Callers. // It returns the number of entries written to pc. +// +// Note that since each slice entry pc[i] is a return program counter, +// looking up the file and line for pc[i] (for example, using (*Func).FileLine) +// will return the file and line number of the instruction immediately +// following the call. +// To look up the file and line number of the call itself, use pc[i]-1. +// As an exception to this rule, if pc[i-1] corresponds to the function +// runtime.sigpanic, then pc[i] is the program counter of a faulting +// instruction and should be used without any subtraction. func Callers(skip int, pc []uintptr) int { // runtime.callers uses pc.array==nil as a signal // to print a stack trace. Pick off 0-length pc here diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c index 5ac37803bb..da14f2d241 100644 --- a/src/runtime/heapdump.c +++ b/src/runtime/heapdump.c @@ -251,7 +251,9 @@ dumpbv(BitVector *bv, uintptr offset) for(i = 0; i < bv->n; i += BitsPerPointer) { switch(bv->bytedata[i/8] >> i%8 & 3) { case BitsDead: - return; + // BitsDead has already been processed in makeheapobjbv. + // We should only see it in stack maps, in which case we should continue processing. + break; case BitsScalar: break; case BitsPointer: @@ -400,7 +402,7 @@ dumpgoroutine(G *gp) child.sp = nil; child.depth = 0; fn = dumpframe; - runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, &fn, &child, false); + runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, &fn, &child, 0); // dump defer & panic records for(d = gp->defer; d != nil; d = d->link) { diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index a18e77421e..fab8cf2695 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -41,7 +41,7 @@ var zerobase uintptr // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. -func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer { +func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { if size == 0 { return unsafe.Pointer(&zerobase) } @@ -391,7 +391,7 @@ func loadPtrMask(typ *_type) []uint8 { // implementation of new builtin func newobject(typ *_type) unsafe.Pointer { - flags := 0 + flags := uint32(0) if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } @@ -400,7 +400,7 @@ func newobject(typ *_type) unsafe.Pointer { // implementation of make builtin for slices func newarray(typ *_type, n uintptr) unsafe.Pointer { - flags := 0 + flags := uint32(0) if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c index 214b9ebc24..3248b0f49a 100644 --- a/src/runtime/mgc0.c +++ b/src/runtime/mgc0.c @@ -1021,7 +1021,7 @@ scanstack(G *gp) runtime·throw("can't scan gchelper stack"); fn = scanframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &fn, nil, false); + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &fn, nil, 0); runtime·tracebackdefers(gp, &fn, nil); } @@ -2635,7 +2635,7 @@ runtime·getgcmask(byte *p, Type *t, byte **mask, uintptr *len) frame.fn = nil; frame.sp = (uintptr)p; cb = getgcmaskcb; - runtime·gentraceback(g->m->curg->sched.pc, g->m->curg->sched.sp, 0, g->m->curg, 0, nil, 1000, &cb, &frame, false); + runtime·gentraceback(g->m->curg->sched.pc, g->m->curg->sched.sp, 0, g->m->curg, 0, nil, 1000, &cb, &frame, 0); if(frame.fn != nil) { Func *f; StackMap *stackmap; diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index 803da56670..d409c6c306 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -528,8 +528,6 @@ var allgs []*g // proc.c // Most clients should use the runtime/pprof package instead // of calling GoroutineProfile directly. func GoroutineProfile(p []StackRecord) (n int, ok bool) { - sp := getcallersp(unsafe.Pointer(&p)) - pc := getcallerpc(unsafe.Pointer(&p)) n = NumGoroutine() if n <= len(p) { @@ -542,7 +540,11 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { if n <= len(p) { ok = true r := p - saveg(pc, sp, gp, &r[0]) + sp := getcallersp(unsafe.Pointer(&p)) + pc := getcallerpc(unsafe.Pointer(&p)) + onM(func() { + saveg(pc, sp, gp, &r[0]) + }) r = r[1:] for _, gp1 := range allgs { if gp1 == gp || readgstatus(gp1) == _Gdead { @@ -562,7 +564,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { } func saveg(pc, sp uintptr, gp *g, r *StackRecord) { - n := gentraceback(pc, sp, 0, gp, 0, &r.Stack0[0], len(r.Stack0), nil, nil, false) + n := gentraceback(pc, sp, 0, gp, 0, &r.Stack0[0], len(r.Stack0), nil, nil, 0) if n < len(r.Stack0) { r.Stack0[n] = 0 } @@ -573,8 +575,6 @@ func saveg(pc, sp uintptr, gp *g, r *StackRecord) { // If all is true, Stack formats stack traces of all other goroutines // into buf after the trace for the current goroutine. func Stack(buf []byte, all bool) int { - sp := getcallersp(unsafe.Pointer(&buf)) - pc := getcallerpc(unsafe.Pointer(&buf)) mp := acquirem() gp := mp.curg if all { @@ -589,14 +589,19 @@ func Stack(buf []byte, all bool) int { n := 0 if len(buf) > 0 { - gp.writebuf = buf[0:0:len(buf)] - goroutineheader(gp) - traceback(pc, sp, 0, gp) - if all { - tracebackothers(gp) - } - n = len(gp.writebuf) - gp.writebuf = nil + sp := getcallersp(unsafe.Pointer(&buf)) + pc := getcallerpc(unsafe.Pointer(&buf)) + onM(func() { + g0 := getg() + g0.writebuf = buf[0:0:len(buf)] + goroutineheader(gp) + traceback(pc, sp, 0, gp) + if all { + tracebackothers(gp) + } + n = len(g0.writebuf) + g0.writebuf = nil + }) } if all { @@ -623,7 +628,11 @@ func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) { } if gp.m.curg == nil || gp == gp.m.curg { goroutineheader(gp) - traceback(getcallerpc(unsafe.Pointer(&p)), getcallersp(unsafe.Pointer(&p)), 0, gp) + pc := getcallerpc(unsafe.Pointer(&p)) + sp := getcallersp(unsafe.Pointer(&p)) + onM(func() { + traceback(pc, sp, 0, gp) + }) } else { goroutineheader(gp.m.curg) traceback(^uintptr(0), ^uintptr(0), 0, gp.m.curg) @@ -639,7 +648,11 @@ func tracefree(p unsafe.Pointer, size uintptr) { gp.m.traceback = 2 print("tracefree(", p, ", ", hex(size), ")\n") goroutineheader(gp) - traceback(getcallerpc(unsafe.Pointer(&p)), getcallersp(unsafe.Pointer(&p)), 0, gp) + pc := getcallerpc(unsafe.Pointer(&p)) + sp := getcallersp(unsafe.Pointer(&p)) + onM(func() { + traceback(pc, sp, 0, gp) + }) print("\n") gp.m.traceback = 0 unlock(&tracelock) diff --git a/src/runtime/os_android.c b/src/runtime/os_android.c index 58e0dac939..5805f68713 100644 --- a/src/runtime/os_android.c +++ b/src/runtime/os_android.c @@ -9,7 +9,7 @@ // Export the runtime entry point symbol. // // Used by the app package to start the Go runtime after loading -// a shared library via JNI. See code.google.com/p/go.mobile/app. +// a shared library via JNI. See golang.org/x/mobile/app. void _rt0_arm_linux1(); #pragma cgo_export_static _rt0_arm_linux1 diff --git a/src/runtime/os_plan9_386.c b/src/runtime/os_plan9_386.c index 3490862244..42c6d161c7 100644 --- a/src/runtime/os_plan9_386.c +++ b/src/runtime/os_plan9_386.c @@ -114,7 +114,7 @@ Throw: if(runtime·gotraceback(&crash)) { runtime·goroutineheader(gp); - runtime·traceback(ureg->pc, ureg->sp, 0, gp); + runtime·tracebacktrap(ureg->pc, ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(ureg); diff --git a/src/runtime/os_plan9_amd64.c b/src/runtime/os_plan9_amd64.c index 6b0f8ae3a2..a9dc0eb966 100644 --- a/src/runtime/os_plan9_amd64.c +++ b/src/runtime/os_plan9_amd64.c @@ -122,7 +122,7 @@ Throw: if(runtime·gotraceback(&crash)) { runtime·goroutineheader(gp); - runtime·traceback(ureg->ip, ureg->sp, 0, gp); + runtime·tracebacktrap(ureg->ip, ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(ureg); diff --git a/src/runtime/os_windows_386.c b/src/runtime/os_windows_386.c index 213582799b..9962f0dc2e 100644 --- a/src/runtime/os_windows_386.c +++ b/src/runtime/os_windows_386.c @@ -97,7 +97,7 @@ runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) runtime·printf("\n"); if(runtime·gotraceback(&crash)){ - runtime·traceback(r->Eip, r->Esp, 0, gp); + runtime·tracebacktrap(r->Eip, r->Esp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } diff --git a/src/runtime/os_windows_amd64.c b/src/runtime/os_windows_amd64.c index b96cf70d1e..e4617e4cef 100644 --- a/src/runtime/os_windows_amd64.c +++ b/src/runtime/os_windows_amd64.c @@ -119,7 +119,7 @@ runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp) runtime·printf("\n"); if(runtime·gotraceback(&crash)){ - runtime·traceback(r->Rip, r->Rsp, 0, gp); + runtime·tracebacktrap(r->Rip, r->Rsp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } diff --git a/src/runtime/proc.c b/src/runtime/proc.c index c1df40d02f..ce39db4abb 100644 --- a/src/runtime/proc.c +++ b/src/runtime/proc.c @@ -994,7 +994,7 @@ runtime·newextram(void) // the goroutine stack ends. mp = runtime·allocm(nil); gp = runtime·malg(4096); - gp->sched.pc = (uintptr)runtime·goexit; + gp->sched.pc = (uintptr)runtime·goexit + PCQuantum; gp->sched.sp = gp->stack.hi; gp->sched.sp -= 4*sizeof(uintreg); // extra space in case of reads slightly beyond frame gp->sched.lr = 0; @@ -1647,12 +1647,10 @@ runtime·gosched_m(G *gp) } // Finishes execution of the current goroutine. -// Need to mark it as nosplit, because it runs with sp > stackbase. -// Since it does not return it does not matter. But if it is preempted -// at the split stack check, GC will complain about inconsistent sp. +// Must be NOSPLIT because it is called from Go. #pragma textflag NOSPLIT void -runtime·goexit(void) +runtime·goexit1(void) { void (*fn)(G*); @@ -2198,7 +2196,7 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp runtime·memclr((byte*)&newg->sched, sizeof newg->sched); newg->sched.sp = (uintptr)sp; - newg->sched.pc = (uintptr)runtime·goexit; + newg->sched.pc = (uintptr)runtime·goexit + PCQuantum; // +PCQuantum so that previous instruction is in same function newg->sched.g = newg; runtime·gostartcallfn(&newg->sched, fn); newg->gopc = (uintptr)callerpc; @@ -2432,9 +2430,10 @@ static struct ProfState { int32 hz; } prof; -static void System(void) {} -static void ExternalCode(void) {} -static void GC(void) {} +static void System(void) { System(); } +static void ExternalCode(void) { ExternalCode(); } +static void GC(void) { GC(); } + extern void runtime·cpuproftick(uintptr*, int32); extern byte runtime·etext[]; @@ -2538,7 +2537,7 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) n = 0; if(traceback) - n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, stk, nelem(stk), nil, nil, TraceTrap); if(!traceback || n <= 0) { // Normal traceback is impossible or has failed. // See if it falls into several common cases. @@ -2548,13 +2547,13 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) // Cgo, we can't unwind and symbolize arbitrary C code, // so instead collect Go stack that leads to the cgo call. // This is especially important on windows, since all syscalls are cgo calls. - n = runtime·gentraceback(mp->curg->syscallpc, mp->curg->syscallsp, 0, mp->curg, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback(mp->curg->syscallpc, mp->curg->syscallsp, 0, mp->curg, 0, stk, nelem(stk), nil, nil, 0); } #ifdef GOOS_windows if(n == 0 && mp->libcallg != nil && mp->libcallpc != 0 && mp->libcallsp != 0) { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. - n = runtime·gentraceback(mp->libcallpc, mp->libcallsp, 0, mp->libcallg, 0, stk, nelem(stk), nil, nil, false); + n = runtime·gentraceback(mp->libcallpc, mp->libcallsp, 0, mp->libcallg, 0, stk, nelem(stk), nil, nil, 0); } #endif if(n == 0) { diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index fec224390c..330ed429b9 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -737,9 +737,15 @@ struct Stkframe BitVector* argmap; // force use of this argmap }; -intgo runtime·gentraceback(uintptr, uintptr, uintptr, G*, intgo, uintptr*, intgo, bool(**)(Stkframe*, void*), void*, bool); +enum +{ + TraceRuntimeFrames = 1<<0, // include frames for internal runtime functions. + TraceTrap = 1<<1, // the initial PC, SP are from a trap, not a return PC from a call +}; +intgo runtime·gentraceback(uintptr, uintptr, uintptr, G*, intgo, uintptr*, intgo, bool(**)(Stkframe*, void*), void*, uintgo); void runtime·tracebackdefers(G*, bool(**)(Stkframe*, void*), void*); void runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G* gp); +void runtime·tracebacktrap(uintptr pc, uintptr sp, uintptr lr, G* gp); void runtime·tracebackothers(G*); bool runtime·haszeroargs(uintptr pc); bool runtime·topofstack(Func*); diff --git a/src/runtime/sema.go b/src/runtime/sema.go index a42a29988a..d2a028c01b 100644 --- a/src/runtime/sema.go +++ b/src/runtime/sema.go @@ -259,6 +259,7 @@ func syncsemrelease(s *syncSema, n uint32) { } s.tail = w goparkunlock(&s.lock, "semarelease") + releaseSudog(w) } else { unlock(&s.lock) } diff --git a/src/runtime/signal_386.c b/src/runtime/signal_386.c index d55e304528..30a7488bd7 100644 --- a/src/runtime/signal_386.c +++ b/src/runtime/signal_386.c @@ -109,7 +109,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_EIP(info, ctxt), SIG_ESP(info, ctxt), 0, gp); + runtime·tracebacktrap(SIG_EIP(info, ctxt), SIG_ESP(info, ctxt), 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/signal_amd64x.c b/src/runtime/signal_amd64x.c index 44e68cecfc..feb4afcce3 100644 --- a/src/runtime/signal_amd64x.c +++ b/src/runtime/signal_amd64x.c @@ -143,7 +143,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_RIP(info, ctxt), SIG_RSP(info, ctxt), 0, gp); + runtime·tracebacktrap(SIG_RIP(info, ctxt), SIG_RSP(info, ctxt), 0, gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/signal_arm.c b/src/runtime/signal_arm.c index 3571cf3ac6..afad5e7d16 100644 --- a/src/runtime/signal_arm.c +++ b/src/runtime/signal_arm.c @@ -108,7 +108,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) if(runtime·gotraceback(&crash)){ runtime·goroutineheader(gp); - runtime·traceback(SIG_PC(info, ctxt), SIG_SP(info, ctxt), SIG_LR(info, ctxt), gp); + runtime·tracebacktrap(SIG_PC(info, ctxt), SIG_SP(info, ctxt), SIG_LR(info, ctxt), gp); runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(info, ctxt); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index a4947a53b3..ffae73a2ab 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -582,7 +582,7 @@ copystack(G *gp, uintptr newsize) adjinfo.old = old; adjinfo.delta = new.hi - old.hi; cb = adjustframe; - runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, false); + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, &cb, &adjinfo, 0); // adjust other miscellaneous things that have pointers into stacks. adjustctxt(gp, &adjinfo); diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 421ab04e50..9889567d61 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -228,6 +228,34 @@ func atomicloaduint(ptr *uint) uint //go:noescape func setcallerpc(argp unsafe.Pointer, pc uintptr) +// getcallerpc returns the program counter (PC) of its caller's caller. +// getcallersp returns the stack pointer (SP) of its caller's caller. +// For both, the argp must be a pointer to the caller's first function argument. +// The implementation may or may not use argp, depending on +// the architecture. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := getcallerpc(unsafe.Pointer(&arg1)) +// sp := getcallerpc(unsafe.Pointer(&arg2)) +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to getcallerpc and getcallersp must be done in the +// frame being asked about. It would not be correct for f to pass &arg1 +// to another function g and let g call getcallerpc/getcallersp. +// The call inside g might return information about g's caller or +// information about f's caller or complete garbage. +// +// The result of getcallersp is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of getcallersp should be used +// immediately and can only be passed to nosplit functions. + //go:noescape func getcallerpc(argp unsafe.Pointer) uintptr diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 24dc3eea95..1c6ce6e644 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -96,11 +96,27 @@ func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v uns // the runtime.Callers function (pcbuf != nil), as well as the garbage // collector (callback != nil). A little clunky to merge these, but avoids // duplicating the code and all its subtlety. -func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max int, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer, printall bool) int { +func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max int, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer, flags uint) int { if goexitPC == 0 { gothrow("gentraceback before goexitPC initialization") } g := getg() + if g == gp && g == g.m.curg { + // The starting sp has been passed in as a uintptr, and the caller may + // have other uintptr-typed stack references as well. + // If during one of the calls that got us here or during one of the + // callbacks below the stack must be grown, all these uintptr references + // to the stack will not be updated, and gentraceback will continue + // to inspect the old stack memory, which may no longer be valid. + // Even if all the variables were updated correctly, it is not clear that + // we want to expose a traceback that begins on one stack and ends + // on another stack. That could confuse callers quite a bit. + // Instead, we require that gentraceback and any other function that + // accepts an sp for the current goroutine (typically obtained by + // calling getcallersp) must not run on that goroutine's stack but + // instead on the g0 stack. + gothrow("gentraceback cannot trace user goroutine on its own stack") + } gotraceback := gotraceback(nil) if pc0 == ^uintptr(0) && sp0 == ^uintptr(0) { // Signal to fetch saved values from gp. if gp.syscallsp != 0 { @@ -297,13 +313,13 @@ func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf } } if printing { - if printall || showframe(f, gp) { + if (flags&_TraceRuntimeFrames) != 0 || showframe(f, gp) { // Print during crash. // main(0x1, 0x2, 0x3) // /home/rsc/go/src/runtime/x.go:23 +0xf // tracepc := frame.pc // back up to CALL instruction for funcline. - if n > 0 && frame.pc > f.entry && !waspanic { + if (n > 0 || flags&_TraceTrap == 0) && frame.pc > f.entry && !waspanic { tracepc-- } print(gofuncname(f), "(") @@ -475,17 +491,32 @@ func printcreatedby(gp *g) { } func traceback(pc uintptr, sp uintptr, lr uintptr, gp *g) { + traceback1(pc, sp, lr, gp, 0) +} + +// tracebacktrap is like traceback but expects that the PC and SP were obtained +// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or getcallerpc/getcallersp. +// Because they are from a trap instead of from a saved pair, +// the initial PC must not be rewound to the previous instruction. +// (All the saved pairs record a PC that is a return address, so we +// rewind it into the CALL instruction.) +func tracebacktrap(pc uintptr, sp uintptr, lr uintptr, gp *g) { + traceback1(pc, sp, lr, gp, _TraceTrap) +} + +func traceback1(pc uintptr, sp uintptr, lr uintptr, gp *g, flags uint) { var n int if readgstatus(gp)&^_Gscan == _Gsyscall { - // Override signal registers if blocked in system call. + // Override registers if blocked in system call. pc = gp.syscallpc sp = gp.syscallsp + flags &^= _TraceTrap } // Print traceback. By default, omits runtime frames. // If that means we print nothing at all, repeat forcing all frames printed. - n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, false) - if n == 0 { - n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, true) + n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, flags) + if n == 0 && (flags&_TraceRuntimeFrames) == 0 { + n = gentraceback(pc, sp, lr, gp, 0, nil, _TracebackMaxFrames, nil, nil, flags|_TraceRuntimeFrames) } if n == _TracebackMaxFrames { print("...additional frames elided...\n") @@ -496,11 +527,15 @@ func traceback(pc uintptr, sp uintptr, lr uintptr, gp *g) { func callers(skip int, pcbuf *uintptr, m int) int { sp := getcallersp(unsafe.Pointer(&skip)) pc := uintptr(getcallerpc(unsafe.Pointer(&skip))) - return gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, false) + var n int + onM(func() { + n = gentraceback(pc, sp, 0, getg(), skip, pcbuf, m, nil, nil, 0) + }) + return n } func gcallers(gp *g, skip int, pcbuf *uintptr, m int) int { - return gentraceback(^uintptr(0), ^uintptr(0), 0, gp, skip, pcbuf, m, nil, nil, false) + return gentraceback(^uintptr(0), ^uintptr(0), 0, gp, skip, pcbuf, m, nil, nil, 0) } func showframe(f *_func, gp *g) bool { diff --git a/src/sync/atomic/value.go b/src/sync/atomic/value.go index ab46d9a240..ab3aa11285 100644 --- a/src/sync/atomic/value.go +++ b/src/sync/atomic/value.go @@ -38,7 +38,7 @@ func (v *Value) Load() (x interface{}) { return } -// Store sets the value of the Value to v. +// Store sets the value of the Value to x. // All calls to Store for a given Value must use values of the same concrete type. // Store of an inconsistent type panics, as does Store(nil). func (v *Value) Store(x interface{}) { diff --git a/test/fixedbugs/issue7690.go b/test/fixedbugs/issue7690.go new file mode 100644 index 0000000000..4ad9e8622a --- /dev/null +++ b/test/fixedbugs/issue7690.go @@ -0,0 +1,49 @@ +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// issue 7690 - Stack and other routines did not back up initial PC +// into CALL instruction, instead reporting line number of next instruction, +// which might be on a different line. + +package main + +import ( + "bytes" + "regexp" + "runtime" + "strconv" +) + +func main() { + buf1 := make([]byte, 1000) + buf2 := make([]byte, 1000) + + runtime.Stack(buf1, false) // CALL is last instruction on this line + n := runtime.Stack(buf2, false) // CALL is followed by load of result from stack + + buf1 = buf1[:bytes.IndexByte(buf1, 0)] + buf2 = buf2[:n] + + re := regexp.MustCompile(`(?m)^main\.main\(\)\n.*/issue7690.go:([0-9]+)`) + m1 := re.FindStringSubmatch(string(buf1)) + if m1 == nil { + println("BUG: cannot find main.main in first trace") + return + } + m2 := re.FindStringSubmatch(string(buf2)) + if m2 == nil { + println("BUG: cannot find main.main in second trace") + return + } + + n1, _ := strconv.Atoi(m1[1]) + n2, _ := strconv.Atoi(m2[1]) + if n1+1 != n2 { + println("BUG: expect runtime.Stack on back to back lines, have", n1, n2) + println(string(buf1)) + println(string(buf2)) + } +} diff --git a/test/linkx.go b/test/linkx.go index 06888a229a..151b6db1ec 100644 --- a/test/linkx.go +++ b/test/linkx.go @@ -1,13 +1,11 @@ -// $G $D/$F.go && $L -X main.tbd hello -X main.overwrite trumped -X main.nosuchsymbol neverseen $F.$A && ./$A.out - -// NOTE: This test is not run by 'run.go' and so not run by all.bash. -// To run this test you must use the ./run shell script. +// skip // Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Test the -X facility of the gc linker (6l etc.). +// This test is run by linkx_run.go. package main @@ -15,10 +13,6 @@ var tbd string var overwrite string = "dibs" func main() { - if tbd != "hello" { - println("BUG: test/linkx tbd", len(tbd), tbd) - } - if overwrite != "trumped" { - println("BUG: test/linkx overwrite", len(overwrite), overwrite) - } + println(tbd) + println(overwrite) } diff --git a/test/linkx_run.go b/test/linkx_run.go new file mode 100644 index 0000000000..5b67ce7d3d --- /dev/null +++ b/test/linkx_run.go @@ -0,0 +1,33 @@ +// +build !nacl +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run the linkx test. + +package main + +import ( + "fmt" + "os" + "os/exec" +) + +func main() { + cmd := exec.Command("go", "run", "-ldflags=-X main.tbd hello -X main.overwrite trumped", "linkx.go") + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Println(string(out)) + fmt.Println(err) + os.Exit(1) + } + + want := "hello\ntrumped\n" + got := string(out) + if got != want { + fmt.Printf("got %q want %q\n", got, want) + os.Exit(1) + } +} diff --git a/test/run.go b/test/run.go index 28882cf54c..e8ec2df9c4 100644 --- a/test/run.go +++ b/test/run.go @@ -907,8 +907,6 @@ func (t *test) wantedErrors(file, short string) (errs []wantedError) { } var skipOkay = map[string]bool{ - "linkx.go": true, // like "run" but wants linker flags - "sinit.go": true, "fixedbugs/bug248.go": true, // combines errorcheckdir and rundir in the same dir. "fixedbugs/bug302.go": true, // tests both .$O and .a imports. "fixedbugs/bug345.go": true, // needs the appropriate flags in gc invocation. diff --git a/test/sinit.go b/test/sinit.go index 5e50e1100a..df1a4cc930 100644 --- a/test/sinit.go +++ b/test/sinit.go @@ -1,7 +1,4 @@ -// $G -S $D/$F.go | egrep initdone >/dev/null && echo BUG sinit || true - -// NOTE: This test is not run by 'run.go' and so not run by all.bash. -// To run this test you must use the ./run shell script. +// skip // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -9,6 +6,7 @@ // Test that many initializations can be done at link time and // generate no executable init functions. +// This test is run by sinit_run.go. package p @@ -106,20 +104,27 @@ var answers = [...]int{ } var ( - copy_zero = zero - copy_one = one - copy_pi = pi - copy_slice = slice + copy_zero = zero + copy_one = one + copy_pi = pi + copy_slice = slice copy_sliceInt = sliceInt - copy_hello = hello - copy_bytes = bytes + copy_hello = hello + + // Could be handled without an initialization function, but + // requires special handling for "a = []byte("..."); b = a" + // which is not a likely case. + // copy_bytes = bytes + // https://codereview.appspot.com/171840043 is one approach to + // make this special case work. + copy_four, copy_five = four, five - copy_x, copy_y = x, y - copy_nilslice = nilslice - copy_nilmap = nilmap - copy_nilfunc = nilfunc - copy_nilchan = nilchan - copy_nilptr = nilptr + copy_x, copy_y = x, y + copy_nilslice = nilslice + copy_nilmap = nilmap + copy_nilfunc = nilfunc + copy_nilchan = nilchan + copy_nilptr = nilptr ) var copy_a = a @@ -172,7 +177,7 @@ var sx []int var s0 = []int{0, 0, 0} var s1 = []int{1, 2, 3} -func fi() int +func fi() int { return 1 } var ax [10]int var a0 = [10]int{0, 0, 0} @@ -202,58 +207,66 @@ var pt0b = &T{X: 0} var pt1 = &T{X: 1, Y: 2} var pt1a = &T{3, 4} -var copy_bx = bx +// The checks similar to +// var copy_bx = bx +// are commented out. The compiler no longer statically initializes them. +// See issue 7665 and https://codereview.appspot.com/93200044. +// If https://codereview.appspot.com/169040043 is submitted, and this +// test is changed to pass -complete to the compiler, then we can +// uncomment the copy lines again. + +// var copy_bx = bx var copy_b0 = b0 var copy_b1 = b1 -var copy_fx = fx +// var copy_fx = fx var copy_f0 = f0 var copy_f1 = f1 -var copy_gx = gx +// var copy_gx = gx var copy_g0 = g0 var copy_g1 = g1 -var copy_ix = ix +// var copy_ix = ix var copy_i0 = i0 var copy_i1 = i1 -var copy_jx = jx +// var copy_jx = jx var copy_j0 = j0 var copy_j1 = j1 -var copy_cx = cx +// var copy_cx = cx var copy_c0 = c0 var copy_c1 = c1 -var copy_dx = dx +// var copy_dx = dx var copy_d0 = d0 var copy_d1 = d1 -var copy_sx = sx +// var copy_sx = sx var copy_s0 = s0 var copy_s1 = s1 -var copy_ax = ax +// var copy_ax = ax var copy_a0 = a0 var copy_a1 = a1 -var copy_tx = tx +// var copy_tx = tx var copy_t0 = t0 var copy_t0a = t0a var copy_t0b = t0b var copy_t1 = t1 var copy_t1a = t1a -var copy_psx = psx +// var copy_psx = psx var copy_ps0 = ps0 var copy_ps1 = ps1 -var copy_pax = pax +// var copy_pax = pax var copy_pa0 = pa0 var copy_pa1 = pa1 -var copy_ptx = ptx +// var copy_ptx = ptx var copy_pt0 = pt0 var copy_pt0a = pt0a var copy_pt0b = pt0b @@ -266,6 +279,8 @@ type T1 int func (t *T1) M() {} -type Mer interface { M() } +type Mer interface { + M() +} var _ Mer = (*T1)(nil) diff --git a/test/sinit_run.go b/test/sinit_run.go new file mode 100644 index 0000000000..b0a91ce5b1 --- /dev/null +++ b/test/sinit_run.go @@ -0,0 +1,40 @@ +// +build !nacl +// run + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run the sinit test. + +package main + +import ( + "bytes" + "fmt" + "go/build" + "os" + "os/exec" +) + +func main() { + letter, err := build.ArchChar(build.Default.GOARCH) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + cmd := exec.Command("go", "tool", letter+"g", "-S", "sinit.go") + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Println(string(out)) + fmt.Println(err) + os.Exit(1) + } + os.Remove("sinit." + letter) + + if bytes.Contains(out, []byte("initdone")) { + fmt.Println("sinit generated an init function") + os.Exit(1) + } +}