From c21ba09bcd83e58e862b1acc676dc36f31444022 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Fri, 2 Nov 2018 00:57:30 -0400 Subject: [PATCH 001/111] crypto/tls: implement TLS 1.3 server handshake (base) Implement a basic TLS 1.3 server handshake, only enabled if explicitly requested with MaxVersion. This CL intentionally leaves for future CLs: - PSK modes and resumption - client authentication - compatibility mode ChangeCipherSpecs - early data skipping - post-handshake messages - downgrade protection - KeyLogWriter support - TLS_FALLBACK_SCSV processing It also leaves a few areas up for a wider refactor (maybe in Go 1.13): - the certificate selection logic can be significantly improved, including supporting and surfacing signature_algorithms_cert, but this isn't new in TLS 1.3 (see comment in processClientHello) - handshake_server_tls13.go can be dried up and broken into more meaningful, smaller functions, but it felt premature to do before PSK and client auth support - the monstrous ClientHello equality check in doHelloRetryRequest can get both cleaner and more complete with collaboration from the parsing layer, which can come at the same time as extension duplicates detection Updates #9671 Change-Id: Id9db2b6ecc2eea21bf9b59b6d1d9c84a7435151c Reviewed-on: https://go-review.googlesource.com/c/147017 Run-TryBot: Filippo Valsorda TryBot-Result: Gobot Gobot Reviewed-by: Adam Langley --- src/crypto/tls/auth.go | 33 ++ src/crypto/tls/common.go | 2 +- src/crypto/tls/handshake_messages.go | 4 +- src/crypto/tls/handshake_server.go | 154 +++--- src/crypto/tls/handshake_server_test.go | 95 +++- src/crypto/tls/handshake_server_tls13.go | 461 ++++++++++++++++++ src/crypto/tls/testdata/Server-TLSv12-P256 | 85 ++++ ...ECDHE-RSA-AES-GCM => Server-TLSv12-X25519} | 0 .../tls/testdata/Server-TLSv13-AES128-SHA256 | 90 ++++ .../tls/testdata/Server-TLSv13-AES256-SHA384 | 92 ++++ src/crypto/tls/testdata/Server-TLSv13-ALPN | 94 ++++ .../tls/testdata/Server-TLSv13-ALPN-NoMatch | 93 ++++ .../testdata/Server-TLSv13-CHACHA20-SHA256 | 90 ++++ .../testdata/Server-TLSv13-ECDHE-ECDSA-AES | 86 ++++ .../Server-TLSv13-ExportKeyingMaterial | 92 ++++ .../testdata/Server-TLSv13-HelloRetryRequest | 118 +++++ src/crypto/tls/testdata/Server-TLSv13-P256 | 95 ++++ .../tls/testdata/Server-TLSv13-RSA-RSAPSS | 90 ++++ src/crypto/tls/testdata/Server-TLSv13-X25519 | 91 ++++ 19 files changed, 1788 insertions(+), 77 deletions(-) create mode 100644 src/crypto/tls/handshake_server_tls13.go create mode 100644 src/crypto/tls/testdata/Server-TLSv12-P256 rename src/crypto/tls/testdata/{Server-TLSv12-X25519-ECDHE-RSA-AES-GCM => Server-TLSv12-X25519} (100%) create mode 100644 src/crypto/tls/testdata/Server-TLSv13-AES128-SHA256 create mode 100644 src/crypto/tls/testdata/Server-TLSv13-AES256-SHA384 create mode 100644 src/crypto/tls/testdata/Server-TLSv13-ALPN create mode 100644 src/crypto/tls/testdata/Server-TLSv13-ALPN-NoMatch create mode 100644 src/crypto/tls/testdata/Server-TLSv13-CHACHA20-SHA256 create mode 100644 src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES create mode 100644 src/crypto/tls/testdata/Server-TLSv13-ExportKeyingMaterial create mode 100644 src/crypto/tls/testdata/Server-TLSv13-HelloRetryRequest create mode 100644 src/crypto/tls/testdata/Server-TLSv13-P256 create mode 100644 src/crypto/tls/testdata/Server-TLSv13-RSA-RSAPSS create mode 100644 src/crypto/tls/testdata/Server-TLSv13-X25519 diff --git a/src/crypto/tls/auth.go b/src/crypto/tls/auth.go index 3e12d9745c..859387ee14 100644 --- a/src/crypto/tls/auth.go +++ b/src/crypto/tls/auth.go @@ -7,6 +7,7 @@ package tls import ( "crypto" "crypto/ecdsa" + "crypto/elliptic" "crypto/rsa" "encoding/asn1" "errors" @@ -131,3 +132,35 @@ func writeSignedMessage(sigHash io.Writer, context string, transcript hash.Hash) io.WriteString(sigHash, context) sigHash.Write(transcript.Sum(nil)) } + +// signatureSchemesForCertificate returns the list of supported SignatureSchemes +// for a given certificate, based on the public key. +func signatureSchemesForCertificate(cert *Certificate) []SignatureScheme { + priv, ok := cert.PrivateKey.(crypto.Signer) + if !ok { + return nil + } + + switch priv := priv.Public().(type) { + case *ecdsa.PublicKey: + switch priv.Curve { + case elliptic.P256(): + return []SignatureScheme{ECDSAWithP256AndSHA256} + case elliptic.P384(): + return []SignatureScheme{ECDSAWithP384AndSHA384} + case elliptic.P521(): + return []SignatureScheme{ECDSAWithP521AndSHA512} + default: + return nil + } + case *rsa.PublicKey: + // RSA keys with RSA-PSS OID are not supported by crypto/x509. + return []SignatureScheme{ + PSSWithSHA256, + PSSWithSHA384, + PSSWithSHA512, + } + default: + return nil + } +} diff --git a/src/crypto/tls/common.go b/src/crypto/tls/common.go index 7e5976ae5f..ddd3da5216 100644 --- a/src/crypto/tls/common.go +++ b/src/crypto/tls/common.go @@ -740,7 +740,7 @@ func (c *Config) supportedVersions(isClient bool) []uint16 { continue } // TLS 1.3 is only supported if explicitly requested while in development. - if v == VersionTLS13 && (!isClient || c == nil || c.MaxVersion != VersionTLS13) { + if v == VersionTLS13 && (c == nil || c.MaxVersion != VersionTLS13) { continue } versions = append(versions, v) diff --git a/src/crypto/tls/handshake_messages.go b/src/crypto/tls/handshake_messages.go index c622e0877a..98b7bd511f 100644 --- a/src/crypto/tls/handshake_messages.go +++ b/src/crypto/tls/handshake_messages.go @@ -131,7 +131,7 @@ func (m *clientHelloMsg) marshal() []byte { }) } if len(m.supportedCurves) > 0 { - // RFC 4492, Section 5.1.1 and RFC 8446, Section 4.2.7 + // RFC 4492, sections 5.1.1 and RFC 8446, Section 4.2.7 b.AddUint16(extensionSupportedCurves) b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { @@ -379,7 +379,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { } m.ocspStapling = statusType == statusTypeOCSP case extensionSupportedCurves: - // RFC 4492, Section 5.1.1 and RFC 8446, Section 4.2.7 + // RFC 4492, sections 5.1.1 and RFC 8446, Section 4.2.7 var curves cryptobyte.String if !extData.ReadUint16LengthPrefixed(&curves) || curves.Empty() { return false diff --git a/src/crypto/tls/handshake_server.go b/src/crypto/tls/handshake_server.go index ae793e2dd4..d1f123cec3 100644 --- a/src/crypto/tls/handshake_server.go +++ b/src/crypto/tls/handshake_server.go @@ -19,20 +19,19 @@ import ( // serverHandshakeState contains details of a server handshake in progress. // It's discarded once the handshake has completed. type serverHandshakeState struct { - c *Conn - clientHello *clientHelloMsg - hello *serverHelloMsg - suite *cipherSuite - ellipticOk bool - ecdsaOk bool - rsaDecryptOk bool - rsaSignOk bool - sessionState *sessionState - finishedHash finishedHash - masterSecret []byte - certsFromClient [][]byte - cert *Certificate - cachedClientHelloInfo *ClientHelloInfo + c *Conn + clientHello *clientHelloMsg + hello *serverHelloMsg + suite *cipherSuite + ellipticOk bool + ecdsaOk bool + rsaDecryptOk bool + rsaSignOk bool + sessionState *sessionState + finishedHash finishedHash + masterSecret []byte + certsFromClient [][]byte + cert *Certificate } // serverHandshake performs a TLS handshake as a server. @@ -41,17 +40,36 @@ func (c *Conn) serverHandshake() error { // encrypt the tickets with. c.config.serverInitOnce.Do(func() { c.config.serverInit(nil) }) - hs := serverHandshakeState{ - c: c, - } - isResume, err := hs.readClientHello() + clientHello, err := c.readClientHello() if err != nil { return err } + if c.vers == VersionTLS13 { + hs := serverHandshakeStateTLS13{ + c: c, + clientHello: clientHello, + } + return hs.handshake() + } + + hs := serverHandshakeState{ + c: c, + clientHello: clientHello, + } + return hs.handshake() +} + +func (hs *serverHandshakeState) handshake() error { + c := hs.c + + if err := hs.processClientHello(); err != nil { + return err + } + // For an overview of TLS handshaking, see RFC 5246, Section 7.3. c.buffering = true - if isResume { + if hs.checkForResumption() { // The client has included a session ticket and so we do an abbreviated handshake. if err := hs.doResumeHandshake(); err != nil { return err @@ -81,6 +99,9 @@ func (c *Conn) serverHandshake() error { } else { // The client didn't include a session ticket, or it wasn't // valid so we do a full handshake. + if err := hs.pickCipherSuite(); err != nil { + return err + } if err := hs.doFullHandshake(); err != nil { return err } @@ -109,42 +130,47 @@ func (c *Conn) serverHandshake() error { return nil } -// readClientHello reads a ClientHello message from the client and decides -// whether we will perform session resumption. -func (hs *serverHandshakeState) readClientHello() (isResume bool, err error) { - c := hs.c - +// readClientHello reads a ClientHello message and selects the protocol version. +func (c *Conn) readClientHello() (*clientHelloMsg, error) { msg, err := c.readHandshake() if err != nil { - return false, err + return nil, err } - var ok bool - hs.clientHello, ok = msg.(*clientHelloMsg) + clientHello, ok := msg.(*clientHelloMsg) if !ok { c.sendAlert(alertUnexpectedMessage) - return false, unexpectedMessageError(hs.clientHello, msg) + return nil, unexpectedMessageError(clientHello, msg) } if c.config.GetConfigForClient != nil { - if newConfig, err := c.config.GetConfigForClient(hs.clientHelloInfo()); err != nil { + chi := clientHelloInfo(c, clientHello) + if newConfig, err := c.config.GetConfigForClient(chi); err != nil { c.sendAlert(alertInternalError) - return false, err + return nil, err } else if newConfig != nil { newConfig.serverInitOnce.Do(func() { newConfig.serverInit(c.config) }) c.config = newConfig } } - clientVersions := hs.clientHello.supportedVersions - if len(hs.clientHello.supportedVersions) == 0 { - clientVersions = supportedVersionsFromMax(hs.clientHello.vers) + clientVersions := clientHello.supportedVersions + if len(clientHello.supportedVersions) == 0 { + clientVersions = supportedVersionsFromMax(clientHello.vers) } c.vers, ok = c.config.mutualVersion(false, clientVersions) if !ok { c.sendAlert(alertProtocolVersion) - return false, fmt.Errorf("tls: client offered only unsupported versions: %x", clientVersions) + return nil, fmt.Errorf("tls: client offered only unsupported versions: %x", clientVersions) } c.haveVers = true + c.in.version = c.vers + c.out.version = c.vers + + return clientHello, nil +} + +func (hs *serverHandshakeState) processClientHello() error { + c := hs.c hs.hello = new(serverHelloMsg) hs.hello.vers = c.vers @@ -181,19 +207,19 @@ Curves: if !foundCompression { c.sendAlert(alertHandshakeFailure) - return false, errors.New("tls: client does not support uncompressed connections") + return errors.New("tls: client does not support uncompressed connections") } hs.hello.random = make([]byte, 32) - _, err = io.ReadFull(c.config.rand(), hs.hello.random) + _, err := io.ReadFull(c.config.rand(), hs.hello.random) if err != nil { c.sendAlert(alertInternalError) - return false, err + return err } if len(hs.clientHello.secureRenegotiation) != 0 { c.sendAlert(alertHandshakeFailure) - return false, errors.New("tls: initial handshake had non-empty renegotiation extension") + return errors.New("tls: initial handshake had non-empty renegotiation extension") } hs.hello.secureRenegotiationSupported = hs.clientHello.secureRenegotiationSupported @@ -218,10 +244,10 @@ Curves: } } - hs.cert, err = c.config.getCertificate(hs.clientHelloInfo()) + hs.cert, err = c.config.getCertificate(clientHelloInfo(c, hs.clientHello)) if err != nil { c.sendAlert(alertInternalError) - return false, err + return err } if hs.clientHello.scts { hs.hello.scts = hs.cert.SignedCertificateTimestamps @@ -235,7 +261,7 @@ Curves: hs.rsaSignOk = true default: c.sendAlert(alertInternalError) - return false, fmt.Errorf("tls: unsupported signing key type (%T)", priv.Public()) + return fmt.Errorf("tls: unsupported signing key type (%T)", priv.Public()) } } if priv, ok := hs.cert.PrivateKey.(crypto.Decrypter); ok { @@ -244,13 +270,15 @@ Curves: hs.rsaDecryptOk = true default: c.sendAlert(alertInternalError) - return false, fmt.Errorf("tls: unsupported decryption key type (%T)", priv.Public()) + return fmt.Errorf("tls: unsupported decryption key type (%T)", priv.Public()) } } - if hs.checkForResumption() { - return true, nil - } + return nil +} + +func (hs *serverHandshakeState) pickCipherSuite() error { + c := hs.c var preferenceList, supportedList []uint16 if c.config.PreferServerCipherSuites { @@ -269,7 +297,7 @@ Curves: if hs.suite == nil { c.sendAlert(alertHandshakeFailure) - return false, errors.New("tls: no cipher suite supported by both client and server") + return errors.New("tls: no cipher suite supported by both client and server") } // See RFC 7507. @@ -278,13 +306,13 @@ Curves: // The client is doing a fallback connection. if hs.clientHello.vers < c.config.supportedVersions(false)[0] { c.sendAlert(alertInappropriateFallback) - return false, errors.New("tls: client using inappropriate protocol fallback") + return errors.New("tls: client using inappropriate protocol fallback") } break } } - return false, nil + return nil } // checkForResumption reports whether we should perform resumption on this connection. @@ -766,26 +794,20 @@ func (hs *serverHandshakeState) setCipherSuite(id uint16, supportedCipherSuites return false } -func (hs *serverHandshakeState) clientHelloInfo() *ClientHelloInfo { - if hs.cachedClientHelloInfo != nil { - return hs.cachedClientHelloInfo +func clientHelloInfo(c *Conn, clientHello *clientHelloMsg) *ClientHelloInfo { + supportedVersions := clientHello.supportedVersions + if len(clientHello.supportedVersions) == 0 { + supportedVersions = supportedVersionsFromMax(clientHello.vers) } - supportedVersions := hs.clientHello.supportedVersions - if len(hs.clientHello.supportedVersions) == 0 { - supportedVersions = supportedVersionsFromMax(hs.clientHello.vers) - } - - hs.cachedClientHelloInfo = &ClientHelloInfo{ - CipherSuites: hs.clientHello.cipherSuites, - ServerName: hs.clientHello.serverName, - SupportedCurves: hs.clientHello.supportedCurves, - SupportedPoints: hs.clientHello.supportedPoints, - SignatureSchemes: hs.clientHello.supportedSignatureAlgorithms, - SupportedProtos: hs.clientHello.alpnProtocols, + return &ClientHelloInfo{ + CipherSuites: clientHello.cipherSuites, + ServerName: clientHello.serverName, + SupportedCurves: clientHello.supportedCurves, + SupportedPoints: clientHello.supportedPoints, + SignatureSchemes: clientHello.supportedSignatureAlgorithms, + SupportedProtos: clientHello.alpnProtocols, SupportedVersions: supportedVersions, - Conn: hs.c.conn, + Conn: c.conn, } - - return hs.cachedClientHelloInfo } diff --git a/src/crypto/tls/handshake_server_test.go b/src/crypto/tls/handshake_server_test.go index f7785ecd1f..a02eae24b3 100644 --- a/src/crypto/tls/handshake_server_test.go +++ b/src/crypto/tls/handshake_server_test.go @@ -79,17 +79,25 @@ func testClientHelloFailure(t *testing.T, serverConfig *Config, m handshakeMessa cli.writeRecord(recordTypeHandshake, m.marshal()) c.Close() }() + conn := Server(s, serverConfig) + ch, err := conn.readClientHello() hs := serverHandshakeState{ - c: Server(s, serverConfig), + c: conn, + clientHello: ch, + } + if err == nil { + err = hs.processClientHello() + } + if err == nil { + err = hs.pickCipherSuite() } - _, err := hs.readClientHello() s.Close() if len(expectedSubStr) == 0 { if err != nil && err != io.EOF { t.Errorf("Got error: %s; expected to succeed", err) } } else if err == nil || !strings.Contains(err.Error(), expectedSubStr) { - t.Errorf("Got error: %s; expected to match substring '%s'", err, expectedSubStr) + t.Errorf("Got error: %v; expected to match substring '%s'", err, expectedSubStr) } } @@ -727,6 +735,16 @@ func runServerTestTLS12(t *testing.T, template *serverTest) { runServerTestForVersion(t, template, "TLSv12", "-tls1_2") } +func runServerTestTLS13(t *testing.T, template *serverTest) { + // TODO(filippo): set MaxVersion to VersionTLS13 instead in testConfig + // while regenerating server tests. + if template.config == nil { + template.config = testConfig.Clone() + } + template.config.MaxVersion = VersionTLS13 + runServerTestForVersion(t, template, "TLSv13", "-tls1_3") +} + func TestHandshakeServerRSARC4(t *testing.T) { test := &serverTest{ name: "RSA-RC4", @@ -774,6 +792,28 @@ func TestHandshakeServerAES256GCMSHA384(t *testing.T) { runServerTestTLS12(t, test) } +func TestHandshakeServerAES128SHA256(t *testing.T) { + test := &serverTest{ + name: "AES128-SHA256", + command: []string{"openssl", "s_client", "-no_ticket", "-ciphersuites", "TLS_AES_128_GCM_SHA256"}, + } + runServerTestTLS13(t, test) +} +func TestHandshakeServerAES256SHA384(t *testing.T) { + test := &serverTest{ + name: "AES256-SHA384", + command: []string{"openssl", "s_client", "-no_ticket", "-ciphersuites", "TLS_AES_256_GCM_SHA384"}, + } + runServerTestTLS13(t, test) +} +func TestHandshakeServerCHACHA20SHA256(t *testing.T) { + test := &serverTest{ + name: "CHACHA20-SHA256", + command: []string{"openssl", "s_client", "-no_ticket", "-ciphersuites", "TLS_CHACHA20_POLY1305_SHA256"}, + } + runServerTestTLS13(t, test) +} + func TestHandshakeServerECDHEECDSAAES(t *testing.T) { config := testConfig.Clone() config.Certificates = make([]Certificate, 1) @@ -783,11 +823,12 @@ func TestHandshakeServerECDHEECDSAAES(t *testing.T) { test := &serverTest{ name: "ECDHE-ECDSA-AES", - command: []string{"openssl", "s_client", "-no_ticket", "-cipher", "ECDHE-ECDSA-AES256-SHA"}, + command: []string{"openssl", "s_client", "-no_ticket", "-cipher", "ECDHE-ECDSA-AES256-SHA", "-ciphersuites", "TLS_AES_128_GCM_SHA256"}, config: config, } runServerTestTLS10(t, test) runServerTestTLS12(t, test) + runServerTestTLS13(t, test) } func TestHandshakeServerX25519(t *testing.T) { @@ -795,11 +836,37 @@ func TestHandshakeServerX25519(t *testing.T) { config.CurvePreferences = []CurveID{X25519} test := &serverTest{ - name: "X25519-ECDHE-RSA-AES-GCM", - command: []string{"openssl", "s_client", "-no_ticket", "-cipher", "ECDHE-RSA-AES128-GCM-SHA256"}, + name: "X25519", + command: []string{"openssl", "s_client", "-no_ticket", "-cipher", "ECDHE-RSA-AES128-GCM-SHA256", "-curves", "X25519"}, config: config, } runServerTestTLS12(t, test) + runServerTestTLS13(t, test) +} + +func TestHandshakeServerP256(t *testing.T) { + config := testConfig.Clone() + config.CurvePreferences = []CurveID{CurveP256} + + test := &serverTest{ + name: "P256", + command: []string{"openssl", "s_client", "-no_ticket", "-cipher", "ECDHE-RSA-AES128-GCM-SHA256", "-curves", "P-256"}, + config: config, + } + runServerTestTLS12(t, test) + runServerTestTLS13(t, test) +} + +func TestHandshakeServerHelloRetryRequest(t *testing.T) { + config := testConfig.Clone() + config.CurvePreferences = []CurveID{CurveP256} + + test := &serverTest{ + name: "HelloRetryRequest", + command: []string{"openssl", "s_client", "-no_ticket", "-curves", "X25519:P-256"}, + config: config, + } + runServerTestTLS13(t, test) } func TestHandshakeServerALPN(t *testing.T) { @@ -821,6 +888,7 @@ func TestHandshakeServerALPN(t *testing.T) { }, } runServerTestTLS12(t, test) + runServerTestTLS13(t, test) } func TestHandshakeServerALPNNoMatch(t *testing.T) { @@ -843,6 +911,7 @@ func TestHandshakeServerALPNNoMatch(t *testing.T) { }, } runServerTestTLS12(t, test) + runServerTestTLS13(t, test) } // TestHandshakeServerSNI involves a client sending an SNI extension of @@ -1052,6 +1121,7 @@ func TestHandshakeServerExportKeyingMaterial(t *testing.T) { } runServerTestTLS10(t, test) runServerTestTLS12(t, test) + runServerTestTLS13(t, test) } func TestHandshakeServerRSAPKCS1v15(t *testing.T) { @@ -1068,6 +1138,7 @@ func TestHandshakeServerRSAPSS(t *testing.T) { command: []string{"openssl", "s_client", "-no_ticket", "-sigalgs", "rsa_pss_rsae_sha256"}, } runServerTestTLS12(t, test) + runServerTestTLS13(t, test) } func benchmarkHandshakeServer(b *testing.B, cipherSuite uint16, curve CurveID, cert []byte, key crypto.PrivateKey) { @@ -1286,10 +1357,18 @@ func TestSNIGivenOnFailure(t *testing.T) { cli.writeRecord(recordTypeHandshake, clientHello.marshal()) c.Close() }() + conn := Server(s, serverConfig) + ch, err := conn.readClientHello() hs := serverHandshakeState{ - c: Server(s, serverConfig), + c: conn, + clientHello: ch, + } + if err == nil { + err = hs.processClientHello() + } + if err == nil { + err = hs.pickCipherSuite() } - _, err := hs.readClientHello() defer s.Close() if err == nil { diff --git a/src/crypto/tls/handshake_server_tls13.go b/src/crypto/tls/handshake_server_tls13.go new file mode 100644 index 0000000000..0ba74d5ff8 --- /dev/null +++ b/src/crypto/tls/handshake_server_tls13.go @@ -0,0 +1,461 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "bytes" + "crypto" + "crypto/hmac" + "crypto/rsa" + "errors" + "fmt" + "hash" + "io" + "sync/atomic" +) + +type serverHandshakeStateTLS13 struct { + c *Conn + clientHello *clientHelloMsg + hello *serverHelloMsg + suite *cipherSuiteTLS13 + cert *Certificate + sigAlg SignatureScheme + handshakeSecret []byte + trafficSecret []byte // client_application_traffic_secret_0 + transcript hash.Hash +} + +func (hs *serverHandshakeStateTLS13) handshake() error { + c := hs.c + + // For an overview of the TLS 1.3 handshake, see RFC 8446, Section 2. + if err := hs.processClientHello(); err != nil { + return err + } + c.buffering = true + if err := hs.sendServerParameters(); err != nil { + return err + } + if err := hs.sendServerCertificate(); err != nil { + return err + } + if err := hs.sendServerFinished(); err != nil { + return err + } + if _, err := c.flush(); err != nil { + return err + } + if err := hs.readClientFinished(); err != nil { + return err + } + + atomic.StoreUint32(&c.handshakeStatus, 1) + + return nil +} + +func (hs *serverHandshakeStateTLS13) processClientHello() error { + c := hs.c + + hs.hello = new(serverHelloMsg) + + // TLS 1.3 froze the ServerHello.legacy_version field, and uses + // supported_versions instead. See RFC 8446, sections 4.1.3 and 4.2.1. + hs.hello.vers = VersionTLS12 + hs.hello.supportedVersion = c.vers + + if len(hs.clientHello.supportedVersions) == 0 { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: client used the legacy version field to negotiate TLS 1.3") + } + + if len(hs.clientHello.compressionMethods) != 1 || + hs.clientHello.compressionMethods[0] != compressionNone { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: TLS 1.3 client supports illegal compression methods") + } + + hs.hello.random = make([]byte, 32) + if _, err := io.ReadFull(c.config.rand(), hs.hello.random); err != nil { + c.sendAlert(alertInternalError) + return err + } + + if len(hs.clientHello.secureRenegotiation) != 0 { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: initial handshake had non-empty renegotiation extension") + } + + if hs.clientHello.earlyData { + return errors.New("tls: early data skipping not implemented") // TODO(filippo) + } + + hs.hello.sessionId = hs.clientHello.sessionId + hs.hello.compressionMethod = compressionNone + + var preferenceList, supportedList []uint16 + if c.config.PreferServerCipherSuites { + preferenceList = defaultCipherSuitesTLS13() + supportedList = hs.clientHello.cipherSuites + } else { + preferenceList = hs.clientHello.cipherSuites + supportedList = defaultCipherSuitesTLS13() + } + for _, suiteID := range preferenceList { + hs.suite = mutualCipherSuiteTLS13(supportedList, suiteID) + if hs.suite != nil { + break + } + } + if hs.suite == nil { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: no cipher suite supported by both client and server") + } + c.cipherSuite = hs.suite.id + hs.hello.cipherSuite = hs.suite.id + hs.transcript = hs.suite.hash.New() + + // Pick the ECDHE group in server preference order, but give priority to + // groups with a key share, to avoid a HelloRetryRequest round-trip. + var selectedGroup CurveID + var clientKeyShare *keyShare +GroupSelection: + for _, preferredGroup := range c.config.curvePreferences() { + for _, ks := range hs.clientHello.keyShares { + if ks.group == preferredGroup { + selectedGroup = ks.group + clientKeyShare = &ks + break GroupSelection + } + } + if selectedGroup != 0 { + continue + } + for _, group := range hs.clientHello.supportedCurves { + if group == preferredGroup { + selectedGroup = group + break + } + } + } + if selectedGroup == 0 { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: no ECDHE curve supported by both client and server") + } + if clientKeyShare == nil { + if err := hs.doHelloRetryRequest(selectedGroup); err != nil { + return err + } + clientKeyShare = &hs.clientHello.keyShares[0] + } + + if _, ok := curveForCurveID(selectedGroup); selectedGroup != X25519 && !ok { + c.sendAlert(alertInternalError) + return errors.New("tls: CurvePreferences includes unsupported curve") + } + params, err := generateECDHEParameters(c.config.rand(), selectedGroup) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + hs.hello.serverShare = keyShare{group: selectedGroup, data: params.PublicKey()} + sharedKey := params.SharedKey(clientKeyShare.data) + if sharedKey == nil { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: invalid client key share") + } + earlySecret := hs.suite.extract(nil, nil) + hs.handshakeSecret = hs.suite.extract(sharedKey, + hs.suite.deriveSecret(earlySecret, "derived", nil)) + + // This implements a very simplistic certificate selection strategy for now: + // getCertificate delegates to the application Config.GetCertificate, or + // selects based on the server_name only. If the selected certificate's + // public key does not match the client signature_algorithms, the handshake + // is aborted. No attention is given to signature_algorithms_cert, and it is + // not passed to the application Config.GetCertificate. This will need to + // improve according to RFC 8446, sections 4.4.2.2 and 4.2.3. + certificate, err := c.config.getCertificate(clientHelloInfo(c, hs.clientHello)) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + supportedAlgs := signatureSchemesForCertificate(certificate) + if supportedAlgs == nil { + c.sendAlert(alertInternalError) + return fmt.Errorf("tls: unsupported certificate key (%T)", certificate.PrivateKey) + } + // Pick signature scheme in client preference order, as the server + // preference order is not configurable. + for _, preferredAlg := range hs.clientHello.supportedSignatureAlgorithms { + if isSupportedSignatureAlgorithm(preferredAlg, supportedAlgs) { + hs.sigAlg = preferredAlg + break + } + } + if hs.sigAlg == 0 { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: client doesn't support selected certificate") + } + hs.cert = certificate + + return nil +} + +func (hs *serverHandshakeStateTLS13) doHelloRetryRequest(selectedGroup CurveID) error { + c := hs.c + + // The first ClientHello gets double-hashed into the transcript upon a + // HelloRetryRequest. See RFC 8446, Section 4.4.1. + hs.transcript.Write(hs.clientHello.marshal()) + chHash := hs.transcript.Sum(nil) + hs.transcript.Reset() + hs.transcript.Write([]byte{typeMessageHash, 0, 0, uint8(len(chHash))}) + hs.transcript.Write(chHash) + + helloRetryRequest := &serverHelloMsg{ + vers: hs.hello.vers, + random: helloRetryRequestRandom, + sessionId: hs.hello.sessionId, + cipherSuite: hs.hello.cipherSuite, + compressionMethod: hs.hello.compressionMethod, + supportedVersion: hs.hello.supportedVersion, + selectedGroup: selectedGroup, + } + + hs.transcript.Write(helloRetryRequest.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, helloRetryRequest.marshal()); err != nil { + return err + } + + msg, err := c.readHandshake() + if err != nil { + return err + } + + clientHello, ok := msg.(*clientHelloMsg) + if !ok { + c.sendAlert(alertUnexpectedMessage) + return unexpectedMessageError(clientHello, msg) + } + + if len(clientHello.keyShares) != 1 || clientHello.keyShares[0].group != selectedGroup { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: client sent invalid key share in second ClientHello") + } + + if clientHello.earlyData { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: client indicated early data in second ClientHello") + } + + if illegalClientHelloChange(clientHello, hs.clientHello) { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: client illegally modified second ClientHello") + } + + hs.clientHello = clientHello + return nil +} + +// illegalClientHelloChange returns whether the two ClientHello messages are +// different, with the exception of the changes allowed before and after a +// HelloRetryRequest. See RFC 8446, Section 4.1.2. +func illegalClientHelloChange(ch, ch1 *clientHelloMsg) bool { + if len(ch.supportedVersions) != len(ch1.supportedVersions) || + len(ch.cipherSuites) != len(ch1.cipherSuites) || + len(ch.supportedCurves) != len(ch1.supportedCurves) || + len(ch.supportedSignatureAlgorithms) != len(ch1.supportedSignatureAlgorithms) || + len(ch.supportedSignatureAlgorithmsCert) != len(ch1.supportedSignatureAlgorithmsCert) || + len(ch.alpnProtocols) != len(ch1.alpnProtocols) { + return true + } + for i := range ch.supportedVersions { + if ch.supportedVersions[i] != ch1.supportedVersions[i] { + return true + } + } + for i := range ch.cipherSuites { + if ch.cipherSuites[i] != ch1.cipherSuites[i] { + return true + } + } + for i := range ch.supportedCurves { + if ch.supportedCurves[i] != ch1.supportedCurves[i] { + return true + } + } + for i := range ch.supportedSignatureAlgorithms { + if ch.supportedSignatureAlgorithms[i] != ch1.supportedSignatureAlgorithms[i] { + return true + } + } + for i := range ch.supportedSignatureAlgorithmsCert { + if ch.supportedSignatureAlgorithmsCert[i] != ch1.supportedSignatureAlgorithmsCert[i] { + return true + } + } + for i := range ch.alpnProtocols { + if ch.alpnProtocols[i] != ch1.alpnProtocols[i] { + return true + } + } + return ch.vers != ch1.vers || + !bytes.Equal(ch.random, ch1.random) || + !bytes.Equal(ch.sessionId, ch1.sessionId) || + !bytes.Equal(ch.compressionMethods, ch1.compressionMethods) || + ch.nextProtoNeg != ch1.nextProtoNeg || + ch.serverName != ch1.serverName || + ch.ocspStapling != ch1.ocspStapling || + !bytes.Equal(ch.supportedPoints, ch1.supportedPoints) || + ch.ticketSupported != ch1.ticketSupported || + !bytes.Equal(ch.sessionTicket, ch1.sessionTicket) || + ch.secureRenegotiationSupported != ch1.secureRenegotiationSupported || + !bytes.Equal(ch.secureRenegotiation, ch1.secureRenegotiation) || + ch.scts != ch1.scts || + !bytes.Equal(ch.cookie, ch1.cookie) || + !bytes.Equal(ch.pskModes, ch1.pskModes) +} + +func (hs *serverHandshakeStateTLS13) sendServerParameters() error { + c := hs.c + + hs.transcript.Write(hs.clientHello.marshal()) + hs.transcript.Write(hs.hello.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, hs.hello.marshal()); err != nil { + return err + } + + clientSecret := hs.suite.deriveSecret(hs.handshakeSecret, + clientHandshakeTrafficLabel, hs.transcript) + c.in.setTrafficSecret(hs.suite, clientSecret) + serverSecret := hs.suite.deriveSecret(hs.handshakeSecret, + serverHandshakeTrafficLabel, hs.transcript) + c.out.setTrafficSecret(hs.suite, serverSecret) + + encryptedExtensions := new(encryptedExtensionsMsg) + + if len(hs.clientHello.alpnProtocols) > 0 { + if selectedProto, fallback := mutualProtocol(hs.clientHello.alpnProtocols, c.config.NextProtos); !fallback { + encryptedExtensions.alpnProtocol = selectedProto + c.clientProtocol = selectedProto + } + } + + hs.transcript.Write(encryptedExtensions.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, encryptedExtensions.marshal()); err != nil { + return err + } + + return nil +} + +func (hs *serverHandshakeStateTLS13) sendServerCertificate() error { + c := hs.c + + certMsg := new(certificateMsgTLS13) + + certMsg.certificate = *hs.cert + certMsg.scts = hs.clientHello.scts && len(hs.cert.SignedCertificateTimestamps) > 0 + certMsg.ocspStapling = hs.clientHello.ocspStapling && len(hs.cert.OCSPStaple) > 0 + + hs.transcript.Write(certMsg.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, certMsg.marshal()); err != nil { + return err + } + + certVerifyMsg := new(certificateVerifyMsg) + certVerifyMsg.hasSignatureAlgorithm = true + certVerifyMsg.signatureAlgorithm = hs.sigAlg + + sigType := signatureFromSignatureScheme(hs.sigAlg) + sigHash, err := hashFromSignatureScheme(hs.sigAlg) + if sigType == 0 || err != nil { + c.sendAlert(alertInternalError) + return err + } + h := sigHash.New() + writeSignedMessage(h, serverSignatureContext, hs.transcript) + + signOpts := crypto.SignerOpts(sigHash) + if sigType == signatureRSAPSS { + signOpts = &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: sigHash} + } + sig, err := hs.cert.PrivateKey.(crypto.Signer).Sign(c.config.rand(), h.Sum(nil), signOpts) + if err != nil { + c.sendAlert(alertInternalError) + return errors.New("tls: failed to sign handshake: " + err.Error()) + } + certVerifyMsg.signature = sig + + hs.transcript.Write(certVerifyMsg.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, certVerifyMsg.marshal()); err != nil { + return err + } + + return nil +} + +func (hs *serverHandshakeStateTLS13) sendServerFinished() error { + c := hs.c + + // See RFC 8446, sections 4.4.4 and 4.4. + finishedKey := hs.suite.expandLabel(c.out.trafficSecret, "finished", nil, hs.suite.hash.Size()) + verifyData := hmac.New(hs.suite.hash.New, finishedKey) + verifyData.Write(hs.transcript.Sum(nil)) + finished := &finishedMsg{ + verifyData: verifyData.Sum(nil), + } + + hs.transcript.Write(finished.marshal()) + if _, err := c.writeRecord(recordTypeHandshake, finished.marshal()); err != nil { + return err + } + + // Derive secrets that take context through the server Finished. + + masterSecret := hs.suite.extract(nil, + hs.suite.deriveSecret(hs.handshakeSecret, "derived", nil)) + + hs.trafficSecret = hs.suite.deriveSecret(masterSecret, + clientApplicationTrafficLabel, hs.transcript) + serverSecret := hs.suite.deriveSecret(masterSecret, + serverApplicationTrafficLabel, hs.transcript) + c.out.setTrafficSecret(hs.suite, serverSecret) + + c.ekm = hs.suite.exportKeyingMaterial(masterSecret, hs.transcript) + + return nil +} + +func (hs *serverHandshakeStateTLS13) readClientFinished() error { + c := hs.c + + msg, err := c.readHandshake() + if err != nil { + return err + } + + finished, ok := msg.(*finishedMsg) + if !ok { + c.sendAlert(alertUnexpectedMessage) + return unexpectedMessageError(finished, msg) + } + + finishedKey := hs.suite.expandLabel(c.in.trafficSecret, "finished", nil, hs.suite.hash.Size()) + expectedMAC := hmac.New(hs.suite.hash.New, finishedKey) + expectedMAC.Write(hs.transcript.Sum(nil)) + if !hmac.Equal(expectedMAC.Sum(nil), finished.verifyData) { + c.sendAlert(alertDecryptError) + return errors.New("tls: invalid client finished hash") + } + + hs.transcript.Write(finished.marshal()) + + c.in.setTrafficSecret(hs.suite, hs.trafficSecret) + + return nil +} diff --git a/src/crypto/tls/testdata/Server-TLSv12-P256 b/src/crypto/tls/testdata/Server-TLSv12-P256 new file mode 100644 index 0000000000..c97bae4a63 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv12-P256 @@ -0,0 +1,85 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 8f 01 00 00 8b 03 03 74 08 f5 ee 24 |...........t...$| +00000010 cb c0 26 fc f1 11 c6 9d fb ac f5 ed d1 05 78 e6 |..&...........x.| +00000020 cf a6 cb f2 ed 1a 46 3a cf 25 8b 00 00 04 c0 2f |......F:.%...../| +00000030 00 ff 01 00 00 5e 00 00 00 0e 00 0c 00 00 09 31 |.....^.........1| +00000040 32 37 2e 30 2e 30 2e 31 00 0b 00 04 03 00 01 02 |27.0.0.1........| +00000050 00 0a 00 04 00 02 00 17 00 16 00 00 00 17 00 00 |................| +00000060 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 |...0............| +00000070 08 09 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 |................| +00000080 06 01 03 03 02 03 03 01 02 01 03 02 02 02 04 02 |................| +00000090 05 02 06 02 |....| +>>> Flow 2 (server to client) +00000000 16 03 03 00 31 02 00 00 2d 03 03 00 00 00 00 00 |....1...-.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 00 c0 2f 00 00 |............./..| +00000030 05 ff 01 00 01 00 16 03 03 02 59 0b 00 02 55 00 |..........Y...U.| +00000040 02 52 00 02 4f 30 82 02 4b 30 82 01 b4 a0 03 02 |.R..O0..K0......| +00000050 01 02 02 09 00 e8 f0 9d 3f e2 5b ea a6 30 0d 06 |........?.[..0..| +00000060 09 2a 86 48 86 f7 0d 01 01 0b 05 00 30 1f 31 0b |.*.H........0.1.| +00000070 30 09 06 03 55 04 0a 13 02 47 6f 31 10 30 0e 06 |0...U....Go1.0..| +00000080 03 55 04 03 13 07 47 6f 20 52 6f 6f 74 30 1e 17 |.U....Go Root0..| +00000090 0d 31 36 30 31 30 31 30 30 30 30 30 30 5a 17 0d |.160101000000Z..| +000000a0 32 35 30 31 30 31 30 30 30 30 30 30 5a 30 1a 31 |250101000000Z0.1| +000000b0 0b 30 09 06 03 55 04 0a 13 02 47 6f 31 0b 30 09 |.0...U....Go1.0.| +000000c0 06 03 55 04 03 13 02 47 6f 30 81 9f 30 0d 06 09 |..U....Go0..0...| +000000d0 2a 86 48 86 f7 0d 01 01 01 05 00 03 81 8d 00 30 |*.H............0| +000000e0 81 89 02 81 81 00 db 46 7d 93 2e 12 27 06 48 bc |.......F}...'.H.| +000000f0 06 28 21 ab 7e c4 b6 a2 5d fe 1e 52 45 88 7a 36 |.(!.~...]..RE.z6| +00000100 47 a5 08 0d 92 42 5b c2 81 c0 be 97 79 98 40 fb |G....B[.....y.@.| +00000110 4f 6d 14 fd 2b 13 8b c2 a5 2e 67 d8 d4 09 9e d6 |Om..+.....g.....| +00000120 22 38 b7 4a 0b 74 73 2b c2 34 f1 d1 93 e5 96 d9 |"8.J.ts+.4......| +00000130 74 7b f3 58 9f 6c 61 3c c0 b0 41 d4 d9 2b 2b 24 |t{.X.la<..A..++$| +00000140 23 77 5b 1c 3b bd 75 5d ce 20 54 cf a1 63 87 1d |#w[.;.u]. T..c..| +00000150 1e 24 c4 f3 1d 1a 50 8b aa b6 14 43 ed 97 a7 75 |.$....P....C...u| +00000160 62 f4 14 c8 52 d7 02 03 01 00 01 a3 81 93 30 81 |b...R.........0.| +00000170 90 30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 05 |.0...U..........| +00000180 a0 30 1d 06 03 55 1d 25 04 16 30 14 06 08 2b 06 |.0...U.%..0...+.| +00000190 01 05 05 07 03 01 06 08 2b 06 01 05 05 07 03 02 |........+.......| +000001a0 30 0c 06 03 55 1d 13 01 01 ff 04 02 30 00 30 19 |0...U.......0.0.| +000001b0 06 03 55 1d 0e 04 12 04 10 9f 91 16 1f 43 43 3e |..U..........CC>| +000001c0 49 a6 de 6d b6 80 d7 9f 60 30 1b 06 03 55 1d 23 |I..m....`0...U.#| +000001d0 04 14 30 12 80 10 48 13 49 4d 13 7e 16 31 bb a3 |..0...H.IM.~.1..| +000001e0 01 d5 ac ab 6e 7b 30 19 06 03 55 1d 11 04 12 30 |....n{0...U....0| +000001f0 10 82 0e 65 78 61 6d 70 6c 65 2e 67 6f 6c 61 6e |...example.golan| +00000200 67 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 |g0...*.H........| +00000210 03 81 81 00 9d 30 cc 40 2b 5b 50 a0 61 cb ba e5 |.....0.@+[P.a...| +00000220 53 58 e1 ed 83 28 a9 58 1a a9 38 a4 95 a1 ac 31 |SX...(.X..8....1| +00000230 5a 1a 84 66 3d 43 d3 2d d9 0b f2 97 df d3 20 64 |Z..f=C.-...... d| +00000240 38 92 24 3a 00 bc cf 9c 7d b7 40 20 01 5f aa d3 |8.$:....}.@ ._..| +00000250 16 61 09 a2 76 fd 13 c3 cc e1 0c 5c ee b1 87 82 |.a..v......\....| +00000260 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 |.l..s..Cw.......| +00000270 40 83 61 c9 4c 72 2b 9d ae db 46 06 06 4d f4 c1 |@.a.Lr+...F..M..| +00000280 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 |.>...B...=.`.\!.| +00000290 3b e9 fa e7 16 03 03 00 cd 0c 00 00 c9 03 00 17 |;...............| +000002a0 41 04 1e 18 37 ef 0d 19 51 88 35 75 71 b5 e5 54 |A...7...Q.5uq..T| +000002b0 5b 12 2e 8f 09 67 fd a7 24 20 3e b2 56 1c ce 97 |[....g..$ >.V...| +000002c0 28 5e f8 2b 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 |(^.+-O....lK[.V.| +000002d0 32 42 e9 58 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc |2B.X..I..h.A.Vk.| +000002e0 5a 89 08 04 00 80 7b f8 9c bb af 20 2b b4 44 40 |Z.....{.... +.D@| +000002f0 0d bc 31 37 53 26 d4 74 b6 0b 5b 79 f5 a9 ea 2f |..17S&.t..[y.../| +00000300 e7 4f 58 42 a1 d1 43 96 bf 74 dd 2e 33 28 bd b1 |.OXB..C..t..3(..| +00000310 0b 8a eb d7 6c 1d 7a 71 3d 61 0b f1 8e 8c f8 32 |....l.zq=a.....2| +00000320 7a ec 60 ed 4a 84 67 ad 23 f0 c8 68 27 4d 82 d9 |z.`.J.g.#..h'M..| +00000330 15 58 38 cf 0a ec 5a 15 b8 14 e2 95 c6 6c b5 f3 |.X8...Z......l..| +00000340 5b ee 01 1c f7 00 83 ae e4 c8 89 77 8b 64 7b 9a |[..........w.d{.| +00000350 52 a3 c0 1c 6c 19 bc 3d 28 19 24 04 8d 2d 1c 7e |R...l..=(.$..-.~| +00000360 4d 56 01 64 f6 3c 16 03 03 00 04 0e 00 00 00 |MV.d.<.........| +>>> Flow 3 (client to server) +00000000 16 03 03 00 46 10 00 00 42 41 04 06 a5 96 8c 26 |....F...BA.....&| +00000010 fb 4e 33 6d 4f 28 6a 84 af 17 7e 6c 36 ca be 3b |.N3mO(j...~l6..;| +00000020 a2 0b b1 82 1d 35 37 02 09 f3 f0 c4 88 a9 82 1f |.....57.........| +00000030 df b5 c2 09 ed a9 7a e5 80 71 76 14 a8 98 03 08 |......z..qv.....| +00000040 25 04 94 03 4a c0 2a cb 77 bd e2 14 03 03 00 01 |%...J.*.w.......| +00000050 01 16 03 03 00 28 f4 25 33 55 25 c7 6d c9 e5 68 |.....(.%3U%.m..h| +00000060 09 5c 05 c5 49 78 82 08 20 7b 96 1b f8 5c 1a 41 |.\..Ix.. {...\.A| +00000070 be a3 4d 4c 01 3d 16 fa 3a f6 dc 37 5e 3b |..ML.=..:..7^;| +>>> Flow 4 (server to client) +00000000 14 03 03 00 01 01 16 03 03 00 28 00 00 00 00 00 |..........(.....| +00000010 00 00 00 24 72 76 66 ac a0 75 a4 5b 80 c8 f8 52 |...$rvf..u.[...R| +00000020 4b e3 8f 5a 02 f1 44 16 18 e6 ef b8 d3 51 50 f0 |K..Z..D......QP.| +00000030 06 b7 22 17 03 03 00 25 00 00 00 00 00 00 00 01 |.."....%........| +00000040 a0 f8 ed 7c ec 72 b8 fb 2a 6f 8b 61 83 ce 10 a0 |...|.r..*o.a....| +00000050 0e 45 39 ce 20 c1 b4 6e e1 a9 a6 e9 13 15 03 03 |.E9. ..n........| +00000060 00 1a 00 00 00 00 00 00 00 02 ca ce 38 1c fe db |............8...| +00000070 f3 53 18 ff fd b2 31 17 07 4d 99 10 |.S....1..M..| diff --git a/src/crypto/tls/testdata/Server-TLSv12-X25519-ECDHE-RSA-AES-GCM b/src/crypto/tls/testdata/Server-TLSv12-X25519 similarity index 100% rename from src/crypto/tls/testdata/Server-TLSv12-X25519-ECDHE-RSA-AES-GCM rename to src/crypto/tls/testdata/Server-TLSv12-X25519 diff --git a/src/crypto/tls/testdata/Server-TLSv13-AES128-SHA256 b/src/crypto/tls/testdata/Server-TLSv13-AES128-SHA256 new file mode 100644 index 0000000000..049d2ff962 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-AES128-SHA256 @@ -0,0 +1,90 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 dc 01 00 00 d8 03 03 f8 75 ee c1 65 |............u..e| +00000010 31 d9 dd 36 00 f6 b0 f0 a8 d6 a0 42 da d1 8a a3 |1..6.......B....| +00000020 17 82 1a 44 14 6e bc 43 e0 4b 89 20 88 b8 53 ac |...D.n.C.K. ..S.| +00000030 16 d5 64 58 23 21 20 c4 0b 8d 96 d7 db 59 44 3d |..dX#! ......YD=| +00000040 9e 67 9b f8 a8 21 6c 6d 02 54 a9 b6 00 04 13 01 |.g...!lm.T......| +00000050 00 ff 01 00 00 8b 00 00 00 0e 00 0c 00 00 09 31 |...............1| +00000060 32 37 2e 30 2e 30 2e 31 00 0b 00 04 03 00 01 02 |27.0.0.1........| +00000070 00 0a 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 |................| +00000080 00 16 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 |................| +00000090 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 |................| +000000a0 08 05 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 |...........+....| +000000b0 04 00 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 |..-.....3.&.$...| +000000c0 20 be 01 5d e4 e9 b4 66 52 bf 5f 5f ab 82 80 be | ..]...fR.__....| +000000d0 25 13 b3 e7 28 5e 00 a6 b0 a6 d5 f1 f0 20 42 e5 |%...(^....... B.| +000000e0 2e |.| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 88 b8 53 ac |........... ..S.| +00000030 16 d5 64 58 23 21 20 c4 0b 8d 96 d7 db 59 44 3d |..dX#! ......YD=| +00000040 9e 67 9b f8 a8 21 6c 6d 02 54 a9 b6 13 01 00 00 |.g...!lm.T......| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 ff de 7b 26 44 70 00 05 64 fc 65 e3 |......{&Dp..d.e.| +00000090 23 fd ea 4b b3 d0 e6 80 db 2b 4c 17 03 03 02 6d |#..K.....+L....m| +000000a0 8f 7b 9c 1f b5 a8 26 b3 89 76 13 7f a2 9a 9b e3 |.{....&..v......| +000000b0 b8 06 c3 ff 02 c2 41 8e 29 fd f4 45 43 d3 57 23 |......A.)..EC.W#| +000000c0 71 32 03 31 24 2b 73 ca 7a df e9 79 4e 96 7c d5 |q2.1$+s.z..yN.|.| +000000d0 f1 19 05 11 1c cf 5f 85 30 e5 32 2c 61 2c d2 40 |......_.0.2,a,.@| +000000e0 25 c0 72 2c f8 ef d2 8d d9 4f 81 88 cc 9f c0 71 |%.r,.....O.....q| +000000f0 97 9e 79 f4 33 e8 75 47 1e 5a c1 02 20 48 6f a4 |..y.3.uG.Z.. Ho.| +00000100 41 d6 37 73 07 ad 90 37 80 d7 f9 c1 59 96 9f a8 |A.7s...7....Y...| +00000110 e4 f5 e4 65 99 02 a3 69 95 e1 39 07 fb a8 ac 6d |...e...i..9....m| +00000120 40 fb de 64 05 5b b4 32 ba 09 da 92 0a ba 1e 11 |@..d.[.2........| +00000130 3f d3 bd 6d 68 f7 15 d3 74 60 18 cd 96 04 2f db |?..mh...t`..../.| +00000140 c5 09 f2 05 5f 82 8b 23 65 00 7c b7 d5 ef 1d 0e |...._..#e.|.....| +00000150 3a 08 2b c9 6d 99 9b 9a a6 55 2c df 08 c0 4a b1 |:.+.m....U,...J.| +00000160 b3 69 9c d9 68 49 43 28 8e 00 c5 e1 60 07 25 4d |.i..hIC(....`.%M| +00000170 aa 61 2e 74 82 49 62 e0 a8 f0 53 6c 64 ea fc cc |.a.t.Ib...Sld...| +00000180 84 ab 26 b6 b8 ef 55 cd 3d 34 1a 65 25 8f 76 f4 |..&...U.=4.e%.v.| +00000190 dc 06 9b 67 98 59 c1 37 86 0f 24 34 86 b5 a3 dd |...g.Y.7..$4....| +000001a0 5b d4 c8 04 cb 73 e1 67 bb 66 d7 94 16 eb 3b 73 |[....s.g.f....;s| +000001b0 7e 67 8d 7f bf e9 f9 89 6d d8 6a a9 3c 97 eb 67 |~g......m.j.<..g| +000001c0 f0 6f fa 9b 4d f0 25 25 a2 30 1d 3c 93 14 6e 33 |.o..M.%%.0.<..n3| +000001d0 fa bf 5b 3c c3 cc f4 0e fa 55 e2 20 46 3f 1c b7 |..[<.....U. F?..| +000001e0 9d d3 ec d8 54 18 0b 4a be 45 bd 5e a1 3a f2 e9 |....T..J.E.^.:..| +000001f0 15 b0 15 a4 b4 a6 f5 52 36 9b 6b 18 ce ac 37 ae |.......R6.k...7.| +00000200 76 e6 2c d1 6b 3f 95 eb 37 79 fe ec cc a6 34 4a |v.,.k?..7y....4J| +00000210 27 68 0e d0 80 d8 5c 1c 9c ac aa 02 18 e7 c1 72 |'h....\........r| +00000220 08 52 07 63 04 65 13 53 23 51 ce 0e f4 1d 4f ca |.R.c.e.S#Q....O.| +00000230 51 13 ad 10 1b f3 a4 c3 69 ce c0 ed d1 25 6c 60 |Q.......i....%l`| +00000240 e7 21 9e d7 9f 8b a1 20 61 75 f6 e0 06 c4 dd bb |.!..... au......| +00000250 8e e2 05 86 ef fe 75 0e 47 ae 54 82 e9 32 9b 87 |......u.G.T..2..| +00000260 fb eb e4 14 e3 f0 90 1d 48 72 00 02 53 52 24 47 |........Hr..SR$G| +00000270 98 a1 cc b9 b3 8d ab a7 db b0 f5 83 db 56 a1 ad |.............V..| +00000280 7d 45 e7 5f 6a bd a9 65 87 8c 48 1f de dc b4 ce |}E._j..e..H.....| +00000290 47 7c ec 63 fb 77 f3 5a a0 3b 84 53 cf 8b 73 30 |G|.c.w.Z.;.S..s0| +000002a0 bd 0f ac 5a 9b e8 a1 88 f6 45 96 ca b9 48 c3 be |...Z.....E...H..| +000002b0 8b 7e f1 1a fd 8a 54 9e 5a 76 e7 9a bc 06 7e 04 |.~....T.Zv....~.| +000002c0 bd e1 a1 a3 4d 52 56 3b 64 29 70 87 89 c5 f5 ce |....MRV;d)p.....| +000002d0 1f 65 7d 55 9f 28 32 3e 6b c7 b6 17 0b dd 7e ea |.e}U.(2>k.....~.| +000002e0 ef 7b a0 f0 6f 84 2a 11 93 d5 d8 99 dc ee 17 57 |.{..o.*........W| +000002f0 3f d7 7f a7 da c0 30 77 13 31 60 9c ca 32 67 09 |?.....0w.1`..2g.| +00000300 70 ce 05 0d c8 b6 e1 a2 df e4 f1 3c 67 17 03 03 |p............d...B| +00000370 39 15 00 13 c3 c8 35 66 21 d0 1f 94 0b 21 79 22 |9.....5f!....!y"| +00000380 5b 32 89 d4 81 28 6c c7 55 74 a7 a4 22 62 a9 cb |[2...(l.Ut.."b..| +00000390 76 9f e9 af 55 d5 1f b5 51 f7 40 2c fb 3b 03 30 |v...U...Q.@,.;.0| +000003a0 13 ca a0 79 ba 80 1b 0e 57 3d 84 17 03 03 00 35 |...y....W=.....5| +000003b0 fb 34 92 6a 46 f2 59 1a 94 39 63 d1 9d 58 40 e8 |.4.jF.Y..9c..X@.| +000003c0 f0 bb e7 fe 4d 1c 42 a2 38 9e cd a2 01 1c b6 a7 |....M.B.8.......| +000003d0 e3 9a cb 28 73 5f 8a 1e b3 40 41 80 85 2c 49 54 |...(s_...@A..,IT| +000003e0 3d 13 dc cc 6c |=...l| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 35 c5 aa 6e cd 44 |..........5..n.D| +00000010 2f f6 09 74 33 13 e1 c1 32 6f 94 cd 55 2c 45 88 |/..t3...2o..U,E.| +00000020 f1 f8 51 c7 3e 64 62 e1 8a 48 cc bd c8 ac 91 a3 |..Q.>db..H......| +00000030 90 ea 45 1b 21 52 d4 81 84 88 0d ed a2 86 d4 64 |..E.!R.........d| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 4c e5 b2 aa 21 9e 56 24 62 da a7 |.....L...!.V$b..| +00000010 af ef 76 b5 8a 9a a1 3b 2e cd e8 68 27 ac 08 e1 |..v....;...h'...| +00000020 c1 37 52 17 03 03 00 13 8e 91 11 5b cf c4 28 e3 |.7R........[..(.| +00000030 a5 ea bb 89 93 fc 94 bc e6 28 32 |.........(2| diff --git a/src/crypto/tls/testdata/Server-TLSv13-AES256-SHA384 b/src/crypto/tls/testdata/Server-TLSv13-AES256-SHA384 new file mode 100644 index 0000000000..ab98f48cf2 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-AES256-SHA384 @@ -0,0 +1,92 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 dc 01 00 00 d8 03 03 35 42 f3 56 6d |...........5B.Vm| +00000010 cd 58 c6 08 54 f1 6c 64 c5 16 eb 39 b0 d9 1b 22 |.X..T.ld...9..."| +00000020 88 d6 36 a7 70 a8 27 c8 e8 1a 20 20 cc c6 13 bc |..6.p.'... ....| +00000030 f4 3a ed 6a d2 59 ac 02 18 68 e9 80 50 4a a3 df |.:.j.Y...h..PJ..| +00000040 1e 6b 54 0c 85 3a 17 5c 23 7f e0 fc 00 04 13 02 |.kT..:.\#.......| +00000050 00 ff 01 00 00 8b 00 00 00 0e 00 0c 00 00 09 31 |...............1| +00000060 32 37 2e 30 2e 30 2e 31 00 0b 00 04 03 00 01 02 |27.0.0.1........| +00000070 00 0a 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 |................| +00000080 00 16 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 |................| +00000090 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 |................| +000000a0 08 05 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 |...........+....| +000000b0 04 00 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 |..-.....3.&.$...| +000000c0 20 de c8 19 7a 66 cb db c3 04 b9 96 f4 5c 2e 52 | ...zf.......\.R| +000000d0 1b 63 0b 11 87 a2 a9 14 f8 54 10 a4 de 5f 9f 13 |.c.......T..._..| +000000e0 09 |.| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 cc c6 13 bc |........... ....| +00000030 f4 3a ed 6a d2 59 ac 02 18 68 e9 80 50 4a a3 df |.:.j.Y...h..PJ..| +00000040 1e 6b 54 0c 85 3a 17 5c 23 7f e0 fc 13 02 00 00 |.kT..:.\#.......| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 de 82 46 36 68 05 6c 4d 4c 79 da 2a |......F6h.lMLy.*| +00000090 a2 de 70 e3 64 66 79 ba 1d 4f d3 17 03 03 02 6d |..p.dfy..O.....m| +000000a0 66 1a 81 1c 38 21 0d 17 ef 4c 0f ad 52 c1 f3 d5 |f...8!...L..R...| +000000b0 d6 3f ee 05 dd bb ec e3 cd 00 a3 e5 42 90 f6 b8 |.?..........B...| +000000c0 2f d3 95 b8 27 3d 70 63 55 ef 32 e8 55 1a 20 0e |/...'=pcU.2.U. .| +000000d0 c8 47 7b 0c 15 30 86 37 72 25 d7 3b 97 a3 d1 90 |.G{..0.7r%.;....| +000000e0 5c a4 07 e4 b8 3c 3b bc 74 ea 93 d5 a8 30 dc bc |\....<;.t....0..| +000000f0 f9 2b 62 4e 15 84 c6 1c 67 a3 85 e2 8e 71 1d 6a |.+bN....g....q.j| +00000100 8d 26 97 96 ba 08 9d 09 e5 21 fa 4c 72 d9 7a df |.&.......!.Lr.z.| +00000110 39 69 02 82 58 0c ba 79 2a c1 73 d7 97 44 62 56 |9i..X..y*.s..DbV| +00000120 b6 7f 08 91 a3 ed 95 18 84 99 31 f6 64 54 59 bc |..........1.dTY.| +00000130 e0 c6 a2 e1 5f 51 5d 03 fd 5b cf 8f 97 1b 41 4b |...._Q]..[....AK| +00000140 c6 fa 78 06 c4 4d 3c e0 ba d9 3c 5f 5f f8 72 79 |..x..M<...<__.ry| +00000150 53 a0 cc bc aa 92 be 78 15 08 35 36 1f 53 91 89 |S......x..56.S..| +00000160 ec 74 0b b4 9e 97 85 86 d0 15 ff b6 62 9c 89 07 |.t..........b...| +00000170 9f 5d e8 9f 9d 7e b9 17 0e 33 78 06 ea 84 db 9e |.]...~...3x.....| +00000180 90 37 8a db c1 66 e2 a1 72 40 91 c5 49 e6 04 d9 |.7...f..r@..I...| +00000190 35 7c 50 1b 2e d6 63 44 c8 44 f9 e3 72 9c 1c 29 |5|P...cD.D..r..)| +000001a0 63 0f 5e fd a8 ab 09 c8 b8 02 7b 6f 40 35 85 3e |c.^.......{o@5.>| +000001b0 bb ae c9 b2 06 6d 0d 70 7e 65 b9 30 20 8f 54 da |.....m.p~e.0 .T.| +000001c0 9d 55 6e 70 7a a3 b0 15 5a 29 72 51 61 84 57 5c |.Unpz...Z)rQa.W\| +000001d0 f0 65 0a 9b 6e fb a4 d9 22 70 45 de c1 a6 5f 95 |.e..n..."pE..._.| +000001e0 3e 95 c8 3f 6b fa 69 01 ab e8 e0 61 6d 76 7b 5b |>..?k.i....amv{[| +000001f0 b2 33 e9 c1 67 99 33 99 03 75 e3 2c 29 b8 0f 75 |.3..g.3..u.,)..u| +00000200 b8 00 c5 dd cf f3 2e 24 fc 55 cc d5 d4 2d cc f4 |.......$.U...-..| +00000210 f1 59 20 d5 7a c1 ef f5 3b 8b b0 2a 0d c0 2a ad |.Y .z...;..*..*.| +00000220 2c ce c6 91 4d 17 1e 5c e8 d1 98 1e 58 cc d9 42 |,...M..\....X..B| +00000230 42 e9 ff b8 f7 33 f0 af df 8f 1e 7f 52 79 7e 4f |B....3......Ry~O| +00000240 48 e1 21 10 8c aa 7c 84 5d 42 f0 01 73 ff bb e1 |H.!...|.]B..s...| +00000250 de cb 61 a9 d0 9a be 2a cd 18 60 91 0c 20 86 db |..a....*..`.. ..| +00000260 4a e7 08 eb ab af 1b 7a bb 04 f6 49 29 b9 20 65 |J......z...I). e| +00000270 da a6 09 f3 8e 57 b4 5f b0 53 07 86 06 cf 44 de |.....W._.S....D.| +00000280 ba bd 16 ca 88 df 65 50 1a 0d 4b 30 41 ee 04 6a |......eP..K0A..j| +00000290 90 71 27 1b 05 b2 5e 0b 9c 77 ec fb 8a 02 82 18 |.q'...^..w......| +000002a0 34 3e af ea 43 71 4b 9c 12 75 f2 59 32 04 76 87 |4>..CqK..u.Y2.v.| +000002b0 2a ea 31 f6 c8 29 30 41 3f cf 4e c5 2b 38 d0 15 |*.1..)0A?.N.+8..| +000002c0 e6 fa 8d de 2a ec 3e 49 b9 bb 2e 46 ef 8b 8f a4 |....*.>I...F....| +000002d0 e0 96 39 67 d6 2f 56 fb 8b 9c 7c 49 65 3d b3 8e |..9g./V...|Ie=..| +000002e0 1f 48 02 24 53 10 43 49 6d 31 fc d6 99 f6 50 73 |.H.$S.CIm1....Ps| +000002f0 b3 fd b7 57 72 c6 81 a2 df 16 13 0a da 67 8f 3c |...Wr........g.<| +00000300 ff 94 b7 8a 26 0c f2 fe c9 f6 67 a2 f2 17 03 03 |....&.....g.....| +00000310 00 99 1d 59 bb 70 37 f6 a7 72 d1 27 3b 5a d5 1d |...Y.p7..r.';Z..| +00000320 f6 e6 02 84 3c 8b 24 0e 78 ed 88 73 e8 e4 d2 8d |....<.$.x..s....| +00000330 97 c4 cc a3 22 b7 c1 ba 2d df ef d4 84 0d 78 ce |...."...-.....x.| +00000340 70 c9 4f 60 dc fb 13 50 2d d1 52 ba e9 d4 5a 5e |p.O`...P-.R...Z^| +00000350 ed eb 93 ae 99 b8 08 91 61 ee a5 9c 3e 63 07 ed |........a...>c..| +00000360 17 b1 29 5f 03 ae 5b 0d a7 0a 4a 3a f5 6b 9c 6f |..)_..[...J:.k.o| +00000370 28 ad 16 6d f5 31 17 87 f5 10 b5 53 13 c8 a2 a1 |(..m.1.....S....| +00000380 7a 9a 12 10 04 0b e7 b6 c7 3b 55 25 05 d3 0c 5e |z........;U%...^| +00000390 32 60 bf 43 e6 62 d0 cc cd a6 61 9b 19 dc 83 34 |2`.C.b....a....4| +000003a0 9f f2 2d 8f ca 9e 96 73 4d e9 52 17 03 03 00 45 |..-....sM.R....E| +000003b0 28 21 db ab 2f da c8 33 af a5 44 5b f4 2f 18 bd |(!../..3..D[./..| +000003c0 72 74 6d a7 c0 16 fb 13 de 8b d4 f8 03 8f ba 28 |rtm............(| +000003d0 06 89 08 0f 82 14 9b 6d 9b 61 7c 6c c3 de 76 2e |.......m.a|l..v.| +000003e0 f3 2b ff db 2d bc 6f 84 75 19 b3 df eb a1 a7 69 |.+..-.o.u......i| +000003f0 a7 a7 70 27 48 |..p'H| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 5b 1d b7 49 6d |..........E[..Im| +00000010 d0 f1 9a 16 cc e8 08 84 c0 80 23 1f 4f fc 15 b1 |..........#.O...| +00000020 20 28 d0 65 1b 58 8b 67 fa 30 c2 37 86 ac b2 47 | (.e.X.g.0.7...G| +00000030 d6 b8 ac 8a 69 bc a3 09 10 c8 47 15 54 b3 a4 79 |....i.....G.T..y| +00000040 1f 12 2d c1 ca e1 ce 54 c0 ce e5 fc 61 f2 7c 07 |..-....T....a.|.| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e ba d7 68 39 2c 76 30 fb 0e 36 47 |.......h9,v0..6G| +00000010 d9 b8 7c 2d e2 1c fd 13 86 9b 2c 07 ab 66 03 2d |..|-......,..f.-| +00000020 1e 7d 69 17 03 03 00 13 eb 7b d8 47 8a 5e 53 83 |.}i......{.G.^S.| +00000030 17 87 b3 aa c5 80 71 ba df 1b 60 |......q...`| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ALPN b/src/crypto/tls/testdata/Server-TLSv13-ALPN new file mode 100644 index 0000000000..c78bf3e9ca --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-ALPN @@ -0,0 +1,94 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 f8 01 00 00 f4 03 03 c6 52 45 ca ae |............RE..| +00000010 9b 43 49 d4 01 e2 af b0 e8 9b f4 9b f6 ef 38 c6 |.CI...........8.| +00000020 71 9f ed 84 a6 2c 01 92 1f 3f 1b 20 5b cb 8a 16 |q....,...?. [...| +00000030 b3 d2 a8 19 41 d5 0c 6a fa 39 0a b2 6d 65 18 d1 |....A..j.9..me..| +00000040 67 88 16 6b e0 9b e4 7a d2 95 f9 93 00 08 13 02 |g..k...z........| +00000050 13 03 13 01 00 ff 01 00 00 a3 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 0c 00 0a 00 1d 00 17 00 1e |................| +00000080 00 19 00 18 00 23 00 00 00 10 00 10 00 0e 06 70 |.....#.........p| +00000090 72 6f 74 6f 32 06 70 72 6f 74 6f 31 00 16 00 00 |roto2.proto1....| +000000a0 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 06 03 |................| +000000b0 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 |................| +000000c0 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 2d 00 |.......+......-.| +000000d0 02 01 01 00 33 00 26 00 24 00 1d 00 20 d8 72 8f |....3.&.$... .r.| +000000e0 a6 3b a1 1c f7 73 ea b0 d5 85 03 31 46 d6 6b 8b |.;...s.....1F.k.| +000000f0 e0 e6 e0 e6 20 09 a7 f8 a2 fd d1 a7 6b |.... .......k| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 5b cb 8a 16 |........... [...| +00000030 b3 d2 a8 19 41 d5 0c 6a fa 39 0a b2 6d 65 18 d1 |....A..j.9..me..| +00000040 67 88 16 6b e0 9b e4 7a d2 95 f9 93 13 02 00 00 |g..k...z........| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 24 9b af d8 42 6e 75 12 c4 10 c3 42 72 |...$...Bnu....Br| +00000090 de e6 50 f9 2b 15 68 6d b1 32 2f fb 63 b1 80 1a |..P.+.hm.2/.c...| +000000a0 f0 b3 3e 22 c9 d4 ed ba 17 03 03 02 6d c9 fc b7 |..>"........m...| +000000b0 3c 02 17 dc 26 56 b6 1f 3d 2c 07 0e 96 52 a8 9f |<...&V..=,...R..| +000000c0 a2 2a 6e e9 c7 93 2e c7 98 af 9b 63 99 25 14 f1 |.*n........c.%..| +000000d0 45 29 63 26 35 62 af d7 37 f3 33 03 2e a8 02 f2 |E)c&5b..7.3.....| +000000e0 a4 a5 9e 85 19 ad a8 20 81 85 6a 19 c5 bb 8f d7 |....... ..j.....| +000000f0 ea 4d af ee 9d 95 78 26 5a 24 da ee 5b 96 97 34 |.M....x&Z$..[..4| +00000100 7c e6 2c c1 b0 e8 d3 3a 7f 4c b5 c9 44 ad d5 35 ||.,....:.L..D..5| +00000110 99 b2 99 d9 c3 10 97 03 ac fe c9 6d cf 1a c0 6a |...........m...j| +00000120 ec d0 b7 ed b8 79 d6 48 8b eb df 0b 42 87 8e 50 |.....y.H....B..P| +00000130 9a c9 47 e7 e3 6a c6 03 85 8e b4 57 58 b2 aa 5f |..G..j.....WX.._| +00000140 53 8d 00 43 7e 36 4a 9b 2a 0a b7 45 e5 8b 4c 71 |S..C~6J.*..E..Lq| +00000150 62 b3 07 e1 1d 07 16 7b 4b c7 4b ac b3 e0 8b 4b |b......{K.K....K| +00000160 ec 3c 48 2f be eb 32 f6 f5 fa 24 86 76 e6 df ec |.r{.....S...|5| +000001f0 54 38 8a 0b 11 35 90 02 3c 2b ef 3c 9c 81 80 0b |T8...5..<+.<....| +00000200 9e a5 5a d0 07 37 ec b2 4b c9 2d 8a 22 45 20 60 |..Z..7..K.-."E `| +00000210 71 58 e0 a0 99 40 06 62 d4 19 cb 9d 62 65 a8 d6 |qX...@.b....be..| +00000220 dd bd 82 c9 01 ba ce 17 04 d2 09 3a 6b f6 12 9c |...........:k...| +00000230 41 1d 1f f9 f5 42 93 36 70 0b 3a ba 49 f9 54 dc |A....B.6p.:.I.T.| +00000240 d2 30 02 7d 04 0b 4a 3b 12 37 4e a7 96 8c 8b 8c |.0.}..J;.7N.....| +00000250 2e a7 84 78 22 18 d4 2d 93 0a da e1 97 bc ad b5 |...x"..-........| +00000260 2b 81 39 8e d3 5e 81 a1 51 c7 3e 40 3a fb ce 89 |+.9..^..Q.>@:...| +00000270 cf f3 dc 3e 18 0f f1 80 19 00 25 ca 5d e5 0e 0d |...>......%.]...| +00000280 76 4f de 4c eb 92 fa bf 4c 72 01 1d 26 38 88 f1 |vO.L....Lr..&8..| +00000290 9f d1 5f e4 ca e8 19 db dc f3 ba 0a 14 d4 63 b3 |.._...........c.| +000002a0 04 f4 4f ea 1c 0e 93 89 ad 6a 00 e0 64 fd 22 ae |..O......j..d.".| +000002b0 a7 58 dd e8 73 1e 89 b5 ed de ae b3 1e fb 54 1e |.X..s.........T.| +000002c0 70 ab 1e 38 2f bc 1d 16 3b e0 51 9f d9 dd 28 f2 |p..8/...;.Q...(.| +000002d0 2c 24 80 e7 76 b9 d2 25 53 5e c0 df 07 19 0a 8c |,$..v..%S^......| +000002e0 13 ed cd d7 0f dc 4d 92 66 49 41 b1 1f a8 92 c3 |......M.fIA.....| +000002f0 80 26 44 4f f8 49 86 b5 28 1e 3c 0c 1f 42 b9 ae |.&DO.I..(.<..B..| +00000300 27 57 f1 46 d3 61 23 8e 39 cf 50 ad fb 10 ba 7d |'W.F.a#.9.P....}| +00000310 a0 94 c5 b3 6e ad a1 15 71 00 17 03 03 00 99 87 |....n...q.......| +00000320 b6 bb fc 79 5f 53 43 73 bc 7f 0a 49 78 70 48 61 |...y_SCs...IxpHa| +00000330 22 e8 a5 f2 64 55 67 8b 68 de f4 12 64 52 2b 22 |"...dUg.h...dR+"| +00000340 e4 55 53 94 30 a4 7e 10 36 fc 98 9f 0a 32 df 02 |.US.0.~.6....2..| +00000350 b5 bf 58 34 cc 30 31 7b b8 79 54 56 38 48 94 1f |..X4.01{.yTV8H..| +00000360 df 98 b9 c6 dc 29 3c 94 12 ef 47 98 91 0a 81 44 |.....)<...G....D| +00000370 93 f4 96 a2 b2 05 09 d9 6e 25 9f 59 17 0f 41 8b |........n%.Y..A.| +00000380 f7 5c 81 e0 71 25 eb 24 eb bd cf 9a d9 24 8d c6 |.\..q%.$.....$..| +00000390 04 4e 10 0f 92 41 59 df 5c 77 c4 4d c8 30 e0 70 |.N...AY.\w.M.0.p| +000003a0 0c 5d 1b 49 b4 ea 06 bd c5 de 1d b5 19 63 46 72 |.].I.........cFr| +000003b0 98 72 48 e0 aa 76 6c d8 17 03 03 00 45 72 a8 f6 |.rH..vl.....Er..| +000003c0 c0 00 8a 52 91 57 bf 8d 63 87 31 df 97 17 af 41 |...R.W..c.1....A| +000003d0 88 44 19 1e 32 b9 32 d5 16 92 1b c7 6b 6e 29 8c |.D..2.2.....kn).| +000003e0 cc 42 59 05 8d 76 e4 4c 7e 46 5f 42 84 0d 28 37 |.BY..v.L~F_B..(7| +000003f0 15 53 d9 fc de 4b 04 d4 9a 14 6e 9f e8 c9 20 13 |.S...K....n... .| +00000400 00 c4 |..| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 ac 55 3c 0a 23 |..........E.U<.#| +00000010 d7 29 fa 16 c2 df a7 42 9e 15 7e 93 a7 0a 31 79 |.).....B..~...1y| +00000020 38 96 95 4a 0b 61 d8 f5 bb e5 51 a8 c3 ea 2b 92 |8..J.a....Q...+.| +00000030 a2 b0 9c 59 e1 52 b9 80 26 1b 84 c3 1a 68 f4 40 |...Y.R..&....h.@| +00000040 82 45 42 47 2f c7 3d ba 77 88 8b 5c 3b 03 a8 a7 |.EBG/.=.w..\;...| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 86 2e cf 5b 32 e1 e0 db 51 51 55 |........[2...QQU| +00000010 ef c3 05 87 ea 90 a8 37 8c 28 5d da 9f 64 70 33 |.......7.(]..dp3| +00000020 8f 94 f0 17 03 03 00 13 ba 81 00 ec 23 57 05 42 |............#W.B| +00000030 aa f4 ca b2 4e 98 d0 22 3b fc 38 |....N..";.8| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ALPN-NoMatch b/src/crypto/tls/testdata/Server-TLSv13-ALPN-NoMatch new file mode 100644 index 0000000000..8d524a9109 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-ALPN-NoMatch @@ -0,0 +1,93 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 f8 01 00 00 f4 03 03 7d 4e fc 70 f4 |...........}N.p.| +00000010 ca f8 bc d4 12 5d 8f 66 37 a0 c8 30 1a 4d 1b 4f |.....].f7..0.M.O| +00000020 0e 0b 86 1a 16 6d 77 2f ff eb a8 20 09 1f f5 06 |.....mw/... ....| +00000030 2c 4d 55 28 be dd 24 02 70 4c 0d 73 c4 c0 a1 d8 |,MU(..$.pL.s....| +00000040 b3 f0 13 26 76 df 47 bd 2e 27 1d 81 00 08 13 02 |...&v.G..'......| +00000050 13 03 13 01 00 ff 01 00 00 a3 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 0c 00 0a 00 1d 00 17 00 1e |................| +00000080 00 19 00 18 00 23 00 00 00 10 00 10 00 0e 06 70 |.....#.........p| +00000090 72 6f 74 6f 32 06 70 72 6f 74 6f 31 00 16 00 00 |roto2.proto1....| +000000a0 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 06 03 |................| +000000b0 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 |................| +000000c0 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 2d 00 |.......+......-.| +000000d0 02 01 01 00 33 00 26 00 24 00 1d 00 20 a9 8a 27 |....3.&.$... ..'| +000000e0 a2 7c b7 26 1d c6 ad f7 3a d0 97 81 ac 05 6c 10 |.|.&....:.....l.| +000000f0 a0 0d 3c 96 24 ce 55 ef 43 76 87 d4 31 |..<.$.U.Cv..1| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 09 1f f5 06 |........... ....| +00000030 2c 4d 55 28 be dd 24 02 70 4c 0d 73 c4 c0 a1 d8 |,MU(..$.pL.s....| +00000040 b3 f0 13 26 76 df 47 bd 2e 27 1d 81 13 02 00 00 |...&v.G..'......| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 74 38 f2 8b ac 52 87 0d e3 a9 77 eb |....t8...R....w.| +00000090 af 7f 53 3e 5c 3c cf 49 0a f3 c1 17 03 03 02 6d |..S>\<.I.......m| +000000a0 1f aa 15 fd 9d 05 77 15 12 24 2d ca 2f 93 18 44 |......w..$-./..D| +000000b0 7a 7c e3 c2 57 33 62 fe dc 95 f7 67 c8 08 95 77 |z|..W3b....g...w| +000000c0 1d f3 17 51 80 bc d9 3a 5d f0 a1 f9 fb 4b 21 00 |...Q...:]....K!.| +000000d0 7a 11 2a 51 3d 83 45 ef 62 ce 31 c2 98 9e 20 95 |z.*Q=.E.b.1... .| +000000e0 66 31 a6 3c 37 3b 91 16 d9 3e 37 0e e4 99 a6 a5 |f1.<7;...>7.....| +000000f0 5c 71 42 79 7d de 3c 9c 51 66 ce 2d 6e 1b e4 1d |\qBy}.<.Qf.-n...| +00000100 d5 ee cd 2c ed f9 94 f9 46 9d d1 91 4d 1c 7f dc |...,....F...M...| +00000110 17 a6 5a 42 87 ee 7a bd f3 6a d4 aa 44 b8 97 6d |..ZB..z..j..D..m| +00000120 a7 a3 9f 2d 14 d1 6d 64 e5 8a ed cf c9 a2 da a9 |...-..md........| +00000130 1b f5 83 d4 c8 aa 53 6c e9 5a ee e3 c6 c3 41 df |......Sl.Z....A.| +00000140 a1 17 44 17 7f ca f0 a8 ac 87 fc 2e 6d ba 0f 85 |..D.........m...| +00000150 d4 6d 58 99 8c 19 c1 0a b1 d8 9f 2c 93 36 e3 d3 |.mX........,.6..| +00000160 b6 0e 16 f1 8f a5 04 ee 3a 4f c5 fb 4c 7d d9 1c |........:O..L}..| +00000170 d4 d0 dd b5 59 9e 11 df 46 6c 6c cb f6 76 7a 03 |....Y...Fll..vz.| +00000180 b3 3e a2 45 38 03 56 e6 5a 23 ff 83 ee 0e d1 51 |.>.E8.V.Z#.....Q| +00000190 cc 94 51 82 6c 1a 8b 15 7e bd cd 09 44 72 42 65 |..Q.l...~...DrBe| +000001a0 45 0c 6c ff d7 4d 32 8a b1 9c 3f fa c0 8e 9e 86 |E.l..M2...?.....| +000001b0 5c d7 23 29 bf dd 40 fa d6 db 25 cb 63 74 33 15 |\.#)..@...%.ct3.| +000001c0 23 79 ab 22 39 e2 9e 41 89 01 9f 2e 58 8c 4b fe |#y."9..A....X.K.| +000001d0 b9 a5 cd 5e df be b2 d1 36 9e 8a c5 3f c4 53 db |...^....6...?.S.| +000001e0 29 95 33 6f 24 dd 84 5d da c2 5c 2c 1d cc de 40 |).3o$..]..\,...@| +000001f0 6c e7 07 a5 e3 95 f9 08 62 75 db dd ca e1 4c ab |l.......bu....L.| +00000200 12 3a 16 14 4a b6 78 58 42 81 91 8e b9 48 e0 a3 |.:..J.xXB....H..| +00000210 e4 1e 5f 18 e1 cf 3b 2f b3 2d 81 4c 4c b5 e4 24 |.._...;/.-.LL..$| +00000220 2c f0 e6 b7 43 9c e2 e4 25 88 5d 07 be e3 7b e5 |,...C...%.]...{.| +00000230 d5 7f 2a ad 28 72 54 92 73 19 39 6e f4 d9 3e 13 |..*.(rT.s.9n..>.| +00000240 f3 ed 37 66 22 8a 89 df 7b 99 b3 34 18 0f 7a 76 |..7f"...{..4..zv| +00000250 86 d4 da bd fd b5 f9 75 70 51 de e9 28 75 72 00 |.......upQ..(ur.| +00000260 f5 3f 11 f7 15 01 7e b3 bd b5 a2 05 33 dc 9d 69 |.?....~.....3..i| +00000270 26 f3 cb 45 a8 68 37 0b c6 6f b1 ca 3c 52 01 3c |&..E.h7..o..*h...| +000002a0 c8 83 75 75 d1 95 84 83 c9 e1 2d 0e 1c 37 9f 14 |..uu......-..7..| +000002b0 3a 78 86 01 c6 bb 04 eb 68 c3 26 ed 96 b7 49 43 |:x......h.&...IC| +000002c0 a5 ae 54 c0 d3 23 40 08 26 2f 3a 7e 63 04 2f 62 |..T..#@.&/:~c./b| +000002d0 3e ce 3d b6 70 bb 79 c6 c9 dc 14 24 de 02 96 08 |>.=.p.y....$....| +000002e0 d9 64 a2 c6 26 29 9d 58 a1 85 63 16 db a6 8d 3b |.d..&).X..c....;| +000002f0 0b 1a 15 9c 13 cb 8c cb ff 22 aa 59 da bd 81 78 |.........".Y...x| +00000300 90 f4 86 52 ad ae fc e4 1c ec ff 08 47 17 03 03 |...R........G...| +00000310 00 99 e6 54 8e c8 05 45 8c cb d1 44 df 88 d8 7d |...T...E...D...}| +00000320 1b 07 e1 06 d1 2b c4 fb 75 dd 63 4c b0 14 ee a4 |.....+..u.cL....| +00000330 7f 05 f3 e6 76 04 04 0b 0a 31 23 27 b1 ee 65 36 |....v....1#'..e6| +00000340 50 d9 cb 12 90 a8 6c 73 37 91 3a 57 fa fd 80 79 |P.....ls7.:W...y| +00000350 17 42 f9 27 36 e3 8a 75 3d 95 d6 17 a3 4e 73 5c |.B.'6..u=....Ns\| +00000360 4b e8 fc 58 d3 56 a2 e2 32 26 a0 2d 69 24 31 1e |K..X.V..2&.-i$1.| +00000370 6a 83 8e 5f 3f c4 9f d9 8b f8 59 3e ec 27 d0 96 |j.._?.....Y>.'..| +00000380 6f d6 28 d1 54 e6 f5 14 b4 55 67 de 90 ee f7 ef |o.(.T....Ug.....| +00000390 69 03 b7 d7 80 bd a9 ee a5 30 89 d6 5b ce 07 9e |i........0..[...| +000003a0 0a 09 03 90 4e 5d 75 8c 02 f2 7d 17 03 03 00 45 |....N]u...}....E| +000003b0 43 81 6f 9c bf c2 3c 14 23 f4 4b 9e 1e 17 0e d4 |C.o...<.#.K.....| +000003c0 bd 3a eb 33 34 e3 59 ea 5c ec 9a 5d 86 27 d8 51 |.:.34.Y.\..].'.Q| +000003d0 1f 0c bb d1 80 e0 2e 41 e5 7d 0a 50 df 26 68 cb |.......A.}.P.&h.| +000003e0 d5 83 aa 94 e9 1a c3 0b ab 0d bd 13 b3 55 d8 89 |.............U..| +000003f0 e1 e2 3b fe 01 |..;..| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 ac 36 2d 53 3e |..........E.6-S>| +00000010 08 c7 41 f3 da 40 ad d6 03 05 c1 b5 71 86 82 df |..A..@......q...| +00000020 6a 84 9e df f4 c5 7f 52 29 b0 a5 97 45 69 6e 6a |j......R)...Einj| +00000030 69 1c 91 32 bc 5b e9 62 b1 b8 af 9b 48 06 bb 37 |i..2.[.b....H..7| +00000040 99 45 f5 a9 cd 25 5a 41 c3 49 10 36 af 4a 56 90 |.E...%ZA.I.6.JV.| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e b4 23 d9 cd 9b 2a a8 76 07 c0 1e |......#...*.v...| +00000010 54 bb 7f 63 d9 31 2f e8 7b 91 dc b6 c9 3c a1 48 |T..c.1/.{....<.H| +00000020 2b 82 11 17 03 03 00 13 ae 1b 32 f4 2f 9c 1d 8c |+.........2./...| +00000030 98 4e 1d 80 68 f0 e2 a9 25 db f7 |.N..h...%..| diff --git a/src/crypto/tls/testdata/Server-TLSv13-CHACHA20-SHA256 b/src/crypto/tls/testdata/Server-TLSv13-CHACHA20-SHA256 new file mode 100644 index 0000000000..4aa6d6b568 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-CHACHA20-SHA256 @@ -0,0 +1,90 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 dc 01 00 00 d8 03 03 1d 2a 53 ec f5 |............*S..| +00000010 20 a6 22 2e 35 9d c4 a8 9b f0 0f 89 70 ef af 2b | .".5.......p..+| +00000020 3c 8c 6b 8a 44 74 71 8f e2 b6 f5 20 34 ab 1d 74 |<.k.Dtq.... 4..t| +00000030 cc cf 78 1f 9a 92 0b 7b 77 6a 67 16 fa 45 d2 d8 |..x....{wjg..E..| +00000040 1f c2 15 c3 91 f7 b6 1f 01 62 3e d1 00 04 13 03 |.........b>.....| +00000050 00 ff 01 00 00 8b 00 00 00 0e 00 0c 00 00 09 31 |...............1| +00000060 32 37 2e 30 2e 30 2e 31 00 0b 00 04 03 00 01 02 |27.0.0.1........| +00000070 00 0a 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 |................| +00000080 00 16 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 |................| +00000090 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 |................| +000000a0 08 05 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 |...........+....| +000000b0 04 00 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 |..-.....3.&.$...| +000000c0 20 71 d3 74 f3 4c 75 f7 29 4d 65 5b 13 b4 ea 4a | q.t.Lu.)Me[...J| +000000d0 ad 25 d2 5c 36 e8 42 f5 1d 4e b3 3e 7c a0 87 48 |.%.\6.B..N.>|..H| +000000e0 29 |)| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 34 ab 1d 74 |........... 4..t| +00000030 cc cf 78 1f 9a 92 0b 7b 77 6a 67 16 fa 45 d2 d8 |..x....{wjg..E..| +00000040 1f c2 15 c3 91 f7 b6 1f 01 62 3e d1 13 03 00 00 |.........b>.....| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 ed 03 2b 35 ba 39 86 ae ee 50 9e 71 |......+5.9...P.q| +00000090 f6 0a 49 2d 6e cc f5 0a 25 00 fc 17 03 03 02 6d |..I-n...%......m| +000000a0 dd 4a 14 ec 14 0e 41 8c 98 3b 02 df cb 33 f1 33 |.J....A..;...3.3| +000000b0 49 04 d2 bb 10 a1 b6 bb 92 17 09 19 4d 1e 18 03 |I...........M...| +000000c0 0c 97 c3 f9 e1 35 fd 5e 09 58 07 e3 15 8c ca 6b |.....5.^.X.....k| +000000d0 19 7a e1 ac 17 84 f8 f3 1d f4 1c f7 53 d6 fc 54 |.z..........S..T| +000000e0 5a c5 a7 4b 67 58 b9 63 85 a8 12 e4 3e 7a 87 7e |Z..KgX.c....>z.~| +000000f0 d6 b0 7f b9 70 b8 6f 57 23 7c 03 69 61 0a 3e bb |....p.oW#|.ia.>.| +00000100 65 b7 e5 48 4c c5 a3 ae 34 46 cf a4 b1 5b a5 26 |e..HL...4F...[.&| +00000110 5b 50 01 0b b7 36 30 06 46 b1 3e ae f7 72 2b 44 |[P...60.F.>..r+D| +00000120 7b e7 4d d0 18 c0 07 53 d4 73 80 02 cf 10 1b c9 |{.M....S.s......| +00000130 10 a8 29 81 c4 89 78 cf ad 3a 90 92 e3 f2 ac 9c |..)...x..:......| +00000140 88 64 f6 1f 96 bf f7 a8 f4 a1 ec 26 52 a7 ab 3b |.d.........&R..;| +00000150 0c 33 0f 46 6b cf c0 07 80 ca 46 61 28 46 9e 5a |.3.Fk.....Fa(F.Z| +00000160 22 98 d7 cf 1f b8 c5 b4 d1 14 3f 71 0b e8 a3 3f |".........?q...?| +00000170 8a 69 a4 1c a7 5a e4 e0 a0 d8 ee e4 5c ef 6b 55 |.i...Z......\.kU| +00000180 55 e5 4e 6f 75 79 d0 53 b8 21 22 1d bf 86 65 15 |U.Nouy.S.!"...e.| +00000190 7a 24 7f 7c 4b 82 84 6d aa 08 9e 45 1a 5f 2b 34 |z$.|K..m...E._+4| +000001a0 66 f2 92 cc f0 7a 7a e5 d1 2a c2 2e cb 78 c5 aa |f....zz..*...x..| +000001b0 18 87 d3 45 6d 39 39 25 3c f3 aa db 6c 10 8c b4 |...Em99%<...l...| +000001c0 f7 f8 ea 49 5f 8a 5b 20 4d f2 e6 53 11 0f a5 3e |...I_.[ M..S...>| +000001d0 ab 0e 0f 6b fb 5f 43 cb 9f ed 0a f1 5e 21 0f 7c |...k._C.....^!.|| +000001e0 86 4f e0 62 cb 2e be 49 2a c0 3b 53 92 8c 58 c2 |.O.b...I*.;S..X.| +000001f0 53 82 bc 1b 84 e1 5f 85 2d 9e 5f 85 9f 4c 31 7d |S....._.-._..L1}| +00000200 20 ad a4 07 4f 08 06 7b e8 47 52 cf a0 4e e5 0f | ...O..{.GR..N..| +00000210 ff d2 5b f6 f2 a0 5d 08 92 98 f0 d9 a0 e9 4f cd |..[...].......O.| +00000220 c9 bb de d9 a0 b5 e1 ec a9 60 7b a9 37 2f 5c 77 |.........`{.7/\w| +00000230 60 62 de dd f8 e1 63 4e 0b 7f 92 ff 81 96 02 ab |`b....cN........| +00000240 86 e4 f8 4e 52 60 91 cf 75 fc 1d c5 b0 74 c2 06 |...NR`..u....t..| +00000250 15 4e be 1b f8 1c 14 ba 8c 2f 0b ab a2 cc 0e 9a |.N......./......| +00000260 ce fc 05 e9 21 e8 08 55 61 61 8e 98 c8 73 63 2c |....!..Uaa...sc,| +00000270 97 71 2e 74 a1 b1 42 dc fe 6e 26 0f 5b 9d 13 96 |.q.t..B..n&.[...| +00000280 47 03 5f 46 4b ae 81 d0 a0 d1 a7 a3 10 de e6 c6 |G._FK...........| +00000290 6c bc 01 65 04 6f e5 6c 35 72 4c 65 74 a9 2c b0 |l..e.o.l5rLet.,.| +000002a0 10 75 7b d5 c1 1a f0 f8 19 f5 7a 7e 28 f2 53 9d |.u{.......z~(.S.| +000002b0 a0 c4 c9 e3 e4 80 71 bc 5d cc 5f 3d 7b 2d 5f f0 |......q.]._={-_.| +000002c0 51 94 8c d8 8e 0f f6 b8 29 44 90 c3 99 9a dc de |Q.......)D......| +000002d0 b6 81 ac 65 70 2b ad 1d df 47 43 a5 a0 b6 51 6f |...ep+...GC...Qo| +000002e0 0e 29 de 9c cf b0 89 93 94 08 d8 f3 a5 05 b9 d3 |.)..............| +000002f0 a0 bd 92 81 82 91 89 2d 4e df 13 98 09 67 31 c1 |.......-N....g1.| +00000300 4d 28 d0 60 28 dc ae 40 c8 0f 33 d8 ac 17 03 03 |M(.`(..@..3.....| +00000310 00 99 22 21 e8 35 f9 9c 29 9d 3a aa 00 ca 0c a7 |.."!.5..).:.....| +00000320 d1 62 94 94 44 81 8f e0 56 13 29 5f c4 a9 2b ac |.b..D...V.)_..+.| +00000330 8a 67 fe 5f 32 4c 40 00 36 db ff 03 41 67 55 9f |.g._2L@.6...AgU.| +00000340 b5 a8 cc 52 89 0d 08 23 af c5 0f 02 70 3a 62 9f |...R...#....p:b.| +00000350 91 82 cc 1d 6f 57 d3 2b f6 3f 75 f3 65 6b c2 ca |....oW.+.?u.ek..| +00000360 e9 5f 0d 1c 5a 31 8f 8f f8 09 8f a8 4a 27 64 f7 |._..Z1......J'd.| +00000370 18 2d fb 29 09 cd 07 fb 7c 32 47 4e 8d e2 e0 47 |.-.)....|2GN...G| +00000380 60 76 f8 9e 2c 81 71 09 ac 7d 3a 10 30 27 07 06 |`v..,.q..}:.0'..| +00000390 c9 7c 9d 57 5c c2 83 be bf 67 28 70 15 4a fa b6 |.|.W\....g(p.J..| +000003a0 56 8b 15 a8 a6 c8 65 ab 8c a4 03 17 03 03 00 35 |V.....e........5| +000003b0 c6 0c 8f e5 57 46 d9 de 3a 13 7e 80 a9 06 76 e7 |....WF..:.~...v.| +000003c0 a7 6d 5d 0e d1 21 dc 20 8f 2c df 86 cf cd e6 6e |.m]..!. .,.....n| +000003d0 71 14 69 a5 63 a1 38 80 d4 28 20 76 31 37 c6 be |q.i.c.8..( v17..| +000003e0 d9 65 b9 0b 74 |.e..t| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 35 65 9c 9f 5e ef |..........5e..^.| +00000010 8f 90 79 3f a5 ff cb 00 55 e9 2d 94 62 d4 04 72 |..y?....U.-.b..r| +00000020 0e 93 d1 5b cf 53 80 0b 72 d1 b4 e9 09 0b 9a 0f |...[.S..r.......| +00000030 10 81 27 bf c5 d2 6c 5a 99 f5 04 b5 3e a7 ab 77 |..'...lZ....>..w| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 24 3a 1a 51 aa f2 9e 47 34 90 c7 |.....$:.Q...G4..| +00000010 d9 cc 02 40 89 15 7b 47 b6 45 4a 8b fb 8b b5 1a |...@..{G.EJ.....| +00000020 43 4c 1e 17 03 03 00 13 01 41 14 2e 85 26 f4 d0 |CL.......A...&..| +00000030 da a5 91 14 52 66 0c 7d 26 82 d0 |....Rf.}&..| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES new file mode 100644 index 0000000000..152eb24854 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES @@ -0,0 +1,86 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 dc 01 00 00 d8 03 03 34 c0 8c 76 f5 |...........4..v.| +00000010 c3 94 de 34 58 f6 fe 27 51 28 2d cf be 74 61 8e |...4X..'Q(-..ta.| +00000020 5c 25 14 22 1b b2 70 4b c6 a5 e7 20 f5 d9 d5 d1 |\%."..pK... ....| +00000030 5c 24 ab df 36 f1 c8 b9 4a 66 aa 52 d3 6c 3b 07 |\$..6...Jf.R.l;.| +00000040 53 e2 e3 a5 6a bf ad 25 1e 93 06 40 00 04 13 01 |S...j..%...@....| +00000050 00 ff 01 00 00 8b 00 00 00 0e 00 0c 00 00 09 31 |...............1| +00000060 32 37 2e 30 2e 30 2e 31 00 0b 00 04 03 00 01 02 |27.0.0.1........| +00000070 00 0a 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 |................| +00000080 00 16 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 |................| +00000090 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 |................| +000000a0 08 05 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 |...........+....| +000000b0 04 00 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 |..-.....3.&.$...| +000000c0 20 09 5c dd 14 86 ba fa c5 6f ab b8 50 d5 1b 6f | .\......o..P..o| +000000d0 1b 49 12 c2 d6 e7 a0 8c ca 95 7e 5d 62 ab 5a c8 |.I........~]b.Z.| +000000e0 73 |s| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 f5 d9 d5 d1 |........... ....| +00000030 5c 24 ab df 36 f1 c8 b9 4a 66 aa 52 d3 6c 3b 07 |\$..6...Jf.R.l;.| +00000040 53 e2 e3 a5 6a bf ad 25 1e 93 06 40 13 01 00 00 |S...j..%...@....| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 57 c3 ea 50 c5 f5 d6 1f 4f e9 7e 30 |....W..P....O.~0| +00000090 79 5b 6e 09 78 a7 f8 0d a4 20 3d 17 03 03 02 22 |y[n.x.... =...."| +000000a0 78 32 99 7b 8b 67 07 bc bf 86 bc 4e 6d ba 2b 2e |x2.{.g.....Nm.+.| +000000b0 ba c8 bb d4 c5 10 dc 5a 9a 38 13 38 92 6f 8b 8c |.......Z.8.8.o..| +000000c0 73 8e 97 5e 39 3e 4e 93 b7 76 c4 c7 75 67 43 c9 |s..^9>N..v..ugC.| +000000d0 80 a3 bb 88 71 e4 46 e2 dd 17 df a1 ba b8 af 2f |....q.F......../| +000000e0 47 e8 16 a5 04 08 69 66 34 f2 22 c9 28 06 70 b5 |G.....if4.".(.p.| +000000f0 f7 c8 90 61 f4 77 32 7a f7 e5 41 d2 dc e7 13 9b |...a.w2z..A.....| +00000100 41 09 df 97 e2 9a 31 33 ef 48 48 2c 64 d9 9c 11 |A.....13.HH,d...| +00000110 e7 13 2b 1e 6d 3f 73 20 c7 5a 2f 2b aa bd f6 29 |..+.m?s .Z/+...)| +00000120 49 c1 de 95 65 dc 7c a4 de d8 9b 80 1c 38 3b a4 |I...e.|......8;.| +00000130 0c 7b 31 9a 27 b4 6f 38 2a 58 15 82 97 f0 99 bd |.{1.'.o8*X......| +00000140 d0 61 ca c9 2f 3f 71 8f 29 a6 ed 74 32 73 10 38 |.a../?q.)..t2s.8| +00000150 c8 33 ba a0 9b c3 27 59 20 24 15 e5 02 27 b4 1c |.3....'Y $...'..| +00000160 62 72 9c 83 da 93 9e 03 a1 4a d3 50 df d0 5c 78 |br.......J.P..\x| +00000170 74 58 63 c7 f6 3b 6e ba da 13 5c 66 6e 3e cb d5 |tXc..;n...\fn>..| +00000180 7f b1 ed 62 11 60 a1 ed 20 d8 3b 07 d6 36 f0 f2 |...b.`.. .;..6..| +00000190 75 9a a4 3a 11 ac fa 6e a1 2e 06 fe 44 90 06 2e |u..:...n....D...| +000001a0 78 8e 93 97 7f 7a 5c e2 ac be 29 cd 0f ea d2 5c |x....z\...)....\| +000001b0 ca 96 a1 7e 6e b2 3e b3 80 79 fb 25 a4 ee 99 29 |...~n.>..y.%...)| +000001c0 85 f7 b7 1f dd 35 d4 3b fb d2 a9 a1 e9 67 0f 7a |.....5.;.....g.z| +000001d0 ee fd 61 65 79 fc b3 8a c2 32 1c b6 54 b9 c3 ab |..aey....2..T...| +000001e0 2b 32 32 2d 88 16 9e 10 60 ee ef 58 85 1a 65 eb |+22-....`..X..e.| +000001f0 5b 9f ae e3 fb da 37 7e ec 73 69 57 29 e1 ab 76 |[.....7~.siW)..v| +00000200 a3 bd fa 02 91 3d f8 6d 95 49 81 4a 44 2e d9 8c |.....=.m.I.JD...| +00000210 c7 00 0d ef 0c 84 a9 b2 16 ff 6a 52 79 99 37 6d |..........jRy.7m| +00000220 dc 5f c6 32 76 ac 3b 92 cc 21 6a 2b 26 03 43 66 |._.2v.;..!j+&.Cf| +00000230 b2 64 79 51 11 08 7f 3f 63 ec 22 79 d7 90 4f 84 |.dyQ...?c."y..O.| +00000240 ed 58 1f c3 fe e9 87 bf d1 80 66 25 5a 95 53 1f |.X........f%Z.S.| +00000250 08 88 0c b8 7e 3b 07 c3 c7 4b 13 80 23 b1 15 3e |....~;...K..#..>| +00000260 e7 94 9b 91 8e 56 37 47 99 5f 5f 45 2d 11 f7 fc |.....V7G.__E-...| +00000270 33 b6 1f 6a 05 d8 8c 9f 1a 5f 38 d7 e8 1a d0 83 |3..j....._8.....| +00000280 11 b6 bc 09 7b a9 31 19 b2 f1 a7 ea 8b a3 be a7 |....{.1.........| +00000290 72 97 fb 09 19 5a d0 87 9c 01 bb e6 6e 50 91 87 |r....Z......nP..| +000002a0 b1 e6 0c e0 83 af b2 e8 b9 fa 71 3e 3c 6e 59 91 |..........q>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 35 c1 5b 7a 5c 1b |..........5.[z\.| +00000010 3c 04 8b 1d 88 fb 64 28 08 47 4b 5e 18 f9 b9 25 |<.....d(.GK^...%| +00000020 39 61 50 21 9b 0a 7a af 2a 26 6e 46 30 66 50 db |9aP!..z.*&nF0fP.| +00000030 8e 3f de c3 9e 65 e3 00 51 cd cc 38 44 92 13 5c |.?...e..Q..8D..\| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 1c 4f 11 b2 e9 66 a3 ed 64 2d 23 |......O...f..d-#| +00000010 b6 11 b0 fa 27 55 53 93 3a 70 01 c8 33 58 f8 48 |....'US.:p..3X.H| +00000020 68 81 50 17 03 03 00 13 9c a6 6d 62 80 e3 55 f6 |h.P.......mb..U.| +00000030 22 d5 84 59 35 8f 79 6f 1f 5e 79 |"..Y5.yo.^y| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ExportKeyingMaterial b/src/crypto/tls/testdata/Server-TLSv13-ExportKeyingMaterial new file mode 100644 index 0000000000..b1a9405448 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-ExportKeyingMaterial @@ -0,0 +1,92 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 e4 01 00 00 e0 03 03 40 35 14 01 11 |...........@5...| +00000010 ea 27 ee ac 2d 8b d9 de 62 2d 94 4e 4f c6 97 09 |.'..-...b-.NO...| +00000020 8a 84 1d 96 ca 1e 1c a2 a5 9f 82 20 cf 5d fb ec |........... .]..| +00000030 d8 3d 23 2d 89 77 a9 7b 1a 9a 72 e6 bd 17 50 53 |.=#-.w.{..r...PS| +00000040 56 32 17 d3 50 38 0c 9d 0b e4 8d 9a 00 08 13 02 |V2..P8..........| +00000050 13 03 13 01 00 ff 01 00 00 8f 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 0c 00 0a 00 1d 00 17 00 1e |................| +00000080 00 19 00 18 00 23 00 00 00 16 00 00 00 17 00 00 |.....#..........| +00000090 00 0d 00 1e 00 1c 04 03 05 03 06 03 08 07 08 08 |................| +000000a0 08 09 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 |................| +000000b0 06 01 00 2b 00 03 02 03 04 00 2d 00 02 01 01 00 |...+......-.....| +000000c0 33 00 26 00 24 00 1d 00 20 99 33 07 c3 2f 4d 4d |3.&.$... .3../MM| +000000d0 f1 3b 8a 93 f4 58 77 2b 69 e6 6e ae e8 1b 0a 30 |.;...Xw+i.n....0| +000000e0 a2 35 f5 1f 9e ed 34 ed 0c |.5....4..| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 cf 5d fb ec |........... .]..| +00000030 d8 3d 23 2d 89 77 a9 7b 1a 9a 72 e6 bd 17 50 53 |.=#-.w.{..r...PS| +00000040 56 32 17 d3 50 38 0c 9d 0b e4 8d 9a 13 02 00 00 |V2..P8..........| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 57 f0 44 ca 0f b9 92 0d 8f 6f 1f 11 |....W.D......o..| +00000090 a1 03 28 76 f0 0c 46 87 5a c7 b5 17 03 03 02 6d |..(v..F.Z......m| +000000a0 78 f0 92 2b 56 5d b9 62 95 1b 04 8b d0 49 63 31 |x..+V].b.....Ic1| +000000b0 1f 1d 9a 37 b3 64 e5 11 21 58 4c e2 d2 da a4 e1 |...7.d..!XL.....| +000000c0 1b 26 e7 11 3f b5 46 18 62 46 35 8a ae b1 88 93 |.&..?.F.bF5.....| +000000d0 5c 97 6c 66 6e a7 5d 63 fb e2 07 07 f5 59 02 38 |\.lfn.]c.....Y.8| +000000e0 78 d5 52 97 9b 2f 06 d3 91 c0 a3 62 9b 74 7e f8 |x.R../.....b.t~.| +000000f0 41 0b 71 df f9 ed 42 1b 51 ec db 66 86 d7 db c0 |A.q...B.Q..f....| +00000100 a1 27 9e 51 95 fe 54 66 4a f2 72 7b d7 91 cd 00 |.'.Q..TfJ.r{....| +00000110 b1 fd 36 a1 6f c5 c4 31 3e b8 d8 ab 0b f5 57 bf |..6.o..1>.....W.| +00000120 63 91 99 5e 04 63 c7 fa c9 73 b5 23 0e e9 45 3a |c..^.c...s.#..E:| +00000130 15 2b a7 d3 f5 04 50 ab 17 65 40 d3 63 da 6e 2a |.+....P..e@.c.n*| +00000140 66 45 2d 41 1f 09 fa 62 67 86 8f e6 c5 7e 02 d0 |fE-A...bg....~..| +00000150 27 bf 43 e8 15 8e 4f 71 67 2e 9b 13 61 44 23 0b |'.C...Oqg...aD#.| +00000160 29 06 81 0c 9c 28 c5 a9 f1 6e 49 84 b3 75 90 93 |)....(...nI..u..| +00000170 0a f7 db 01 29 9f 73 5d 00 f5 41 a7 cb 0b b0 97 |....).s]..A.....| +00000180 d7 b6 d0 71 31 56 88 88 7b 16 3b 54 5e 82 2b 87 |...q1V..{.;T^.+.| +00000190 2c 74 2c 8f 0a ec 5f 2b ef a6 86 55 49 d9 a2 af |,t,..._+...UI...| +000001a0 4a 34 48 a4 65 b6 4f f3 7f b6 30 e8 c2 f1 03 f4 |J4H.e.O...0.....| +000001b0 89 90 02 b6 f2 6e 27 e8 33 5f c7 34 91 a7 fd 96 |.....n'.3_.4....| +000001c0 58 a1 4b 3a e5 73 92 1c ed 01 dd 15 a2 b5 61 01 |X.K:.s........a.| +000001d0 1b 1d 52 0d 10 1f 1e a8 3b a4 b8 cf 50 0b ff e6 |..R.....;...P...| +000001e0 cf b7 59 cf 60 55 f7 2d ad 1d 7a 76 dc c0 4d d3 |..Y.`U.-..zv..M.| +000001f0 5f 06 d2 1e 02 a8 23 12 6c ae 3a 90 d4 1a ef b1 |_.....#.l.:.....| +00000200 31 c8 82 5c ca 92 1d db c3 0c 5e 9e 80 1c 1d b2 |1..\......^.....| +00000210 f5 55 b5 55 92 94 9a 43 ef 60 86 ee f0 65 68 bd |.U.U...C.`...eh.| +00000220 ad f8 5d f2 06 3f 2d b5 52 26 71 33 bb 0a f2 31 |..]..?-.R&q3...1| +00000230 8a 98 41 8d 8d 59 d1 b7 c9 b1 3c e1 37 9e 70 0b |..A..Y....<.7.p.| +00000240 da ae 25 34 49 93 ce f3 a0 c9 b6 7e 06 34 53 07 |..%4I......~.4S.| +00000250 ef 61 43 9d 79 2c d0 02 5a 64 bd 4a 46 98 3c 42 |.aC.y,..Zd.JF.g.........x| +000003a0 3f eb 55 57 d9 7b 3a 31 c8 b9 d2 17 03 03 00 45 |?.UW.{:1.......E| +000003b0 ec 76 02 21 7b 96 9c ff a5 ea e9 2e 92 a7 d6 f4 |.v.!{...........| +000003c0 8e c0 a0 bc 21 11 44 df 84 dd 3c 21 5d cc 1b 2c |....!.D....| +000003f0 f0 29 1b 4b 6e |.).Kn| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 4a 89 ee af 15 |..........EJ....| +00000010 db cf 58 6f f9 5a cd 28 a7 57 08 38 7a a4 1a a6 |..Xo.Z.(.W.8z...| +00000020 d9 ed 5f f8 b1 f8 a9 aa 19 70 5f 8d 87 b9 d8 5c |.._......p_....\| +00000030 b6 4d d6 04 4b 66 1b 6a 57 25 58 bf a1 b6 72 81 |.M..Kf.jW%X...r.| +00000040 74 b9 c9 6f 02 c9 0b f6 b2 7f 70 72 78 a6 06 bd |t..o......prx...| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e ee 68 ad 7a 47 ba 93 80 26 01 37 |......h.zG...&.7| +00000010 1b 3b b8 61 aa 60 a1 0f 21 a4 81 51 16 a5 b1 36 |.;.a.`..!..Q...6| +00000020 b2 39 ca 17 03 03 00 13 5e 1c 9b a5 d1 02 68 96 |.9......^.....h.| +00000030 99 41 8c a1 9e 49 38 1d 97 b6 c8 |.A...I8....| diff --git a/src/crypto/tls/testdata/Server-TLSv13-HelloRetryRequest b/src/crypto/tls/testdata/Server-TLSv13-HelloRetryRequest new file mode 100644 index 0000000000..b63a5f8ab5 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-HelloRetryRequest @@ -0,0 +1,118 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 da 01 00 00 d6 03 03 29 a2 7e 24 c3 |...........).~$.| +00000010 02 bf 2c 29 7b 47 08 06 bf 75 ef c5 59 2d a4 f0 |..,){G...u..Y-..| +00000020 2f fc 53 62 5d b8 4d 3c f1 31 2d 20 96 7c 00 da |/.Sb].M<.1- .|..| +00000030 35 8c 39 0a f8 1c 61 b8 4a a8 28 b4 a2 de 56 7b |5.9...a.J.(...V{| +00000040 a9 f3 ab 5d db 30 ca 2c d1 82 9e e2 00 08 13 02 |...].0.,........| +00000050 13 03 13 01 00 ff 01 00 00 85 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 06 00 04 00 1d 00 17 00 16 |................| +00000080 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 |................| +00000090 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 |................| +000000a0 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 |.........+......| +000000b0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 dd |-.....3.&.$... .| +000000c0 c7 c6 35 0b c9 5f f0 7e 40 e8 f1 f7 6f e7 84 8f |..5.._.~@...o...| +000000d0 7f 32 2f 8c a3 22 c6 c1 b9 34 1c ef 17 e7 25 |.2/.."...4....%| +>>> Flow 2 (server to client) +00000000 16 03 03 00 58 02 00 00 54 03 03 cf 21 ad 74 e5 |....X...T...!.t.| +00000010 9a 61 11 be 1d 8c 02 1e 65 b8 91 c2 a2 11 16 7a |.a......e......z| +00000020 bb 8c 5e 07 9e 09 e2 c8 a8 33 9c 20 96 7c 00 da |..^......3. .|..| +00000030 35 8c 39 0a f8 1c 61 b8 4a a8 28 b4 a2 de 56 7b |5.9...a.J.(...V{| +00000040 a9 f3 ab 5d db 30 ca 2c d1 82 9e e2 13 02 00 00 |...].0.,........| +00000050 0c 00 2b 00 02 03 04 00 33 00 02 00 17 |..+.....3....| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 16 03 03 00 fb 01 00 00 f7 03 |................| +00000010 03 29 a2 7e 24 c3 02 bf 2c 29 7b 47 08 06 bf 75 |.).~$...,){G...u| +00000020 ef c5 59 2d a4 f0 2f fc 53 62 5d b8 4d 3c f1 31 |..Y-../.Sb].M<.1| +00000030 2d 20 96 7c 00 da 35 8c 39 0a f8 1c 61 b8 4a a8 |- .|..5.9...a.J.| +00000040 28 b4 a2 de 56 7b a9 f3 ab 5d db 30 ca 2c d1 82 |(...V{...].0.,..| +00000050 9e e2 00 08 13 02 13 03 13 01 00 ff 01 00 00 a6 |................| +00000060 00 00 00 0e 00 0c 00 00 09 31 32 37 2e 30 2e 30 |.........127.0.0| +00000070 2e 31 00 0b 00 04 03 00 01 02 00 0a 00 06 00 04 |.1..............| +00000080 00 1d 00 17 00 16 00 00 00 17 00 00 00 0d 00 1e |................| +00000090 00 1c 04 03 05 03 06 03 08 07 08 08 08 09 08 0a |................| +000000a0 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 00 2b |...............+| +000000b0 00 03 02 03 04 00 2d 00 02 01 01 00 33 00 47 00 |......-.....3.G.| +000000c0 45 00 17 00 41 04 30 30 c4 62 c9 14 6b f7 28 88 |E...A.00.b..k.(.| +000000d0 ff c8 3f 87 2b 9f 24 a3 46 9b 2f 86 c7 df 3f 05 |..?.+.$.F./...?.| +000000e0 6b 8e a7 80 64 ff 66 b7 4d 80 62 fd b4 ba de 27 |k...d.f.M.b....'| +000000f0 44 7f 7a a4 c7 4a f0 81 25 52 3f 8a 8a 48 1e ff |D.z..J..%R?..H..| +00000100 6f 11 ad a9 fe bd |o.....| +>>> Flow 4 (server to client) +00000000 16 03 03 00 9b 02 00 00 97 03 03 00 00 00 00 00 |................| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 96 7c 00 da |........... .|..| +00000030 35 8c 39 0a f8 1c 61 b8 4a a8 28 b4 a2 de 56 7b |5.9...a.J.(...V{| +00000040 a9 f3 ab 5d db 30 ca 2c d1 82 9e e2 13 02 00 00 |...].0.,........| +00000050 4f 00 2b 00 02 03 04 00 33 00 45 00 17 00 41 04 |O.+.....3.E...A.| +00000060 1e 18 37 ef 0d 19 51 88 35 75 71 b5 e5 54 5b 12 |..7...Q.5uq..T[.| +00000070 2e 8f 09 67 fd a7 24 20 3e b2 56 1c ce 97 28 5e |...g..$ >.V...(^| +00000080 f8 2b 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 32 42 |.+-O....lK[.V.2B| +00000090 e9 58 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc 5a 89 |.X..I..h.A.Vk.Z.| +000000a0 17 03 03 00 17 74 86 c5 4d 7f f0 06 2b fa a3 03 |.....t..M...+...| +000000b0 4d 27 46 e7 33 ce 70 32 ba 61 55 61 17 03 03 02 |M'F.3.p2.aUa....| +000000c0 6d c5 ac cc 02 19 b0 92 c1 bc 6a fe 1d 94 a6 75 |m.........j....u| +000000d0 66 6a 88 b0 87 76 da 3f f9 07 40 67 c7 e5 b8 b7 |fj...v.?..@g....| +000000e0 70 38 41 0e 11 3b cf db 0b 76 73 58 10 93 0e 06 |p8A..;...vsX....| +000000f0 cb 31 75 d0 a8 2f 20 35 b4 c2 87 5f 42 94 e1 18 |.1u../ 5..._B...| +00000100 d5 f8 bc 11 d7 7c 9b ff 0b fe 59 8b 78 1e ef 52 |.....|....Y.x..R| +00000110 6f ea 39 e5 f5 55 93 8b 0b 65 87 05 d3 0d a3 6d |o.9..U...e.....m| +00000120 2e 10 79 1d 60 61 ba 43 13 cd 5a 58 28 cb 32 df |..y.`a.C..ZX(.2.| +00000130 74 3a 58 4e e0 21 d4 d0 2f 6f 63 72 1c 82 18 16 |t:XN.!../ocr....| +00000140 12 4c e7 0b 5e 21 ce e4 de 26 6c d2 91 c1 fd 85 |.L..^!...&l.....| +00000150 5a f4 b7 5d 5a c6 fe 8c 05 fe f7 2a ae a1 67 73 |Z..]Z......*..gs| +00000160 8a 66 60 07 57 35 46 ed cf e1 58 f3 22 94 57 26 |.f`.W5F...X.".W&| +00000170 ae 81 3c ea 42 1c ef 56 ca 0e 35 5a 4c 97 49 36 |..<.B..V..5ZL.I6| +00000180 73 e2 be 8b ad 55 42 a3 8f 50 65 8a 7a 9d cf 7c |s....UB..Pe.z..|| +00000190 4f d9 4b db 0d a4 0e 57 99 08 72 0a 1f cf 9d 19 |O.K....W..r.....| +000001a0 ad 24 7b 64 9b ef d2 9f 8f 42 d4 ca a8 84 2f 15 |.${d.....B..../.| +000001b0 3a 10 8e 1e 22 b8 fc 9f 77 0b 8e 82 22 bd 08 f2 |:..."...w..."...| +000001c0 3e 4c a7 f1 d0 46 a8 fb fb 5e f6 0d 32 8e 2d e4 |>L...F...^..2.-.| +000001d0 3b 17 b8 da 71 03 cb d9 b8 12 9b 70 a0 3a 07 19 |;...q......p.:..| +000001e0 f6 c1 66 1e b8 e2 b2 5a 50 50 c5 51 8f f3 91 e7 |..f....ZPP.Q....| +000001f0 bc ba a2 ee ca a8 71 24 5f f2 25 79 c3 a2 23 70 |......q$_.%y..#p| +00000200 03 0b 8b 99 4d 33 92 f1 ff 64 cd cc 31 b2 13 a9 |....M3...d..1...| +00000210 d3 c2 37 b7 11 17 7f d7 64 ba da 6d 46 b1 a2 03 |..7.....d..mF...| +00000220 fe 8d 4d e3 cd eb a3 2f af 89 f4 a4 af 82 0e e3 |..M..../........| +00000230 65 c3 64 38 33 6a bd c9 13 77 9d 37 c7 c8 1d 55 |e.d83j...w.7...U| +00000240 f9 a6 b8 fc 57 0f f7 94 76 1f bd 4f 4a 74 fe 0a |....W...v..OJt..| +00000250 98 be e9 d9 2e d2 c0 c9 fb d2 3b 27 fb 37 14 f5 |..........;'.7..| +00000260 29 d5 f4 88 a5 b0 98 1d 0c 85 9e 1f ad 29 cf 36 |)............).6| +00000270 3d 2f c0 54 93 1f 14 8b 1e a3 93 aa 53 af da d1 |=/.T........S...| +00000280 da 0b 2b e4 01 fe 8e 48 df 8b 97 fe 92 ab 32 80 |..+....H......2.| +00000290 c3 d2 84 1a 45 a7 0d f3 f8 07 e3 7d a3 27 62 67 |....E......}.'bg| +000002a0 10 4b 3e 09 ee 22 77 0d 54 71 8b 6f 68 6a a9 cc |.K>.."w.Tq.ohj..| +000002b0 80 34 5e 35 36 b0 cb d5 8b 04 6f 0f 28 82 5f 69 |.4^56.....o.(._i| +000002c0 d3 66 cd 19 db 39 1d 73 c2 28 36 94 5f 1b 24 c6 |.f...9.s.(6._.$.| +000002d0 9a 62 34 0f ec 9a f9 c5 8c 72 5c c3 a0 c7 d6 5b |.b4......r\....[| +000002e0 1d 2e 4b 31 cd 2e 8b 37 cc 9a b2 ca e9 5d b0 f9 |..K1...7.....]..| +000002f0 b4 2c ba 27 08 c3 d6 90 1b 51 0c 83 72 0b 8a 72 |.,.'.....Q..r..r| +00000300 a1 12 ab 5c 91 a8 b2 76 f1 c1 09 d3 74 fb bc e5 |...\...v....t...| +00000310 33 96 59 91 3e e0 89 cf 99 7c 40 c4 af 0e 8d 58 |3.Y.>....|@....X| +00000320 20 ea d9 0d 0e 64 10 2c c8 ad ed 38 b0 fa 17 03 | ....d.,...8....| +00000330 03 00 99 f0 80 72 33 a3 53 2f c6 7f 68 e5 42 ee |.....r3.S/..h.B.| +00000340 d0 81 00 07 8b b7 69 ec 0d 9e 5a dc f5 0b 40 82 |......i...Z...@.| +00000350 0e a5 bc ce dd 13 1e 15 4c 10 d8 62 00 42 45 eb |........L..b.BE.| +00000360 2b f1 aa d4 43 4e 29 02 a8 0e b8 3d 17 88 84 0c |+...CN)....=....| +00000370 2d d0 49 48 ff f5 83 8f 0d da 7f 81 d6 e7 93 d5 |-.IH............| +00000380 12 c0 59 1c ed b7 35 7f 9e 1f 9c 39 e6 56 ce a2 |..Y...5....9.V..| +00000390 98 ca 74 72 49 65 7f 69 16 a7 13 67 b3 11 fe 32 |..trIe.i...g...2| +000003a0 99 23 c6 8f 37 e5 18 e8 5e 3b e3 25 84 cd f5 9f |.#..7...^;.%....| +000003b0 de 4c f0 b1 cb 25 31 86 73 07 48 f8 30 7e 7c 7b |.L...%1.s.H.0~|{| +000003c0 04 0f a5 5c 15 c2 25 00 43 18 6e 35 17 03 03 00 |...\..%.C.n5....| +000003d0 45 e0 8a a3 17 a4 5d 29 0d da b8 c1 e8 b9 19 cf |E.....])........| +000003e0 4b 08 e6 d8 5c 0e 9e ed b4 ea cc 95 68 54 6b 4b |K...\.......hTkK| +000003f0 0f 1f 32 3e f0 68 10 a1 9f aa 4b 44 86 3f 3b 66 |..2>.h....KD.?;f| +00000400 08 cd cb d0 92 a2 07 df 64 f9 f2 88 f9 c2 63 9f |........d.....c.| +00000410 01 98 a7 58 41 e6 |...XA.| +>>> Flow 5 (client to server) +00000000 17 03 03 00 45 64 f9 84 9e cc d0 58 bd 27 48 4f |....Ed.....X.'HO| +00000010 5c fb 26 21 50 37 f7 eb 80 fc 86 a9 a7 00 d7 0f |\.&!P7..........| +00000020 c4 4e e8 2d 4a cf 0b 57 34 10 f4 ea df a0 91 31 |.N.-J..W4......1| +00000030 80 45 78 20 38 1c cd dd 16 5d f5 6c 83 a6 ee f4 |.Ex 8....].l....| +00000040 24 c7 3f f1 0a f8 2d 78 7e b4 |$.?...-x~.| +>>> Flow 6 (server to client) +00000000 17 03 03 00 1e fe 3e da ac 07 b3 a8 37 ca 6d 3a |......>.....7.m:| +00000010 21 3a 2e 52 e4 1c ee 1c 49 e8 c5 a4 ef 10 bb 92 |!:.R....I.......| +00000020 7b 45 ba 17 03 03 00 13 4d f6 22 39 83 19 74 a8 |{E......M."9..t.| +00000030 86 55 bb 07 4d db 37 a6 f1 9f 13 |.U..M.7....| diff --git a/src/crypto/tls/testdata/Server-TLSv13-P256 b/src/crypto/tls/testdata/Server-TLSv13-P256 new file mode 100644 index 0000000000..433327be21 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-P256 @@ -0,0 +1,95 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 f9 01 00 00 f5 03 03 9b 17 e9 d1 ef |................| +00000010 4d bc 9c 08 44 87 d5 43 09 b5 bd f3 7c a6 52 59 |M...D..C....|.RY| +00000020 ae d3 af 09 c4 24 18 12 7f 4a 03 20 af 3e 04 1d |.....$...J. .>..| +00000030 3a 26 d4 bb 9c 02 71 32 c4 19 a6 ef 46 26 0f d6 |:&....q2....F&..| +00000040 ab 9e 49 4f b5 06 37 92 7a 0a 37 5e 00 08 13 02 |..IO..7.z.7^....| +00000050 13 03 13 01 00 ff 01 00 00 a4 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 04 00 02 00 17 00 16 00 00 |................| +00000080 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 06 03 |................| +00000090 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 |................| +000000a0 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 2d 00 |.......+......-.| +000000b0 02 01 01 00 33 00 47 00 45 00 17 00 41 04 ec c3 |....3.G.E...A...| +000000c0 f4 ab c7 58 b2 1c b4 8d d9 62 5f eb b9 b8 8f 33 |...X.....b_....3| +000000d0 e3 77 4a e5 57 1d 03 ce 2b bb 9d e6 b3 e4 b6 b6 |.wJ.W...+.......| +000000e0 10 01 03 df c6 b4 ac 26 c8 58 9a a8 97 1d e7 92 |.......&.X......| +000000f0 15 d3 78 a4 40 12 8f e1 c0 0a 80 2b 06 c5 |..x.@......+..| +>>> Flow 2 (server to client) +00000000 16 03 03 00 9b 02 00 00 97 03 03 00 00 00 00 00 |................| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 af 3e 04 1d |........... .>..| +00000030 3a 26 d4 bb 9c 02 71 32 c4 19 a6 ef 46 26 0f d6 |:&....q2....F&..| +00000040 ab 9e 49 4f b5 06 37 92 7a 0a 37 5e 13 02 00 00 |..IO..7.z.7^....| +00000050 4f 00 2b 00 02 03 04 00 33 00 45 00 17 00 41 04 |O.+.....3.E...A.| +00000060 1e 18 37 ef 0d 19 51 88 35 75 71 b5 e5 54 5b 12 |..7...Q.5uq..T[.| +00000070 2e 8f 09 67 fd a7 24 20 3e b2 56 1c ce 97 28 5e |...g..$ >.V...(^| +00000080 f8 2b 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 32 42 |.+-O....lK[.V.2B| +00000090 e9 58 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc 5a 89 |.X..I..h.A.Vk.Z.| +000000a0 17 03 03 00 17 42 79 da 18 8e 38 c1 2c a4 2a 9c |.....By...8.,.*.| +000000b0 fd 94 6a a6 b3 08 51 aa 7b 2d 19 a7 17 03 03 02 |..j...Q.{-......| +000000c0 6d 57 f7 92 34 01 09 13 e8 f8 ef 75 ad ba e8 a5 |mW..4......u....| +000000d0 9b cd e9 6c ab 48 b5 4d 33 1e 0b 95 47 74 78 9d |...l.H.M3...Gtx.| +000000e0 3c bb 27 a8 87 1c fb 11 80 91 93 ad 0f 0d 1c 98 |<.'.............| +000000f0 44 cc 36 bc 00 0d eb c0 8b 73 40 11 dd c5 9a a9 |D.6......s@.....| +00000100 90 55 81 e8 55 69 e0 73 97 49 c3 2b ee 56 3c c0 |.U..Ui.s.I.+.V<.| +00000110 c3 b7 4a 18 a3 e4 45 c9 7f 0c 7d ba b4 52 9a a8 |..J...E...}..R..| +00000120 b1 26 4e 57 3b fb 5b 30 28 b0 95 c9 72 35 2d 10 |.&NW;.[0(...r5-.| +00000130 24 8e 70 bb c6 2a 33 83 1c 78 c1 91 c4 6b 06 c0 |$.p..*3..x...k..| +00000140 d0 65 b5 d5 21 21 02 21 86 df 24 d0 99 90 24 12 |.e..!!.!..$...$.| +00000150 9f c8 a4 8d e9 29 5c 84 52 82 4b 11 a6 de 7f 88 |.....)\.R.K.....| +00000160 d9 35 b3 1b d5 c9 0a 54 f2 64 a7 43 13 19 61 0f |.5.....T.d.C..a.| +00000170 28 11 39 3f b1 2a 49 f7 0f de cb f8 ff ad b6 90 |(.9?.*I.........| +00000180 5a af 2a 17 9a 40 c7 c8 32 88 5d e1 af 54 92 3a |Z.*..@..2.]..T.:| +00000190 8b b9 8a 50 93 dd 73 89 cf 0f bf ae 7e ad ba bf |...P..s.....~...| +000001a0 ed 2a f1 47 e0 e5 77 92 27 e5 5b bd 7f 7e a5 47 |.*.G..w.'.[..~.G| +000001b0 af dd d7 ab 72 db 1a bb 83 f8 18 ba 46 92 74 a2 |....r.......F.t.| +000001c0 9e 7a 5c 4d e8 b9 c0 6e ed 20 6d ec 8f 3d 65 d6 |.z\M...n. m..=e.| +000001d0 d5 48 4b 24 14 00 10 2c f2 31 c1 94 ce b6 f8 e2 |.HK$...,.1......| +000001e0 e1 d3 8c e7 38 3c 4b 34 1f 78 8d e0 9b d3 9d 21 |....8>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 b1 15 d9 f0 b0 |..........E.....| +00000010 cf 81 8f de 00 84 97 be 1b 6f af 52 3b d2 34 1b |.........o.R;.4.| +00000020 4d bb d3 b8 c2 09 60 21 e1 61 d3 d9 a1 80 68 77 |M.....`!.a....hw| +00000030 6c 37 6a 87 0e e7 b7 de fe b6 70 eb d3 36 92 c3 |l7j.......p..6..| +00000040 e9 e9 02 e9 cf d0 d4 be 75 9b f3 84 51 fe da 78 |........u...Q..x| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 7f 61 7b 92 ca 8f 9e c7 96 b0 a5 |......a{........| +00000010 9b 3f cf c3 0e 7a 8d 2c 9f 5c 40 8a 99 6d 71 45 |.?...z.,.\@..mqE| +00000020 9a 9c 96 17 03 03 00 13 01 42 62 3a 49 1a b5 5b |.........Bb:I..[| +00000030 43 0f ec 5b 46 0a e9 1c 6f a6 e2 |C..[F...o..| diff --git a/src/crypto/tls/testdata/Server-TLSv13-RSA-RSAPSS b/src/crypto/tls/testdata/Server-TLSv13-RSA-RSAPSS new file mode 100644 index 0000000000..01cc23032a --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-RSA-RSAPSS @@ -0,0 +1,90 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 c6 01 00 00 c2 03 03 f0 a4 89 8f b9 |................| +00000010 fd ba d7 9f 2d ca bf f0 7a 6e 04 6a b4 54 9d f4 |....-...zn.j.T..| +00000020 dd b5 e5 c9 f7 4f e1 a4 0c a9 72 20 57 d4 f7 3a |.....O....r W..:| +00000030 88 0a d3 95 c7 3a 4c 7c e3 0c ac 99 bc 24 d7 ad |.....:L|.....$..| +00000040 3f 53 07 08 00 aa c3 e5 2c 2b d3 2f 00 08 13 02 |?S......,+./....| +00000050 13 03 13 01 00 ff 01 00 00 71 00 00 00 0e 00 0c |.........q......| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 0c 00 0a 00 1d 00 17 00 1e |................| +00000080 00 19 00 18 00 16 00 00 00 17 00 00 00 0d 00 04 |................| +00000090 00 02 08 04 00 2b 00 03 02 03 04 00 2d 00 02 01 |.....+......-...| +000000a0 01 00 33 00 26 00 24 00 1d 00 20 ef 5d 96 0f 42 |..3.&.$... .]..B| +000000b0 9a fe c1 40 ee 31 cd 34 6d 11 d2 ad e9 99 9b b6 |...@.1.4m.......| +000000c0 70 27 8f dc 81 c2 a6 ac 90 3f 4b |p'.......?K| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 57 d4 f7 3a |........... W..:| +00000030 88 0a d3 95 c7 3a 4c 7c e3 0c ac 99 bc 24 d7 ad |.....:L|.....$..| +00000040 3f 53 07 08 00 aa c3 e5 2c 2b d3 2f 13 02 00 00 |?S......,+./....| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 e9 da 29 cf f2 21 32 b3 92 c0 18 5b |......)..!2....[| +00000090 26 f6 c8 ed eb 67 5c 9c df 5c f2 17 03 03 02 6d |&....g\..\.....m| +000000a0 c8 d6 15 36 38 ef 64 62 1b 49 c8 c9 bc ce f7 99 |...68.db.I......| +000000b0 da da 8e d0 2f 98 b1 cf 9a fa 53 cf 8a 20 11 6c |..../.....S.. .l| +000000c0 25 10 fc fd b2 f1 22 ea 42 64 21 b5 2f 94 9a 0a |%.....".Bd!./...| +000000d0 d3 3e 3b 14 4e b1 d1 3e fb 40 03 54 af d3 9e ea |.>;.N..>.@.T....| +000000e0 c8 fb b2 ed ff fa c1 1d 7e 72 fb bd dd 04 b2 0c |........~r......| +000000f0 ac 33 4c 1b e4 65 2f 91 c1 3e f1 1a 8f e0 92 23 |.3L..e/..>.....#| +00000100 3a 4d 04 f2 4f 14 d4 bb fe f7 fe 6a 69 8b aa 15 |:M..O......ji...| +00000110 c4 97 dd 06 3c 79 1c c9 aa 77 51 ac 95 8e 50 f8 |....| +00000150 2a be aa 06 e7 22 cd 8c f1 46 2a 54 6e 54 bf 90 |*...."...F*TnT..| +00000160 89 01 5e e4 e2 aa 7b d7 bc e9 37 d4 ee 75 18 57 |..^...{...7..u.W| +00000170 0c f7 8b fb 70 b2 cc 1c e2 ed 64 20 22 56 7a d3 |....p.....d "Vz.| +00000180 24 eb 25 0d 29 6f 8b be 5a 99 89 eb aa 04 18 8c |$.%.)o..Z.......| +00000190 bd c8 b3 95 57 5d 5b 00 55 d8 ef a0 22 f8 cb 26 |....W][.U..."..&| +000001a0 8d e0 9a 25 a7 77 6d 25 27 c3 aa 75 f7 51 15 c5 |...%.wm%'..u.Q..| +000001b0 30 72 57 ef d4 41 3e cf dd fc 77 d9 d8 08 41 87 |0rW..A>...w...A.| +000001c0 b6 9a 06 c6 f2 00 c5 a2 14 e9 f3 52 91 65 db 69 |...........R.e.i| +000001d0 a7 2e fb 32 5c 3f 13 c8 ea 65 3a 3a 4d 65 a9 69 |...2\?...e::Me.i| +000001e0 3f 7b f8 7c ee 1e a2 87 81 10 5c 7f 8d 37 1a 75 |?{.|......\..7.u| +000001f0 29 8a 78 58 8a d7 f7 af 75 ee 3d f2 58 c2 de a5 |).xX....u.=.X...| +00000200 60 e7 f9 a3 a1 66 cf df 76 2f 2a cf 5e 6e 80 a3 |`....f..v/*.^n..| +00000210 47 16 c9 c9 d1 b5 02 38 63 0a 86 fc f2 e8 0b 16 |G......8c.......| +00000220 d6 43 8e d1 68 b8 01 a0 63 69 01 c8 d3 1e eb 48 |.C..h...ci.....H| +00000230 28 5e 82 24 3a 19 62 f2 48 65 13 82 77 8c 9a ca |(^.$:.b.He..w...| +00000240 69 84 27 4d fd 88 81 79 74 8a a7 ef 2e be 6c c7 |i.'M...yt.....l.| +00000250 89 9d 19 ab 44 fa 35 99 cc 8e af c3 c0 98 85 85 |....D.5.........| +00000260 e7 92 45 84 39 c3 2d 27 a1 55 98 81 3a 09 1f f3 |..E.9.-'.U..:...| +00000270 9a 86 e5 53 f4 7d 35 8d 1f 2e 06 ef ed bf eb cb |...S.}5.........| +00000280 54 41 b7 9c d9 df 1b 75 00 f3 81 29 27 e4 ed 90 |TA.....u...)'...| +00000290 bd 16 39 96 15 22 c3 19 35 5c 88 db 00 05 38 77 |..9.."..5\....8w| +000002a0 8e ef 42 f5 b2 8c 0a 6c 7f 38 7a f7 c1 13 a6 3d |..B....l.8z....=| +000002b0 80 3c 62 aa 26 d0 80 08 31 b0 f4 84 36 c1 8d 51 |.>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 93 78 a6 3d 50 |..........E.x.=P| +00000010 bd bf cb d5 f5 2d af d7 1a 04 5a 7c 18 57 5f ce |.....-....Z|.W_.| +00000020 bf aa f3 25 1d 19 29 3a 90 06 9b 9a ad bb 03 92 |...%..):........| +00000030 58 62 1f db 30 3b db 83 bf 21 dc 32 50 7a cc c1 |Xb..0;...!.2Pz..| +00000040 51 2d 46 a4 41 eb 07 b3 91 54 55 23 bd 1b c9 82 |Q-F.A....TU#....| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e 94 17 fa b4 46 cf 05 18 36 1e 2c |.........F...6.,| +00000010 fa b8 76 fd 66 4f d0 6c df 7d 35 ea cc 5f 8c fb |..v.fO.l.}5.._..| +00000020 6f 31 cc 17 03 03 00 13 7e 91 d1 39 ce 2c 4d e4 |o1......~..9.,M.| +00000030 49 90 a1 a6 c1 12 bf 0b 12 80 40 |I.........@| diff --git a/src/crypto/tls/testdata/Server-TLSv13-X25519 b/src/crypto/tls/testdata/Server-TLSv13-X25519 new file mode 100644 index 0000000000..e2ede18235 --- /dev/null +++ b/src/crypto/tls/testdata/Server-TLSv13-X25519 @@ -0,0 +1,91 @@ +>>> Flow 1 (client to server) +00000000 16 03 01 00 d8 01 00 00 d4 03 03 7b 32 49 3a 30 |...........{2I:0| +00000010 09 44 1d e3 d6 4e c0 2e ec e1 ce fc d9 70 6d 47 |.D...N.......pmG| +00000020 32 61 92 e3 4b 0e 02 96 0a b4 b6 20 18 b5 42 4e |2a..K...... ..BN| +00000030 a4 06 40 82 76 bc 30 6b 5c ef 16 94 e4 bb fa 0b |..@.v.0k\.......| +00000040 49 d4 b1 c5 df 0d 01 92 be 99 6f f2 00 08 13 02 |I.........o.....| +00000050 13 03 13 01 00 ff 01 00 00 83 00 00 00 0e 00 0c |................| +00000060 00 00 09 31 32 37 2e 30 2e 30 2e 31 00 0b 00 04 |...127.0.0.1....| +00000070 03 00 01 02 00 0a 00 04 00 02 00 1d 00 16 00 00 |................| +00000080 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 06 03 |................| +00000090 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 |................| +000000a0 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 2d 00 |.......+......-.| +000000b0 02 01 01 00 33 00 26 00 24 00 1d 00 20 32 8d 5d |....3.&.$... 2.]| +000000c0 3b f5 8f b3 7b 41 92 90 e9 3f 4f aa 61 a8 91 f6 |;...{A...?O.a...| +000000d0 85 c1 70 d6 1a 94 8d 16 1d 4e c8 ea 54 |..p......N..T| +>>> Flow 2 (server to client) +00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| +00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 18 b5 42 4e |........... ..BN| +00000030 a4 06 40 82 76 bc 30 6b 5c ef 16 94 e4 bb fa 0b |..@.v.0k\.......| +00000040 49 d4 b1 c5 df 0d 01 92 be 99 6f f2 13 02 00 00 |I.........o.....| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| +00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| +00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 17 |.........._X.;t.| +00000080 03 03 00 17 7c ee 15 67 75 36 92 d7 4a d6 56 f5 |....|..gu6..J.V.| +00000090 6d ef 3d e1 2e 94 de 7d 72 dc 2f 17 03 03 02 6d |m.=....}r./....m| +000000a0 a4 38 eb 50 27 ef fa 47 59 74 d0 ad 55 e2 19 15 |.8.P'..GYt..U...| +000000b0 a8 9e 12 dc c5 70 7c fd b7 1c cd 56 b7 44 cc 9c |.....p|....V.D..| +000000c0 4c b7 6c 99 0a 20 8d db fc 5b 4d 97 27 b3 64 61 |L.l.. ...[M.'.da| +000000d0 fd e6 99 9b d7 1f 2c 86 a3 b7 23 16 8c f0 f0 a4 |......,...#.....| +000000e0 70 e1 0e 70 61 db 2d c2 60 10 30 21 eb 3c f4 d4 |p..pa.-.`.0!.<..| +000000f0 07 27 f2 54 bd e7 b8 7a 13 10 47 10 41 46 3d 6f |.'.T...z..G.AF=o| +00000100 20 3b 16 07 d6 0d 16 d0 06 34 a4 b9 eb 6a 3d 1b | ;.......4...j=.| +00000110 69 fd 7b f2 2d a8 8d 56 51 cf 0e 58 c3 19 ce 88 |i.{.-..VQ..X....| +00000120 9f 6c c6 38 11 24 81 2f da c1 f2 57 32 2f 5c 1e |.l.8.$./...W2/\.| +00000130 e1 04 56 58 fd 5c ca b8 c6 f1 bd 66 84 a9 2d 48 |..VX.\.....f..-H| +00000140 4f 4c 08 b3 92 ba 5d 89 95 ce 3d b2 de 76 00 01 |OL....]...=..v..| +00000150 95 56 56 2c 39 1a 65 68 f5 28 4e d2 e9 cc 85 2e |.VV,9.eh.(N.....| +00000160 f0 d9 e6 40 f3 f2 88 10 24 5b 92 5e 18 2d 4c e5 |...@....$[.^.-L.| +00000170 36 dd c6 09 d1 ca ae 44 84 7a 9b dd f3 c9 ac 44 |6......D.z.....D| +00000180 7d 77 e9 41 8b b2 26 93 16 e9 d4 06 8b f5 ab c5 |}w.A..&.........| +00000190 8d ad f4 61 5f 1c ab f0 ff 86 20 f0 4d 90 f5 cb |...a_..... .M...| +000001a0 0f d8 fe 3f 2e 78 5e 11 bf 82 0d 55 a2 9a 7b 4b |...?.x^....U..{K| +000001b0 5d a0 b7 6c b1 d3 98 67 81 c2 ef fa 22 2f a4 68 |]..l...g...."/.h| +000001c0 b4 bd 38 08 68 b8 08 2e 95 70 64 e1 c0 4a 29 2b |..8.h....pd..J)+| +000001d0 d0 5d 0c 2a f0 df 87 e9 7c bd f5 bc a5 e2 56 56 |.].*....|.....VV| +000001e0 e6 79 0e f9 ce 42 9a 7c d9 e4 3a b2 c9 c7 54 f7 |.y...B.|..:...T.| +000001f0 a2 f8 af b0 8d 56 2b 9d 3a de da 22 7a f6 56 b9 |.....V+.:.."z.V.| +00000200 6f cf 61 b1 9f 49 91 1e aa da 27 cb bd 81 f9 a0 |o.a..I....'.....| +00000210 01 8c e0 ad d6 3f ef 8a 2e 8c 4b da 77 83 b7 7a |.....?....K.w..z| +00000220 69 83 62 2e 48 00 d2 dc 96 46 ef e1 c5 1a 49 eb |i.b.H....F....I.| +00000230 3c 6d b9 a6 c1 45 21 a6 7a cb 46 23 57 b1 21 d9 || +00000340 e7 dd b0 e3 9f ee 2d e7 f4 fa 11 56 01 27 86 04 |......-....V.'..| +00000350 4d 76 58 61 e5 1e b3 61 07 1e 50 2f 38 2e 2e 86 |MvXa...a..P/8...| +00000360 b9 c5 77 c5 c5 bf 54 4d b8 50 9f 03 fb 5f 8d 60 |..w...TM.P..._.`| +00000370 49 ed 98 90 34 d5 66 e1 0a 17 9c 55 3f 7a 7e 31 |I...4.f....U?z~1| +00000380 dd b5 d9 12 9d 8c 00 6e 37 d4 3e 45 ad da 6d 30 |.......n7.>E..m0| +00000390 47 2c 70 95 10 a4 d2 1e 20 37 04 6a 58 96 34 0d |G,p..... 7.jX.4.| +000003a0 2c 93 09 de 7b 1b a0 5c 25 19 c5 17 03 03 00 45 |,...{..\%......E| +000003b0 ba 0f cd 55 4f 62 24 0a f9 7d 58 93 69 4f 5f c4 |...UOb$..}X.iO_.| +000003c0 d2 bd 95 3e 10 15 57 1d c1 16 e3 4d 46 8b fd d1 |...>..W....MF...| +000003d0 16 d1 4a 83 4b d5 ed d2 00 90 90 c3 8f 91 43 81 |..J.K.........C.| +000003e0 a1 90 16 3b c9 54 6c 3c 0f 71 c7 5f 2c b0 3f c3 |...;.Tl<.q._,.?.| +000003f0 9b 69 ce 6c 60 |.i.l`| +>>> Flow 3 (client to server) +00000000 14 03 03 00 01 01 17 03 03 00 45 bf 5b 81 15 24 |..........E.[..$| +00000010 3b be 70 41 4d ea 7c 04 a7 31 84 d3 38 b0 e4 cb |;.pAM.|..1..8...| +00000020 42 05 b7 a8 07 1a d5 4f f3 e7 50 56 e8 73 63 1a |B......O..PV.sc.| +00000030 d8 a0 76 5e 10 c8 73 07 fa 88 86 65 69 40 81 37 |..v^..s....ei@.7| +00000040 a2 00 89 b6 ff f1 49 1d 69 e5 63 2e bc 5a eb e6 |......I.i.c..Z..| +>>> Flow 4 (server to client) +00000000 17 03 03 00 1e ca eb a0 6a b4 a5 eb ca 28 14 25 |........j....(.%| +00000010 07 18 47 3d f0 f6 22 2e 7f 9c 09 73 8c a7 f8 63 |..G=.."....s...c| +00000020 e7 41 fd 17 03 03 00 13 ba f6 ed 7e 36 b5 f2 c6 |.A.........~6...| +00000030 15 27 34 2d 8d bd ea 99 29 81 ee |.'4-....)..| From e25823edcea364df70b8db7462f53e7dac2b8fca Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 1 Nov 2018 13:46:50 -0700 Subject: [PATCH 002/111] cmd/compile/internal/gc: add tracing support to debug type checking The compiler must first be built with the constant enableTrace set to true (typecheck.go). After that, the -t flag becomes available which enables tracing output of type-checking functions. With enableTrace == false, the tracing code becomes dead code and won't affect the compiler. Typical output might look like this: path/y.go:4:6: typecheck 0xc00033e180 DCLTYPE tc=0 path/y.go:4:6: . typecheck1 0xc00033e180 DCLTYPE tc=2 path/y.go:4:6: . . typecheck 0xc000331a40 TYPE T tc=1 path/y.go:4:6: . . . typecheck1 0xc000331a40 TYPE T tc=2 path/y.go:4:6: . . . . typecheckdef 0xc000331a40 TYPE T tc=2 path/y.go:4:6: . . . . => 0xc000331a40 TYPE T tc=2 type=*T path/y.go:4:6: . . . => 0xc000331a40 TYPE T tc=2 type=*T path/y.go:4:6: . . => 0xc000331a40 TYPE T tc=1 type=*T path/y.go:4:6: . => 0xc00033e180 DCLTYPE tc=2 type= path/y.go:4:6: => 0xc00033e180 DCLTYPE tc=1 type= Disabled by default. Change-Id: Ifd8385290d1cf0d3fc5e8468b2f4ab84e8eff338 Reviewed-on: https://go-review.googlesource.com/c/146782 Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/main.go | 3 + src/cmd/compile/internal/gc/typecheck.go | 96 ++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 059bf5d1fc..49a4e05d99 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -229,6 +229,9 @@ func Main(archInit func(*Arch)) { flag.BoolVar(&flag_race, "race", false, "enable race detector") } objabi.Flagcount("s", "warn about composite literals that can be simplified", &Debug['s']) + if enableTrace { + flag.BoolVar(&trace, "t", false, "trace type-checking") + } flag.StringVar(&pathPrefix, "trimpath", "", "remove `prefix` from recorded source file paths") flag.BoolVar(&Debug_vlog, "v", false, "increase debug verbosity") objabi.Flagcount("w", "debug type checking", &Debug['w']) diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index 6ee52eae84..be11a9841f 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -12,6 +12,50 @@ import ( "strings" ) +// To enable tracing support (-t flag), set enableTrace to true. +const enableTrace = false + +var trace bool +var traceIndent []byte + +func tracePrint(title string, n *Node) func(np **Node) { + indent := traceIndent + + // guard against nil + var pos, op string + var tc uint8 + if n != nil { + pos = linestr(n.Pos) + op = n.Op.String() + tc = n.Typecheck() + } + + fmt.Printf("%s: %s%s %p %s %v tc=%d\n", pos, indent, title, n, op, n, tc) + traceIndent = append(traceIndent, ". "...) + + return func(np **Node) { + traceIndent = traceIndent[:len(traceIndent)-2] + + // if we have a result, use that + if np != nil { + n = *np + } + + // guard against nil + // use outer pos, op so we don't get empty pos/op if n == nil (nicer output) + var tc uint8 + var typ *types.Type + if n != nil { + pos = linestr(n.Pos) + op = n.Op.String() + tc = n.Typecheck() + typ = n.Type + } + + fmt.Printf("%s: %s=> %p %s %v tc=%d type=%#L\n", pos, indent, n, op, n, tc, typ) + } +} + const ( Etop = 1 << iota // evaluated at statement level Erv // evaluated in value context @@ -31,11 +75,16 @@ const ( var typecheckdefstack []*Node // resolve ONONAME to definition, if any. -func resolve(n *Node) *Node { +func resolve(n *Node) (res *Node) { if n == nil || n.Op != ONONAME { return n } + // only trace if there's work to do + if enableTrace && trace { + defer tracePrint("resolve", n)(&res) + } + if n.Sym.Pkg != localpkg { if inimport { Fatalf("recursive inimport") @@ -150,7 +199,7 @@ var typecheck_tcstack []*Node // typecheck type checks node n. // The result of typecheck MUST be assigned back to n, e.g. // n.Left = typecheck(n.Left, top) -func typecheck(n *Node, top int) *Node { +func typecheck(n *Node, top int) (res *Node) { // cannot type check until all the source has been parsed if !typecheckok { Fatalf("early typecheck") @@ -160,6 +209,11 @@ func typecheck(n *Node, top int) *Node { return nil } + // only trace if there's work to do + if enableTrace && trace { + defer tracePrint("typecheck", n)(&res) + } + lno := setlineno(n) // Skip over parens. @@ -294,7 +348,11 @@ func indexlit(n *Node) *Node { // The result of typecheck1 MUST be assigned back to n, e.g. // n.Left = typecheck1(n.Left, top) -func typecheck1(n *Node, top int) *Node { +func typecheck1(n *Node, top int) (res *Node) { + if enableTrace && trace { + defer tracePrint("typecheck1", n)(&res) + } + switch n.Op { case OLITERAL, ONAME, ONONAME, OTYPE: if n.Sym == nil { @@ -2381,7 +2439,11 @@ func lookdot1(errnode *Node, s *types.Sym, t *types.Type, fs *types.Fields, dost // typecheckMethodExpr checks selector expressions (ODOT) where the // base expression is a type expression (OTYPE). -func typecheckMethodExpr(n *Node) *Node { +func typecheckMethodExpr(n *Node) (res *Node) { + if enableTrace && trace { + defer tracePrint("typecheckMethodExpr", n)(&res) + } + t := n.Left.Type // Compute the method set for t. @@ -2924,7 +2986,11 @@ func pushtype(n *Node, t *types.Type) { // The result of typecheckcomplit MUST be assigned back to n, e.g. // n.Left = typecheckcomplit(n.Left) -func typecheckcomplit(n *Node) *Node { +func typecheckcomplit(n *Node) (res *Node) { + if enableTrace && trace { + defer tracePrint("typecheckcomplit", n)(&res) + } + lno := lineno defer func() { lineno = lno @@ -3337,6 +3403,10 @@ func samesafeexpr(l *Node, r *Node) bool { // if this assignment is the definition of a var on the left side, // fill in the var's type. func typecheckas(n *Node) { + if enableTrace && trace { + defer tracePrint("typecheckas", n)(nil) + } + // delicate little dance. // the definition of n may refer to this assignment // as its definition, in which case it will call typecheckas. @@ -3393,6 +3463,10 @@ func checkassignto(src *types.Type, dst *Node) { } func typecheckas2(n *Node) { + if enableTrace && trace { + defer tracePrint("typecheckas2", n)(nil) + } + ls := n.List.Slice() for i1, n1 := range ls { // delicate little dance. @@ -3521,6 +3595,10 @@ out: // type check function definition func typecheckfunc(n *Node) { + if enableTrace && trace { + defer tracePrint("typecheckfunc", n)(nil) + } + for _, ln := range n.Func.Dcl { if ln.Op == ONAME && (ln.Class() == PPARAM || ln.Class() == PPARAMOUT) { ln.Name.Decldepth = 1 @@ -3637,6 +3715,10 @@ func copytype(n *Node, t *types.Type) { } func typecheckdeftype(n *Node) { + if enableTrace && trace { + defer tracePrint("typecheckdeftype", n)(nil) + } + n.Type.Sym = n.Sym n.SetTypecheck(1) n.Name.Param.Ntype = typecheck(n.Name.Param.Ntype, Etype) @@ -3654,6 +3736,10 @@ func typecheckdeftype(n *Node) { } func typecheckdef(n *Node) { + if enableTrace && trace { + defer tracePrint("typecheckdef", n)(nil) + } + lno := setlineno(n) if n.Op == ONONAME { From 3813edf26edb78620632dc9c7d66096e5b2b5019 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 2 Nov 2018 15:18:43 +0000 Subject: [PATCH 003/111] all: use "reports whether" consistently in the few places that didn't Go documentation style for boolean funcs is to say: // Foo reports whether ... func Foo() bool (rather than "returns true if") This CL also replaces 4 uses of "iff" with the same "reports whether" wording, which doesn't lose any meaning, and will prevent people from sending typo fixes when they don't realize it's "if and only if". In the past I think we've had the typo CLs updated to just say "reports whether". So do them all at once. (Inspired by the addition of another "returns true if" in CL 146938 in fd_plan9.go) Created with: $ perl -i -npe 's/returns true if/reports whether/' $(git grep -l "returns true iff" | grep -v vendor) $ perl -i -npe 's/returns true if/reports whether/' $(git grep -l "returns true if" | grep -v vendor) Change-Id: Ided502237f5ab0d25cb625dbab12529c361a8b9f Reviewed-on: https://go-review.googlesource.com/c/147037 Reviewed-by: Ian Lance Taylor --- src/cmd/asm/internal/lex/input.go | 2 +- src/cmd/compile/internal/gc/closure.go | 2 +- src/cmd/compile/internal/gc/syntax.go | 2 +- src/cmd/compile/internal/ssa/config.go | 2 +- src/cmd/compile/internal/ssa/debug.go | 2 +- src/cmd/compile/internal/ssa/func.go | 2 +- src/cmd/compile/internal/ssa/gen/genericOps.go | 2 +- src/cmd/compile/internal/ssa/gen/rulegen.go | 8 ++++---- src/cmd/compile/internal/ssa/poset.go | 8 ++++---- src/cmd/compile/internal/ssa/rewrite.go | 2 +- src/cmd/compile/internal/ssa/trim.go | 4 ++-- src/cmd/fix/fix.go | 2 +- src/cmd/internal/dwarf/dwarf.go | 2 +- src/cmd/trace/annotations.go | 2 +- src/cmd/vet/print.go | 2 +- src/crypto/x509/x509.go | 2 +- src/database/sql/sql.go | 2 +- src/encoding/asn1/asn1.go | 2 +- src/go/printer/nodes.go | 2 +- src/go/types/builtins.go | 2 +- src/go/types/scope.go | 2 +- src/go/types/type.go | 2 +- src/html/template/js.go | 2 +- src/html/template/url.go | 2 +- src/internal/poll/fd_plan9.go | 2 +- src/internal/poll/fd_poll_nacljs.go | 2 +- src/internal/poll/fd_poll_runtime.go | 2 +- src/math/big/float.go | 2 +- src/math/cmplx/isinf.go | 2 +- src/math/cmplx/isnan.go | 2 +- src/math/signbit.go | 2 +- src/os/stat_windows.go | 2 +- src/path/filepath/match_test.go | 2 +- src/reflect/type.go | 2 +- src/regexp/syntax/regexp.go | 2 +- src/runtime/mbitmap.go | 2 +- src/runtime/mcentral.go | 2 +- src/runtime/mgc.go | 6 +++--- src/runtime/mgcwork.go | 4 ++-- src/runtime/mwbbuf.go | 2 +- src/runtime/netpoll.go | 2 +- src/runtime/panic.go | 4 ++-- src/runtime/pprof/internal/profile/profile.go | 2 +- src/runtime/proc.go | 8 ++++---- src/runtime/signal_windows.go | 2 +- src/sync/runtime.go | 2 +- src/syscall/mksyscall_windows.go | 2 +- 47 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/cmd/asm/internal/lex/input.go b/src/cmd/asm/internal/lex/input.go index 5186635fe7..a43953b515 100644 --- a/src/cmd/asm/internal/lex/input.go +++ b/src/cmd/asm/internal/lex/input.go @@ -139,7 +139,7 @@ func (in *Input) Text() string { return in.text } -// hash processes a # preprocessor directive. It returns true iff it completes. +// hash processes a # preprocessor directive. It reports whether it completes. func (in *Input) hash() bool { // We have a '#'; it must be followed by a known word (define, include, etc.). tok := in.Stack.Next() diff --git a/src/cmd/compile/internal/gc/closure.go b/src/cmd/compile/internal/gc/closure.go index ec19f5c112..5123df8e9d 100644 --- a/src/cmd/compile/internal/gc/closure.go +++ b/src/cmd/compile/internal/gc/closure.go @@ -314,7 +314,7 @@ func transformclosure(xfunc *Node) { lineno = lno } -// hasemptycvars returns true iff closure clo has an +// hasemptycvars reports whether closure clo has an // empty list of captured vars. func hasemptycvars(clo *Node) bool { xfunc := clo.Func.Closure diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index 0fe6defe99..e29a3d7657 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -929,7 +929,7 @@ type nodeQueue struct { head, tail int } -// empty returns true if q contains no Nodes. +// empty reports whether q contains no Nodes. func (q *nodeQueue) empty() bool { return q.head == q.tail } diff --git a/src/cmd/compile/internal/ssa/config.go b/src/cmd/compile/internal/ssa/config.go index e79629695a..558c4b7db8 100644 --- a/src/cmd/compile/internal/ssa/config.go +++ b/src/cmd/compile/internal/ssa/config.go @@ -112,7 +112,7 @@ type Logger interface { // Logf logs a message from the compiler. Logf(string, ...interface{}) - // Log returns true if logging is not a no-op + // Log reports whether logging is not a no-op // some logging calls account for more than a few heap allocations. Log() bool diff --git a/src/cmd/compile/internal/ssa/debug.go b/src/cmd/compile/internal/ssa/debug.go index 8df8a94b76..3d0be0fe1c 100644 --- a/src/cmd/compile/internal/ssa/debug.go +++ b/src/cmd/compile/internal/ssa/debug.go @@ -790,7 +790,7 @@ func (e *pendingEntry) clear() { } } -// canMerge returns true if the location description for new is the same as +// canMerge reports whether the location description for new is the same as // pending. func canMerge(pending, new VarLoc) bool { if pending.absent() && new.absent() { diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 2ed4086fd1..d73d39ce28 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -621,7 +621,7 @@ func (f *Func) invalidateCFG() { f.cachedLoopnest = nil } -// DebugHashMatch returns true if environment variable evname +// DebugHashMatch reports whether environment variable evname // 1) is empty (this is a special more-quickly implemented case of 3) // 2) is "y" or "Y" // 3) is a suffix of the sha1 hash of name diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go index 522ccbf893..ba8d93cf2c 100644 --- a/src/cmd/compile/internal/ssa/gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/gen/genericOps.go @@ -529,7 +529,7 @@ var genericOps = []opData{ {name: "AtomicAdd64", argLength: 3, typ: "(UInt64,Mem)", hasSideEffects: true}, // Do *arg0 += arg1. arg2=memory. Returns sum and new memory. {name: "AtomicCompareAndSwap32", argLength: 4, typ: "(Bool,Mem)", hasSideEffects: true}, // if *arg0==arg1, then set *arg0=arg2. Returns true if store happens and new memory. {name: "AtomicCompareAndSwap64", argLength: 4, typ: "(Bool,Mem)", hasSideEffects: true}, // if *arg0==arg1, then set *arg0=arg2. Returns true if store happens and new memory. - {name: "AtomicCompareAndSwapRel32", argLength: 4, typ: "(Bool,Mem)", hasSideEffects: true}, // if *arg0==arg1, then set *arg0=arg2. Lock release, returns true if store happens and new memory. + {name: "AtomicCompareAndSwapRel32", argLength: 4, typ: "(Bool,Mem)", hasSideEffects: true}, // if *arg0==arg1, then set *arg0=arg2. Lock release, reports whether store happens and new memory. {name: "AtomicAnd8", argLength: 3, typ: "Mem", hasSideEffects: true}, // *arg0 &= arg1. arg2=memory. Returns memory. {name: "AtomicOr8", argLength: 3, typ: "Mem", hasSideEffects: true}, // *arg0 |= arg1. arg2=memory. Returns memory. diff --git a/src/cmd/compile/internal/ssa/gen/rulegen.go b/src/cmd/compile/internal/ssa/gen/rulegen.go index faaad974c4..34517b4cb9 100644 --- a/src/cmd/compile/internal/ssa/gen/rulegen.go +++ b/src/cmd/compile/internal/ssa/gen/rulegen.go @@ -6,7 +6,7 @@ // This program generates Go code that applies rewrite rules to a Value. // The generated code implements a function of type func (v *Value) bool -// which returns true iff if did something. +// which reports whether if did something. // Ideas stolen from Swift: http://www.hpl.hp.com/techreports/Compaq-DEC/WRL-2000-2.html package main @@ -386,7 +386,7 @@ func genRules(arch arch) { } } -// genMatch returns true if the match can fail. +// genMatch reports whether the match can fail. func genMatch(w io.Writer, arch arch, match string, loc string) bool { return genMatch0(w, arch, match, "v", map[string]struct{}{}, true, loc) } @@ -623,7 +623,7 @@ outer: return r } -// isBlock returns true if this op is a block opcode. +// isBlock reports whether this op is a block opcode. func isBlock(name string, arch arch) bool { for _, b := range genericBlocks { if b.name == name { @@ -768,7 +768,7 @@ func typeName(typ string) string { } } -// unbalanced returns true if there aren't the same number of ( and ) in the string. +// unbalanced reports whether there aren't the same number of ( and ) in the string. func unbalanced(s string) bool { var left, right int for _, c := range s { diff --git a/src/cmd/compile/internal/ssa/poset.go b/src/cmd/compile/internal/ssa/poset.go index 0e0e2789b1..4ebfb89e52 100644 --- a/src/cmd/compile/internal/ssa/poset.go +++ b/src/cmd/compile/internal/ssa/poset.go @@ -781,7 +781,7 @@ func (po *poset) DotDump(fn string, title string) error { return nil } -// Ordered returns true if n1 sliceLength, or if the +// invalidLength reports whether offset + length > sliceLength, or if the // addition would overflow. func invalidLength(offset, length, sliceLength int) bool { return offset+length < offset || offset+length > sliceLength diff --git a/src/go/printer/nodes.go b/src/go/printer/nodes.go index 1de7cd81b2..a307d8395e 100644 --- a/src/go/printer/nodes.go +++ b/src/go/printer/nodes.go @@ -1134,7 +1134,7 @@ func (p *printer) controlClause(isForStmt bool, init ast.Stmt, expr ast.Expr, po // than starting at the first line break). // func (p *printer) indentList(list []ast.Expr) bool { - // Heuristic: indentList returns true if there are more than one multi- + // Heuristic: indentList reports whether there are more than one multi- // line element in the list, or if there is any element that is not // starting on the same line as the previous one ends. if len(list) >= 2 { diff --git a/src/go/types/builtins.go b/src/go/types/builtins.go index 882c773db4..ece6d4f530 100644 --- a/src/go/types/builtins.go +++ b/src/go/types/builtins.go @@ -13,7 +13,7 @@ import ( ) // builtin type-checks a call to the built-in specified by id and -// returns true if the call is valid, with *x holding the result; +// reports whether the call is valid, with *x holding the result; // but x.expr is not set. If the call is invalid, the result is // false, and *x is undefined. // diff --git a/src/go/types/scope.go b/src/go/types/scope.go index 6cf5cc66f9..b50ee2fd5f 100644 --- a/src/go/types/scope.go +++ b/src/go/types/scope.go @@ -115,7 +115,7 @@ func (s *Scope) Insert(obj Object) Object { func (s *Scope) Pos() token.Pos { return s.pos } func (s *Scope) End() token.Pos { return s.end } -// Contains returns true if pos is within the scope's extent. +// Contains reports whether pos is within the scope's extent. // The result is guaranteed to be valid only if the type-checked // AST has complete position information. func (s *Scope) Contains(pos token.Pos) bool { diff --git a/src/go/types/type.go b/src/go/types/type.go index 77426ba618..3dd9eb97f5 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -340,7 +340,7 @@ func (t *Interface) NumMethods() int { return len(t.allMethods) } // The methods are ordered by their unique Id. func (t *Interface) Method(i int) *Func { return t.allMethods[i] } -// Empty returns true if t is the empty interface. +// Empty reports whether t is the empty interface. func (t *Interface) Empty() bool { return len(t.allMethods) == 0 } // Complete computes the interface's method set. It must be called by users of diff --git a/src/html/template/js.go b/src/html/template/js.go index 2291f47c33..98e821b73c 100644 --- a/src/html/template/js.go +++ b/src/html/template/js.go @@ -371,7 +371,7 @@ func isJSIdentPart(r rune) bool { return false } -// isJSType returns true if the given MIME type should be considered JavaScript. +// isJSType reports whether the given MIME type should be considered JavaScript. // // It is used to determine whether a script tag with a type attribute is a javascript container. func isJSType(mimeType string) bool { diff --git a/src/html/template/url.go b/src/html/template/url.go index 8a4f727e50..6f8185a4e9 100644 --- a/src/html/template/url.go +++ b/src/html/template/url.go @@ -86,7 +86,7 @@ func urlProcessor(norm bool, args ...interface{}) string { } // processURLOnto appends a normalized URL corresponding to its input to b -// and returns true if the appended content differs from s. +// and reports whether the appended content differs from s. func processURLOnto(s string, norm bool, b *bytes.Buffer) bool { b.Grow(len(s) + 16) written := 0 diff --git a/src/internal/poll/fd_plan9.go b/src/internal/poll/fd_plan9.go index fce2285931..0fce32915e 100644 --- a/src/internal/poll/fd_plan9.go +++ b/src/internal/poll/fd_plan9.go @@ -193,7 +193,7 @@ func isInterrupted(err error) bool { return err != nil && stringsHasSuffix(err.Error(), "interrupted") } -// IsPollDescriptor returns true if fd is the descriptor being used by the poller. +// IsPollDescriptor reports whether fd is the descriptor being used by the poller. // This is only used for testing. func IsPollDescriptor(fd uintptr) bool { return false diff --git a/src/internal/poll/fd_poll_nacljs.go b/src/internal/poll/fd_poll_nacljs.go index e0d3f976f1..0871f342d4 100644 --- a/src/internal/poll/fd_poll_nacljs.go +++ b/src/internal/poll/fd_poll_nacljs.go @@ -92,7 +92,7 @@ func setDeadlineImpl(fd *FD, t time.Time, mode int) error { return nil } -// IsPollDescriptor returns true if fd is the descriptor being used by the poller. +// IsPollDescriptor reports whether fd is the descriptor being used by the poller. // This is only used for testing. func IsPollDescriptor(fd uintptr) bool { return false diff --git a/src/internal/poll/fd_poll_runtime.go b/src/internal/poll/fd_poll_runtime.go index 2ee8e7c2c9..687f702556 100644 --- a/src/internal/poll/fd_poll_runtime.go +++ b/src/internal/poll/fd_poll_runtime.go @@ -154,7 +154,7 @@ func setDeadlineImpl(fd *FD, t time.Time, mode int) error { return nil } -// IsPollDescriptor returns true if fd is the descriptor being used by the poller. +// IsPollDescriptor reports whether fd is the descriptor being used by the poller. // This is only used for testing. func IsPollDescriptor(fd uintptr) bool { return runtime_isPollServerDescriptor(fd) diff --git a/src/math/big/float.go b/src/math/big/float.go index d5e801b2c8..b3c3295201 100644 --- a/src/math/big/float.go +++ b/src/math/big/float.go @@ -327,7 +327,7 @@ func (z *Float) SetMantExp(mant *Float, exp int) *Float { return z } -// Signbit returns true if x is negative or negative zero. +// Signbit reports whether x is negative or negative zero. func (x *Float) Signbit() bool { return x.neg } diff --git a/src/math/cmplx/isinf.go b/src/math/cmplx/isinf.go index d5a65b44b3..6273cd3a6c 100644 --- a/src/math/cmplx/isinf.go +++ b/src/math/cmplx/isinf.go @@ -6,7 +6,7 @@ package cmplx import "math" -// IsInf returns true if either real(x) or imag(x) is an infinity. +// IsInf reports whether either real(x) or imag(x) is an infinity. func IsInf(x complex128) bool { if math.IsInf(real(x), 0) || math.IsInf(imag(x), 0) { return true diff --git a/src/math/cmplx/isnan.go b/src/math/cmplx/isnan.go index 05d0cce633..d3382c05ee 100644 --- a/src/math/cmplx/isnan.go +++ b/src/math/cmplx/isnan.go @@ -6,7 +6,7 @@ package cmplx import "math" -// IsNaN returns true if either real(x) or imag(x) is NaN +// IsNaN reports whether either real(x) or imag(x) is NaN // and neither is an infinity. func IsNaN(x complex128) bool { switch { diff --git a/src/math/signbit.go b/src/math/signbit.go index 670cc1a667..f6e61d660e 100644 --- a/src/math/signbit.go +++ b/src/math/signbit.go @@ -4,7 +4,7 @@ package math -// Signbit returns true if x is negative or negative zero. +// Signbit reports whether x is negative or negative zero. func Signbit(x float64) bool { return Float64bits(x)&(1<<63) != 0 } diff --git a/src/os/stat_windows.go b/src/os/stat_windows.go index f4700f5818..271ff5f843 100644 --- a/src/os/stat_windows.go +++ b/src/os/stat_windows.go @@ -10,7 +10,7 @@ import ( "unsafe" ) -// isNulName returns true if name is NUL file name. +// isNulName reports whether name is NUL file name. // For example, it returns true for both "NUL" and "nul". func isNulName(name string) bool { if len(name) != 3 { diff --git a/src/path/filepath/match_test.go b/src/path/filepath/match_test.go index 1d91c274c7..b8657626bc 100644 --- a/src/path/filepath/match_test.go +++ b/src/path/filepath/match_test.go @@ -106,7 +106,7 @@ func TestMatch(t *testing.T) { } } -// contains returns true if vector contains the string s. +// contains reports whether vector contains the string s. func contains(vector []string, s string) bool { for _, elem := range vector { if elem == s { diff --git a/src/reflect/type.go b/src/reflect/type.go index d8971d620e..a04234ca69 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -2314,7 +2314,7 @@ type structTypeFixed32 struct { m [32]method } -// isLetter returns true if a given 'rune' is classified as a Letter. +// isLetter reports whether a given 'rune' is classified as a Letter. func isLetter(ch rune) bool { return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) } diff --git a/src/regexp/syntax/regexp.go b/src/regexp/syntax/regexp.go index a3f56f8c90..ae5fa053f9 100644 --- a/src/regexp/syntax/regexp.go +++ b/src/regexp/syntax/regexp.go @@ -59,7 +59,7 @@ const ( const opPseudo Op = 128 // where pseudo-ops start -// Equal returns true if x and y have identical structure. +// Equal reports whether x and y have identical structure. func (x *Regexp) Equal(y *Regexp) bool { if x == nil || y == nil { return x == y diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 4854c0e632..67d99900a2 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -517,7 +517,7 @@ func (h heapBits) bits() uint32 { return uint32(*h.bitp) >> (h.shift & 31) } -// morePointers returns true if this word and all remaining words in this object +// morePointers reports whether this word and all remaining words in this object // are scalars. // h must not describe the second word of the object. func (h heapBits) morePointers() bool { diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index d9bc8b4719..f108bfc31e 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -203,7 +203,7 @@ func (c *mcentral) uncacheSpan(s *mspan) { // and, based on the number of free objects in s, // moves s to the appropriate list of c or returns it // to the heap. -// freeSpan returns true if s was returned to the heap. +// freeSpan reports whether s was returned to the heap. // If preserve=true, it does not move s (the caller // must take care of it). func (c *mcentral) freeSpan(s *mspan, preserve bool, wasempty bool) bool { diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index e12df7f7d2..f4646db67a 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -735,7 +735,7 @@ func (c *gcControllerState) findRunnableGCWorker(_p_ *p) *g { return gp } -// pollFractionalWorkerExit returns true if a fractional mark worker +// pollFractionalWorkerExit reports whether a fractional mark worker // should self-preempt. It assumes it is called from the fractional // worker. func pollFractionalWorkerExit() bool { @@ -1157,7 +1157,7 @@ const ( gcTriggerCycle ) -// test returns true if the trigger condition is satisfied, meaning +// test reports whether the trigger condition is satisfied, meaning // that the exit condition for the _GCoff phase has been met. The exit // condition should be tested when allocating. func (t gcTrigger) test() bool { @@ -1867,7 +1867,7 @@ func gcBgMarkWorker(_p_ *p) { } } -// gcMarkWorkAvailable returns true if executing a mark worker +// gcMarkWorkAvailable reports whether executing a mark worker // on p is potentially useful. p may be nil, in which case it only // checks the global sources of work. func gcMarkWorkAvailable(p *p) bool { diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index c32c5eddd7..f2f20fcdac 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -138,7 +138,7 @@ func (w *gcWork) put(obj uintptr) { } } -// putFast does a put and returns true if it can be done quickly +// putFast does a put and reports whether it can be done quickly // otherwise it returns false and the caller needs to call put. //go:nowritebarrierrec func (w *gcWork) putFast(obj uintptr) bool { @@ -299,7 +299,7 @@ func (w *gcWork) balance() { } } -// empty returns true if w has no mark work available. +// empty reports whether w has no mark work available. //go:nowritebarrierrec func (w *gcWork) empty() bool { return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0) diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index f35f7286ac..c91cea254e 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -107,7 +107,7 @@ func (b *wbBuf) discard() { b.next = uintptr(unsafe.Pointer(&b.buf[0])) } -// empty returns true if b contains no pointers. +// empty reports whether b contains no pointers. func (b *wbBuf) empty() bool { return b.next == uintptr(unsafe.Pointer(&b.buf[0])) } diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 75db8c6c2f..71ca993cc0 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -95,7 +95,7 @@ func netpollinited() bool { //go:linkname poll_runtime_isPollServerDescriptor internal/poll.runtime_isPollServerDescriptor -// poll_runtime_isPollServerDescriptor returns true if fd is a +// poll_runtime_isPollServerDescriptor reports whether fd is a // descriptor being used by netpoll. func poll_runtime_isPollServerDescriptor(fd uintptr) bool { fds := netpolldescriptor() diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 45be886196..5b989d28e9 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -849,7 +849,7 @@ func canpanic(gp *g) bool { return true } -// shouldPushSigpanic returns true if pc should be used as sigpanic's +// shouldPushSigpanic reports whether pc should be used as sigpanic's // return PC (pushing a frame for the call). Otherwise, it should be // left alone so that LR is used as sigpanic's return PC, effectively // replacing the top-most frame with sigpanic. This is used by @@ -887,7 +887,7 @@ func shouldPushSigpanic(gp *g, pc, lr uintptr) bool { return true } -// isAbortPC returns true if pc is the program counter at which +// isAbortPC reports whether pc is the program counter at which // runtime.abort raises a signal. // // It is nosplit because it's part of the isgoexception diff --git a/src/runtime/pprof/internal/profile/profile.go b/src/runtime/pprof/internal/profile/profile.go index 84e607e9a8..a6f8354b1e 100644 --- a/src/runtime/pprof/internal/profile/profile.go +++ b/src/runtime/pprof/internal/profile/profile.go @@ -573,7 +573,7 @@ func (p *Profile) Demangle(d Demangler) error { return nil } -// Empty returns true if the profile contains no samples. +// Empty reports whether the profile contains no samples. func (p *Profile) Empty() bool { return len(p.Sample) == 0 } diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 542cf1ed70..864efcdfed 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2400,7 +2400,7 @@ stop: goto top } -// pollWork returns true if there is non-background work this P could +// pollWork reports whether there is non-background work this P could // be doing. This is a fairly lightweight check to be used for // background work loops, like idle GC. It checks a subset of the // conditions checked by the actual scheduler. @@ -4713,7 +4713,7 @@ func pidleget() *p { return _p_ } -// runqempty returns true if _p_ has no Gs on its local run queue. +// runqempty reports whether _p_ has no Gs on its local run queue. // It never returns true spuriously. func runqempty(_p_ *p) bool { // Defend against a race where 1) _p_ has G1 in runqnext but runqhead == runqtail, @@ -4934,7 +4934,7 @@ type gQueue struct { tail guintptr } -// empty returns true if q is empty. +// empty reports whether q is empty. func (q *gQueue) empty() bool { return q.head == 0 } @@ -5000,7 +5000,7 @@ type gList struct { head guintptr } -// empty returns true if l is empty. +// empty reports whether l is empty. func (l *gList) empty() bool { return l.head == 0 } diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index 873ce66abe..e8a64da657 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -38,7 +38,7 @@ func initExceptionHandler() { } } -// isgoexception returns true if this exception should be translated +// isgoexception reports whether this exception should be translated // into a Go panic. // // It is nosplit to avoid growing the stack in case we're aborting diff --git a/src/sync/runtime.go b/src/sync/runtime.go index a13d9f6cf1..b6b9e480a4 100644 --- a/src/sync/runtime.go +++ b/src/sync/runtime.go @@ -54,7 +54,7 @@ func init() { } // Active spinning runtime support. -// runtime_canSpin returns true if spinning makes sense at the moment. +// runtime_canSpin reports whether spinning makes sense at the moment. func runtime_canSpin(i int) bool // runtime_doSpin does active spinning. diff --git a/src/syscall/mksyscall_windows.go b/src/syscall/mksyscall_windows.go index dd84e33c0f..ee2123f939 100644 --- a/src/syscall/mksyscall_windows.go +++ b/src/syscall/mksyscall_windows.go @@ -694,7 +694,7 @@ func (src *Source) ParseFile(path string) error { return nil } -// IsStdRepo returns true if src is part of standard library. +// IsStdRepo reports whether src is part of standard library. func (src *Source) IsStdRepo() (bool, error) { if len(src.Files) == 0 { return false, errors.New("no input files provided") From 1645dfa23fbb1d1bab258d1c458f08d9f2741295 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Thu, 25 Oct 2018 22:03:43 -0700 Subject: [PATCH 004/111] net/http: speed up ServeMux matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanning through all path patterns is not necessary, since the paths do not change frequently. Instead, maintain a sorted list of path prefixes and return the first match. name old time/op new time/op delta ServerMatch-12 134ns ± 3% 17ns ± 4% -86.95% (p=0.000 n=19+20) Change-Id: I15b4483dc30db413321435ee6815fc9bf2bcc546 Reviewed-on: https://go-review.googlesource.com/c/144937 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/net/http/server.go | 55 +++++++++++++++++++------------------ src/net/http/server_test.go | 45 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 26 deletions(-) create mode 100644 src/net/http/server_test.go diff --git a/src/net/http/server.go b/src/net/http/server.go index 6e1ccff4cd..a7e79c2d91 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -22,6 +22,7 @@ import ( "os" "path" "runtime" + "sort" "strconv" "strings" "sync" @@ -2179,7 +2180,8 @@ func RedirectHandler(url string, code int) Handler { type ServeMux struct { mu sync.RWMutex m map[string]muxEntry - hosts bool // whether any patterns contain hostnames + es []muxEntry // slice of entries sorted from longest to shortest. + hosts bool // whether any patterns contain hostnames } type muxEntry struct { @@ -2195,19 +2197,6 @@ var DefaultServeMux = &defaultServeMux var defaultServeMux ServeMux -// Does path match pattern? -func pathMatch(pattern, path string) bool { - if len(pattern) == 0 { - // should not happen - return false - } - n := len(pattern) - if pattern[n-1] != '/' { - return pattern == path - } - return len(path) >= n && path[0:n] == pattern -} - // cleanPath returns the canonical path for p, eliminating . and .. elements. func cleanPath(p string) string { if p == "" { @@ -2252,19 +2241,14 @@ func (mux *ServeMux) match(path string) (h Handler, pattern string) { return v.h, v.pattern } - // Check for longest valid match. - var n = 0 - for k, v := range mux.m { - if !pathMatch(k, path) { - continue - } - if h == nil || len(k) > n { - n = len(k) - h = v.h - pattern = v.pattern + // Check for longest valid match. mux.es contains all patterns + // that end in / sorted from longest to shortest. + for _, e := range mux.es { + if strings.HasPrefix(path, e.pattern) { + return e.h, e.pattern } } - return + return nil, "" } // redirectToPathSlash determines if the given path needs appending "/" to it. @@ -2410,13 +2394,32 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { if mux.m == nil { mux.m = make(map[string]muxEntry) } - mux.m[pattern] = muxEntry{h: handler, pattern: pattern} + e := muxEntry{h: handler, pattern: pattern} + mux.m[pattern] = e + if pattern[len(pattern)-1] == '/' { + mux.es = appendSorted(mux.es, e) + } if pattern[0] != '/' { mux.hosts = true } } +func appendSorted(es []muxEntry, e muxEntry) []muxEntry { + n := len(es) + i := sort.Search(n, func(i int) bool { + return len(es[i].pattern) < len(e.pattern) + }) + if i == n { + return append(es, e) + } + // we now know that i points at where we want to insert + es = append(es, muxEntry{}) // try to grow the slice in place, any entry works. + copy(es[i+1:], es[i:]) // Move shorter entries down + es[i] = e + return es +} + // HandleFunc registers the handler function for the given pattern. func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { if handler == nil { diff --git a/src/net/http/server_test.go b/src/net/http/server_test.go new file mode 100644 index 0000000000..0132f3ba5f --- /dev/null +++ b/src/net/http/server_test.go @@ -0,0 +1,45 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Server unit tests + +package http + +import ( + "fmt" + "testing" +) + +func BenchmarkServerMatch(b *testing.B) { + fn := func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "OK") + } + mux := NewServeMux() + mux.HandleFunc("/", fn) + mux.HandleFunc("/index", fn) + mux.HandleFunc("/home", fn) + mux.HandleFunc("/about", fn) + mux.HandleFunc("/contact", fn) + mux.HandleFunc("/robots.txt", fn) + mux.HandleFunc("/products/", fn) + mux.HandleFunc("/products/1", fn) + mux.HandleFunc("/products/2", fn) + mux.HandleFunc("/products/3", fn) + mux.HandleFunc("/products/3/image.jpg", fn) + mux.HandleFunc("/admin", fn) + mux.HandleFunc("/admin/products/", fn) + mux.HandleFunc("/admin/products/create", fn) + mux.HandleFunc("/admin/products/update", fn) + mux.HandleFunc("/admin/products/delete", fn) + + paths := []string{"/", "/notfound", "/admin/", "/admin/foo", "/contact", "/products", + "/products/", "/products/3/image.jpg"} + b.StartTimer() + for i := 0; i < b.N; i++ { + if h, p := mux.match(paths[i%len(paths)]); h != nil && p == "" { + b.Error("impossible") + } + } + b.StopTimer() +} From 441cb988b4c63f4232edd7930758eb0e8cc8599b Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 2 Nov 2018 12:03:31 -0400 Subject: [PATCH 005/111] cmd/internal/obj/arm64: fix encoding of 32-bit negated logical instructions 32-bit negated logical instructions (BICW, ORNW, EONW) with constants were mis-encoded, because they were missing in the cases where we handle 32-bit logical instructions. This CL adds the missing cases. Fixes #28548 Change-Id: I3d6acde7d3b72bb7d3d5d00a9df698a72c806ad5 Reviewed-on: https://go-review.googlesource.com/c/147077 Run-TryBot: Cherry Zhang Run-TryBot: Ben Shi Reviewed-by: Ben Shi --- src/cmd/asm/internal/asm/testdata/arm64.s | 7 +++++++ src/cmd/internal/obj/arm64/asm7.go | 23 ++++++++--------------- src/cmd/internal/obj/arm64/obj7.go | 9 +++------ 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s index 12c7adbd04..b851ba411e 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64.s +++ b/src/cmd/asm/internal/asm/testdata/arm64.s @@ -222,6 +222,13 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 EOR $(1<<63), R1 // EOR $-9223372036854775808, R1 // 210041d2 EOR $(1<<63-1), R1 // EOR $9223372036854775807, R1 // 21f840d2 + ANDW $0x3ff00000, R2 // ANDW $1072693248, R2 // 42240c12 + BICW $0x3ff00000, R2 // BICW $1072693248, R2 // 42540212 + ORRW $0x3ff00000, R2 // ORRW $1072693248, R2 // 42240c32 + ORNW $0x3ff00000, R2 // ORNW $1072693248, R2 // 42540232 + EORW $0x3ff00000, R2 // EORW $1072693248, R2 // 42240c52 + EONW $0x3ff00000, R2 // EONW $1072693248, R2 // 42540252 + AND $0x22220000, R3, R4 // AND $572653568, R3, R4 // 5b44a4d264001b8a ORR $0x22220000, R3, R4 // ORR $572653568, R3, R4 // 5b44a4d264001baa EOR $0x22220000, R3, R4 // EOR $572653568, R3, R4 // 5b44a4d264001bca diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 6a6e81807a..770b4b6fc3 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -1689,21 +1689,14 @@ func (c *ctxt7) oplook(p *obj.Prog) *Optab { a1 = ra0 + 1 p.From.Class = int8(a1) } - if isANDWop(p.As) { - switch p.As { - case AANDW, AORRW, AEORW, AANDSW, ATSTW: - // For 32-bit logical instruction with constant, - // rewrite the high 32-bit to be a copy of the low - // 32-bit, so that the BITCON test can be shared - // for both 32-bit and 64-bit. - if a0 == C_BITCON { - break - } - fallthrough - default: - a1 = c.con32class(&p.From) + 1 - p.From.Class = int8(a1) - } + if isANDWop(p.As) && a0 != C_BITCON { + // For 32-bit logical instruction with constant, + // the BITCON test is special in that it looks at + // the 64-bit which has the high 32-bit as a copy + // of the low 32-bit. We have handled that and + // don't pass it to con32class. + a1 = c.con32class(&p.From) + 1 + p.From.Class = int8(a1) } } } diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index 4476dad071..d0e354eabd 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -311,12 +311,9 @@ func progedit(ctxt *obj.Link, p *obj.Prog, newprog obj.ProgAlloc) { // shared for both 32-bit and 64-bit. 32-bit ops // will zero the high 32-bit of the destination // register anyway. - switch p.As { - case AANDW, AORRW, AEORW, AANDSW, ATSTW: - if p.From.Type == obj.TYPE_CONST { - v := p.From.Offset & 0xffffffff - p.From.Offset = v | v<<32 - } + if isANDWop(p.As) && p.From.Type == obj.TYPE_CONST { + v := p.From.Offset & 0xffffffff + p.From.Offset = v | v<<32 } if c.ctxt.Flag_dynlink { From 15265ec4217f8d9497d8d5079ac8996302dfd007 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 29 Oct 2018 18:21:00 -0400 Subject: [PATCH 006/111] cmd/compile: avoid duplicate GC bitmap symbols Currently, liveness produces a distinct obj.LSym for each GC bitmap for each function. These are then named by content hash and only ultimately deduplicated by WriteObjFile. For various reasons (see next commit), we want to remove this deduplication behavior from WriteObjFile. Furthermore, it's inefficient to produce these duplicate symbols in the first place. GC bitmaps are the only source of duplicate symbols in the compiler. This commit eliminates these duplicate symbols by declaring them in the Ctxt symbol hash just like every other obj.LSym. As a result, all GC bitmaps with the same content now refer to the same obj.LSym. The next commit will remove deduplication from WriteObjFile. For #27539. Change-Id: I4f15e3d99530122cdf473b7a838c69ef5f79db59 Reviewed-on: https://go-review.googlesource.com/c/146557 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/gc/gsubr.go | 18 --------- src/cmd/compile/internal/gc/obj.go | 12 +++--- src/cmd/compile/internal/gc/plive.go | 59 ++++++++++++++++++++-------- src/cmd/compile/internal/gc/ssa.go | 2 +- src/cmd/internal/obj/link.go | 6 +-- src/cmd/internal/obj/plist.go | 12 ------ 6 files changed, 52 insertions(+), 57 deletions(-) diff --git a/src/cmd/compile/internal/gc/gsubr.go b/src/cmd/compile/internal/gc/gsubr.go index f39ffc7365..16602b9988 100644 --- a/src/cmd/compile/internal/gc/gsubr.go +++ b/src/cmd/compile/internal/gc/gsubr.go @@ -185,24 +185,6 @@ func (pp *Progs) settext(fn *Node) { ptxt.From.Type = obj.TYPE_MEM ptxt.From.Name = obj.NAME_EXTERN ptxt.From.Sym = fn.Func.lsym - - p := pp.Prog(obj.AFUNCDATA) - Addrconst(&p.From, objabi.FUNCDATA_ArgsPointerMaps) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = &fn.Func.lsym.Func.GCArgs - - p = pp.Prog(obj.AFUNCDATA) - Addrconst(&p.From, objabi.FUNCDATA_LocalsPointerMaps) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = &fn.Func.lsym.Func.GCLocals - - p = pp.Prog(obj.AFUNCDATA) - Addrconst(&p.From, objabi.FUNCDATA_RegPointerMaps) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = &fn.Func.lsym.Func.GCRegs } func (f *Func) initLSym() { diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index a9dd092b67..e3c8e07ffa 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -277,18 +277,18 @@ func dumpglobls() { // Though the object file format handles duplicates efficiently, // storing only a single copy of the data, // failure to remove these duplicates adds a few percent to object file size. +// +// This is done during the sequential phase after compilation, since +// global symbols can't be declared during parallel compilation. func addGCLocals() { - seen := make(map[string]bool) for _, s := range Ctxt.Text { if s.Func == nil { continue } - for _, gcsym := range []*obj.LSym{&s.Func.GCArgs, &s.Func.GCLocals, &s.Func.GCRegs} { - if seen[gcsym.Name] { - continue + for _, gcsym := range []*obj.LSym{s.Func.GCArgs, s.Func.GCLocals, s.Func.GCRegs} { + if gcsym != nil && !gcsym.OnList() { + ggloblsym(gcsym, int32(len(gcsym.P)), obj.RODATA|obj.DUPOK) } - Ctxt.Data = append(Ctxt.Data, gcsym) - seen[gcsym.Name] = true } if x := s.Func.StackObjects; x != nil { ggloblsym(x, int32(len(x.P)), obj.RODATA|obj.LOCAL) diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index a38c33647e..601815f7c5 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -1461,7 +1461,7 @@ func (lv *Liveness) printDebug() { // first word dumped is the total number of bitmaps. The second word is the // length of the bitmaps. All bitmaps are assumed to be of equal length. The // remaining bytes are the raw bitmaps. -func (lv *Liveness) emit(argssym, livesym, regssym *obj.LSym) { +func (lv *Liveness) emit() (argsSym, liveSym, regsSym *obj.LSym) { // Size args bitmaps to be just large enough to hold the largest pointer. // First, find the largest Xoffset node we care about. // (Nodes without pointers aren't in lv.vars; see livenessShouldTrack.) @@ -1489,13 +1489,16 @@ func (lv *Liveness) emit(argssym, livesym, regssym *obj.LSym) { // This would require shifting all bitmaps. maxLocals := lv.stkptrsize + // Temporary symbols for encoding bitmaps. + var argsSymTmp, liveSymTmp, regsSymTmp obj.LSym + args := bvalloc(int32(maxArgs / int64(Widthptr))) - aoff := duint32(argssym, 0, uint32(len(lv.stackMaps))) // number of bitmaps - aoff = duint32(argssym, aoff, uint32(args.n)) // number of bits in each bitmap + aoff := duint32(&argsSymTmp, 0, uint32(len(lv.stackMaps))) // number of bitmaps + aoff = duint32(&argsSymTmp, aoff, uint32(args.n)) // number of bits in each bitmap locals := bvalloc(int32(maxLocals / int64(Widthptr))) - loff := duint32(livesym, 0, uint32(len(lv.stackMaps))) // number of bitmaps - loff = duint32(livesym, loff, uint32(locals.n)) // number of bits in each bitmap + loff := duint32(&liveSymTmp, 0, uint32(len(lv.stackMaps))) // number of bitmaps + loff = duint32(&liveSymTmp, loff, uint32(locals.n)) // number of bits in each bitmap for _, live := range lv.stackMaps { args.Clear() @@ -1503,13 +1506,13 @@ func (lv *Liveness) emit(argssym, livesym, regssym *obj.LSym) { lv.pointerMap(live, lv.vars, args, locals) - aoff = dbvec(argssym, aoff, args) - loff = dbvec(livesym, loff, locals) + aoff = dbvec(&argsSymTmp, aoff, args) + loff = dbvec(&liveSymTmp, loff, locals) } regs := bvalloc(lv.usedRegs()) - roff := duint32(regssym, 0, uint32(len(lv.regMaps))) // number of bitmaps - roff = duint32(regssym, roff, uint32(regs.n)) // number of bits in each bitmap + roff := duint32(®sSymTmp, 0, uint32(len(lv.regMaps))) // number of bitmaps + roff = duint32(®sSymTmp, roff, uint32(regs.n)) // number of bits in each bitmap if regs.n > 32 { // Our uint32 conversion below won't work. Fatalf("GP registers overflow uint32") @@ -1519,25 +1522,29 @@ func (lv *Liveness) emit(argssym, livesym, regssym *obj.LSym) { for _, live := range lv.regMaps { regs.Clear() regs.b[0] = uint32(live) - roff = dbvec(regssym, roff, regs) + roff = dbvec(®sSymTmp, roff, regs) } } // Give these LSyms content-addressable names, // so that they can be de-duplicated. // This provides significant binary size savings. - // It is safe to rename these LSyms because - // they are tracked separately from ctxt.hash. - argssym.Name = fmt.Sprintf("gclocals·%x", md5.Sum(argssym.P)) - livesym.Name = fmt.Sprintf("gclocals·%x", md5.Sum(livesym.P)) - regssym.Name = fmt.Sprintf("gclocals·%x", md5.Sum(regssym.P)) + // + // These symbols will be added to Ctxt.Data by addGCLocals + // after parallel compilation is done. + makeSym := func(tmpSym *obj.LSym) *obj.LSym { + return Ctxt.LookupInit(fmt.Sprintf("gclocals·%x", md5.Sum(tmpSym.P)), func(lsym *obj.LSym) { + lsym.P = tmpSym.P + }) + } + return makeSym(&argsSymTmp), makeSym(&liveSymTmp), makeSym(®sSymTmp) } // Entry pointer for liveness analysis. Solves for the liveness of // pointer variables in the function and emits a runtime data // structure read by the garbage collector. // Returns a map from GC safe points to their corresponding stack map index. -func liveness(e *ssafn, f *ssa.Func) LivenessMap { +func liveness(e *ssafn, f *ssa.Func, pp *Progs) LivenessMap { // Construct the global liveness state. vars, idx := getvariables(e.curfn) lv := newliveness(e.curfn, f, vars, idx, e.stkptrsize) @@ -1577,7 +1584,25 @@ func liveness(e *ssafn, f *ssa.Func) LivenessMap { // Emit the live pointer map data structures if ls := e.curfn.Func.lsym; ls != nil { - lv.emit(&ls.Func.GCArgs, &ls.Func.GCLocals, &ls.Func.GCRegs) + ls.Func.GCArgs, ls.Func.GCLocals, ls.Func.GCRegs = lv.emit() + + p := pp.Prog(obj.AFUNCDATA) + Addrconst(&p.From, objabi.FUNCDATA_ArgsPointerMaps) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = ls.Func.GCArgs + + p = pp.Prog(obj.AFUNCDATA) + Addrconst(&p.From, objabi.FUNCDATA_LocalsPointerMaps) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = ls.Func.GCLocals + + p = pp.Prog(obj.AFUNCDATA) + Addrconst(&p.From, objabi.FUNCDATA_RegPointerMaps) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = ls.Func.GCRegs } return lv.livenessMap } diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 4607cf1912..b0ccd01752 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -5051,7 +5051,7 @@ func genssa(f *ssa.Func, pp *Progs) { e := f.Frontend().(*ssafn) - s.livenessMap = liveness(e, f) + s.livenessMap = liveness(e, f, pp) emitStackObjects(e, pp) // Remember where each block starts. diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index f983d5277e..d924cbc214 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -403,9 +403,9 @@ type FuncInfo struct { dwarfAbsFnSym *LSym dwarfIsStmtSym *LSym - GCArgs LSym - GCLocals LSym - GCRegs LSym + GCArgs *LSym + GCLocals *LSym + GCRegs *LSym StackObjects *LSym } diff --git a/src/cmd/internal/obj/plist.go b/src/cmd/internal/obj/plist.go index a8675055d9..6710b375f1 100644 --- a/src/cmd/internal/obj/plist.go +++ b/src/cmd/internal/obj/plist.go @@ -147,18 +147,6 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { isstmt.Type = objabi.SDWARFMISC isstmt.Set(AttrDuplicateOK, s.DuplicateOK()) ctxt.Data = append(ctxt.Data, isstmt) - - // Set up the function's gcargs and gclocals. - // They will be filled in later if needed. - gcargs := &s.Func.GCArgs - gcargs.Set(AttrDuplicateOK, true) - gcargs.Type = objabi.SRODATA - gclocals := &s.Func.GCLocals - gclocals.Set(AttrDuplicateOK, true) - gclocals.Type = objabi.SRODATA - gcregs := &s.Func.GCRegs - gcregs.Set(AttrDuplicateOK, true) - gcregs.Type = objabi.SRODATA } func (ctxt *Link) Globl(s *LSym, size int64, flag int) { From c0281afd870f15f117ad1bcb2c46a3a3c3fffb0b Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 29 Oct 2018 18:27:51 -0400 Subject: [PATCH 007/111] cmd/internal/obj: don't dedup symbols in WriteObjFile Currently, WriteObjFile deduplicates symbols by name. This is a strange and unexpected place to do this. But, worse, there's no checking that it's reasonable to deduplicate two symbols, so this makes it incredibly easy to mask errors involving duplicate symbols. Dealing with duplicate symbols is better left to the linker. We're also about to introduce multiple symbols with the same name but different ABIs/versions, which would make this deduplication more complicated. We just removed the only part of the compiler that actually depended on this behavior. This CL removes symbol deduplication from WriteObjFile, since it is no longer needed. For #27539. Change-Id: I650c550e46e83f95c67cb6c6646f9b2f7f10df30 Reviewed-on: https://go-review.googlesource.com/c/146558 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/gc/obj.go | 4 ---- src/cmd/internal/obj/objfile.go | 24 ++---------------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index e3c8e07ffa..5630e12ace 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -273,10 +273,6 @@ func dumpglobls() { } // addGCLocals adds gcargs, gclocals, gcregs, and stack object symbols to Ctxt.Data. -// It takes care not to add any duplicates. -// Though the object file format handles duplicates efficiently, -// storing only a single copy of the data, -// failure to remove these duplicates adds a few percent to object file size. // // This is done during the sequential phase after compilation, since // global symbols can't be declared during parallel compilation. diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index b6cfec3b3e..3c72f543cc 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -25,12 +25,6 @@ type objWriter struct { // Temporary buffer for zigzag int writing. varintbuf [10]uint8 - // Provide the index of a symbol reference by symbol name. - // One map for versioned symbols and one for unversioned symbols. - // Used for deduplicating the symbol reference list. - refIdx map[string]int - vrefIdx map[string]int - // Number of objects written of each type. nRefs int nData int @@ -79,10 +73,8 @@ func (w *objWriter) writeLengths() { func newObjWriter(ctxt *Link, b *bufio.Writer) *objWriter { return &objWriter{ - ctxt: ctxt, - wr: b, - vrefIdx: make(map[string]int), - refIdx: make(map[string]int), + ctxt: ctxt, + wr: b, } } @@ -157,17 +149,6 @@ func (w *objWriter) writeRef(s *LSym, isPath bool) { if s == nil || s.RefIdx != 0 { return } - var m map[string]int - if !s.Static() { - m = w.refIdx - } else { - m = w.vrefIdx - } - - if idx := m[s.Name]; idx != 0 { - s.RefIdx = idx - return - } w.wr.WriteByte(symPrefix) if isPath { w.writeString(filepath.ToSlash(s.Name)) @@ -178,7 +159,6 @@ func (w *objWriter) writeRef(s *LSym, isPath bool) { w.writeBool(s.Static()) w.nRefs++ s.RefIdx = w.nRefs - m[s.Name] = w.nRefs } func (w *objWriter) writeRefs(s *LSym) { From 6151a6d7396691a205fc9086561643f1bcb1f62c Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 2 Nov 2018 21:51:01 -0700 Subject: [PATCH 008/111] cmd/compile: fix fmt_test (fix long test build) Follow-up on https://golang.org/cl/146782. TBR=bradfitz Change-Id: Idaf5488fedfc05d6ff71706fa0bcd70bf98ab25a Reviewed-on: https://go-review.googlesource.com/c/147283 Reviewed-by: Robert Griesemer --- src/cmd/compile/fmt_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go index 6dfdea1a34..1195117c2d 100644 --- a/src/cmd/compile/fmt_test.go +++ b/src/cmd/compile/fmt_test.go @@ -587,6 +587,7 @@ var knownFormats = map[string]string{ "*cmd/compile/internal/types.Sym %S": "", "*cmd/compile/internal/types.Sym %p": "", "*cmd/compile/internal/types.Sym %v": "", + "*cmd/compile/internal/types.Type %#L": "", "*cmd/compile/internal/types.Type %#v": "", "*cmd/compile/internal/types.Type %+v": "", "*cmd/compile/internal/types.Type %-S": "", From f2cd0fa7b773eabcce1b99874188aedcf9acb1ab Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Sat, 3 Nov 2018 09:33:31 -0700 Subject: [PATCH 009/111] cmd/compile/internal/gc: don't print "internal error" twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Fatalf mechanism already prints "compiler internal error:" when reporting an error. There's no need to have "internal error" in the error message passed to Fatalf calls. Removed them. Fixes #28575. Change-Id: I12b1bea37bc839780f257c27ef9e2005bf334925 Reviewed-on: https://go-review.googlesource.com/c/147287 Run-TryBot: Robert Griesemer TryBot-Result: Gobot Gobot Reviewed-by: Martin Möhrmann --- src/cmd/compile/fmt_test.go | 60 +++++++++++++------------- src/cmd/compile/internal/gc/dwinl.go | 2 +- src/cmd/compile/internal/gc/iexport.go | 2 +- src/cmd/compile/internal/gc/plive.go | 6 +-- 4 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go index 1195117c2d..eaa2aa8dbd 100644 --- a/src/cmd/compile/fmt_test.go +++ b/src/cmd/compile/fmt_test.go @@ -599,6 +599,7 @@ var knownFormats = map[string]string{ "*cmd/compile/internal/types.Type %v": "", "*cmd/internal/obj.Addr %v": "", "*cmd/internal/obj.LSym %v": "", + "*math/big.Float %f": "", "*math/big.Int %#x": "", "*math/big.Int %s": "", "*math/big.Int %v": "", @@ -706,33 +707,34 @@ var knownFormats = map[string]string{ "interface{} %v": "", "map[*cmd/compile/internal/gc.Node]*cmd/compile/internal/ssa.Value %v": "", "map[cmd/compile/internal/ssa.ID]uint32 %v": "", - "reflect.Type %s": "", - "rune %#U": "", - "rune %c": "", - "string %-*s": "", - "string %-16s": "", - "string %-6s": "", - "string %.*s": "", - "string %q": "", - "string %s": "", - "string %v": "", - "time.Duration %d": "", - "time.Duration %v": "", - "uint %04x": "", - "uint %5d": "", - "uint %d": "", - "uint %x": "", - "uint16 %d": "", - "uint16 %v": "", - "uint16 %x": "", - "uint32 %#x": "", - "uint32 %d": "", - "uint32 %v": "", - "uint32 %x": "", - "uint64 %08x": "", - "uint64 %d": "", - "uint64 %x": "", - "uint8 %d": "", - "uint8 %x": "", - "uintptr %d": "", + "math/big.Accuracy %s": "", + "reflect.Type %s": "", + "rune %#U": "", + "rune %c": "", + "string %-*s": "", + "string %-16s": "", + "string %-6s": "", + "string %.*s": "", + "string %q": "", + "string %s": "", + "string %v": "", + "time.Duration %d": "", + "time.Duration %v": "", + "uint %04x": "", + "uint %5d": "", + "uint %d": "", + "uint %x": "", + "uint16 %d": "", + "uint16 %v": "", + "uint16 %x": "", + "uint32 %#x": "", + "uint32 %d": "", + "uint32 %v": "", + "uint32 %x": "", + "uint64 %08x": "", + "uint64 %d": "", + "uint64 %x": "", + "uint8 %d": "", + "uint8 %x": "", + "uintptr %d": "", } diff --git a/src/cmd/compile/internal/gc/dwinl.go b/src/cmd/compile/internal/gc/dwinl.go index 51251c9139..ade76f40f8 100644 --- a/src/cmd/compile/internal/gc/dwinl.go +++ b/src/cmd/compile/internal/gc/dwinl.go @@ -300,7 +300,7 @@ func beginRange(calls []dwarf.InlCall, p *obj.Prog, ii int, imap map[int]int) *d } callIdx, found := imap[ii] if !found { - Fatalf("internal error: can't find inlIndex %d in imap for prog at %d\n", ii, p.Pc) + Fatalf("can't find inlIndex %d in imap for prog at %d\n", ii, p.Pc) } call := &calls[callIdx] diff --git a/src/cmd/compile/internal/gc/iexport.go b/src/cmd/compile/internal/gc/iexport.go index d21378df4a..e77ca9a6c1 100644 --- a/src/cmd/compile/internal/gc/iexport.go +++ b/src/cmd/compile/internal/gc/iexport.go @@ -909,7 +909,7 @@ func (w *exportWriter) mpfloat(f *big.Float, typ *types.Type) { manti, acc := mant.Int(nil) if acc != big.Exact { - Fatalf("exporter: internal error") + Fatalf("mantissa scaling failed for %f (%s)", f, acc) } w.mpint(manti, typ) if manti.Sign() != 0 { diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index 601815f7c5..2c31d5feb9 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -1019,7 +1019,7 @@ func (lv *Liveness) epilogue() { live := lv.livevars[index] if v.Op.IsCall() && live.regs != 0 { lv.printDebug() - v.Fatalf("internal error: %v register %s recorded as live at call", lv.fn.Func.Nname, live.regs.niceString(lv.f.Config)) + v.Fatalf("%v register %s recorded as live at call", lv.fn.Func.Nname, live.regs.niceString(lv.f.Config)) } index++ } @@ -1038,7 +1038,7 @@ func (lv *Liveness) epilogue() { // input parameters. for j, n := range lv.vars { if n.Class() != PPARAM && lv.stackMaps[0].Get(int32(j)) { - Fatalf("internal error: %v %L recorded as live on entry", lv.fn.Func.Nname, n) + lv.f.Fatalf("%v %L recorded as live on entry", lv.fn.Func.Nname, n) } } // Check that no registers are live at function entry. @@ -1047,7 +1047,7 @@ func (lv *Liveness) epilogue() { // so it doesn't appear live at entry. if regs := lv.regMaps[0]; regs != 0 { lv.printDebug() - lv.f.Fatalf("internal error: %v register %s recorded as live on entry", lv.fn.Func.Nname, regs.niceString(lv.f.Config)) + lv.f.Fatalf("%v register %s recorded as live on entry", lv.fn.Func.Nname, regs.niceString(lv.f.Config)) } } From 6fe8ee78e9b4870ebc1de2b5cfd6170a78a56c00 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 2 Nov 2018 22:21:23 -0700 Subject: [PATCH 010/111] cmd/compile/internal/gc: remove isforw predicate table (cleanup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Was only ever filled with one Etype (TFORW) and only used in one place. Easier to just check for TFORW. Change-Id: Icc96da3a22b0af1d7e60bc5841c744916c53341e Reviewed-on: https://go-review.googlesource.com/c/147285 Reviewed-by: Martin Möhrmann --- src/cmd/compile/internal/gc/go.go | 1 - src/cmd/compile/internal/gc/reflect.go | 2 +- src/cmd/compile/internal/gc/universe.go | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cmd/compile/internal/gc/go.go b/src/cmd/compile/internal/gc/go.go index cfd695097f..471746ed7d 100644 --- a/src/cmd/compile/internal/gc/go.go +++ b/src/cmd/compile/internal/gc/go.go @@ -147,7 +147,6 @@ var asmhdr string var simtype [NTYPE]types.EType var ( - isforw [NTYPE]bool isInt [NTYPE]bool isFloat [NTYPE]bool isComplex [NTYPE]bool diff --git a/src/cmd/compile/internal/gc/reflect.go b/src/cmd/compile/internal/gc/reflect.go index e4008bd7e7..415d3cd594 100644 --- a/src/cmd/compile/internal/gc/reflect.go +++ b/src/cmd/compile/internal/gc/reflect.go @@ -1137,7 +1137,7 @@ func dtypesym(t *types.Type) *obj.LSym { return lsym } // TODO(mdempsky): Investigate whether this can happen. - if isforw[tbase.Etype] { + if tbase.Etype == TFORW { return lsym } } diff --git a/src/cmd/compile/internal/gc/universe.go b/src/cmd/compile/internal/gc/universe.go index 96980ad500..760a8e40b0 100644 --- a/src/cmd/compile/internal/gc/universe.go +++ b/src/cmd/compile/internal/gc/universe.go @@ -200,8 +200,6 @@ func typeinit() { isComplex[TCOMPLEX64] = true isComplex[TCOMPLEX128] = true - isforw[TFORW] = true - // initialize okfor for et := types.EType(0); et < NTYPE; et++ { if isInt[et] || et == TIDEAL { From 4bb9b61677e937d1f473c27607da3608d0fa7099 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 3 Nov 2018 22:57:52 -0700 Subject: [PATCH 011/111] strings: lower running time of TestCompareStrings At each comparison, we're making a copy of the whole string. Instead, use unsafe to share the string backing store with a []byte. It reduces the test time from ~4sec to ~1sec on my machine (darwin/amd64). Some builders were having much more trouble with this test (>3min), it may help more there. Fixes #26174 Fixes #28573 Fixes #26155 Update #26473 Change-Id: Id5856fd26faf6ff46e763a088f039230556a4116 Reviewed-on: https://go-review.googlesource.com/c/147358 Reviewed-by: Brad Fitzpatrick --- src/strings/compare_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/strings/compare_test.go b/src/strings/compare_test.go index 5d5334461c..94554e0af7 100644 --- a/src/strings/compare_test.go +++ b/src/strings/compare_test.go @@ -11,6 +11,7 @@ import ( "internal/testenv" . "strings" "testing" + "unsafe" ) var compareTests = []struct { @@ -53,6 +54,12 @@ func TestCompareIdenticalString(t *testing.T) { } func TestCompareStrings(t *testing.T) { + // unsafeString converts a []byte to a string with no allocation. + // The caller must not modify b while the result string is in use. + unsafeString := func(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) + } + lengths := make([]int, 0) // lengths to test in ascending order for i := 0; i <= 128; i++ { lengths = append(lengths, i) @@ -79,7 +86,7 @@ func TestCompareStrings(t *testing.T) { b[i] = 9 } - sa, sb := string(a), string(b) + sa, sb := unsafeString(a), unsafeString(b) cmp := Compare(sa[:len], sb[:len]) if cmp != 0 { t.Errorf(`CompareIdentical(%d) = %d`, len, cmp) @@ -96,12 +103,12 @@ func TestCompareStrings(t *testing.T) { } for k := lastLen; k < len; k++ { b[k] = a[k] - 1 - cmp = Compare(string(a[:len]), string(b[:len])) + cmp = Compare(unsafeString(a[:len]), unsafeString(b[:len])) if cmp != 1 { t.Errorf(`CompareAbigger(%d,%d) = %d`, len, k, cmp) } b[k] = a[k] + 1 - cmp = Compare(string(a[:len]), string(b[:len])) + cmp = Compare(unsafeString(a[:len]), unsafeString(b[:len])) if cmp != -1 { t.Errorf(`CompareBbigger(%d,%d) = %d`, len, k, cmp) } From efd229238aceefdee6a00ea28b61c924c2b5f1a5 Mon Sep 17 00:00:00 2001 From: Elias Naur Date: Sun, 4 Nov 2018 09:36:25 +0100 Subject: [PATCH 012/111] runtime: avoid arm64 8.1 atomics on Android The kernel on some Samsung S9+ models reports support for arm64 8.1 atomics, but in reality only some of the cores support them. Go programs scheduled to cores without support will crash with SIGILL. This change unconditionally disables the optimization on Android. A better fix is to precisely detect the offending chipset. Fixes #28431 Change-Id: I35a1273e5660603824d30ebef2ce7e429241bf1f Reviewed-on: https://go-review.googlesource.com/c/147377 Run-TryBot: Elias Naur TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/os_linux_arm64.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/runtime/os_linux_arm64.go b/src/runtime/os_linux_arm64.go index cbe528b4af..2d6f68bdd9 100644 --- a/src/runtime/os_linux_arm64.go +++ b/src/runtime/os_linux_arm64.go @@ -22,7 +22,15 @@ func archauxv(tag, val uintptr) { case _AT_HWCAP: // arm64 doesn't have a 'cpuid' instruction equivalent and relies on // HWCAP/HWCAP2 bits for hardware capabilities. - cpu.HWCap = uint(val) + hwcap := uint(val) + if GOOS == "android" { + // The Samsung S9+ kernel reports support for atomics, but not all cores + // actually support them, resulting in SIGILL. See issue #28431. + // TODO(elias.naur): Only disable the optimization on bad chipsets. + const hwcap_ATOMICS = 1 << 8 + hwcap &= ^uint(hwcap_ATOMICS) + } + cpu.HWCap = hwcap case _AT_HWCAP2: cpu.HWCap2 = uint(val) } From 817f567fa8be4753acfad0b795ec41ab1c0cdb65 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 5 Nov 2018 12:39:11 +0100 Subject: [PATCH 013/111] runtime/internal/sys: regenerate zgoos_*.go files zgoos_aix.go is missing GoosJs, the order of GoosAndroid and GoosAix is mixed up in all files and GoosHurd was added after CL 146023 introduced GOOS=hurd. Change-Id: I7e2f5a15645272e9020cfca86e44c364fc072a2b Reviewed-on: https://go-review.googlesource.com/c/147397 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/internal/sys/zgoos_aix.go | 4 +++- src/runtime/internal/sys/zgoos_android.go | 3 ++- src/runtime/internal/sys/zgoos_darwin.go | 3 ++- src/runtime/internal/sys/zgoos_dragonfly.go | 3 ++- src/runtime/internal/sys/zgoos_freebsd.go | 3 ++- src/runtime/internal/sys/zgoos_hurd.go | 23 +++++++++++++++++++++ src/runtime/internal/sys/zgoos_js.go | 2 ++ src/runtime/internal/sys/zgoos_linux.go | 3 ++- src/runtime/internal/sys/zgoos_nacl.go | 3 ++- src/runtime/internal/sys/zgoos_netbsd.go | 3 ++- src/runtime/internal/sys/zgoos_openbsd.go | 3 ++- src/runtime/internal/sys/zgoos_plan9.go | 3 ++- src/runtime/internal/sys/zgoos_solaris.go | 3 ++- src/runtime/internal/sys/zgoos_windows.go | 3 ++- src/runtime/internal/sys/zgoos_zos.go | 3 ++- 15 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 src/runtime/internal/sys/zgoos_hurd.go diff --git a/src/runtime/internal/sys/zgoos_aix.go b/src/runtime/internal/sys/zgoos_aix.go index 9ce5b3434f..909bfc5e93 100644 --- a/src/runtime/internal/sys/zgoos_aix.go +++ b/src/runtime/internal/sys/zgoos_aix.go @@ -6,11 +6,13 @@ package sys const GOOS = `aix` -const GoosAndroid = 0 const GoosAix = 1 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 +const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 const GoosNetbsd = 0 diff --git a/src/runtime/internal/sys/zgoos_android.go b/src/runtime/internal/sys/zgoos_android.go index 36a5768ab6..434ce46712 100644 --- a/src/runtime/internal/sys/zgoos_android.go +++ b/src/runtime/internal/sys/zgoos_android.go @@ -6,11 +6,12 @@ package sys const GOOS = `android` -const GoosAndroid = 1 const GoosAix = 0 +const GoosAndroid = 1 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_darwin.go b/src/runtime/internal/sys/zgoos_darwin.go index 10c0e88e9a..b645d1cf5f 100644 --- a/src/runtime/internal/sys/zgoos_darwin.go +++ b/src/runtime/internal/sys/zgoos_darwin.go @@ -6,11 +6,12 @@ package sys const GOOS = `darwin` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 1 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_dragonfly.go b/src/runtime/internal/sys/zgoos_dragonfly.go index 5cb47cb84e..154cec370f 100644 --- a/src/runtime/internal/sys/zgoos_dragonfly.go +++ b/src/runtime/internal/sys/zgoos_dragonfly.go @@ -6,11 +6,12 @@ package sys const GOOS = `dragonfly` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 1 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_freebsd.go b/src/runtime/internal/sys/zgoos_freebsd.go index 470406ce5f..5f41c03445 100644 --- a/src/runtime/internal/sys/zgoos_freebsd.go +++ b/src/runtime/internal/sys/zgoos_freebsd.go @@ -6,11 +6,12 @@ package sys const GOOS = `freebsd` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 1 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_hurd.go b/src/runtime/internal/sys/zgoos_hurd.go new file mode 100644 index 0000000000..53f7fc384b --- /dev/null +++ b/src/runtime/internal/sys/zgoos_hurd.go @@ -0,0 +1,23 @@ +// Code generated by gengoos.go using 'go generate'. DO NOT EDIT. + +// +build hurd + +package sys + +const GOOS = `hurd` + +const GoosAix = 0 +const GoosAndroid = 0 +const GoosDarwin = 0 +const GoosDragonfly = 0 +const GoosFreebsd = 0 +const GoosHurd = 1 +const GoosJs = 0 +const GoosLinux = 0 +const GoosNacl = 0 +const GoosNetbsd = 0 +const GoosOpenbsd = 0 +const GoosPlan9 = 0 +const GoosSolaris = 0 +const GoosWindows = 0 +const GoosZos = 0 diff --git a/src/runtime/internal/sys/zgoos_js.go b/src/runtime/internal/sys/zgoos_js.go index cc8eef080f..c6cca49bd9 100644 --- a/src/runtime/internal/sys/zgoos_js.go +++ b/src/runtime/internal/sys/zgoos_js.go @@ -6,10 +6,12 @@ package sys const GOOS = `js` +const GoosAix = 0 const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 1 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_linux.go b/src/runtime/internal/sys/zgoos_linux.go index 76235b748c..088dbc105b 100644 --- a/src/runtime/internal/sys/zgoos_linux.go +++ b/src/runtime/internal/sys/zgoos_linux.go @@ -7,11 +7,12 @@ package sys const GOOS = `linux` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 1 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_nacl.go b/src/runtime/internal/sys/zgoos_nacl.go index 6d28b59667..65bec4af9e 100644 --- a/src/runtime/internal/sys/zgoos_nacl.go +++ b/src/runtime/internal/sys/zgoos_nacl.go @@ -6,11 +6,12 @@ package sys const GOOS = `nacl` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 1 diff --git a/src/runtime/internal/sys/zgoos_netbsd.go b/src/runtime/internal/sys/zgoos_netbsd.go index ef8d938ddb..93d0fa7e11 100644 --- a/src/runtime/internal/sys/zgoos_netbsd.go +++ b/src/runtime/internal/sys/zgoos_netbsd.go @@ -6,11 +6,12 @@ package sys const GOOS = `netbsd` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_openbsd.go b/src/runtime/internal/sys/zgoos_openbsd.go index 2e43847396..79193593f5 100644 --- a/src/runtime/internal/sys/zgoos_openbsd.go +++ b/src/runtime/internal/sys/zgoos_openbsd.go @@ -6,11 +6,12 @@ package sys const GOOS = `openbsd` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_plan9.go b/src/runtime/internal/sys/zgoos_plan9.go index ed598dcaac..2b95e08080 100644 --- a/src/runtime/internal/sys/zgoos_plan9.go +++ b/src/runtime/internal/sys/zgoos_plan9.go @@ -6,11 +6,12 @@ package sys const GOOS = `plan9` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_solaris.go b/src/runtime/internal/sys/zgoos_solaris.go index fe690df6c2..6e3988aed0 100644 --- a/src/runtime/internal/sys/zgoos_solaris.go +++ b/src/runtime/internal/sys/zgoos_solaris.go @@ -6,11 +6,12 @@ package sys const GOOS = `solaris` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_windows.go b/src/runtime/internal/sys/zgoos_windows.go index ea7c43bdf4..a56e12544a 100644 --- a/src/runtime/internal/sys/zgoos_windows.go +++ b/src/runtime/internal/sys/zgoos_windows.go @@ -6,11 +6,12 @@ package sys const GOOS = `windows` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 diff --git a/src/runtime/internal/sys/zgoos_zos.go b/src/runtime/internal/sys/zgoos_zos.go index d4027cf876..0f56e46002 100644 --- a/src/runtime/internal/sys/zgoos_zos.go +++ b/src/runtime/internal/sys/zgoos_zos.go @@ -6,11 +6,12 @@ package sys const GOOS = `zos` -const GoosAndroid = 0 const GoosAix = 0 +const GoosAndroid = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 +const GoosHurd = 0 const GoosJs = 0 const GoosLinux = 0 const GoosNacl = 0 From f999576dd8df6e4d09e3c67c23ba4d8dc18d53d3 Mon Sep 17 00:00:00 2001 From: Muhammad Falak R Wani Date: Mon, 5 Nov 2018 22:04:46 +0530 Subject: [PATCH 014/111] cmd/addr2line: defer closing objfile Change-Id: I19ff9d231c4cc779b0737802c3c40ee2e00934dd Reviewed-on: https://go-review.googlesource.com/c/147477 Reviewed-by: Brad Fitzpatrick --- src/cmd/addr2line/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/addr2line/main.go b/src/cmd/addr2line/main.go index 267f4170a8..018802940b 100644 --- a/src/cmd/addr2line/main.go +++ b/src/cmd/addr2line/main.go @@ -61,6 +61,7 @@ func main() { if err != nil { log.Fatal(err) } + defer f.Close() tab, err := f.PCLineTable() if err != nil { From 3053788cac0343c2fd29806ebc358d2f63976695 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 24 Jul 2017 11:37:59 -0400 Subject: [PATCH 015/111] cmd/trace: add minimum mutator utilization (MMU) plot This adds an endpoint to the trace tool that plots the minimum mutator utilization curve using information on mark assists and GC pauses from the trace. This commit implements a fairly straightforward O(nm) algorithm for computing the MMU (and tests against an even more direct but slower algorithm). Future commits will extend and optimize this algorithm. This should be useful for debugging and understanding mutator utilization issues like #14951, #14812, #18155. #18534, #21107, particularly once follow-up CLs add trace cross-referencing. Change-Id: Ic2866869e7da1e6c56ba3e809abbcb2eb9c4923a Reviewed-on: https://go-review.googlesource.com/c/60790 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/main.go | 1 + src/cmd/trace/mmu.go | 160 ++++++++++++++++++ src/internal/traceparser/gc.go | 248 ++++++++++++++++++++++++++++ src/internal/traceparser/gc_test.go | 158 ++++++++++++++++++ 4 files changed, 567 insertions(+) create mode 100644 src/cmd/trace/mmu.go create mode 100644 src/internal/traceparser/gc.go create mode 100644 src/internal/traceparser/gc_test.go diff --git a/src/cmd/trace/main.go b/src/cmd/trace/main.go index a33d2f4679..f6ec38d673 100644 --- a/src/cmd/trace/main.go +++ b/src/cmd/trace/main.go @@ -202,6 +202,7 @@ var templMain = template.Must(template.New("").Parse(` Scheduler latency profile (⬇)
User-defined tasks
User-defined regions
+Minimum mutator utilization
`)) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go new file mode 100644 index 0000000000..cc14025d38 --- /dev/null +++ b/src/cmd/trace/mmu.go @@ -0,0 +1,160 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimum mutator utilization (MMU) graphing. + +package main + +import ( + "encoding/json" + "fmt" + trace "internal/traceparser" + "log" + "math" + "net/http" + "strings" + "sync" + "time" +) + +func init() { + http.HandleFunc("/mmu", httpMMU) + http.HandleFunc("/mmuPlot", httpMMUPlot) +} + +var mmuCache struct { + init sync.Once + util []trace.MutatorUtil + mmuCurve *trace.MMUCurve + err error +} + +func getMMUCurve() ([]trace.MutatorUtil, *trace.MMUCurve, error) { + mmuCache.init.Do(func() { + tr, err := parseTrace() + if err != nil { + mmuCache.err = err + } else { + mmuCache.util = tr.MutatorUtilization() + mmuCache.mmuCurve = trace.NewMMUCurve(mmuCache.util) + } + }) + return mmuCache.util, mmuCache.mmuCurve, mmuCache.err +} + +// httpMMU serves the MMU plot page. +func httpMMU(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "", time.Time{}, strings.NewReader(templMMU)) +} + +// httpMMUPlot serves the JSON data for the MMU plot. +func httpMMUPlot(w http.ResponseWriter, r *http.Request) { + mu, mmuCurve, err := getMMUCurve() + if err != nil { + http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError) + return + } + + // Find a nice starting point for the plot. + xMin := time.Second + for xMin > 1 { + if mmu := mmuCurve.MMU(xMin); mmu < 0.0001 { + break + } + xMin /= 1000 + } + // Cover six orders of magnitude. + xMax := xMin * 1e6 + // But no more than the length of the trace. + if maxMax := time.Duration(mu[len(mu)-1].Time - mu[0].Time); xMax > maxMax { + xMax = maxMax + } + // Compute MMU curve. + logMin, logMax := math.Log(float64(xMin)), math.Log(float64(xMax)) + const samples = 100 + plot := make([][2]float64, samples) + for i := 0; i < samples; i++ { + window := time.Duration(math.Exp(float64(i)/(samples-1)*(logMax-logMin) + logMin)) + y := mmuCurve.MMU(window) + plot[i] = [2]float64{float64(window), y} + } + + // Create JSON response. + err = json.NewEncoder(w).Encode(map[string]interface{}{"xMin": int64(xMin), "xMax": int64(xMax), "curve": plot}) + if err != nil { + log.Printf("failed to serialize response: %v", err) + return + } +} + +var templMMU = ` + + + + + + + + +
Loading plot...
+ + +` diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go new file mode 100644 index 0000000000..7e349308d7 --- /dev/null +++ b/src/internal/traceparser/gc.go @@ -0,0 +1,248 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package traceparser + +import ( + "strings" + "time" +) + +// MutatorUtil is a change in mutator utilization at a particular +// time. Mutator utilization functions are represented as a +// time-ordered []MutatorUtil. +type MutatorUtil struct { + Time int64 + // Util is the mean mutator utilization starting at Time. This + // is in the range [0, 1]. + Util float64 +} + +// MutatorUtilization returns the mutator utilization function for the +// given trace. This function will always end with 0 utilization. The +// bounds of the function are implicit in the first and last event; +// outside of these bounds the function is undefined. +func (p *Parsed) MutatorUtilization() []MutatorUtil { + events := p.Events + if len(events) == 0 { + return nil + } + + gomaxprocs, gcPs, stw := 1, 0, 0 + out := []MutatorUtil{{events[0].Ts, 1}} + assists := map[uint64]bool{} + block := map[uint64]*Event{} + bgMark := map[uint64]bool{} + for _, ev := range events { + switch ev.Type { + case EvGomaxprocs: + gomaxprocs = int(ev.Args[0]) + case EvGCSTWStart: + stw++ + case EvGCSTWDone: + stw-- + case EvGCMarkAssistStart: + gcPs++ + assists[ev.G] = true + case EvGCMarkAssistDone: + gcPs-- + delete(assists, ev.G) + case EvGoStartLabel: + if strings.HasPrefix(ev.SArgs[0], "GC ") && ev.SArgs[0] != "GC (idle)" { + // Background mark worker. + bgMark[ev.G] = true + gcPs++ + } + fallthrough + case EvGoStart: + if assists[ev.G] { + // Unblocked during assist. + gcPs++ + } + block[ev.G] = ev.Link + default: + if ev != block[ev.G] { + continue + } + + if assists[ev.G] { + // Blocked during assist. + gcPs-- + } + if bgMark[ev.G] { + // Background mark worker done. + gcPs-- + delete(bgMark, ev.G) + } + delete(block, ev.G) + } + + ps := gcPs + if stw > 0 { + ps = gomaxprocs + } + mu := MutatorUtil{ev.Ts, 1 - float64(ps)/float64(gomaxprocs)} + if mu.Util == out[len(out)-1].Util { + // No change. + continue + } + if mu.Time == out[len(out)-1].Time { + // Take the lowest utilization at a time stamp. + if mu.Util < out[len(out)-1].Util { + out[len(out)-1] = mu + } + } else { + out = append(out, mu) + } + } + + // Add final 0 utilization event. This is important to mark + // the end of the trace. The exact value shouldn't matter + // since no window should extend beyond this, but using 0 is + // symmetric with the start of the trace. + endTime := events[len(events)-1].Ts + if out[len(out)-1].Time == endTime { + out[len(out)-1].Util = 0 + } else { + out = append(out, MutatorUtil{endTime, 0}) + } + + return out +} + +// totalUtil is total utilization, measured in nanoseconds. This is a +// separate type primarily to distinguish it from mean utilization, +// which is also a float64. +type totalUtil float64 + +func totalUtilOf(meanUtil float64, dur int64) totalUtil { + return totalUtil(meanUtil * float64(dur)) +} + +// mean returns the mean utilization over dur. +func (u totalUtil) mean(dur time.Duration) float64 { + return float64(u) / float64(dur) +} + +// An MMUCurve is the minimum mutator utilization curve across +// multiple window sizes. +type MMUCurve struct { + util []MutatorUtil + // sums[j] is the cumulative sum of util[:j]. + sums []totalUtil +} + +// NewMMUCurve returns an MMU curve for the given mutator utilization +// function. +func NewMMUCurve(util []MutatorUtil) *MMUCurve { + // Compute cumulative sum. + sums := make([]totalUtil, len(util)) + var prev MutatorUtil + var sum totalUtil + for j, u := range util { + sum += totalUtilOf(prev.Util, u.Time-prev.Time) + sums[j] = sum + prev = u + } + + return &MMUCurve{util, sums} +} + +// MMU returns the minimum mutator utilization for the given time +// window. This is the minimum utilization for all windows of this +// duration across the execution. The returned value is in the range +// [0, 1]. +func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { + if window <= 0 { + return 0 + } + util := c.util + if max := time.Duration(util[len(util)-1].Time - util[0].Time); window > max { + window = max + } + + mmu = 1.0 + + // We think of the mutator utilization over time as the + // box-filtered utilization function, which we call the + // "windowed mutator utilization function". The resulting + // function is continuous and piecewise linear (unless + // window==0, which we handle elsewhere), where the boundaries + // between segments occur when either edge of the window + // encounters a change in the instantaneous mutator + // utilization function. Hence, the minimum of this function + // will always occur when one of the edges of the window + // aligns with a utilization change, so these are the only + // points we need to consider. + // + // We compute the mutator utilization function incrementally + // by tracking the integral from t=0 to the left edge of the + // window and to the right edge of the window. + left := integrator{c, 0} + right := left + time := util[0].Time + for { + // Advance edges to time and time+window. + mu := (right.advance(time+int64(window)) - left.advance(time)).mean(window) + if mu < mmu { + mmu = mu + if mmu == 0 { + // The minimum can't go any lower than + // zero, so stop early. + break + } + } + + // Advance the window to the next time where either + // the left or right edge of the window encounters a + // change in the utilization curve. + if t1, t2 := left.next(time), right.next(time+int64(window))-int64(window); t1 < t2 { + time = t1 + } else { + time = t2 + } + if time > util[len(util)-1].Time-int64(window) { + break + } + } + return mmu +} + +// An integrator tracks a position in a utilization function and +// integrates it. +type integrator struct { + u *MMUCurve + // pos is the index in u.util of the current time's non-strict + // predecessor. + pos int +} + +// advance returns the integral of the utilization function from 0 to +// time. advance must be called on monotonically increasing values of +// times. +func (in *integrator) advance(time int64) totalUtil { + util, pos := in.u.util, in.pos + // Advance pos until pos+1 is time's strict successor (making + // pos time's non-strict predecessor). + for pos+1 < len(util) && util[pos+1].Time <= time { + pos++ + } + in.pos = pos + var partial totalUtil + if time != util[pos].Time { + partial = totalUtilOf(util[pos].Util, time-util[pos].Time) + } + return in.u.sums[pos] + partial +} + +// next returns the smallest time t' > time of a change in the +// utilization function. +func (in *integrator) next(time int64) int64 { + for _, u := range in.u.util[in.pos:] { + if u.Time > time { + return u.Time + } + } + return 1<<63 - 1 +} diff --git a/src/internal/traceparser/gc_test.go b/src/internal/traceparser/gc_test.go new file mode 100644 index 0000000000..821b0f217c --- /dev/null +++ b/src/internal/traceparser/gc_test.go @@ -0,0 +1,158 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package traceparser + +import ( + "math" + "testing" + "time" +) + +// aeq returns true if x and y are equal up to 8 digits (1 part in 100 +// million). +func aeq(x, y float64) bool { + if x < 0 && y < 0 { + x, y = -x, -y + } + const digits = 8 + factor := 1 - math.Pow(10, -digits+1) + return x*factor <= y && y*factor <= x +} + +func TestMMU(t *testing.T) { + t.Parallel() + + // MU + // 1.0 ***** ***** ***** + // 0.5 * * * * + // 0.0 ***** ***** + // 0 1 2 3 4 5 + util := []MutatorUtil{ + {0e9, 1}, + {1e9, 0}, + {2e9, 1}, + {3e9, 0}, + {4e9, 1}, + {5e9, 0}, + } + mmuCurve := NewMMUCurve(util) + + for _, test := range []struct { + window time.Duration + want float64 + }{ + {0, 0}, + {time.Millisecond, 0}, + {time.Second, 0}, + {2 * time.Second, 0.5}, + {3 * time.Second, 1 / 3.0}, + {4 * time.Second, 0.5}, + {5 * time.Second, 3 / 5.0}, + {6 * time.Second, 3 / 5.0}, + } { + if got := mmuCurve.MMU(test.window); !aeq(test.want, got) { + t.Errorf("for %s window, want mu = %f, got %f", test.window, test.want, got) + } + } +} + +func TestMMUTrace(t *testing.T) { + t.Parallel() + + p, err := New("../trace/testdata/stress_1_10_good") + if err != nil { + t.Fatalf("failed to read input file: %v", err) + } + if err := p.Parse(0, 1<<62, nil); err != nil { + t.Fatalf("failed to parse trace: %s", err) + } + mu := p.MutatorUtilization() + mmuCurve := NewMMUCurve(mu) + + // Test the optimized implementation against the "obviously + // correct" implementation. + for window := time.Nanosecond; window < 10*time.Second; window *= 10 { + want := mmuSlow(mu, window) + got := mmuCurve.MMU(window) + if !aeq(want, got) { + t.Errorf("want %f, got %f mutator utilization in window %s", want, got, window) + } + } +} + +func BenchmarkMMU(b *testing.B) { + p, err := New("../trace/testdata/stress_1_10_good") + if err != nil { + b.Fatalf("failed to read input file: %v", err) + } + if err := p.Parse(0, 1<<62, nil); err != nil { + b.Fatalf("failed to parse trace: %s", err) + } + mu := p.MutatorUtilization() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + mmuCurve := NewMMUCurve(mu) + xMin, xMax := time.Microsecond, time.Second + logMin, logMax := math.Log(float64(xMin)), math.Log(float64(xMax)) + const samples = 100 + for i := 0; i < samples; i++ { + window := time.Duration(math.Exp(float64(i)/(samples-1)*(logMax-logMin) + logMin)) + mmuCurve.MMU(window) + } + } +} + +func mmuSlow(util []MutatorUtil, window time.Duration) (mmu float64) { + if max := time.Duration(util[len(util)-1].Time - util[0].Time); window > max { + window = max + } + + mmu = 1.0 + + // muInWindow returns the mean mutator utilization between + // util[0].Time and end. + muInWindow := func(util []MutatorUtil, end int64) float64 { + total := 0.0 + var prevU MutatorUtil + for _, u := range util { + if u.Time > end { + total += prevU.Util * float64(end-prevU.Time) + break + } + total += prevU.Util * float64(u.Time-prevU.Time) + prevU = u + } + return total / float64(end-util[0].Time) + } + update := func() { + for i, u := range util { + if u.Time+int64(window) > util[len(util)-1].Time { + break + } + mmu = math.Min(mmu, muInWindow(util[i:], u.Time+int64(window))) + } + } + + // Consider all left-aligned windows. + update() + // Reverse the trace. Slightly subtle because each MutatorUtil + // is a *change*. + rutil := make([]MutatorUtil, len(util)) + if util[len(util)-1].Util != 0 { + panic("irreversible trace") + } + for i, u := range util { + util1 := 0.0 + if i != 0 { + util1 = util[i-1].Util + } + rutil[len(rutil)-i-1] = MutatorUtil{Time: -u.Time, Util: util1} + } + util = rutil + // Consider all right-aligned windows. + update() + return +} From c6c602a92612a855d8eb2f649f3dff75bb5fb9ad Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 28 Aug 2017 12:29:07 -0400 Subject: [PATCH 016/111] internal/trace: use MU slope to optimize MMU This commit speeds up MMU construction by ~10X (and reduces the number of windows considered by ~20X) by using an observation about the maximum slope of the windowed mutator utilization function to advance the window time in jumps if the window's current mean mutator utilization is much larger than the current minimum. Change-Id: If3cba5da0c4adc37b568740f940793e491e96a51 Reviewed-on: https://go-review.googlesource.com/c/60791 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/internal/traceparser/gc.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index 7e349308d7..66c68cb450 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -194,6 +194,12 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { } } + // The maximum slope of the windowed mutator + // utilization function is 1/window, so we can always + // advance the time by at least (mu - mmu) * window + // without dropping below mmu. + minTime := time + int64((mu-mmu)*float64(window)) + // Advance the window to the next time where either // the left or right edge of the window encounters a // change in the utilization curve. @@ -202,6 +208,9 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { } else { time = t2 } + if time < minTime { + time = minTime + } if time > util[len(util)-1].Time-int64(window) { break } @@ -225,8 +234,28 @@ func (in *integrator) advance(time int64) totalUtil { util, pos := in.u.util, in.pos // Advance pos until pos+1 is time's strict successor (making // pos time's non-strict predecessor). - for pos+1 < len(util) && util[pos+1].Time <= time { - pos++ + // + // Very often, this will be nearby, so we optimize that case, + // but it may be arbitrarily far away, so we handled that + // efficiently, too. + const maxSeq = 8 + if pos+maxSeq < len(util) && util[pos+maxSeq].Time > time { + // Nearby. Use a linear scan. + for pos+1 < len(util) && util[pos+1].Time <= time { + pos++ + } + } else { + // Far. Binary search for time's strict successor. + l, r := pos, len(util) + for l < r { + h := int(uint(l+r) >> 1) + if util[h].Time <= time { + l = h + 1 + } else { + r = h + } + } + pos = l - 1 // Non-strict predecessor. } in.pos = pos var partial totalUtil From 52ee654b25970a0b8c0ce9d2c8eda604df2026bb Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 8 Aug 2017 18:15:32 -0400 Subject: [PATCH 017/111] internal/trace: use banding to optimize MMU computation This further optimizes MMU construction by first computing a low-resolution summary of the utilization curve. This "band" summary lets us compute the worst-possible window starting in each of these low-resolution bands (even without knowing where in the band the window falls). This in turn lets us compute precise minimum mutator utilization only in the worst low-resolution bands until we can show that any remaining bands can't possibly contain a worse window. This slows down MMU construction for small traces, but these are reasonably fast to compute either way. For large traces (e.g., 150,000+ utilization changes) it's significantly faster. Change-Id: Ie66454e71f3fb06be3f6173b6d91ad75c61bda48 Reviewed-on: https://go-review.googlesource.com/c/60792 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/go/build/deps_test.go | 2 +- src/internal/traceparser/gc.go | 182 ++++++++++++++++++++++++++++++++- 2 files changed, 179 insertions(+), 5 deletions(-) diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index ec6e6b4890..d632954d0c 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -273,7 +273,7 @@ var pkgDeps = map[string][]string{ "internal/goroot": {"L4", "OS"}, "internal/singleflight": {"sync"}, "internal/trace": {"L4", "OS"}, - "internal/traceparser": {"L4", "internal/traceparser/filebuf"}, + "internal/traceparser": {"L4", "internal/traceparser/filebuf", "container/heap"}, "internal/traceparser/filebuf": {"L4", "OS"}, "math/big": {"L4"}, "mime": {"L4", "OS", "syscall", "internal/syscall/windows/registry"}, diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index 66c68cb450..0be78e71e3 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -5,6 +5,8 @@ package traceparser import ( + "container/heap" + "math" "strings" "time" ) @@ -131,6 +133,24 @@ type MMUCurve struct { util []MutatorUtil // sums[j] is the cumulative sum of util[:j]. sums []totalUtil + // bands summarizes util in non-overlapping bands of duration + // bandDur. + bands []mmuBand + // bandDur is the duration of each band. + bandDur int64 +} + +type mmuBand struct { + // minUtil is the minimum instantaneous mutator utilization in + // this band. + minUtil float64 + // cumUtil is the cumulative total mutator utilization between + // time 0 and the left edge of this band. + cumUtil totalUtil + + // integrator is the integrator for the left edge of this + // band. + integrator integrator } // NewMMUCurve returns an MMU curve for the given mutator utilization @@ -146,7 +166,77 @@ func NewMMUCurve(util []MutatorUtil) *MMUCurve { prev = u } - return &MMUCurve{util, sums} + // Divide the utilization curve up into equal size + // non-overlapping "bands" and compute a summary for each of + // these bands. + // + // Compute the duration of each band. + numBands := 1000 + if numBands > len(util) { + // There's no point in having lots of bands if there + // aren't many events. + numBands = len(util) + } + dur := util[len(util)-1].Time - util[0].Time + bandDur := (dur + int64(numBands) - 1) / int64(numBands) + if bandDur < 1 { + bandDur = 1 + } + // Compute the bands. There are numBands+1 bands in order to + // record the final cumulative sum. + bands := make([]mmuBand, numBands+1) + c := MMUCurve{util, sums, bands, bandDur} + leftSum := integrator{&c, 0} + for i := range bands { + startTime, endTime := c.bandTime(i) + cumUtil := leftSum.advance(startTime) + predIdx := leftSum.pos + minUtil := 1.0 + for i := predIdx; i < len(util) && util[i].Time < endTime; i++ { + minUtil = math.Min(minUtil, util[i].Util) + } + bands[i] = mmuBand{minUtil, cumUtil, leftSum} + } + + return &c +} + +func (c *MMUCurve) bandTime(i int) (start, end int64) { + start = int64(i)*c.bandDur + c.util[0].Time + end = start + c.bandDur + return +} + +type bandUtil struct { + // Band index + i int + // Lower bound of mutator utilization for all windows + // with a left edge in this band. + utilBound float64 +} + +type bandUtilHeap []bandUtil + +func (h bandUtilHeap) Len() int { + return len(h) +} + +func (h bandUtilHeap) Less(i, j int) bool { + return h[i].utilBound < h[j].utilBound +} + +func (h bandUtilHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *bandUtilHeap) Push(x interface{}) { + *h = append(*h, x.(bandUtil)) +} + +func (h *bandUtilHeap) Pop() interface{} { + x := (*h)[len(*h)-1] + *h = (*h)[:len(*h)-1] + return x } // MMU returns the minimum mutator utilization for the given time @@ -162,7 +252,88 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { window = max } + bandU := bandUtilHeap(c.mkBandUtil(window)) + + // Process bands from lowest utilization bound to highest. + heap.Init(&bandU) + + // Refine each band into a precise window and MMU until the + // precise MMU is less than the lowest band bound. mmu = 1.0 + for len(bandU) > 0 && bandU[0].utilBound < mmu { + mmu = c.bandMMU(bandU[0].i, window, mmu) + heap.Pop(&bandU) + } + return mmu +} + +func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { + // For each band, compute the worst-possible total mutator + // utilization for all windows that start in that band. + + // minBands is the minimum number of bands a window can span + // and maxBands is the maximum number of bands a window can + // span in any alignment. + minBands := int((int64(window) + c.bandDur - 1) / c.bandDur) + maxBands := int((int64(window) + 2*(c.bandDur-1)) / c.bandDur) + if window > 1 && maxBands < 2 { + panic("maxBands < 2") + } + tailDur := int64(window) % c.bandDur + nUtil := len(c.bands) - maxBands + 1 + if nUtil < 0 { + nUtil = 0 + } + bandU := make([]bandUtil, nUtil) + for i := range bandU { + // To compute the worst-case MU, we assume the minimum + // for any bands that are only partially overlapped by + // some window and the mean for any bands that are + // completely covered by all windows. + var util totalUtil + + // Find the lowest and second lowest of the partial + // bands. + l := c.bands[i].minUtil + r1 := c.bands[i+minBands-1].minUtil + r2 := c.bands[i+maxBands-1].minUtil + minBand := math.Min(l, math.Min(r1, r2)) + // Assume the worst window maximally overlaps the + // worst minimum and then the rest overlaps the second + // worst minimum. + if minBands == 1 { + util += totalUtilOf(minBand, int64(window)) + } else { + util += totalUtilOf(minBand, c.bandDur) + midBand := 0.0 + switch { + case minBand == l: + midBand = math.Min(r1, r2) + case minBand == r1: + midBand = math.Min(l, r2) + case minBand == r2: + midBand = math.Min(l, r1) + } + util += totalUtilOf(midBand, tailDur) + } + + // Add the total mean MU of bands that are completely + // overlapped by all windows. + if minBands > 2 { + util += c.bands[i+minBands-1].cumUtil - c.bands[i+1].cumUtil + } + + bandU[i] = bandUtil{i, util.mean(window)} + } + + return bandU +} + +// bandMMU computes the precise minimum mutator utilization for +// windows with a left edge in band bandIdx. +func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, curMMU float64) (mmu float64) { + util := c.util + mmu = curMMU // We think of the mutator utilization over time as the // box-filtered utilization function, which we call the @@ -179,9 +350,12 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { // We compute the mutator utilization function incrementally // by tracking the integral from t=0 to the left edge of the // window and to the right edge of the window. - left := integrator{c, 0} + left := c.bands[bandIdx].integrator right := left - time := util[0].Time + time, endTime := c.bandTime(bandIdx) + if utilEnd := util[len(util)-1].Time - int64(window); utilEnd < endTime { + endTime = utilEnd + } for { // Advance edges to time and time+window. mu := (right.advance(time+int64(window)) - left.advance(time)).mean(window) @@ -211,7 +385,7 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { if time < minTime { time = minTime } - if time > util[len(util)-1].Time-int64(window) { + if time >= endTime { break } } From cbe04e8d70c32b9477ebcde5265bc17cc41af4a7 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 26 Jul 2017 12:21:15 -0400 Subject: [PATCH 018/111] internal/trace: track worst N mutator utilization windows This will let the trace viewer show specifically when poor utilization happened and link to specific instances in the trace. Change-Id: I1f03a0f9d9a7570009bb15762e7b8b6f215e9423 Reviewed-on: https://go-review.googlesource.com/c/60793 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/internal/traceparser/gc.go | 151 ++++++++++++++++++++++++---- src/internal/traceparser/gc_test.go | 32 ++++-- 2 files changed, 157 insertions(+), 26 deletions(-) diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index 0be78e71e3..313e23edf6 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -7,6 +7,7 @@ package traceparser import ( "container/heap" "math" + "sort" "strings" "time" ) @@ -239,13 +240,135 @@ func (h *bandUtilHeap) Pop() interface{} { return x } +// UtilWindow is a specific window at Time. +type UtilWindow struct { + Time int64 + // MutatorUtil is the mean mutator utilization in this window. + MutatorUtil float64 +} + +type utilHeap []UtilWindow + +func (h utilHeap) Len() int { + return len(h) +} + +func (h utilHeap) Less(i, j int) bool { + if h[i].MutatorUtil != h[j].MutatorUtil { + return h[i].MutatorUtil > h[j].MutatorUtil + } + return h[i].Time > h[j].Time +} + +func (h utilHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *utilHeap) Push(x interface{}) { + *h = append(*h, x.(UtilWindow)) +} + +func (h *utilHeap) Pop() interface{} { + x := (*h)[len(*h)-1] + *h = (*h)[:len(*h)-1] + return x +} + +// An accumulator collects different MMU-related statistics depending +// on what's desired. +type accumulator struct { + mmu float64 + + // bound is the mutator utilization bound where adding any + // mutator utilization above this bound cannot affect the + // accumulated statistics. + bound float64 + + // Worst N window tracking + nWorst int + wHeap utilHeap +} + +// addMU records mutator utilization mu over the given window starting +// at time. +// +// It returns true if further calls to addMU would be pointless. +func (acc *accumulator) addMU(time int64, mu float64, window time.Duration) bool { + if mu < acc.mmu { + acc.mmu = mu + } + acc.bound = acc.mmu + + if acc.nWorst == 0 { + // If the minimum has reached zero, it can't go any + // lower, so we can stop early. + return mu == 0 + } + + // Consider adding this window to the n worst. + if len(acc.wHeap) < acc.nWorst || mu < acc.wHeap[0].MutatorUtil { + // This window is lower than the K'th worst window. + // + // Check if there's any overlapping window + // already in the heap and keep whichever is + // worse. + for i, ui := range acc.wHeap { + if time+int64(window) > ui.Time && ui.Time+int64(window) > time { + if ui.MutatorUtil <= mu { + // Keep the first window. + goto keep + } else { + // Replace it with this window. + heap.Remove(&acc.wHeap, i) + break + } + } + } + + heap.Push(&acc.wHeap, UtilWindow{time, mu}) + if len(acc.wHeap) > acc.nWorst { + heap.Pop(&acc.wHeap) + } + keep: + } + if len(acc.wHeap) < acc.nWorst { + // We don't have N windows yet, so keep accumulating. + acc.bound = 1.0 + } else { + // Anything above the least worst window has no effect. + acc.bound = math.Max(acc.bound, acc.wHeap[0].MutatorUtil) + } + + // If we've found enough 0 utilizations, we can stop immediately. + return len(acc.wHeap) == acc.nWorst && acc.wHeap[0].MutatorUtil == 0 +} + // MMU returns the minimum mutator utilization for the given time // window. This is the minimum utilization for all windows of this // duration across the execution. The returned value is in the range // [0, 1]. func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { + acc := accumulator{mmu: 1.0, bound: 1.0} + c.mmu(window, &acc) + return acc.mmu +} + +// Examples returns n specific examples of the lowest mutator +// utilization for the given window size. The returned windows will be +// disjoint (otherwise there would be a huge number of +// mostly-overlapping windows at the single lowest point). There are +// no guarantees on which set of disjoint windows this returns. +func (c *MMUCurve) Examples(window time.Duration, n int) (worst []UtilWindow) { + acc := accumulator{mmu: 1.0, bound: 1.0, nWorst: n} + c.mmu(window, &acc) + sort.Sort(sort.Reverse(acc.wHeap)) + return ([]UtilWindow)(acc.wHeap) +} + +func (c *MMUCurve) mmu(window time.Duration, acc *accumulator) { if window <= 0 { - return 0 + acc.mmu = 0 + return } util := c.util if max := time.Duration(util[len(util)-1].Time - util[0].Time); window > max { @@ -257,14 +380,13 @@ func (c *MMUCurve) MMU(window time.Duration) (mmu float64) { // Process bands from lowest utilization bound to highest. heap.Init(&bandU) - // Refine each band into a precise window and MMU until the - // precise MMU is less than the lowest band bound. - mmu = 1.0 - for len(bandU) > 0 && bandU[0].utilBound < mmu { - mmu = c.bandMMU(bandU[0].i, window, mmu) + // Refine each band into a precise window and MMU until + // refining the next lowest band can no longer affect the MMU + // or windows. + for len(bandU) > 0 && bandU[0].utilBound < acc.bound { + c.bandMMU(bandU[0].i, window, acc) heap.Pop(&bandU) } - return mmu } func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { @@ -331,9 +453,8 @@ func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { // bandMMU computes the precise minimum mutator utilization for // windows with a left edge in band bandIdx. -func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, curMMU float64) (mmu float64) { +func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, acc *accumulator) { util := c.util - mmu = curMMU // We think of the mutator utilization over time as the // box-filtered utilization function, which we call the @@ -359,20 +480,15 @@ func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, curMMU float64) (m for { // Advance edges to time and time+window. mu := (right.advance(time+int64(window)) - left.advance(time)).mean(window) - if mu < mmu { - mmu = mu - if mmu == 0 { - // The minimum can't go any lower than - // zero, so stop early. - break - } + if acc.addMU(time, mu, window) { + break } // The maximum slope of the windowed mutator // utilization function is 1/window, so we can always // advance the time by at least (mu - mmu) * window // without dropping below mmu. - minTime := time + int64((mu-mmu)*float64(window)) + minTime := time + int64((mu-acc.bound)*float64(window)) // Advance the window to the next time where either // the left or right edge of the window encounters a @@ -389,7 +505,6 @@ func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, curMMU float64) (m break } } - return mmu } // An integrator tracks a position in a utilization function and diff --git a/src/internal/traceparser/gc_test.go b/src/internal/traceparser/gc_test.go index 821b0f217c..65772be717 100644 --- a/src/internal/traceparser/gc_test.go +++ b/src/internal/traceparser/gc_test.go @@ -42,19 +42,35 @@ func TestMMU(t *testing.T) { for _, test := range []struct { window time.Duration want float64 + worst []float64 }{ - {0, 0}, - {time.Millisecond, 0}, - {time.Second, 0}, - {2 * time.Second, 0.5}, - {3 * time.Second, 1 / 3.0}, - {4 * time.Second, 0.5}, - {5 * time.Second, 3 / 5.0}, - {6 * time.Second, 3 / 5.0}, + {0, 0, []float64{}}, + {time.Millisecond, 0, []float64{0, 0}}, + {time.Second, 0, []float64{0, 0}}, + {2 * time.Second, 0.5, []float64{0.5, 0.5}}, + {3 * time.Second, 1 / 3.0, []float64{1 / 3.0}}, + {4 * time.Second, 0.5, []float64{0.5}}, + {5 * time.Second, 3 / 5.0, []float64{3 / 5.0}}, + {6 * time.Second, 3 / 5.0, []float64{3 / 5.0}}, } { if got := mmuCurve.MMU(test.window); !aeq(test.want, got) { t.Errorf("for %s window, want mu = %f, got %f", test.window, test.want, got) } + worst := mmuCurve.Examples(test.window, 2) + // Which exact windows are returned is unspecified + // (and depends on the exact banding), so we just + // check that we got the right number with the right + // utilizations. + if len(worst) != len(test.worst) { + t.Errorf("for %s window, want worst %v, got %v", test.window, test.worst, worst) + } else { + for i := range worst { + if worst[i].MutatorUtil != test.worst[i] { + t.Errorf("for %s window, want worst %v, got %v", test.window, test.worst, worst) + break + } + } + } } } From 603af813d6b93cf734b67551c2e776a1417e4603 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Jul 2017 10:24:39 -0400 Subject: [PATCH 019/111] cmd/trace: list and link to worst mutator utilization windows This adds the ability to click a point on the MMU graph to show a list of the worst 10 mutator utilization windows of the selected size. This list in turn links to the trace viewer to drill down on specifically what happened in each specific window. Change-Id: Ic1b72d8b37fbf2212211c513cf36b34788b30133 Reviewed-on: https://go-review.googlesource.com/c/60795 Run-TryBot: Austin Clements Reviewed-by: Hyang-Ah Hana Kim TryBot-Result: Gobot Gobot --- src/cmd/trace/main.go | 2 +- src/cmd/trace/mmu.go | 81 ++++++++++++++++++++++++++++++++++++++++-- src/cmd/trace/trace.go | 30 +++++++++++----- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/src/cmd/trace/main.go b/src/cmd/trace/main.go index f6ec38d673..2f71a3d4bd 100644 --- a/src/cmd/trace/main.go +++ b/src/cmd/trace/main.go @@ -189,7 +189,7 @@ var templMain = template.Must(template.New("").Parse(` {{if $}} {{range $e := $}} - View trace ({{$e.Name}})
+ View trace ({{$e.Name}})
{{end}}
{{else}} diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index cc14025d38..f76e0d0e5f 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -13,6 +13,7 @@ import ( "log" "math" "net/http" + "strconv" "strings" "sync" "time" @@ -21,6 +22,7 @@ import ( func init() { http.HandleFunc("/mmu", httpMMU) http.HandleFunc("/mmuPlot", httpMMUPlot) + http.HandleFunc("/mmuDetails", httpMMUDetails) } var mmuCache struct { @@ -98,6 +100,9 @@ var templMMU = ` google.charts.load('current', {'packages':['corechart']}); google.charts.setOnLoadCallback(refreshChart); + var chart; + var curve; + function niceDuration(ns) { if (ns < 1e3) { return ns + 'ns'; } else if (ns < 1e6) { return ns / 1e3 + 'µs'; } @@ -114,7 +119,7 @@ var templMMU = ` } function drawChart(plotData) { - var curve = plotData.curve; + curve = plotData.curve; var data = new google.visualization.DataTable(); data.addColumn('number', 'Window duration'); data.addColumn('number', 'Minimum mutator utilization'); @@ -148,13 +153,85 @@ var templMMU = ` var container = $('#mmu_chart'); container.empty(); - var chart = new google.visualization.LineChart(container[0]); + chart = new google.visualization.LineChart(container[0]); + chart = new google.visualization.LineChart(document.getElementById('mmu_chart')); chart.draw(data, options); + + google.visualization.events.addListener(chart, 'select', selectHandler); + } + + function selectHandler() { + var items = chart.getSelection(); + if (items.length === 0) { + return; + } + var details = $('#details'); + details.empty(); + var windowNS = curve[items[0].row][0]; + var url = '/mmuDetails?window=' + windowNS; + $.getJSON(url) + .fail(function(xhr, status, error) { + details.text(status + ': ' + url + ' could not be loaded'); + }) + .done(function(worst) { + details.text('Lowest mutator utilization in ' + niceDuration(windowNS) + ' windows:'); + for (var i = 0; i < worst.length; i++) { + details.append($('
')); + var text = worst[i].MutatorUtil.toFixed(3) + ' at time ' + niceDuration(worst[i].Time); + details.append($('').text(text).attr('href', worst[i].URL)); + } + }); }
Loading plot...
+
Select a point for details.
` + +// httpMMUDetails serves details of an MMU graph at a particular window. +func httpMMUDetails(w http.ResponseWriter, r *http.Request) { + _, mmuCurve, err := getMMUCurve() + if err != nil { + http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError) + return + } + + windowStr := r.FormValue("window") + window, err := strconv.ParseUint(windowStr, 10, 64) + if err != nil { + http.Error(w, fmt.Sprintf("failed to parse window parameter %q: %v", windowStr, err), http.StatusBadRequest) + return + } + worst := mmuCurve.Examples(time.Duration(window), 10) + + // Construct a link for each window. + var links []linkedUtilWindow + for _, ui := range worst { + links = append(links, newLinkedUtilWindow(ui, time.Duration(window))) + } + + err = json.NewEncoder(w).Encode(links) + if err != nil { + log.Printf("failed to serialize trace: %v", err) + return + } +} + +type linkedUtilWindow struct { + trace.UtilWindow + URL string +} + +func newLinkedUtilWindow(ui trace.UtilWindow, window time.Duration) linkedUtilWindow { + // Find the range containing this window. + var r Range + for _, r = range ranges { + if r.EndTime > ui.Time { + break + } + } + return linkedUtilWindow{ui, fmt.Sprintf("%s#%v:%v", r.URL(), float64(ui.Time)/1e6, float64(ui.Time+int64(window))/1e6)} +} diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go index d0e0acd78c..d467f371fa 100644 --- a/src/cmd/trace/trace.go +++ b/src/cmd/trace/trace.go @@ -272,9 +272,15 @@ func httpJSONTrace(w http.ResponseWriter, r *http.Request) { // Range is a named range type Range struct { - Name string - Start int - End int + Name string + Start int + End int + StartTime int64 + EndTime int64 +} + +func (r Range) URL() string { + return fmt.Sprintf("/trace?start=%d&end=%d", r.Start, r.End) } // splitTrace splits the trace into a number of ranges, @@ -345,10 +351,14 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { start := 0 for i, ev := range sizes { if sum+ev.Sz > max { + startTime := time.Duration(sizes[start].Time * 1000) + endTime := time.Duration(ev.Time * 1000) ranges = append(ranges, Range{ - Name: fmt.Sprintf("%v-%v", time.Duration(sizes[start].Time*1000), time.Duration(ev.Time*1000)), - Start: start, - End: i + 1, + Name: fmt.Sprintf("%v-%v", startTime, endTime), + Start: start, + End: i + 1, + StartTime: int64(startTime), + EndTime: int64(endTime), }) start = i + 1 sum = minSize @@ -363,9 +373,11 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { if end := len(sizes) - 1; start < end { ranges = append(ranges, Range{ - Name: fmt.Sprintf("%v-%v", time.Duration(sizes[start].Time*1000), time.Duration(sizes[end].Time*1000)), - Start: start, - End: end, + Name: fmt.Sprintf("%v-%v", time.Duration(sizes[start].Time*1000), time.Duration(sizes[end].Time*1000)), + Start: start, + End: end, + StartTime: int64(sizes[start].Time * 1000), + EndTime: int64(sizes[end].Time * 1000), }) } s.Ranges = ranges From 27920c8ddc609662540deaf5a3d3b4fce03abeea Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Jul 2017 13:51:58 -0400 Subject: [PATCH 020/111] internal/trace: flags for what to include in GC utilization Change-Id: I4ba963b003cb25b39d7575d423f17930d84f3f69 Reviewed-on: https://go-review.googlesource.com/c/60796 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/mmu.go | 2 +- src/internal/traceparser/gc.go | 48 ++++++++++++++++++++++++----- src/internal/traceparser/gc_test.go | 4 +-- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index f76e0d0e5f..2a07be4ba2 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -38,7 +38,7 @@ func getMMUCurve() ([]trace.MutatorUtil, *trace.MMUCurve, error) { if err != nil { mmuCache.err = err } else { - mmuCache.util = tr.MutatorUtilization() + mmuCache.util = tr.MutatorUtilization(trace.UtilSTW | trace.UtilBackground | trace.UtilAssist) mmuCache.mmuCurve = trace.NewMMUCurve(mmuCache.util) } }) diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index 313e23edf6..ab0c640e26 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -22,11 +22,27 @@ type MutatorUtil struct { Util float64 } +// UtilFlags controls the behavior of MutatorUtilization. +type UtilFlags int + +const ( + // UtilSTW means utilization should account for STW events. + UtilSTW UtilFlags = 1 << iota + // UtilBackground means utilization should account for + // background mark workers. + UtilBackground + // UtilAssist means utilization should account for mark + // assists. + UtilAssist + // UtilSweep means utilization should account for sweeping. + UtilSweep +) + // MutatorUtilization returns the mutator utilization function for the // given trace. This function will always end with 0 utilization. The // bounds of the function are implicit in the first and last event; // outside of these bounds the function is undefined. -func (p *Parsed) MutatorUtilization() []MutatorUtil { +func (p *Parsed) MutatorUtilization(flags UtilFlags) []MutatorUtil { events := p.Events if len(events) == 0 { return nil @@ -42,17 +58,33 @@ func (p *Parsed) MutatorUtilization() []MutatorUtil { case EvGomaxprocs: gomaxprocs = int(ev.Args[0]) case EvGCSTWStart: - stw++ + if flags&UtilSTW != 0 { + stw++ + } case EvGCSTWDone: - stw-- + if flags&UtilSTW != 0 { + stw-- + } case EvGCMarkAssistStart: - gcPs++ - assists[ev.G] = true + if flags&UtilAssist != 0 { + gcPs++ + assists[ev.G] = true + } case EvGCMarkAssistDone: - gcPs-- - delete(assists, ev.G) + if flags&UtilAssist != 0 { + gcPs-- + delete(assists, ev.G) + } + case EvGCSweepStart: + if flags&UtilSweep != 0 { + gcPs++ + } + case EvGCSweepDone: + if flags&UtilSweep != 0 { + gcPs-- + } case EvGoStartLabel: - if strings.HasPrefix(ev.SArgs[0], "GC ") && ev.SArgs[0] != "GC (idle)" { + if flags&UtilBackground != 0 && strings.HasPrefix(ev.SArgs[0], "GC ") && ev.SArgs[0] != "GC (idle)" { // Background mark worker. bgMark[ev.G] = true gcPs++ diff --git a/src/internal/traceparser/gc_test.go b/src/internal/traceparser/gc_test.go index 65772be717..f1416fa9f9 100644 --- a/src/internal/traceparser/gc_test.go +++ b/src/internal/traceparser/gc_test.go @@ -84,7 +84,7 @@ func TestMMUTrace(t *testing.T) { if err := p.Parse(0, 1<<62, nil); err != nil { t.Fatalf("failed to parse trace: %s", err) } - mu := p.MutatorUtilization() + mu := p.MutatorUtilization(UtilSTW | UtilBackground | UtilAssist) mmuCurve := NewMMUCurve(mu) // Test the optimized implementation against the "obviously @@ -106,7 +106,7 @@ func BenchmarkMMU(b *testing.B) { if err := p.Parse(0, 1<<62, nil); err != nil { b.Fatalf("failed to parse trace: %s", err) } - mu := p.MutatorUtilization() + mu := p.MutatorUtilization(UtilSTW | UtilBackground | UtilAssist | UtilSweep) b.ResetTimer() for i := 0; i < b.N; i++ { From bef4efc822794ea2e7310756bc546bf6930fc066 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Jul 2017 16:26:51 -0400 Subject: [PATCH 021/111] internal/trace: add "per-P" MMU analysis The current MMU analysis considers all Ps together, so if, for example, one of four Ps is blocked, mutator utilization is 75%. However, this is less useful for understanding the impact on individual goroutines because that one blocked goroutine could be blocked for a very long time, but we still appear to have good utilization. Hence, this introduces a new flag that does a "per-P" analysis where the utilization of each P is considered independently. The MMU is then the combination of the MMU for each P's utilization function. Change-Id: Id67b980d4d82b511d28300cdf92ccbb5ae8f0c78 Reviewed-on: https://go-review.googlesource.com/c/60797 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/mmu.go | 15 +- src/internal/traceparser/gc.go | 216 ++++++++++++++++++++-------- src/internal/traceparser/gc_test.go | 6 +- 3 files changed, 172 insertions(+), 65 deletions(-) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index 2a07be4ba2..d3b6768686 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -27,12 +27,12 @@ func init() { var mmuCache struct { init sync.Once - util []trace.MutatorUtil + util [][]trace.MutatorUtil mmuCurve *trace.MMUCurve err error } -func getMMUCurve() ([]trace.MutatorUtil, *trace.MMUCurve, error) { +func getMMUCurve() ([][]trace.MutatorUtil, *trace.MMUCurve, error) { mmuCache.init.Do(func() { tr, err := parseTrace() if err != nil { @@ -69,7 +69,16 @@ func httpMMUPlot(w http.ResponseWriter, r *http.Request) { // Cover six orders of magnitude. xMax := xMin * 1e6 // But no more than the length of the trace. - if maxMax := time.Duration(mu[len(mu)-1].Time - mu[0].Time); xMax > maxMax { + minEvent, maxEvent := mu[0][0].Time, mu[0][len(mu[0])-1].Time + for _, mu1 := range mu[1:] { + if mu1[0].Time < minEvent { + minEvent = mu1[0].Time + } + if mu1[len(mu1)-1].Time > maxEvent { + maxEvent = mu1[len(mu1)-1].Time + } + } + if maxMax := time.Duration(maxEvent - minEvent); xMax > maxMax { xMax = maxMax } // Compute MMU curve. diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index ab0c640e26..569ab86b82 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -36,27 +36,64 @@ const ( UtilAssist // UtilSweep means utilization should account for sweeping. UtilSweep + + // UtilPerProc means each P should be given a separate + // utilization function. Otherwise, there is a single function + // and each P is given a fraction of the utilization. + UtilPerProc ) -// MutatorUtilization returns the mutator utilization function for the -// given trace. This function will always end with 0 utilization. The -// bounds of the function are implicit in the first and last event; -// outside of these bounds the function is undefined. -func (p *Parsed) MutatorUtilization(flags UtilFlags) []MutatorUtil { +// MutatorUtilization returns a set of mutator utilization functions +// for the given trace. Each function will always end with 0 +// utilization. The bounds of each function are implicit in the first +// and last event; outside of these bounds each function is undefined. +// +// If the UtilPerProc flag is not given, this always returns a single +// utilization function. Otherwise, it returns one function per P. +func (p *Parsed) MutatorUtilization(flags UtilFlags) [][]MutatorUtil { events := p.Events if len(events) == 0 { return nil } - gomaxprocs, gcPs, stw := 1, 0, 0 - out := []MutatorUtil{{events[0].Ts, 1}} + type perP struct { + // gc > 0 indicates that GC is active on this P. + gc int + // series the logical series number for this P. This + // is necessary because Ps may be removed and then + // re-added, and then the new P needs a new series. + series int + } + ps := []perP{} + stw := 0 + + out := [][]MutatorUtil{} assists := map[uint64]bool{} block := map[uint64]*Event{} bgMark := map[uint64]bool{} + for _, ev := range events { switch ev.Type { case EvGomaxprocs: - gomaxprocs = int(ev.Args[0]) + gomaxprocs := int(ev.Args[0]) + if len(ps) > gomaxprocs { + if flags&UtilPerProc != 0 { + // End each P's series. + for _, p := range ps[gomaxprocs:] { + out[p.series] = addUtil(out[p.series], MutatorUtil{ev.Ts, 0}) + } + } + ps = ps[:gomaxprocs] + } + for len(ps) < gomaxprocs { + // Start new P's series. + series := 0 + if flags&UtilPerProc != 0 || len(out) == 0 { + series = len(out) + out = append(out, []MutatorUtil{{ev.Ts, 1}}) + } + ps = append(ps, perP{series: series}) + } case EvGCSTWStart: if flags&UtilSTW != 0 { stw++ @@ -67,33 +104,41 @@ func (p *Parsed) MutatorUtilization(flags UtilFlags) []MutatorUtil { } case EvGCMarkAssistStart: if flags&UtilAssist != 0 { - gcPs++ + ps[ev.P].gc++ assists[ev.G] = true } case EvGCMarkAssistDone: if flags&UtilAssist != 0 { - gcPs-- + ps[ev.P].gc-- delete(assists, ev.G) } case EvGCSweepStart: if flags&UtilSweep != 0 { - gcPs++ + ps[ev.P].gc++ } case EvGCSweepDone: if flags&UtilSweep != 0 { - gcPs-- + ps[ev.P].gc-- } case EvGoStartLabel: if flags&UtilBackground != 0 && strings.HasPrefix(ev.SArgs[0], "GC ") && ev.SArgs[0] != "GC (idle)" { // Background mark worker. - bgMark[ev.G] = true - gcPs++ + // + // If we're in per-proc mode, we don't + // count dedicated workers because + // they kick all of the goroutines off + // that P, so don't directly + // contribute to goroutine latency. + if !(flags&UtilPerProc != 0 && ev.SArgs[0] == "GC (dedicated)") { + bgMark[ev.G] = true + ps[ev.P].gc++ + } } fallthrough case EvGoStart: if assists[ev.G] { // Unblocked during assist. - gcPs++ + ps[ev.P].gc++ } block[ev.G] = ev.Link default: @@ -103,49 +148,77 @@ func (p *Parsed) MutatorUtilization(flags UtilFlags) []MutatorUtil { if assists[ev.G] { // Blocked during assist. - gcPs-- + ps[ev.P].gc-- } if bgMark[ev.G] { // Background mark worker done. - gcPs-- + ps[ev.P].gc-- delete(bgMark, ev.G) } delete(block, ev.G) } - ps := gcPs - if stw > 0 { - ps = gomaxprocs - } - mu := MutatorUtil{ev.Ts, 1 - float64(ps)/float64(gomaxprocs)} - if mu.Util == out[len(out)-1].Util { - // No change. - continue - } - if mu.Time == out[len(out)-1].Time { - // Take the lowest utilization at a time stamp. - if mu.Util < out[len(out)-1].Util { - out[len(out)-1] = mu + if flags&UtilPerProc == 0 { + // Compute the current average utilization. + if len(ps) == 0 { + continue } + gcPs := 0 + if stw > 0 { + gcPs = len(ps) + } else { + for i := range ps { + if ps[i].gc > 0 { + gcPs++ + } + } + } + mu := MutatorUtil{ev.Ts, 1 - float64(gcPs)/float64(len(ps))} + + // Record the utilization change. (Since + // len(ps) == len(out), we know len(out) > 0.) + out[0] = addUtil(out[0], mu) } else { - out = append(out, mu) + // Check for per-P utilization changes. + for i := range ps { + p := &ps[i] + util := 1.0 + if stw > 0 || p.gc > 0 { + util = 0.0 + } + out[p.series] = addUtil(out[p.series], MutatorUtil{ev.Ts, util}) + } } } - // Add final 0 utilization event. This is important to mark - // the end of the trace. The exact value shouldn't matter - // since no window should extend beyond this, but using 0 is - // symmetric with the start of the trace. - endTime := events[len(events)-1].Ts - if out[len(out)-1].Time == endTime { - out[len(out)-1].Util = 0 - } else { - out = append(out, MutatorUtil{endTime, 0}) + // Add final 0 utilization event to any remaining series. This + // is important to mark the end of the trace. The exact value + // shouldn't matter since no window should extend beyond this, + // but using 0 is symmetric with the start of the trace. + mu := MutatorUtil{events[len(events)-1].Ts, 0} + for i := range ps { + out[ps[i].series] = addUtil(out[ps[i].series], mu) } - return out } +func addUtil(util []MutatorUtil, mu MutatorUtil) []MutatorUtil { + if len(util) > 0 { + if mu.Util == util[len(util)-1].Util { + // No change. + return util + } + if mu.Time == util[len(util)-1].Time { + // Take the lowest utilization at a time stamp. + if mu.Util < util[len(util)-1].Util { + util[len(util)-1] = mu + } + return util + } + } + return append(util, mu) +} + // totalUtil is total utilization, measured in nanoseconds. This is a // separate type primarily to distinguish it from mean utilization, // which is also a float64. @@ -163,6 +236,10 @@ func (u totalUtil) mean(dur time.Duration) float64 { // An MMUCurve is the minimum mutator utilization curve across // multiple window sizes. type MMUCurve struct { + series []mmuSeries +} + +type mmuSeries struct { util []MutatorUtil // sums[j] is the cumulative sum of util[:j]. sums []totalUtil @@ -188,7 +265,15 @@ type mmuBand struct { // NewMMUCurve returns an MMU curve for the given mutator utilization // function. -func NewMMUCurve(util []MutatorUtil) *MMUCurve { +func NewMMUCurve(utils [][]MutatorUtil) *MMUCurve { + series := make([]mmuSeries, len(utils)) + for i, util := range utils { + series[i] = newMMUSeries(util) + } + return &MMUCurve{series} +} + +func newMMUSeries(util []MutatorUtil) mmuSeries { // Compute cumulative sum. sums := make([]totalUtil, len(util)) var prev MutatorUtil @@ -218,10 +303,10 @@ func NewMMUCurve(util []MutatorUtil) *MMUCurve { // Compute the bands. There are numBands+1 bands in order to // record the final cumulative sum. bands := make([]mmuBand, numBands+1) - c := MMUCurve{util, sums, bands, bandDur} - leftSum := integrator{&c, 0} + s := mmuSeries{util, sums, bands, bandDur} + leftSum := integrator{&s, 0} for i := range bands { - startTime, endTime := c.bandTime(i) + startTime, endTime := s.bandTime(i) cumUtil := leftSum.advance(startTime) predIdx := leftSum.pos minUtil := 1.0 @@ -231,16 +316,18 @@ func NewMMUCurve(util []MutatorUtil) *MMUCurve { bands[i] = mmuBand{minUtil, cumUtil, leftSum} } - return &c + return s } -func (c *MMUCurve) bandTime(i int) (start, end int64) { - start = int64(i)*c.bandDur + c.util[0].Time - end = start + c.bandDur +func (s *mmuSeries) bandTime(i int) (start, end int64) { + start = int64(i)*s.bandDur + s.util[0].Time + end = start + s.bandDur return } type bandUtil struct { + // Utilization series index + series int // Band index i int // Lower bound of mutator utilization for all windows @@ -402,12 +489,22 @@ func (c *MMUCurve) mmu(window time.Duration, acc *accumulator) { acc.mmu = 0 return } - util := c.util - if max := time.Duration(util[len(util)-1].Time - util[0].Time); window > max { - window = max - } - bandU := bandUtilHeap(c.mkBandUtil(window)) + var bandU bandUtilHeap + windows := make([]time.Duration, len(c.series)) + for i, s := range c.series { + windows[i] = window + if max := time.Duration(s.util[len(s.util)-1].Time - s.util[0].Time); window > max { + windows[i] = max + } + + bandU1 := bandUtilHeap(s.mkBandUtil(i, windows[i])) + if bandU == nil { + bandU = bandU1 + } else { + bandU = append(bandU, bandU1...) + } + } // Process bands from lowest utilization bound to highest. heap.Init(&bandU) @@ -416,12 +513,13 @@ func (c *MMUCurve) mmu(window time.Duration, acc *accumulator) { // refining the next lowest band can no longer affect the MMU // or windows. for len(bandU) > 0 && bandU[0].utilBound < acc.bound { - c.bandMMU(bandU[0].i, window, acc) + i := bandU[0].series + c.series[i].bandMMU(bandU[0].i, windows[i], acc) heap.Pop(&bandU) } } -func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { +func (c *mmuSeries) mkBandUtil(series int, window time.Duration) []bandUtil { // For each band, compute the worst-possible total mutator // utilization for all windows that start in that band. @@ -477,7 +575,7 @@ func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { util += c.bands[i+minBands-1].cumUtil - c.bands[i+1].cumUtil } - bandU[i] = bandUtil{i, util.mean(window)} + bandU[i] = bandUtil{series, i, util.mean(window)} } return bandU @@ -485,7 +583,7 @@ func (c *MMUCurve) mkBandUtil(window time.Duration) []bandUtil { // bandMMU computes the precise minimum mutator utilization for // windows with a left edge in band bandIdx. -func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, acc *accumulator) { +func (c *mmuSeries) bandMMU(bandIdx int, window time.Duration, acc *accumulator) { util := c.util // We think of the mutator utilization over time as the @@ -542,7 +640,7 @@ func (c *MMUCurve) bandMMU(bandIdx int, window time.Duration, acc *accumulator) // An integrator tracks a position in a utilization function and // integrates it. type integrator struct { - u *MMUCurve + u *mmuSeries // pos is the index in u.util of the current time's non-strict // predecessor. pos int diff --git a/src/internal/traceparser/gc_test.go b/src/internal/traceparser/gc_test.go index f1416fa9f9..b438a2931f 100644 --- a/src/internal/traceparser/gc_test.go +++ b/src/internal/traceparser/gc_test.go @@ -29,14 +29,14 @@ func TestMMU(t *testing.T) { // 0.5 * * * * // 0.0 ***** ***** // 0 1 2 3 4 5 - util := []MutatorUtil{ + util := [][]MutatorUtil{{ {0e9, 1}, {1e9, 0}, {2e9, 1}, {3e9, 0}, {4e9, 1}, {5e9, 0}, - } + }} mmuCurve := NewMMUCurve(util) for _, test := range []struct { @@ -90,7 +90,7 @@ func TestMMUTrace(t *testing.T) { // Test the optimized implementation against the "obviously // correct" implementation. for window := time.Nanosecond; window < 10*time.Second; window *= 10 { - want := mmuSlow(mu, window) + want := mmuSlow(mu[0], window) got := mmuCurve.MMU(window) if !aeq(want, got) { t.Errorf("want %f, got %f mutator utilization in window %s", want, got, window) From b2e8dd187343cf3059e373374426833d1a676a3e Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Jul 2017 16:30:05 -0400 Subject: [PATCH 022/111] cmd/trace: expose MMU analysis flags in web UI Change-Id: I672240487172380c9eef61837b41698021aaf834 Reviewed-on: https://go-review.googlesource.com/c/60798 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/mmu.go | 133 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 119 insertions(+), 14 deletions(-) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index d3b6768686..3fae3d6645 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -25,24 +25,54 @@ func init() { http.HandleFunc("/mmuDetails", httpMMUDetails) } -var mmuCache struct { +var utilFlagNames = map[string]trace.UtilFlags{ + "perProc": trace.UtilPerProc, + "stw": trace.UtilSTW, + "background": trace.UtilBackground, + "assist": trace.UtilAssist, + "sweep": trace.UtilSweep, +} + +type mmuCacheEntry struct { init sync.Once util [][]trace.MutatorUtil mmuCurve *trace.MMUCurve err error } -func getMMUCurve() ([][]trace.MutatorUtil, *trace.MMUCurve, error) { - mmuCache.init.Do(func() { +var mmuCache struct { + m map[trace.UtilFlags]*mmuCacheEntry + lock sync.Mutex +} + +func init() { + mmuCache.m = make(map[trace.UtilFlags]*mmuCacheEntry) +} + +func getMMUCurve(r *http.Request) ([][]trace.MutatorUtil, *trace.MMUCurve, error) { + var flags trace.UtilFlags + for _, flagStr := range strings.Split(r.FormValue("flags"), "|") { + flags |= utilFlagNames[flagStr] + } + + mmuCache.lock.Lock() + c := mmuCache.m[flags] + if c == nil { + c = new(mmuCacheEntry) + mmuCache.m[flags] = c + } + mmuCache.lock.Unlock() + + c.init.Do(func() { tr, err := parseTrace() if err != nil { - mmuCache.err = err + c.err = err } else { - mmuCache.util = tr.MutatorUtilization(trace.UtilSTW | trace.UtilBackground | trace.UtilAssist) - mmuCache.mmuCurve = trace.NewMMUCurve(mmuCache.util) + c.util = tr.MutatorUtilization(flags) + c.mmuCurve = trace.NewMMUCurve(c.util) } }) - return mmuCache.util, mmuCache.mmuCurve, mmuCache.err + return c.util, c.mmuCurve, c.err } // httpMMU serves the MMU plot page. @@ -52,7 +82,7 @@ func httpMMU(w http.ResponseWriter, r *http.Request) { // httpMMUPlot serves the JSON data for the MMU plot. func httpMMUPlot(w http.ResponseWriter, r *http.Request) { - mu, mmuCurve, err := getMMUCurve() + mu, mmuCurve, err := getMMUCurve(r) if err != nil { http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError) return @@ -107,7 +137,8 @@ var templMMU = ` + -
Loading plot...
+
+
Loading plot...
+
+

+ View
+ + ?Consider whole system utilization. For example, if one of four procs is available to the mutator, mutator utilization will be 0.25. This is the standard definition of an MMU.
+ + ?Consider per-goroutine utilization. When even one goroutine is interrupted by GC, mutator utilization is 0.
+

+

+ Include
+ + ?Stop-the-world stops all goroutines simultaneously.
+ + ?Background workers are GC-specific goroutines. 25% of the CPU is dedicated to background workers during GC.
+ + ?Mark assists are performed by allocation to prevent the mutator from outpacing GC.
+ + ?Sweep reclaims unused memory between GCs. (Enabling this may be very slow.).
+

+
+
Select a point for details.
@@ -202,7 +307,7 @@ var templMMU = ` // httpMMUDetails serves details of an MMU graph at a particular window. func httpMMUDetails(w http.ResponseWriter, r *http.Request) { - _, mmuCurve, err := getMMUCurve() + _, mmuCurve, err := getMMUCurve(r) if err != nil { http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError) return From 33563d1cfc7939d99d18e58cea7eedd6fb1c6ed6 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 17 Aug 2017 11:31:03 -0400 Subject: [PATCH 023/111] internal/trace: support for mutator utilization distributions This adds support for computing the quantiles of a mutator utilization distribution. Change-Id: Ia8b3ed14bf415c234e2f567360fd1b361d28bd40 Reviewed-on: https://go-review.googlesource.com/c/60799 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/internal/traceparser/gc.go | 141 ++++++++++++++++- src/internal/traceparser/gc_test.go | 22 ++- src/internal/traceparser/mud.go | 223 +++++++++++++++++++++++++++ src/internal/traceparser/mud_test.go | 87 +++++++++++ 4 files changed, 466 insertions(+), 7 deletions(-) create mode 100644 src/internal/traceparser/mud.go create mode 100644 src/internal/traceparser/mud_test.go diff --git a/src/internal/traceparser/gc.go b/src/internal/traceparser/gc.go index 569ab86b82..a5b29112b3 100644 --- a/src/internal/traceparser/gc.go +++ b/src/internal/traceparser/gc.go @@ -273,6 +273,10 @@ func NewMMUCurve(utils [][]MutatorUtil) *MMUCurve { return &MMUCurve{series} } +// bandsPerSeries is the number of bands to divide each series into. +// This is only changed by tests. +var bandsPerSeries = 1000 + func newMMUSeries(util []MutatorUtil) mmuSeries { // Compute cumulative sum. sums := make([]totalUtil, len(util)) @@ -289,7 +293,7 @@ func newMMUSeries(util []MutatorUtil) mmuSeries { // these bands. // // Compute the duration of each band. - numBands := 1000 + numBands := bandsPerSeries if numBands > len(util) { // There's no point in having lots of bands if there // aren't many events. @@ -393,8 +397,8 @@ func (h *utilHeap) Pop() interface{} { return x } -// An accumulator collects different MMU-related statistics depending -// on what's desired. +// An accumulator takes a windowed mutator utilization function and +// tracks various statistics for that function. type accumulator struct { mmu float64 @@ -406,10 +410,30 @@ type accumulator struct { // Worst N window tracking nWorst int wHeap utilHeap + + // Mutator utilization distribution tracking + mud *mud + // preciseMass is the distribution mass that must be precise + // before accumulation is stopped. + preciseMass float64 + // lastTime and lastMU are the previous point added to the + // windowed mutator utilization function. + lastTime int64 + lastMU float64 } -// addMU records mutator utilization mu over the given window starting -// at time. +// resetTime declares a discontinuity in the windowed mutator +// utilization function by resetting the current time. +func (acc *accumulator) resetTime() { + // This only matters for distribution collection, since that's + // the only thing that depends on the progression of the + // windowed mutator utilization function. + acc.lastTime = math.MaxInt64 +} + +// addMU adds a point to the windowed mutator utilization function at +// (time, mu). This must be called for monotonically increasing values +// of time. // // It returns true if further calls to addMU would be pointless. func (acc *accumulator) addMU(time int64, mu float64, window time.Duration) bool { @@ -458,6 +482,25 @@ func (acc *accumulator) addMU(time int64, mu float64, window time.Duration) bool acc.bound = math.Max(acc.bound, acc.wHeap[0].MutatorUtil) } + if acc.mud != nil { + if acc.lastTime != math.MaxInt64 { + // Update distribution. + acc.mud.add(acc.lastMU, mu, float64(time-acc.lastTime)) + } + acc.lastTime, acc.lastMU = time, mu + if _, mudBound, ok := acc.mud.approxInvCumulativeSum(); ok { + acc.bound = math.Max(acc.bound, mudBound) + } else { + // We haven't accumulated enough total precise + // mass yet to even reach our goal, so keep + // accumulating. + acc.bound = 1 + } + // It's not worth checking percentiles every time, so + // just keep accumulating this band. + return false + } + // If we've found enough 0 utilizations, we can stop immediately. return len(acc.wHeap) == acc.nWorst && acc.wHeap[0].MutatorUtil == 0 } @@ -484,6 +527,85 @@ func (c *MMUCurve) Examples(window time.Duration, n int) (worst []UtilWindow) { return ([]UtilWindow)(acc.wHeap) } +// MUD returns mutator utilization distribution quantiles for the +// given window size. +// +// The mutator utilization distribution is the distribution of mean +// mutator utilization across all windows of the given window size in +// the trace. +// +// The minimum mutator utilization is the minimum (0th percentile) of +// this distribution. (However, if only the minimum is desired, it's +// more efficient to use the MMU method.) +func (c *MMUCurve) MUD(window time.Duration, quantiles []float64) []float64 { + if len(quantiles) == 0 { + return []float64{} + } + + // Each unrefined band contributes a known total mass to the + // distribution (bandDur except at the end), but in an unknown + // way. However, we know that all the mass it contributes must + // be at or above its worst-case mean mutator utilization. + // + // Hence, we refine bands until the highest desired + // distribution quantile is less than the next worst-case mean + // mutator utilization. At this point, all further + // contributions to the distribution must be beyond the + // desired quantile and hence cannot affect it. + // + // First, find the highest desired distribution quantile. + maxQ := quantiles[0] + for _, q := range quantiles { + if q > maxQ { + maxQ = q + } + } + // The distribution's mass is in units of time (it's not + // normalized because this would make it more annoying to + // account for future contributions of unrefined bands). The + // total final mass will be the duration of the trace itself + // minus the window size. Using this, we can compute the mass + // corresponding to quantile maxQ. + var duration int64 + for _, s := range c.series { + duration1 := s.util[len(s.util)-1].Time - s.util[0].Time + if duration1 >= int64(window) { + duration += duration1 - int64(window) + } + } + qMass := float64(duration) * maxQ + + // Accumulate the MUD until we have precise information for + // everything to the left of qMass. + acc := accumulator{mmu: 1.0, bound: 1.0, preciseMass: qMass, mud: new(mud)} + acc.mud.setTrackMass(qMass) + c.mmu(window, &acc) + + // Evaluate the quantiles on the accumulated MUD. + out := make([]float64, len(quantiles)) + for i := range out { + mu, _ := acc.mud.invCumulativeSum(float64(duration) * quantiles[i]) + if math.IsNaN(mu) { + // There are a few legitimate ways this can + // happen: + // + // 1. If the window is the full trace + // duration, then the windowed MU function is + // only defined at a single point, so the MU + // distribution is not well-defined. + // + // 2. If there are no events, then the MU + // distribution has no mass. + // + // Either way, all of the quantiles will have + // converged toward the MMU at this point. + mu = acc.mmu + } + out[i] = mu + } + return out +} + func (c *MMUCurve) mmu(window time.Duration, acc *accumulator) { if window <= 0 { acc.mmu = 0 @@ -607,12 +729,16 @@ func (c *mmuSeries) bandMMU(bandIdx int, window time.Duration, acc *accumulator) if utilEnd := util[len(util)-1].Time - int64(window); utilEnd < endTime { endTime = utilEnd } + acc.resetTime() for { // Advance edges to time and time+window. mu := (right.advance(time+int64(window)) - left.advance(time)).mean(window) if acc.addMU(time, mu, window) { break } + if time == endTime { + break + } // The maximum slope of the windowed mutator // utilization function is 1/window, so we can always @@ -632,7 +758,10 @@ func (c *mmuSeries) bandMMU(bandIdx int, window time.Duration, acc *accumulator) time = minTime } if time >= endTime { - break + // For MMUs we could stop here, but for MUDs + // it's important that we span the entire + // band. + time = endTime } } } diff --git a/src/internal/traceparser/gc_test.go b/src/internal/traceparser/gc_test.go index b438a2931f..1cd8fb6f78 100644 --- a/src/internal/traceparser/gc_test.go +++ b/src/internal/traceparser/gc_test.go @@ -75,7 +75,8 @@ func TestMMU(t *testing.T) { } func TestMMUTrace(t *testing.T) { - t.Parallel() + // Can't be t.Parallel() because it modifies the + // testingOneBand package variable. p, err := New("../trace/testdata/stress_1_10_good") if err != nil { @@ -96,6 +97,25 @@ func TestMMUTrace(t *testing.T) { t.Errorf("want %f, got %f mutator utilization in window %s", want, got, window) } } + + // Test MUD with band optimization against MUD without band + // optimization. We don't have a simple testing implementation + // of MUDs (the simplest implementation is still quite + // complex), but this is still a pretty good test. + defer func(old int) { bandsPerSeries = old }(bandsPerSeries) + bandsPerSeries = 1 + mmuCurve2 := NewMMUCurve(mu) + quantiles := []float64{0, 1 - .999, 1 - .99} + for window := time.Microsecond; window < time.Second; window *= 10 { + mud1 := mmuCurve.MUD(window, quantiles) + mud2 := mmuCurve2.MUD(window, quantiles) + for i := range mud1 { + if !aeq(mud1[i], mud2[i]) { + t.Errorf("for quantiles %v at window %v, want %v, got %v", quantiles, window, mud2, mud1) + break + } + } + } } func BenchmarkMMU(b *testing.B) { diff --git a/src/internal/traceparser/mud.go b/src/internal/traceparser/mud.go new file mode 100644 index 0000000000..8eed89ff36 --- /dev/null +++ b/src/internal/traceparser/mud.go @@ -0,0 +1,223 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package traceparser + +import ( + "math" + "sort" +) + +// mud is an updatable mutator utilization distribution. +// +// This is a continuous distribution of duration over mutator +// utilization. For example, the integral from mutator utilization a +// to b is the total duration during which the mutator utilization was +// in the range [a, b]. +// +// This distribution is *not* normalized (it is not a probability +// distribution). This makes it easier to work with as it's being +// updated. +// +// It is represented as the sum of scaled uniform distribution +// functions and Dirac delta functions (which are treated as +// degenerate uniform distributions). +type mud struct { + sorted, unsorted []edge + + // trackMass is the inverse cumulative sum to track as the + // distribution is updated. + trackMass float64 + // trackBucket is the bucket in which trackMass falls. If the + // total mass of the distribution is < trackMass, this is + // len(hist). + trackBucket int + // trackSum is the cumulative sum of hist[:trackBucket]. Once + // trackSum >= trackMass, trackBucket must be recomputed. + trackSum float64 + + // hist is a hierarchical histogram of distribution mass. + hist [mudDegree]float64 +} + +const ( + // mudDegree is the number of buckets in the MUD summary + // histogram. + mudDegree = 1024 +) + +type edge struct { + // At x, the function increases by y. + x, delta float64 + // Additionally at x is a Dirac delta function with area dirac. + dirac float64 +} + +// add adds a uniform function over [l, r] scaled so the total weight +// of the uniform is area. If l==r, this adds a Dirac delta function. +func (d *mud) add(l, r, area float64) { + if area == 0 { + return + } + + if r < l { + l, r = r, l + } + + // Add the edges. + if l == r { + d.unsorted = append(d.unsorted, edge{l, 0, area}) + } else { + delta := area / (r - l) + d.unsorted = append(d.unsorted, edge{l, delta, 0}, edge{r, -delta, 0}) + } + + // Update the histogram. + h := &d.hist + lbFloat, lf := math.Modf(l * mudDegree) + lb := int(lbFloat) + if lb >= mudDegree { + lb, lf = mudDegree-1, 1 + } + if l == r { + h[lb] += area + } else { + rbFloat, rf := math.Modf(r * mudDegree) + rb := int(rbFloat) + if rb >= mudDegree { + rb, rf = mudDegree-1, 1 + } + if lb == rb { + h[lb] += area + } else { + perBucket := area / (r - l) / mudDegree + h[lb] += perBucket * (1 - lf) + h[rb] += perBucket * rf + for i := lb + 1; i < rb; i++ { + h[i] += perBucket + } + } + } + + // Update mass tracking. + if thresh := float64(d.trackBucket) / mudDegree; l < thresh { + if r < thresh { + d.trackSum += area + } else { + d.trackSum += area * (thresh - l) / (r - l) + } + if d.trackSum >= d.trackMass { + // The tracked mass now falls in a different + // bucket. Recompute the inverse cumulative sum. + d.setTrackMass(d.trackMass) + } + } +} + +// setTrackMass sets the mass to track the inverse cumulative sum for. +// +// Specifically, mass is a cumulative duration, and the mutator +// utilization bounds for this duration can be queried using +// approxInvCumulativeSum. +func (d *mud) setTrackMass(mass float64) { + d.trackMass = mass + + // Find the bucket currently containing trackMass by computing + // the cumulative sum. + sum := 0.0 + for i, val := range d.hist[:] { + newSum := sum + val + if newSum > mass { + // mass falls in bucket i. + d.trackBucket = i + d.trackSum = sum + return + } + sum = newSum + } + d.trackBucket = len(d.hist) + d.trackSum = sum +} + +// approxInvCumulativeSum is like invCumulativeSum, but specifically +// operates on the tracked mass and returns an upper and lower bound +// approximation of the inverse cumulative sum. +// +// The true inverse cumulative sum will be in the range [lower, upper). +func (d *mud) approxInvCumulativeSum() (float64, float64, bool) { + if d.trackBucket == len(d.hist) { + return math.NaN(), math.NaN(), false + } + return float64(d.trackBucket) / mudDegree, float64(d.trackBucket+1) / mudDegree, true +} + +// invCumulativeSum returns x such that the integral of d from -∞ to x +// is y. If the total weight of d is less than y, it returns the +// maximum of the distribution and false. +// +// Specifically, y is a cumulative duration, and invCumulativeSum +// returns the mutator utilization x such that at least y time has +// been spent with mutator utilization <= x. +func (d *mud) invCumulativeSum(y float64) (float64, bool) { + if len(d.sorted) == 0 && len(d.unsorted) == 0 { + return math.NaN(), false + } + + // Sort edges. + edges := d.unsorted + sort.Slice(edges, func(i, j int) bool { + return edges[i].x < edges[j].x + }) + // Merge with sorted edges. + d.unsorted = nil + if d.sorted == nil { + d.sorted = edges + } else { + oldSorted := d.sorted + newSorted := make([]edge, len(oldSorted)+len(edges)) + i, j := 0, 0 + for o := range newSorted { + if i >= len(oldSorted) { + copy(newSorted[o:], edges[j:]) + break + } else if j >= len(edges) { + copy(newSorted[o:], oldSorted[i:]) + break + } else if oldSorted[i].x < edges[j].x { + newSorted[o] = oldSorted[i] + i++ + } else { + newSorted[o] = edges[j] + j++ + } + } + d.sorted = newSorted + } + + // Traverse edges in order computing a cumulative sum. + csum, rate, prevX := 0.0, 0.0, 0.0 + for _, e := range d.sorted { + newCsum := csum + (e.x-prevX)*rate + if newCsum >= y { + // y was exceeded between the previous edge + // and this one. + if rate == 0 { + // Anywhere between prevX and + // e.x will do. We return e.x + // because that takes care of + // the y==0 case naturally. + return e.x, true + } + return (y-csum)/rate + prevX, true + } + newCsum += e.dirac + if newCsum >= y { + // y was exceeded by the Dirac delta at e.x. + return e.x, true + } + csum, prevX = newCsum, e.x + rate += e.delta + } + return prevX, false +} diff --git a/src/internal/traceparser/mud_test.go b/src/internal/traceparser/mud_test.go new file mode 100644 index 0000000000..6e048fcf19 --- /dev/null +++ b/src/internal/traceparser/mud_test.go @@ -0,0 +1,87 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package traceparser + +import ( + "math/rand" + "testing" +) + +func TestMUD(t *testing.T) { + // Insert random uniforms and check histogram mass and + // cumulative sum approximations. + rnd := rand.New(rand.NewSource(42)) + mass := 0.0 + var mud mud + for i := 0; i < 100; i++ { + area, l, r := rnd.Float64(), rnd.Float64(), rnd.Float64() + if rnd.Intn(10) == 0 { + r = l + } + t.Log(l, r, area) + mud.add(l, r, area) + mass += area + + // Check total histogram weight. + hmass := 0.0 + for _, val := range mud.hist { + hmass += val + } + if !aeq(mass, hmass) { + t.Fatalf("want mass %g, got %g", mass, hmass) + } + + // Check inverse cumulative sum approximations. + for j := 0.0; j < mass; j += mass * 0.099 { + mud.setTrackMass(j) + l, u, ok := mud.approxInvCumulativeSum() + inv, ok2 := mud.invCumulativeSum(j) + if !ok || !ok2 { + t.Fatalf("inverse cumulative sum failed: approx %v, exact %v", ok, ok2) + } + if !(l <= inv && inv < u) { + t.Fatalf("inverse(%g) = %g, not ∈ [%g, %g)", j, inv, l, u) + } + } + } +} + +func TestMUDTracking(t *testing.T) { + // Test that the tracked mass is tracked correctly across + // updates. + rnd := rand.New(rand.NewSource(42)) + const uniforms = 100 + for trackMass := 0.0; trackMass < uniforms; trackMass += uniforms / 50 { + var mud mud + mass := 0.0 + mud.setTrackMass(trackMass) + for i := 0; i < uniforms; i++ { + area, l, r := rnd.Float64(), rnd.Float64(), rnd.Float64() + mud.add(l, r, area) + mass += area + l, u, ok := mud.approxInvCumulativeSum() + inv, ok2 := mud.invCumulativeSum(trackMass) + + if mass < trackMass { + if ok { + t.Errorf("approx(%g) = [%g, %g), but mass = %g", trackMass, l, u, mass) + } + if ok2 { + t.Errorf("exact(%g) = %g, but mass = %g", trackMass, inv, mass) + } + } else { + if !ok { + t.Errorf("approx(%g) failed, but mass = %g", trackMass, mass) + } + if !ok2 { + t.Errorf("exact(%g) failed, but mass = %g", trackMass, mass) + } + if ok && ok2 && !(l <= inv && inv < u) { + t.Errorf("inverse(%g) = %g, not ∈ [%g, %g)", trackMass, inv, l, u) + } + } + } + } +} From b251d7fbe6d69e1ce81baf7959062ae489858f31 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 17 Aug 2017 18:11:01 -0400 Subject: [PATCH 024/111] cmd/trace: display p99.9, p99 and p95 MUT This uses the mutator utilization distribution to compute the p99.9, p99, and p95 mutator utilization topograph lines and display them along with the MMU. Change-Id: I8c7e0ec326aa4bc00619ec7562854253f01cc802 Reviewed-on: https://go-review.googlesource.com/c/60800 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/mmu.go | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index 3fae3d6645..062e5ad2ca 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -88,6 +88,14 @@ func httpMMUPlot(w http.ResponseWriter, r *http.Request) { return } + var quantiles []float64 + for _, flagStr := range strings.Split(r.FormValue("flags"), "|") { + if flagStr == "mut" { + quantiles = []float64{0, 1 - .999, 1 - .99, 1 - .95} + break + } + } + // Find a nice starting point for the plot. xMin := time.Second for xMin > 1 { @@ -114,15 +122,21 @@ func httpMMUPlot(w http.ResponseWriter, r *http.Request) { // Compute MMU curve. logMin, logMax := math.Log(float64(xMin)), math.Log(float64(xMax)) const samples = 100 - plot := make([][2]float64, samples) + plot := make([][]float64, samples) for i := 0; i < samples; i++ { window := time.Duration(math.Exp(float64(i)/(samples-1)*(logMax-logMin) + logMin)) - y := mmuCurve.MMU(window) - plot[i] = [2]float64{float64(window), y} + if quantiles == nil { + plot[i] = make([]float64, 2) + plot[i][1] = mmuCurve.MMU(window) + } else { + plot[i] = make([]float64, 1+len(quantiles)) + copy(plot[i][1:], mmuCurve.MUD(window, quantiles)) + } + plot[i][0] = float64(window) } // Create JSON response. - err = json.NewEncoder(w).Encode(map[string]interface{}{"xMin": int64(xMin), "xMax": int64(xMax), "curve": plot}) + err = json.NewEncoder(w).Encode(map[string]interface{}{"xMin": int64(xMin), "xMax": int64(xMax), "quantiles": quantiles, "curve": plot}) if err != nil { log.Printf("failed to serialize response: %v", err) return @@ -150,6 +164,10 @@ var templMMU = ` else { return ns / 1e9 + 's'; } } + function niceQuantile(q) { + return 'p' + q*100; + } + function mmuFlags() { var flags = ""; $("#options input").each(function(i, elt) { @@ -181,6 +199,11 @@ var templMMU = ` var data = new google.visualization.DataTable(); data.addColumn('number', 'Window duration'); data.addColumn('number', 'Minimum mutator utilization'); + if (plotData.quantiles) { + for (var i = 1; i < plotData.quantiles.length; i++) { + data.addColumn('number', niceQuantile(1 - plotData.quantiles[i]) + ' MU'); + } + } data.addRows(curve); for (var i = 0; i < curve.length; i++) { data.setFormattedValue(i, 0, niceDuration(curve[i][0])); @@ -201,6 +224,7 @@ var templMMU = ` maxValue: 1.0, }, legend: { position: 'none' }, + focusTarget: 'category', width: 900, height: 500, chartArea: { width: '80%', height: '80%' }, @@ -208,6 +232,10 @@ var templMMU = ` for (var v = plotData.xMin; v <= plotData.xMax; v *= 10) { options.hAxis.ticks.push({v:v, f:niceDuration(v)}); } + if (plotData.quantiles) { + options.vAxis.title = 'Mutator utilization'; + options.legend.position = 'in'; + } var container = $('#mmu_chart'); container.empty(); @@ -298,6 +326,11 @@ var templMMU = ` ?Sweep reclaims unused memory between GCs. (Enabling this may be very slow.).

+

+ Display
+ + ?Display percentile mutator utilization in addition to minimum. E.g., p99 MU drops the worst 1% of windows.
+

Select a point for details.
From e72595ee0f97746be3ce594834a7003d5e804795 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 25 Jul 2017 15:03:44 -0400 Subject: [PATCH 025/111] cmd/trace: notes on MMU view improvements Change-Id: Ib9dcdc76095f6718f1cdc83349503f52567c76d4 Reviewed-on: https://go-review.googlesource.com/c/60801 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/cmd/trace/mmu.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/cmd/trace/mmu.go b/src/cmd/trace/mmu.go index 062e5ad2ca..6a7d28e61d 100644 --- a/src/cmd/trace/mmu.go +++ b/src/cmd/trace/mmu.go @@ -4,6 +4,25 @@ // Minimum mutator utilization (MMU) graphing. +// TODO: +// +// In worst window list, show break-down of GC utilization sources +// (STW, assist, etc). Probably requires a different MutatorUtil +// representation. +// +// When a window size is selected, show a second plot of the mutator +// utilization distribution for that window size. +// +// Render plot progressively so rough outline is visible quickly even +// for very complex MUTs. Start by computing just a few window sizes +// and then add more window sizes. +// +// Consider using sampling to compute an approximate MUT. This would +// work by sampling the mutator utilization at randomly selected +// points in time in the trace to build an empirical distribution. We +// could potentially put confidence intervals on these estimates and +// render this progressively as we refine the distributions. + package main import ( From 2ae8bf7054b3320682e547396d9b6b5e51f5ade1 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 17 Oct 2018 20:16:45 +0000 Subject: [PATCH 026/111] runtime: fix stale comments about mheap and mspan As of 07e738e all spans are allocated out of a treap, and not just large spans or spans for large objects. Also, now we have a separate treap for spans that have been scavenged. Change-Id: I9c2cb7b6798fc536bbd34835da2e888224fd7ed4 Reviewed-on: https://go-review.googlesource.com/c/142958 Reviewed-by: Austin Clements --- src/runtime/mheap.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 8f6db8eec5..56ec3d4465 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -21,7 +21,7 @@ import ( const minPhysPageSize = 4096 // Main malloc heap. -// The heap itself is the "free[]" and "large" arrays, +// The heap itself is the "free" and "scav" treaps, // but all the other global data is here too. // // mheap must not be heap-allocated because it contains mSpanLists, @@ -147,7 +147,7 @@ type mheap struct { spanalloc fixalloc // allocator for span* cachealloc fixalloc // allocator for mcache* - treapalloc fixalloc // allocator for treapNodes* used by large objects + treapalloc fixalloc // allocator for treapNodes* specialfinalizeralloc fixalloc // allocator for specialfinalizer* specialprofilealloc fixalloc // allocator for specialprofile* speciallock mutex // lock for special record allocators. @@ -198,15 +198,16 @@ type arenaHint struct { // An MSpan is a run of pages. // -// When a MSpan is in the heap free list, state == mSpanFree +// When a MSpan is in the heap free treap, state == mSpanFree // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. +// If the MSpan is in the heap scav treap, then in addition to the +// above scavenged == true. scavenged == false in all other cases. // // When a MSpan is allocated, state == mSpanInUse or mSpanManual // and heapmap(i) == span for all s->start <= i < s->start+s->npages. -// Every MSpan is in one doubly-linked list, -// either one of the MHeap's free lists or one of the -// MCentral's span lists. +// Every MSpan is in one doubly-linked list, either in the MHeap's +// busy list or one of the MCentral's span lists. // An MSpan representing actual memory has state mSpanInUse, // mSpanManual, or mSpanFree. Transitions between these states are @@ -848,7 +849,7 @@ func (h *mheap) setSpans(base, npage uintptr, s *mspan) { // Allocates a span of the given size. h must be locked. // The returned span has been removed from the -// free list, but its state is still mSpanFree. +// free structures, but its state is still mSpanFree. func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { var s *mspan From 3377b4673d6e0ca1a9bba1c7196d7e673ddb8108 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 2 Nov 2018 21:41:41 -0700 Subject: [PATCH 027/111] cmd/compile: encapsulate and document two types.Type internal fields Change-Id: I5f7d2155c2c3a47dabdf16fe46b122ede81de4fc Reviewed-on: https://go-review.googlesource.com/c/147284 Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/reflect.go | 2 +- src/cmd/compile/internal/gc/typecheck.go | 6 ++---- src/cmd/compile/internal/types/type.go | 20 ++++++++++++++------ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/cmd/compile/internal/gc/reflect.go b/src/cmd/compile/internal/gc/reflect.go index 415d3cd594..50b741358f 100644 --- a/src/cmd/compile/internal/gc/reflect.go +++ b/src/cmd/compile/internal/gc/reflect.go @@ -812,7 +812,7 @@ func dcommontype(lsym *obj.LSym, t *types.Type) int { sptrWeak := true var sptr *obj.LSym - if !t.IsPtr() || t.PtrBase != nil { + if !t.IsPtr() || t.IsPtrElem() { tptr := types.NewPtr(t) if t.Sym != nil || methods(tptr) != nil { sptrWeak = false diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index be11a9841f..2a59521484 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -3671,8 +3671,7 @@ func copytype(n *Node, t *types.Type) { embedlineno := n.Type.ForwardType().Embedlineno l := n.Type.ForwardType().Copyto - ptrBase := n.Type.PtrBase - sliceOf := n.Type.SliceOf + cache := n.Type.Cache // TODO(mdempsky): Fix Type rekinding. *n.Type = *t @@ -3693,8 +3692,7 @@ func copytype(n *Node, t *types.Type) { t.Nod = asTypesNode(n) t.SetDeferwidth(false) - t.PtrBase = ptrBase - t.SliceOf = sliceOf + t.Cache = cache // Propagate go:notinheap pragma from the Name to the Type. if n.Name != nil && n.Name.Param != nil && n.Name.Param.Pragma&NotInHeap != 0 { diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index 39f4d2aa7b..b20039239b 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -149,8 +149,11 @@ type Type struct { Nod *Node // canonical OTYPE node Orig *Type // original type (type literal or predefined type) - SliceOf *Type - PtrBase *Type + // Cache of composite types, with this type being the element type. + Cache struct { + ptr *Type // *T, or nil + slice *Type // []T, or nil + } Sym *Sym // symbol containing name, for named types Vargen int32 // unique name for OTYPE/ONAME @@ -488,7 +491,7 @@ func NewArray(elem *Type, bound int64) *Type { // NewSlice returns the slice Type with element type elem. func NewSlice(elem *Type) *Type { - if t := elem.SliceOf; t != nil { + if t := elem.Cache.slice; t != nil { if t.Elem() != elem { Fatalf("elem mismatch") } @@ -497,7 +500,7 @@ func NewSlice(elem *Type) *Type { t := New(TSLICE) t.Extra = Slice{Elem: elem} - elem.SliceOf = t + elem.Cache.slice = t return t } @@ -551,7 +554,7 @@ func NewPtr(elem *Type) *Type { Fatalf("NewPtr: pointer to elem Type is nil") } - if t := elem.PtrBase; t != nil { + if t := elem.Cache.ptr; t != nil { if t.Elem() != elem { Fatalf("NewPtr: elem mismatch") } @@ -563,7 +566,7 @@ func NewPtr(elem *Type) *Type { t.Width = int64(Widthptr) t.Align = uint8(Widthptr) if NewPtrCacheEnabled { - elem.PtrBase = t + elem.Cache.ptr = t } return t } @@ -1258,6 +1261,11 @@ func (t *Type) IsPtr() bool { return t.Etype == TPTR } +// IsPtrElem reports whether t is the element of a pointer (to t). +func (t *Type) IsPtrElem() bool { + return t.Cache.ptr != nil +} + // IsUnsafePtr reports whether t is an unsafe pointer. func (t *Type) IsUnsafePtr() bool { return t.Etype == TUNSAFEPTR From e6305380a067c51223a59baf8a77575595a5f1e6 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 2 Nov 2018 23:28:26 -0700 Subject: [PATCH 028/111] cmd/compile: reintroduce work-around for cyclic alias declarations This change re-introduces (temporarily) a work-around for recursive alias type declarations, originally in https://golang.org/cl/35831/ (intended as fix for #18640). The work-around was removed later for a more comprehensive cycle detection check. That check contained a subtle error which made the code appear to work, while in fact creating incorrect types internally. See #25838 for details. By re-introducing the original work-around, we eliminate problems with many simple recursive type declarations involving aliases; specifically cases such as #27232 and #27267. However, the more general problem remains. This CL also fixes the subtle error (incorrect variable use when analyzing a type cycle) mentioned above and now issues a fatal error with a reference to the relevant issue (rather than crashing later during the compilation). While not great, this is better than the current status. The long-term solution will need to address these cycles (see #25838). As a consequence, several old test cases are not accepted anymore by the compiler since they happened to work accidentally only. This CL disables parts or all code of those test cases. The issues are: #18640, #23823, and #24939. One of the new test cases (fixedbugs/issue27232.go) exposed a go/types issue. The test case is excluded from the go/types test suite and an issue was filed (#28576). Updates #18640. Updates #23823. Updates #24939. Updates #25838. Updates #28576. Fixes #27232. Fixes #27267. Change-Id: I6c2d10da98bfc6f4f445c755fcaab17fc7b214c5 Reviewed-on: https://go-review.googlesource.com/c/147286 Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/main.go | 8 ++++++-- src/cmd/compile/internal/gc/typecheck.go | 12 ++++++++++-- src/go/types/stdlib_test.go | 1 + test/fixedbugs/issue18640.go | 5 ++--- test/fixedbugs/issue23823.go | 8 ++++++-- test/fixedbugs/issue24939.go | 4 +++- test/fixedbugs/issue27232.go | 19 +++++++++++++++++++ test/fixedbugs/issue27267.go | 21 +++++++++++++++++++++ 8 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 test/fixedbugs/issue27232.go create mode 100644 test/fixedbugs/issue27267.go diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 49a4e05d99..78142d3bf8 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -491,13 +491,17 @@ func Main(archInit func(*Arch)) { // Phase 1: const, type, and names and types of funcs. // This will gather all the information about types // and methods but doesn't depend on any of it. + // + // We also defer type alias declarations until phase 2 + // to avoid cycles like #18640. + // TODO(gri) Remove this again once we have a fix for #25838. defercheckwidth() // Don't use range--typecheck can add closures to xtop. timings.Start("fe", "typecheck", "top1") for i := 0; i < len(xtop); i++ { n := xtop[i] - if op := n.Op; op != ODCL && op != OAS && op != OAS2 { + if op := n.Op; op != ODCL && op != OAS && op != OAS2 && (op != ODCLTYPE || !n.Left.Name.Param.Alias) { xtop[i] = typecheck(n, Etop) } } @@ -509,7 +513,7 @@ func Main(archInit func(*Arch)) { timings.Start("fe", "typecheck", "top2") for i := 0; i < len(xtop); i++ { n := xtop[i] - if op := n.Op; op == ODCL || op == OAS || op == OAS2 { + if op := n.Op; op == ODCL || op == OAS || op == OAS2 || op == ODCLTYPE && n.Left.Name.Param.Alias { xtop[i] = typecheck(n, Etop) } } diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index 2a59521484..06dd176b37 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -255,8 +255,16 @@ func typecheck(n *Node, top int) (res *Node) { // since it would expand indefinitely when aliases // are substituted. cycle := cycleFor(n) - for _, n := range cycle { - if n.Name != nil && !n.Name.Param.Alias { + for _, n1 := range cycle { + if n1.Name != nil && !n1.Name.Param.Alias { + // Cycle is ok. But if n is an alias type and doesn't + // have a type yet, we have a recursive type declaration + // with aliases that we can't handle properly yet. + // Report an error rather than crashing later. + if n.Name != nil && n.Name.Param.Alias && n.Type == nil { + lineno = n.Pos + Fatalf("cannot handle alias type declaration (issue #25838): %v", n) + } lineno = lno return n } diff --git a/src/go/types/stdlib_test.go b/src/go/types/stdlib_test.go index 84908fd190..a4ff1ab9a8 100644 --- a/src/go/types/stdlib_test.go +++ b/src/go/types/stdlib_test.go @@ -180,6 +180,7 @@ func TestStdFixed(t *testing.T) { "issue22200b.go", // go/types does not have constraints on stack size "issue25507.go", // go/types does not have constraints on stack size "issue20780.go", // go/types does not have constraints on stack size + "issue27232.go", // go/types has a bug with alias type (issue #28576) ) } diff --git a/test/fixedbugs/issue18640.go b/test/fixedbugs/issue18640.go index 60abd31f76..091bbe596b 100644 --- a/test/fixedbugs/issue18640.go +++ b/test/fixedbugs/issue18640.go @@ -20,8 +20,7 @@ type ( d = c ) -// The compiler reports an incorrect (non-alias related) -// type cycle here (via dowith()). Disabled for now. +// The compiler cannot handle these cases. Disabled for now. // See issue #25838. /* type ( @@ -32,7 +31,6 @@ type ( i = j j = e ) -*/ type ( a1 struct{ *b1 } @@ -45,3 +43,4 @@ type ( b2 = c2 c2 struct{ *b2 } ) +*/ diff --git a/test/fixedbugs/issue23823.go b/test/fixedbugs/issue23823.go index 2f802d0988..9297966cbd 100644 --- a/test/fixedbugs/issue23823.go +++ b/test/fixedbugs/issue23823.go @@ -1,4 +1,4 @@ -// errorcheck +// compile // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -6,10 +6,14 @@ package p +// The compiler cannot handle this. Disabled for now. +// See issue #25838. +/* type I1 = interface { I2 } -type I2 interface { // ERROR "invalid recursive type" +type I2 interface { I1 } +*/ diff --git a/test/fixedbugs/issue24939.go b/test/fixedbugs/issue24939.go index 26530e95b2..0ae6f2b9f2 100644 --- a/test/fixedbugs/issue24939.go +++ b/test/fixedbugs/issue24939.go @@ -15,7 +15,9 @@ type M interface { } type P = interface { - I() M + // The compiler cannot handle this case. Disabled for now. + // See issue #25838. + // I() M } func main() {} diff --git a/test/fixedbugs/issue27232.go b/test/fixedbugs/issue27232.go new file mode 100644 index 0000000000..3a1cc87e4c --- /dev/null +++ b/test/fixedbugs/issue27232.go @@ -0,0 +1,19 @@ +// compile + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type F = func(T) + +type T interface { + m(F) +} + +type t struct{} + +func (t) m(F) {} + +var _ T = &t{} diff --git a/test/fixedbugs/issue27267.go b/test/fixedbugs/issue27267.go new file mode 100644 index 0000000000..ebae44f48f --- /dev/null +++ b/test/fixedbugs/issue27267.go @@ -0,0 +1,21 @@ +// compile + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +// 1st test case from issue +type F = func(E) // compiles if not type alias or moved below E struct +type E struct { + f F +} + +var x = E{func(E) {}} + +// 2nd test case from issue +type P = *S +type S struct { + p P +} From a540aa338a3145ab32ca4409919c82722f8724f3 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Mon, 5 Nov 2018 10:33:28 -0800 Subject: [PATCH 029/111] test: add test that gccgo failed to compile Updates #28601 Change-Id: I734fc5ded153126d384f0df912ecd4d208005e49 Reviewed-on: https://go-review.googlesource.com/c/147537 Run-TryBot: Ian Lance Taylor Reviewed-by: Cherry Zhang --- test/fixedbugs/issue28601.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/fixedbugs/issue28601.go diff --git a/test/fixedbugs/issue28601.go b/test/fixedbugs/issue28601.go new file mode 100644 index 0000000000..ec367e9282 --- /dev/null +++ b/test/fixedbugs/issue28601.go @@ -0,0 +1,15 @@ +// compile + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Failed to compile with gccgo. + +package p + +import "unsafe" + +const w int = int(unsafe.Sizeof(0)) + +var a [w]byte From f1a9f1df5070f69685e269de940c6218f899d228 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Wed, 31 Oct 2018 00:19:35 +0200 Subject: [PATCH 030/111] go/doc: inspect function signature for building playground examples This documentation example was broken: https://golang.org/pkg/image/png/#example_Decode. It did not have the "io" package imported, The package was referenced in the result type of the function. The "playExample" function did not inspect the result types of declared functions. This CL adds inspecting of parameters and result types of functions. Fixes #28492 Updates #9679 Change-Id: I6d8b11bad2db8ea8ba69039cfaa914093bdd5132 Reviewed-on: https://go-review.googlesource.com/c/146118 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Robert Griesemer --- src/go/doc/example.go | 12 ++++++++ src/go/doc/example_test.go | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/go/doc/example.go b/src/go/doc/example.go index d6d4ece3a8..cf3547810a 100644 --- a/src/go/doc/example.go +++ b/src/go/doc/example.go @@ -219,6 +219,18 @@ func playExample(file *ast.File, f *ast.FuncDecl) *ast.File { for i := 0; i < len(depDecls); i++ { switch d := depDecls[i].(type) { case *ast.FuncDecl: + // Inspect types of parameters and results. See #28492. + if d.Type.Params != nil { + for _, p := range d.Type.Params.List { + ast.Inspect(p.Type, inspectFunc) + } + } + if d.Type.Results != nil { + for _, r := range d.Type.Results.List { + ast.Inspect(r.Type, inspectFunc) + } + } + ast.Inspect(d.Body, inspectFunc) case *ast.GenDecl: for _, spec := range d.Specs { diff --git a/src/go/doc/example_test.go b/src/go/doc/example_test.go index 552a51bf74..0d2bf72e31 100644 --- a/src/go/doc/example_test.go +++ b/src/go/doc/example_test.go @@ -351,6 +351,68 @@ func TestExamplesWholeFile(t *testing.T) { } } +const exampleInspectSignature = `package foo_test + +import ( + "bytes" + "io" +) + +func getReader() io.Reader { return nil } + +func do(b bytes.Reader) {} + +func Example() { + getReader() + do() + // Output: +} + +func ExampleIgnored() { +} +` + +const exampleInspectSignatureOutput = `package main + +import ( + "bytes" + "io" +) + +func getReader() io.Reader { return nil } + +func do(b bytes.Reader) {} + +func main() { + getReader() + do() +} +` + +func TestExampleInspectSignature(t *testing.T) { + // Verify that "bytes" and "io" are imported. See issue #28492. + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "test.go", strings.NewReader(exampleInspectSignature), parser.ParseComments) + if err != nil { + t.Fatal(err) + } + es := doc.Examples(file) + if len(es) != 2 { + t.Fatalf("wrong number of examples; got %d want 2", len(es)) + } + // We are interested in the first example only. + e := es[0] + if e.Name != "" { + t.Errorf("got Name == %q, want %q", e.Name, "") + } + if g, w := formatFile(t, fset, e.Play), exampleInspectSignatureOutput; g != w { + t.Errorf("got Play == %q, want %q", g, w) + } + if g, w := e.Output, ""; g != w { + t.Errorf("got Output == %q, want %q", g, w) + } +} + func formatFile(t *testing.T, fset *token.FileSet, n *ast.File) string { if n == nil { return "" From 9c89923266a372e9357dc3296b6c53bb931dd4a9 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 5 Nov 2018 12:36:42 -0500 Subject: [PATCH 031/111] runtime: deflake TestTracebackAncestors TestTracebackAncestors has a ~0.1% chance of failing with more goroutines in the traceback than expected. This happens because there's a window between each goroutine starting its child and that goroutine actually exiting. The test captures its own stack trace after everything is "done", but if this happens during that window, it will include the goroutine that's in the process of being torn down. Here's an example of such a failure: https://build.golang.org/log/fad10d0625295eb79fa879f53b8b32b9d0596af8 This CL fixes this by recording the goroutines that are expected to exit and removing them from the stack trace. With this fix, this test passed 15,000 times with no failures. Change-Id: I71e7c6282987a15e8b74188b9c585aa2ca97cbcd Reviewed-on: https://go-review.googlesource.com/c/147517 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- .../testdata/testprog/traceback_ancestors.go | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/runtime/testdata/testprog/traceback_ancestors.go b/src/runtime/testdata/testprog/traceback_ancestors.go index fe57c1c157..0ee402c4bd 100644 --- a/src/runtime/testdata/testprog/traceback_ancestors.go +++ b/src/runtime/testdata/testprog/traceback_ancestors.go @@ -5,8 +5,10 @@ package main import ( + "bytes" "fmt" "runtime" + "strings" ) func init() { @@ -18,25 +20,50 @@ const numFrames = 2 func TracebackAncestors() { w := make(chan struct{}) - recurseThenCallGo(w, numGoroutines, numFrames) + recurseThenCallGo(w, numGoroutines, numFrames, true) <-w printStack() close(w) } +var ignoreGoroutines = make(map[string]bool) + func printStack() { buf := make([]byte, 1024) for { n := runtime.Stack(buf, true) if n < len(buf) { - fmt.Print(string(buf[:n])) + tb := string(buf[:n]) + + // Delete any ignored goroutines, if present. + pos := 0 + for pos < len(tb) { + next := pos + strings.Index(tb[pos:], "\n\n") + if next < pos { + next = len(tb) + } else { + next += len("\n\n") + } + + if strings.HasPrefix(tb[pos:], "goroutine ") { + id := tb[pos+len("goroutine "):] + id = id[:strings.IndexByte(id, ' ')] + if ignoreGoroutines[id] { + tb = tb[:pos] + tb[next:] + next = pos + } + } + pos = next + } + + fmt.Print(tb) return } buf = make([]byte, 2*len(buf)) } } -func recurseThenCallGo(w chan struct{}, frames int, goroutines int) { +func recurseThenCallGo(w chan struct{}, frames int, goroutines int, main bool) { if frames == 0 { // Signal to TracebackAncestors that we are done recursing and starting goroutines. w <- struct{}{} @@ -44,10 +71,29 @@ func recurseThenCallGo(w chan struct{}, frames int, goroutines int) { return } if goroutines == 0 { + // Record which goroutine this is so we can ignore it + // in the traceback if it hasn't finished exiting by + // the time we printStack. + if !main { + ignoreGoroutines[goroutineID()] = true + } + // Start the next goroutine now that there are no more recursions left // for this current goroutine. - go recurseThenCallGo(w, frames-1, numFrames) + go recurseThenCallGo(w, frames-1, numFrames, false) return } - recurseThenCallGo(w, frames, goroutines-1) + recurseThenCallGo(w, frames, goroutines-1, main) +} + +func goroutineID() string { + buf := make([]byte, 128) + runtime.Stack(buf, false) + const prefix = "goroutine " + if !bytes.HasPrefix(buf, []byte(prefix)) { + panic(fmt.Sprintf("expected %q at beginning of traceback:\n%s", prefix, buf)) + } + buf = buf[len(prefix):] + n := bytes.IndexByte(buf, ' ') + return string(buf[:n]) } From 44dcb5cb61aee5435e0b3c78544a1d3352a4cc98 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Mon, 5 Nov 2018 19:26:25 +0000 Subject: [PATCH 032/111] runtime: clean up MSpan* MCache* MCentral* in docs This change cleans up references to MSpan, MCache, and MCentral in the docs via a bunch of sed invocations to better reflect the Go names for the equivalent structures (i.e. mspan, mcache, mcentral) and their methods (i.e. MSpan_Sweep -> mspan.sweep). Change-Id: Ie911ac975a24bd25200a273086dd835ab78b1711 Reviewed-on: https://go-review.googlesource.com/c/147557 Reviewed-by: Austin Clements Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot --- src/runtime/heapdump.go | 4 ++-- src/runtime/mcache.go | 2 +- src/runtime/mcentral.go | 10 ++++----- src/runtime/mfixalloc.go | 2 +- src/runtime/mgcmark.go | 2 +- src/runtime/mgcsweep.go | 20 +++++++++--------- src/runtime/mheap.go | 44 ++++++++++++++++++++-------------------- src/runtime/mstats.go | 2 +- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index eadbcaeee1..ca56708a04 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -428,7 +428,7 @@ func dumproots() { dumpmemrange(unsafe.Pointer(firstmoduledata.bss), firstmoduledata.ebss-firstmoduledata.bss) dumpfields(firstmoduledata.gcbssmask) - // MSpan.types + // mspan.types for _, s := range mheap_.allspans { if s.state == mSpanInUse { // Finalizers @@ -661,7 +661,7 @@ func writeheapdump_m(fd uintptr) { _g_.waitreason = waitReasonDumpingHeap // Update stats so we can dump them. - // As a side effect, flushes all the MCaches so the MSpan.freelist + // As a side effect, flushes all the mcaches so the mspan.freelist // lists contain all the free objects. updatememstats() diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index e20e92cdf4..7895e489bc 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -79,7 +79,7 @@ type stackfreelist struct { size uintptr // total size of stacks in list } -// dummy MSpan that contains no free objects. +// dummy mspan that contains no free objects. var emptymspan mspan func allocmcache() *mcache { diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index f108bfc31e..a60eb9fd0c 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -6,8 +6,8 @@ // // See malloc.go for an overview. // -// The MCentral doesn't actually contain the list of free objects; the MSpan does. -// Each MCentral is two lists of MSpans: those with free objects (c->nonempty) +// The mcentral doesn't actually contain the list of free objects; the mspan does. +// Each mcentral is two lists of mspans: those with free objects (c->nonempty) // and those that are completely allocated (c->empty). package runtime @@ -36,7 +36,7 @@ func (c *mcentral) init(spc spanClass) { c.empty.init() } -// Allocate a span to use in an MCache. +// Allocate a span to use in an mcache. func (c *mcentral) cacheSpan() *mspan { // Deduct credit for this span allocation and sweep if necessary. spanBytes := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) * _PageSize @@ -146,7 +146,7 @@ havespan: return s } -// Return span from an MCache. +// Return span from an mcache. func (c *mcentral) uncacheSpan(s *mspan) { if s.allocCount == 0 { throw("uncaching span but s.allocCount == 0") @@ -231,7 +231,7 @@ func (c *mcentral) freeSpan(s *mspan, preserve bool, wasempty bool) bool { } // delay updating sweepgen until here. This is the signal that - // the span may be used in an MCache, so it must come after the + // the span may be used in an mcache, so it must come after the // linked list operations above (actually, just after the // lock of c above.) atomic.Store(&s.sweepgen, mheap_.sweepgen) diff --git a/src/runtime/mfixalloc.go b/src/runtime/mfixalloc.go index 1febe782bb..f9dd6ca474 100644 --- a/src/runtime/mfixalloc.go +++ b/src/runtime/mfixalloc.go @@ -12,7 +12,7 @@ import "unsafe" // FixAlloc is a simple free-list allocator for fixed size objects. // Malloc uses a FixAlloc wrapped around sysAlloc to manage its -// MCache and MSpan objects. +// mcache and mspan objects. // // Memory returned by fixalloc.alloc is zeroed by default, but the // caller may take responsibility for zeroing allocations by setting diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 14f09700ee..28260ab706 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -178,7 +178,7 @@ func markroot(gcw *gcWork, i uint32) { systemstack(markrootFreeGStacks) case baseSpans <= i && i < baseStacks: - // mark MSpan.specials + // mark mspan.specials markrootSpans(gcw, int(i-baseSpans)) default: diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 627a6a023f..6733aa9b4a 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -152,7 +152,7 @@ func (s *mspan) ensureSwept() { // (if GC is triggered on another goroutine). _g_ := getg() if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { - throw("MSpan_EnsureSwept: m is not locked") + throw("mspan.ensureSwept: m is not locked") } sg := mheap_.sweepgen @@ -178,7 +178,7 @@ func (s *mspan) ensureSwept() { // Sweep frees or collects finalizers for blocks not marked in the mark phase. // It clears the mark bits in preparation for the next GC round. // Returns true if the span was returned to heap. -// If preserve=true, don't return it to heap nor relink in MCentral lists; +// If preserve=true, don't return it to heap nor relink in mcentral lists; // caller takes care of it. //TODO go:nowritebarrier func (s *mspan) sweep(preserve bool) bool { @@ -186,12 +186,12 @@ func (s *mspan) sweep(preserve bool) bool { // GC must not start while we are in the middle of this function. _g_ := getg() if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { - throw("MSpan_Sweep: m is not locked") + throw("mspan.sweep: m is not locked") } sweepgen := mheap_.sweepgen if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { - print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") - throw("MSpan_Sweep: bad span state") + print("mspan.sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") + throw("mspan.sweep: bad span state") } if trace.enabled { @@ -327,8 +327,8 @@ func (s *mspan) sweep(preserve bool) bool { // The span must be in our exclusive ownership until we update sweepgen, // check for potential races. if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { - print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") - throw("MSpan_Sweep: bad span state after sweep") + print("mspan.sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") + throw("mspan.sweep: bad span state after sweep") } // Serialization point. // At this point the mark bits are cleared and allocation ready @@ -339,7 +339,7 @@ func (s *mspan) sweep(preserve bool) bool { if nfreed > 0 && spc.sizeclass() != 0 { c.local_nsmallfree[spc.sizeclass()] += uintptr(nfreed) res = mheap_.central[spc].mcentral.freeSpan(s, preserve, wasempty) - // MCentral_FreeSpan updates sweepgen + // mcentral.freeSpan updates sweepgen } else if freeToHeap { // Free large span to heap @@ -351,12 +351,12 @@ func (s *mspan) sweep(preserve bool) bool { // calling sysFree here without any kind of adjustment of the // heap data structures means that when the memory does // come back to us, we have the wrong metadata for it, either in - // the MSpan structures or in the garbage collection bitmap. + // the mspan structures or in the garbage collection bitmap. // Using sysFault here means that the program will run out of // memory fairly quickly in efence mode, but at least it won't // have mysterious crashes due to confused memory reuse. // It should be possible to switch back to sysFree if we also - // implement and then call some kind of MHeap_DeleteSpan. + // implement and then call some kind of mheap.deleteSpan. if debug.efence > 0 { s.limit = 0 // prevent mlookup from finding this span sysFault(unsafe.Pointer(s.base()), size) diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 56ec3d4465..43f59adb8a 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -136,8 +136,8 @@ type mheap struct { // _ uint32 // ensure 64-bit alignment of central // central free lists for small size classes. - // the padding makes sure that the MCentrals are - // spaced CacheLinePadSize bytes apart, so that each MCentral.lock + // the padding makes sure that the mcentrals are + // spaced CacheLinePadSize bytes apart, so that each mcentral.lock // gets its own cache line. // central is indexed by spanClass. central [numSpanClasses]struct { @@ -196,20 +196,20 @@ type arenaHint struct { next *arenaHint } -// An MSpan is a run of pages. +// An mspan is a run of pages. // -// When a MSpan is in the heap free treap, state == mSpanFree +// When a mspan is in the heap free treap, state == mSpanFree // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. -// If the MSpan is in the heap scav treap, then in addition to the +// If the mspan is in the heap scav treap, then in addition to the // above scavenged == true. scavenged == false in all other cases. // -// When a MSpan is allocated, state == mSpanInUse or mSpanManual +// When a mspan is allocated, state == mSpanInUse or mSpanManual // and heapmap(i) == span for all s->start <= i < s->start+s->npages. -// Every MSpan is in one doubly-linked list, either in the MHeap's -// busy list or one of the MCentral's span lists. +// Every mspan is in one doubly-linked list, either in the mheap's +// busy list or one of the mcentral's span lists. -// An MSpan representing actual memory has state mSpanInUse, +// An mspan representing actual memory has state mSpanInUse, // mSpanManual, or mSpanFree. Transitions between these states are // constrained as follows: // @@ -880,10 +880,10 @@ func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { HaveSpan: // Mark span in use. if s.state != mSpanFree { - throw("MHeap_AllocLocked - MSpan not free") + throw("mheap.allocLocked - mspan not free") } if s.npages < npage { - throw("MHeap_AllocLocked - bad npages") + throw("mheap.allocLocked - bad npages") } // First, subtract any memory that was released back to @@ -1022,16 +1022,16 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i switch s.state { case mSpanManual: if s.allocCount != 0 { - throw("MHeap_FreeSpanLocked - invalid stack free") + throw("mheap.freeSpanLocked - invalid stack free") } case mSpanInUse: if s.allocCount != 0 || s.sweepgen != h.sweepgen { - print("MHeap_FreeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") - throw("MHeap_FreeSpanLocked - invalid free") + print("mheap.freeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") + throw("mheap.freeSpanLocked - invalid free") } h.pagesInUse -= uint64(s.npages) default: - throw("MHeap_FreeSpanLocked - invalid span state") + throw("mheap.freeSpanLocked - invalid span state") } if acctinuse { @@ -1251,9 +1251,9 @@ func (list *mSpanList) init() { func (list *mSpanList) remove(span *mspan) { if span.list != list { - print("runtime: failed MSpanList_Remove span.npages=", span.npages, + print("runtime: failed mSpanList.remove span.npages=", span.npages, " span=", span, " prev=", span.prev, " span.list=", span.list, " list=", list, "\n") - throw("MSpanList_Remove") + throw("mSpanList.remove") } if list.first == span { list.first = span.next @@ -1276,8 +1276,8 @@ func (list *mSpanList) isEmpty() bool { func (list *mSpanList) insert(span *mspan) { if span.next != nil || span.prev != nil || span.list != nil { - println("runtime: failed MSpanList_Insert", span, span.next, span.prev, span.list) - throw("MSpanList_Insert") + println("runtime: failed mSpanList.insert", span, span.next, span.prev, span.list) + throw("mSpanList.insert") } span.next = list.first if list.first != nil { @@ -1294,8 +1294,8 @@ func (list *mSpanList) insert(span *mspan) { func (list *mSpanList) insertBack(span *mspan) { if span.next != nil || span.prev != nil || span.list != nil { - println("runtime: failed MSpanList_InsertBack", span, span.next, span.prev, span.list) - throw("MSpanList_InsertBack") + println("runtime: failed mSpanList.insertBack", span, span.next, span.prev, span.list) + throw("mSpanList.insertBack") } span.prev = list.last if list.last != nil { @@ -1523,7 +1523,7 @@ func setprofilebucket(p unsafe.Pointer, b *bucket) { } // Do whatever cleanup needs to be done to deallocate s. It has -// already been unlinked from the MSpan specials list. +// already been unlinked from the mspan specials list. func freespecial(s *special, p unsafe.Pointer, size uintptr) { switch s.kind { case _KindSpecialFinalizer: diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index fd576b7ae0..9250865ed1 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -529,7 +529,7 @@ func updatememstats() { memstats.by_size[i].nfree = 0 } - // Flush MCache's to MCentral. + // Flush mcache's to mcentral. systemstack(flushallmcaches) // Aggregate local stats. From 9e619739fdc4cbaeb00a10ef95ce3e5d6996e8a7 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Sat, 3 Nov 2018 09:09:28 -0700 Subject: [PATCH 033/111] cmd/compile: copy all fields during SubstAny MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consider these functions: func f(a any) int func g(a any) int Prior to this change, since f and g have identical signatures, they would share a single generated func type. types.SubstAny makes a shallow type copy, even after instantiation, f and g share a single generated Result type. So if you instantiate f with any=T, call dowidth, instantiate g with any=U, and call dowidth, and if sizeof(T) != sizeof(U), then the Offset of the result for f is now wrong. I don't believe this happens at all right now, but it bit me hard when experimenting with some other compiler changes. And it's hard to debug. It results in rare stack corruption, causing problems far from the actual source of the problem. To fix this, change SubstAny to make deep copies of TSTRUCTs. name old alloc/op new alloc/op delta Template 35.3MB ± 0% 35.4MB ± 0% +0.23% (p=0.008 n=5+5) Unicode 29.1MB ± 0% 29.1MB ± 0% +0.16% (p=0.008 n=5+5) GoTypes 122MB ± 0% 122MB ± 0% +0.16% (p=0.008 n=5+5) Compiler 513MB ± 0% 514MB ± 0% +0.19% (p=0.008 n=5+5) SSA 1.94GB ± 0% 1.94GB ± 0% +0.01% (p=0.008 n=5+5) Flate 24.2MB ± 0% 24.2MB ± 0% +0.08% (p=0.008 n=5+5) GoParser 28.5MB ± 0% 28.5MB ± 0% +0.24% (p=0.008 n=5+5) Reflect 86.2MB ± 0% 86.3MB ± 0% +0.09% (p=0.008 n=5+5) Tar 34.9MB ± 0% 34.9MB ± 0% +0.13% (p=0.008 n=5+5) XML 47.0MB ± 0% 47.1MB ± 0% +0.18% (p=0.008 n=5+5) [Geo mean] 80.9MB 81.0MB +0.15% name old allocs/op new allocs/op delta Template 348k ± 0% 349k ± 0% +0.38% (p=0.008 n=5+5) Unicode 340k ± 0% 340k ± 0% +0.21% (p=0.008 n=5+5) GoTypes 1.27M ± 0% 1.28M ± 0% +0.27% (p=0.008 n=5+5) Compiler 4.90M ± 0% 4.92M ± 0% +0.36% (p=0.008 n=5+5) SSA 15.3M ± 0% 15.3M ± 0% +0.03% (p=0.008 n=5+5) Flate 232k ± 0% 233k ± 0% +0.14% (p=0.008 n=5+5) GoParser 291k ± 0% 292k ± 0% +0.42% (p=0.008 n=5+5) Reflect 1.05M ± 0% 1.05M ± 0% +0.14% (p=0.008 n=5+5) Tar 343k ± 0% 344k ± 0% +0.22% (p=0.008 n=5+5) XML 428k ± 0% 430k ± 0% +0.36% (p=0.008 n=5+5) [Geo mean] 807k 809k +0.25% Change-Id: I62134db642206cded01920dc1d8a7da61f7ca0ac Reviewed-on: https://go-review.googlesource.com/c/147038 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/types/type.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index b20039239b..45355e5798 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -665,23 +665,18 @@ func SubstAny(t *Type, types *[]*Type) *Type { } case TSTRUCT: + // Make a copy of all fields, including ones whose type does not change. + // This prevents aliasing across functions, which can lead to later + // fields getting their Offset incorrectly overwritten. fields := t.FieldSlice() - var nfs []*Field + nfs := make([]*Field, len(fields)) for i, f := range fields { nft := SubstAny(f.Type, types) - if nft == f.Type { - continue - } - if nfs == nil { - nfs = append([]*Field(nil), fields...) - } nfs[i] = f.Copy() nfs[i].Type = nft } - if nfs != nil { - t = t.copy() - t.SetFields(nfs) - } + t = t.copy() + t.SetFields(nfs) } return t From 5848b6c9b854546473814c8752ee117a71bb8b54 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Mon, 29 Oct 2018 13:54:24 -0700 Subject: [PATCH 034/111] cmd/compile: shrink specialized convT2x call sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convT2E16 and other specialized type-to-interface routines accept a type/itab argument and return a complete interface value. However, we know enough in the routine to do without the type. And the caller can construct the interface value using the type. Doing so shrinks the call sites of ten of the specialized convT2x routines. It also lets us unify the empty and non-empty interface routines. Cuts 12k off cmd/go. name old time/op new time/op delta ConvT2ESmall-8 2.96ns ± 2% 2.34ns ± 4% -21.01% (p=0.000 n=175+189) ConvT2EUintptr-8 3.00ns ± 3% 2.34ns ± 4% -22.02% (p=0.000 n=189+187) ConvT2ELarge-8 21.3ns ± 7% 21.5ns ± 5% +1.02% (p=0.000 n=200+197) ConvT2ISmall-8 2.99ns ± 4% 2.33ns ± 3% -21.95% (p=0.000 n=193+184) ConvT2IUintptr-8 3.02ns ± 3% 2.33ns ± 3% -22.82% (p=0.000 n=198+190) ConvT2ILarge-8 21.7ns ± 5% 22.2ns ± 4% +2.31% (p=0.000 n=199+198) ConvT2Ezero/zero/16-8 2.96ns ± 2% 2.33ns ± 3% -21.11% (p=0.000 n=174+187) ConvT2Ezero/zero/32-8 2.96ns ± 1% 2.35ns ± 4% -20.62% (p=0.000 n=163+193) ConvT2Ezero/zero/64-8 2.99ns ± 2% 2.34ns ± 4% -21.78% (p=0.000 n=183+188) ConvT2Ezero/zero/str-8 3.27ns ± 3% 2.54ns ± 3% -22.32% (p=0.000 n=195+192) ConvT2Ezero/zero/slice-8 3.46ns ± 4% 2.81ns ± 3% -18.96% (p=0.000 n=197+164) ConvT2Ezero/zero/big-8 88.4ns ±20% 90.0ns ±20% +1.84% (p=0.000 n=196+198) ConvT2Ezero/nonzero/16-8 12.6ns ± 3% 12.3ns ± 3% -2.34% (p=0.000 n=167+196) ConvT2Ezero/nonzero/32-8 12.3ns ± 4% 11.9ns ± 3% -2.95% (p=0.000 n=187+193) ConvT2Ezero/nonzero/64-8 14.2ns ± 6% 13.8ns ± 5% -2.94% (p=0.000 n=198+199) ConvT2Ezero/nonzero/str-8 27.2ns ± 5% 26.8ns ± 5% -1.33% (p=0.000 n=200+198) ConvT2Ezero/nonzero/slice-8 33.3ns ± 8% 33.1ns ± 6% -0.82% (p=0.000 n=199+200) ConvT2Ezero/nonzero/big-8 88.8ns ±22% 90.2ns ±18% +1.58% (p=0.000 n=200+199) Neligible toolspeed impact. name old alloc/op new alloc/op delta Template 35.4MB ± 0% 35.3MB ± 0% -0.06% (p=0.008 n=5+5) Unicode 29.1MB ± 0% 29.1MB ± 0% ~ (p=0.310 n=5+5) GoTypes 122MB ± 0% 122MB ± 0% -0.08% (p=0.008 n=5+5) Compiler 514MB ± 0% 513MB ± 0% -0.02% (p=0.008 n=5+5) SSA 1.94GB ± 0% 1.94GB ± 0% -0.01% (p=0.008 n=5+5) Flate 24.2MB ± 0% 24.2MB ± 0% ~ (p=0.548 n=5+5) GoParser 28.5MB ± 0% 28.5MB ± 0% -0.05% (p=0.016 n=5+5) Reflect 86.3MB ± 0% 86.2MB ± 0% -0.02% (p=0.008 n=5+5) Tar 34.9MB ± 0% 34.9MB ± 0% ~ (p=0.095 n=5+5) XML 47.1MB ± 0% 47.1MB ± 0% -0.05% (p=0.008 n=5+5) [Geo mean] 81.0MB 81.0MB -0.03% name old allocs/op new allocs/op delta Template 349k ± 0% 349k ± 0% -0.08% (p=0.008 n=5+5) Unicode 340k ± 0% 340k ± 0% ~ (p=0.111 n=5+5) GoTypes 1.28M ± 0% 1.28M ± 0% -0.09% (p=0.008 n=5+5) Compiler 4.92M ± 0% 4.92M ± 0% -0.08% (p=0.008 n=5+5) SSA 15.3M ± 0% 15.3M ± 0% -0.03% (p=0.008 n=5+5) Flate 233k ± 0% 233k ± 0% ~ (p=0.500 n=5+5) GoParser 292k ± 0% 292k ± 0% -0.06% (p=0.008 n=5+5) Reflect 1.05M ± 0% 1.05M ± 0% -0.02% (p=0.008 n=5+5) Tar 344k ± 0% 343k ± 0% -0.06% (p=0.008 n=5+5) XML 430k ± 0% 429k ± 0% -0.08% (p=0.008 n=5+5) [Geo mean] 809k 809k -0.05% name old object-bytes new object-bytes delta Template 507kB ± 0% 507kB ± 0% -0.04% (p=0.008 n=5+5) Unicode 225kB ± 0% 225kB ± 0% ~ (all equal) GoTypes 1.85MB ± 0% 1.85MB ± 0% -0.08% (p=0.008 n=5+5) Compiler 6.75MB ± 0% 6.75MB ± 0% +0.01% (p=0.008 n=5+5) SSA 21.4MB ± 0% 21.4MB ± 0% -0.02% (p=0.008 n=5+5) Flate 328kB ± 0% 328kB ± 0% -0.03% (p=0.008 n=5+5) GoParser 403kB ± 0% 402kB ± 0% -0.06% (p=0.008 n=5+5) Reflect 1.41MB ± 0% 1.41MB ± 0% -0.03% (p=0.008 n=5+5) Tar 457kB ± 0% 457kB ± 0% -0.05% (p=0.008 n=5+5) XML 601kB ± 0% 600kB ± 0% -0.16% (p=0.008 n=5+5) [Geo mean] 1.05MB 1.04MB -0.05% Change-Id: I677a4108c0ecd32617549294036aa84f9214c4fe Reviewed-on: https://go-review.googlesource.com/c/147360 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Martin Möhrmann --- src/cmd/compile/internal/gc/builtin.go | 302 +++++++++--------- .../compile/internal/gc/builtin/runtime.go | 22 +- src/cmd/compile/internal/gc/walk.go | 71 ++-- src/runtime/iface.go | 134 +++----- 4 files changed, 242 insertions(+), 287 deletions(-) diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 325bf4aa0e..4e9f11c8b3 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -51,110 +51,105 @@ var runtimeDecls = [...]struct { {"decoderune", funcTag, 50}, {"countrunes", funcTag, 51}, {"convI2I", funcTag, 52}, - {"convT2E", funcTag, 53}, - {"convT2E16", funcTag, 52}, - {"convT2E32", funcTag, 52}, - {"convT2E64", funcTag, 52}, - {"convT2Estring", funcTag, 52}, - {"convT2Eslice", funcTag, 52}, - {"convT2Enoptr", funcTag, 53}, - {"convT2I", funcTag, 53}, - {"convT2I16", funcTag, 52}, - {"convT2I32", funcTag, 52}, - {"convT2I64", funcTag, 52}, - {"convT2Istring", funcTag, 52}, - {"convT2Islice", funcTag, 52}, - {"convT2Inoptr", funcTag, 53}, + {"convT16", funcTag, 54}, + {"convT32", funcTag, 54}, + {"convT64", funcTag, 54}, + {"convTstring", funcTag, 54}, + {"convTslice", funcTag, 54}, + {"convT2E", funcTag, 55}, + {"convT2Enoptr", funcTag, 55}, + {"convT2I", funcTag, 55}, + {"convT2Inoptr", funcTag, 55}, {"assertE2I", funcTag, 52}, - {"assertE2I2", funcTag, 54}, + {"assertE2I2", funcTag, 56}, {"assertI2I", funcTag, 52}, - {"assertI2I2", funcTag, 54}, - {"panicdottypeE", funcTag, 55}, - {"panicdottypeI", funcTag, 55}, - {"panicnildottype", funcTag, 56}, - {"ifaceeq", funcTag, 59}, - {"efaceeq", funcTag, 59}, - {"fastrand", funcTag, 61}, - {"makemap64", funcTag, 63}, - {"makemap", funcTag, 64}, - {"makemap_small", funcTag, 65}, - {"mapaccess1", funcTag, 66}, - {"mapaccess1_fast32", funcTag, 67}, - {"mapaccess1_fast64", funcTag, 67}, - {"mapaccess1_faststr", funcTag, 67}, - {"mapaccess1_fat", funcTag, 68}, - {"mapaccess2", funcTag, 69}, - {"mapaccess2_fast32", funcTag, 70}, - {"mapaccess2_fast64", funcTag, 70}, - {"mapaccess2_faststr", funcTag, 70}, - {"mapaccess2_fat", funcTag, 71}, - {"mapassign", funcTag, 66}, - {"mapassign_fast32", funcTag, 67}, - {"mapassign_fast32ptr", funcTag, 67}, - {"mapassign_fast64", funcTag, 67}, - {"mapassign_fast64ptr", funcTag, 67}, - {"mapassign_faststr", funcTag, 67}, - {"mapiterinit", funcTag, 72}, - {"mapdelete", funcTag, 72}, - {"mapdelete_fast32", funcTag, 73}, - {"mapdelete_fast64", funcTag, 73}, - {"mapdelete_faststr", funcTag, 73}, - {"mapiternext", funcTag, 74}, - {"mapclear", funcTag, 75}, - {"makechan64", funcTag, 77}, - {"makechan", funcTag, 78}, - {"chanrecv1", funcTag, 80}, - {"chanrecv2", funcTag, 81}, - {"chansend1", funcTag, 83}, + {"assertI2I2", funcTag, 56}, + {"panicdottypeE", funcTag, 57}, + {"panicdottypeI", funcTag, 57}, + {"panicnildottype", funcTag, 58}, + {"ifaceeq", funcTag, 60}, + {"efaceeq", funcTag, 60}, + {"fastrand", funcTag, 62}, + {"makemap64", funcTag, 64}, + {"makemap", funcTag, 65}, + {"makemap_small", funcTag, 66}, + {"mapaccess1", funcTag, 67}, + {"mapaccess1_fast32", funcTag, 68}, + {"mapaccess1_fast64", funcTag, 68}, + {"mapaccess1_faststr", funcTag, 68}, + {"mapaccess1_fat", funcTag, 69}, + {"mapaccess2", funcTag, 70}, + {"mapaccess2_fast32", funcTag, 71}, + {"mapaccess2_fast64", funcTag, 71}, + {"mapaccess2_faststr", funcTag, 71}, + {"mapaccess2_fat", funcTag, 72}, + {"mapassign", funcTag, 67}, + {"mapassign_fast32", funcTag, 68}, + {"mapassign_fast32ptr", funcTag, 68}, + {"mapassign_fast64", funcTag, 68}, + {"mapassign_fast64ptr", funcTag, 68}, + {"mapassign_faststr", funcTag, 68}, + {"mapiterinit", funcTag, 73}, + {"mapdelete", funcTag, 73}, + {"mapdelete_fast32", funcTag, 74}, + {"mapdelete_fast64", funcTag, 74}, + {"mapdelete_faststr", funcTag, 74}, + {"mapiternext", funcTag, 75}, + {"mapclear", funcTag, 76}, + {"makechan64", funcTag, 78}, + {"makechan", funcTag, 79}, + {"chanrecv1", funcTag, 81}, + {"chanrecv2", funcTag, 82}, + {"chansend1", funcTag, 84}, {"closechan", funcTag, 23}, - {"writeBarrier", varTag, 85}, - {"typedmemmove", funcTag, 86}, - {"typedmemclr", funcTag, 87}, - {"typedslicecopy", funcTag, 88}, - {"selectnbsend", funcTag, 89}, - {"selectnbrecv", funcTag, 90}, - {"selectnbrecv2", funcTag, 92}, - {"selectsetpc", funcTag, 56}, - {"selectgo", funcTag, 93}, + {"writeBarrier", varTag, 86}, + {"typedmemmove", funcTag, 87}, + {"typedmemclr", funcTag, 88}, + {"typedslicecopy", funcTag, 89}, + {"selectnbsend", funcTag, 90}, + {"selectnbrecv", funcTag, 91}, + {"selectnbrecv2", funcTag, 93}, + {"selectsetpc", funcTag, 58}, + {"selectgo", funcTag, 94}, {"block", funcTag, 5}, - {"makeslice", funcTag, 94}, - {"makeslice64", funcTag, 95}, - {"growslice", funcTag, 97}, - {"memmove", funcTag, 98}, - {"memclrNoHeapPointers", funcTag, 99}, - {"memclrHasPointers", funcTag, 99}, - {"memequal", funcTag, 100}, - {"memequal8", funcTag, 101}, - {"memequal16", funcTag, 101}, - {"memequal32", funcTag, 101}, - {"memequal64", funcTag, 101}, - {"memequal128", funcTag, 101}, - {"int64div", funcTag, 102}, - {"uint64div", funcTag, 103}, - {"int64mod", funcTag, 102}, - {"uint64mod", funcTag, 103}, - {"float64toint64", funcTag, 104}, - {"float64touint64", funcTag, 105}, - {"float64touint32", funcTag, 106}, - {"int64tofloat64", funcTag, 107}, - {"uint64tofloat64", funcTag, 108}, - {"uint32tofloat64", funcTag, 109}, - {"complex128div", funcTag, 110}, - {"racefuncenter", funcTag, 111}, + {"makeslice", funcTag, 95}, + {"makeslice64", funcTag, 96}, + {"growslice", funcTag, 98}, + {"memmove", funcTag, 99}, + {"memclrNoHeapPointers", funcTag, 100}, + {"memclrHasPointers", funcTag, 100}, + {"memequal", funcTag, 101}, + {"memequal8", funcTag, 102}, + {"memequal16", funcTag, 102}, + {"memequal32", funcTag, 102}, + {"memequal64", funcTag, 102}, + {"memequal128", funcTag, 102}, + {"int64div", funcTag, 103}, + {"uint64div", funcTag, 104}, + {"int64mod", funcTag, 103}, + {"uint64mod", funcTag, 104}, + {"float64toint64", funcTag, 105}, + {"float64touint64", funcTag, 106}, + {"float64touint32", funcTag, 107}, + {"int64tofloat64", funcTag, 108}, + {"uint64tofloat64", funcTag, 109}, + {"uint32tofloat64", funcTag, 110}, + {"complex128div", funcTag, 111}, + {"racefuncenter", funcTag, 112}, {"racefuncenterfp", funcTag, 5}, {"racefuncexit", funcTag, 5}, - {"raceread", funcTag, 111}, - {"racewrite", funcTag, 111}, - {"racereadrange", funcTag, 112}, - {"racewriterange", funcTag, 112}, - {"msanread", funcTag, 112}, - {"msanwrite", funcTag, 112}, + {"raceread", funcTag, 112}, + {"racewrite", funcTag, 112}, + {"racereadrange", funcTag, 113}, + {"racewriterange", funcTag, 113}, + {"msanread", funcTag, 113}, + {"msanwrite", funcTag, 113}, {"support_popcnt", varTag, 11}, {"support_sse41", varTag, 11}, } func runtimeTypes() []*types.Type { - var typs [113]*types.Type + var typs [114]*types.Type typs[0] = types.Bytetype typs[1] = types.NewPtr(typs[0]) typs[2] = types.Types[TANY] @@ -208,65 +203,66 @@ func runtimeTypes() []*types.Type { typs[50] = functype(nil, []*Node{anonfield(typs[21]), anonfield(typs[32])}, []*Node{anonfield(typs[40]), anonfield(typs[32])}) typs[51] = functype(nil, []*Node{anonfield(typs[21])}, []*Node{anonfield(typs[32])}) typs[52] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2])}, []*Node{anonfield(typs[2])}) - typs[53] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, []*Node{anonfield(typs[2])}) - typs[54] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2])}, []*Node{anonfield(typs[2]), anonfield(typs[11])}) - typs[55] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[1])}, nil) - typs[56] = functype(nil, []*Node{anonfield(typs[1])}, nil) - typs[57] = types.NewPtr(typs[47]) - typs[58] = types.Types[TUNSAFEPTR] - typs[59] = functype(nil, []*Node{anonfield(typs[57]), anonfield(typs[58]), anonfield(typs[58])}, []*Node{anonfield(typs[11])}) - typs[60] = types.Types[TUINT32] - typs[61] = functype(nil, nil, []*Node{anonfield(typs[60])}) - typs[62] = types.NewMap(typs[2], typs[2]) - typs[63] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[3])}, []*Node{anonfield(typs[62])}) - typs[64] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[3])}, []*Node{anonfield(typs[62])}) - typs[65] = functype(nil, nil, []*Node{anonfield(typs[62])}) - typs[66] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[3])}, []*Node{anonfield(typs[3])}) - typs[67] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[2])}, []*Node{anonfield(typs[3])}) - typs[68] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[3]), anonfield(typs[1])}, []*Node{anonfield(typs[3])}) - typs[69] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[3])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) - typs[70] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[2])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) - typs[71] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[3]), anonfield(typs[1])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) - typs[72] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[3])}, nil) - typs[73] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62]), anonfield(typs[2])}, nil) - typs[74] = functype(nil, []*Node{anonfield(typs[3])}, nil) - typs[75] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[62])}, nil) - typs[76] = types.NewChan(typs[2], types.Cboth) - typs[77] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[76])}) - typs[78] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32])}, []*Node{anonfield(typs[76])}) - typs[79] = types.NewChan(typs[2], types.Crecv) - typs[80] = functype(nil, []*Node{anonfield(typs[79]), anonfield(typs[3])}, nil) - typs[81] = functype(nil, []*Node{anonfield(typs[79]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[82] = types.NewChan(typs[2], types.Csend) - typs[83] = functype(nil, []*Node{anonfield(typs[82]), anonfield(typs[3])}, nil) - typs[84] = types.NewArray(typs[0], 3) - typs[85] = tostruct([]*Node{namedfield("enabled", typs[11]), namedfield("pad", typs[84]), namedfield("needed", typs[11]), namedfield("cgo", typs[11]), namedfield("alignme", typs[17])}) - typs[86] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[3])}, nil) - typs[87] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, nil) - typs[88] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2]), anonfield(typs[2])}, []*Node{anonfield(typs[32])}) - typs[89] = functype(nil, []*Node{anonfield(typs[82]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[90] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[79])}, []*Node{anonfield(typs[11])}) - typs[91] = types.NewPtr(typs[11]) - typs[92] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[91]), anonfield(typs[79])}, []*Node{anonfield(typs[11])}) - typs[93] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[32])}, []*Node{anonfield(typs[32]), anonfield(typs[11])}) - typs[94] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[32])}, []*Node{anonfield(typs[58])}) - typs[95] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[58])}) - typs[96] = types.NewSlice(typs[2]) - typs[97] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[96]), anonfield(typs[32])}, []*Node{anonfield(typs[96])}) - typs[98] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[47])}, nil) - typs[99] = functype(nil, []*Node{anonfield(typs[58]), anonfield(typs[47])}, nil) - typs[100] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[47])}, []*Node{anonfield(typs[11])}) - typs[101] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[102] = functype(nil, []*Node{anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[15])}) - typs[103] = functype(nil, []*Node{anonfield(typs[17]), anonfield(typs[17])}, []*Node{anonfield(typs[17])}) - typs[104] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[15])}) - typs[105] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[17])}) - typs[106] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[60])}) - typs[107] = functype(nil, []*Node{anonfield(typs[15])}, []*Node{anonfield(typs[13])}) - typs[108] = functype(nil, []*Node{anonfield(typs[17])}, []*Node{anonfield(typs[13])}) - typs[109] = functype(nil, []*Node{anonfield(typs[60])}, []*Node{anonfield(typs[13])}) - typs[110] = functype(nil, []*Node{anonfield(typs[19]), anonfield(typs[19])}, []*Node{anonfield(typs[19])}) - typs[111] = functype(nil, []*Node{anonfield(typs[47])}, nil) - typs[112] = functype(nil, []*Node{anonfield(typs[47]), anonfield(typs[47])}, nil) + typs[53] = types.Types[TUNSAFEPTR] + typs[54] = functype(nil, []*Node{anonfield(typs[2])}, []*Node{anonfield(typs[53])}) + typs[55] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, []*Node{anonfield(typs[2])}) + typs[56] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2])}, []*Node{anonfield(typs[2]), anonfield(typs[11])}) + typs[57] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[1])}, nil) + typs[58] = functype(nil, []*Node{anonfield(typs[1])}, nil) + typs[59] = types.NewPtr(typs[47]) + typs[60] = functype(nil, []*Node{anonfield(typs[59]), anonfield(typs[53]), anonfield(typs[53])}, []*Node{anonfield(typs[11])}) + typs[61] = types.Types[TUINT32] + typs[62] = functype(nil, nil, []*Node{anonfield(typs[61])}) + typs[63] = types.NewMap(typs[2], typs[2]) + typs[64] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[3])}, []*Node{anonfield(typs[63])}) + typs[65] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[3])}, []*Node{anonfield(typs[63])}) + typs[66] = functype(nil, nil, []*Node{anonfield(typs[63])}) + typs[67] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[3])}, []*Node{anonfield(typs[3])}) + typs[68] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[2])}, []*Node{anonfield(typs[3])}) + typs[69] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[3]), anonfield(typs[1])}, []*Node{anonfield(typs[3])}) + typs[70] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[3])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) + typs[71] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[2])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) + typs[72] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[3]), anonfield(typs[1])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) + typs[73] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[3])}, nil) + typs[74] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[2])}, nil) + typs[75] = functype(nil, []*Node{anonfield(typs[3])}, nil) + typs[76] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[63])}, nil) + typs[77] = types.NewChan(typs[2], types.Cboth) + typs[78] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[77])}) + typs[79] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32])}, []*Node{anonfield(typs[77])}) + typs[80] = types.NewChan(typs[2], types.Crecv) + typs[81] = functype(nil, []*Node{anonfield(typs[80]), anonfield(typs[3])}, nil) + typs[82] = functype(nil, []*Node{anonfield(typs[80]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[83] = types.NewChan(typs[2], types.Csend) + typs[84] = functype(nil, []*Node{anonfield(typs[83]), anonfield(typs[3])}, nil) + typs[85] = types.NewArray(typs[0], 3) + typs[86] = tostruct([]*Node{namedfield("enabled", typs[11]), namedfield("pad", typs[85]), namedfield("needed", typs[11]), namedfield("cgo", typs[11]), namedfield("alignme", typs[17])}) + typs[87] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[3])}, nil) + typs[88] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, nil) + typs[89] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2]), anonfield(typs[2])}, []*Node{anonfield(typs[32])}) + typs[90] = functype(nil, []*Node{anonfield(typs[83]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[91] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[80])}, []*Node{anonfield(typs[11])}) + typs[92] = types.NewPtr(typs[11]) + typs[93] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[92]), anonfield(typs[80])}, []*Node{anonfield(typs[11])}) + typs[94] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[32])}, []*Node{anonfield(typs[32]), anonfield(typs[11])}) + typs[95] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[32])}, []*Node{anonfield(typs[53])}) + typs[96] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[53])}) + typs[97] = types.NewSlice(typs[2]) + typs[98] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[97]), anonfield(typs[32])}, []*Node{anonfield(typs[97])}) + typs[99] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[47])}, nil) + typs[100] = functype(nil, []*Node{anonfield(typs[53]), anonfield(typs[47])}, nil) + typs[101] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[47])}, []*Node{anonfield(typs[11])}) + typs[102] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[103] = functype(nil, []*Node{anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[15])}) + typs[104] = functype(nil, []*Node{anonfield(typs[17]), anonfield(typs[17])}, []*Node{anonfield(typs[17])}) + typs[105] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[15])}) + typs[106] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[17])}) + typs[107] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[61])}) + typs[108] = functype(nil, []*Node{anonfield(typs[15])}, []*Node{anonfield(typs[13])}) + typs[109] = functype(nil, []*Node{anonfield(typs[17])}, []*Node{anonfield(typs[13])}) + typs[110] = functype(nil, []*Node{anonfield(typs[61])}, []*Node{anonfield(typs[13])}) + typs[111] = functype(nil, []*Node{anonfield(typs[19]), anonfield(typs[19])}, []*Node{anonfield(typs[19])}) + typs[112] = functype(nil, []*Node{anonfield(typs[47])}, nil) + typs[113] = functype(nil, []*Node{anonfield(typs[47]), anonfield(typs[47])}, nil) return typs[:] } diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index e6d174bc4b..1eaf332e50 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -61,23 +61,23 @@ func slicestringcopy(to any, fr any) int func decoderune(string, int) (retv rune, retk int) func countrunes(string) int -// interface conversions +// Non-empty-interface to non-empty-interface conversion. func convI2I(typ *byte, elem any) (ret any) +// Specialized type-to-interface conversion. +// These return only a data pointer. +func convT16(val any) unsafe.Pointer // val must be uint16-like (same size and alignment as a uint16) +func convT32(val any) unsafe.Pointer // val must be uint32-like (same size and alignment as a uint32) +func convT64(val any) unsafe.Pointer // val must be uint64-like (same size and alignment as a uint64 and contains no pointers) +func convTstring(val any) unsafe.Pointer // val must be a string +func convTslice(val any) unsafe.Pointer // val must be a slice + +// Type to empty-interface conversion. func convT2E(typ *byte, elem *any) (ret any) -func convT2E16(typ *byte, val any) (ret any) -func convT2E32(typ *byte, val any) (ret any) -func convT2E64(typ *byte, val any) (ret any) -func convT2Estring(typ *byte, val any) (ret any) // val must be a string -func convT2Eslice(typ *byte, val any) (ret any) // val must be a slice func convT2Enoptr(typ *byte, elem *any) (ret any) +// Type to non-empty-interface conversion. func convT2I(tab *byte, elem *any) (ret any) -func convT2I16(tab *byte, val any) (ret any) -func convT2I32(tab *byte, val any) (ret any) -func convT2I64(tab *byte, val any) (ret any) -func convT2Istring(tab *byte, val any) (ret any) // val must be a string -func convT2Islice(tab *byte, val any) (ret any) // val must be a slice func convT2Inoptr(tab *byte, elem *any) (ret any) // interface type assertions x.(T) diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 0e07efa0d9..fd484a6472 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -384,41 +384,31 @@ func convFuncName(from, to *types.Type) (fnname string, needsaddr bool) { tkind := to.Tie() switch from.Tie() { case 'I': - switch tkind { - case 'I': + if tkind == 'I' { return "convI2I", false } case 'T': + switch { + case from.Size() == 2 && from.Align == 2: + return "convT16", false + case from.Size() == 4 && from.Align == 4 && !types.Haspointers(from): + return "convT32", false + case from.Size() == 8 && from.Align == types.Types[TUINT64].Align && !types.Haspointers(from): + return "convT64", false + case from.IsString(): + return "convTstring", false + case from.IsSlice(): + return "convTslice", false + } + switch tkind { case 'E': - switch { - case from.Size() == 2 && from.Align == 2: - return "convT2E16", false - case from.Size() == 4 && from.Align == 4 && !types.Haspointers(from): - return "convT2E32", false - case from.Size() == 8 && from.Align == types.Types[TUINT64].Align && !types.Haspointers(from): - return "convT2E64", false - case from.IsString(): - return "convT2Estring", false - case from.IsSlice(): - return "convT2Eslice", false - case !types.Haspointers(from): + if !types.Haspointers(from) { return "convT2Enoptr", true } return "convT2E", true case 'I': - switch { - case from.Size() == 2 && from.Align == 2: - return "convT2I16", false - case from.Size() == 4 && from.Align == 4 && !types.Haspointers(from): - return "convT2I32", false - case from.Size() == 8 && from.Align == types.Types[TUINT64].Align && !types.Haspointers(from): - return "convT2I64", false - case from.IsString(): - return "convT2Istring", false - case from.IsSlice(): - return "convT2Islice", false - case !types.Haspointers(from): + if !types.Haspointers(from) { return "convT2Inoptr", true } return "convT2I", true @@ -925,6 +915,34 @@ opswitch: break } + fnname, needsaddr := convFuncName(n.Left.Type, n.Type) + + if !needsaddr && !n.Left.Type.IsInterface() { + // Use a specialized conversion routine that only returns a data pointer. + // ptr = convT2X(val) + // e = iface{typ/tab, ptr} + fn := syslook(fnname) + dowidth(n.Left.Type) + fn = substArgTypes(fn, n.Left.Type) + dowidth(fn.Type) + call := nod(OCALL, fn, nil) + call.List.Set1(n.Left) + call = typecheck(call, Erv) + call = walkexpr(call, init) + call = safeexpr(call, init) + var tab *Node + if n.Type.IsEmptyInterface() { + tab = typename(n.Left.Type) + } else { + tab = itabname(n.Left.Type, n.Type) + } + e := nod(OEFACE, tab, call) + e.Type = n.Type + e.SetTypecheck(1) + n = e + break + } + var ll []*Node if n.Type.IsEmptyInterface() { if !n.Left.Type.IsInterface() { @@ -938,7 +956,6 @@ opswitch: } } - fnname, needsaddr := convFuncName(n.Left.Type, n.Type) v := n.Left if needsaddr { // Types of large or unknown size are passed by reference. diff --git a/src/runtime/iface.go b/src/runtime/iface.go index 1ef9825a48..8eca2e849d 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -267,6 +267,34 @@ func panicnildottype(want *_type) { // Just to match other nil conversion errors, we don't for now. } +// The specialized convTx routines need a type descriptor to use when calling mallocgc. +// We don't need the type to be exact, just to have the correct size, alignment, and pointer-ness. +// However, when debugging, it'd be nice to have some indication in mallocgc where the types came from, +// so we use named types here. +// We then construct interface values of these types, +// and then extract the type word to use as needed. +type ( + uint16InterfacePtr uint16 + uint32InterfacePtr uint32 + uint64InterfacePtr uint64 + stringInterfacePtr string + sliceInterfacePtr []byte +) + +var ( + uint16Eface interface{} = uint16InterfacePtr(0) + uint32Eface interface{} = uint32InterfacePtr(0) + uint64Eface interface{} = uint64InterfacePtr(0) + stringEface interface{} = stringInterfacePtr("") + sliceEface interface{} = sliceInterfacePtr(nil) + + uint16Type *_type = (*eface)(unsafe.Pointer(&uint16Eface))._type + uint32Type *_type = (*eface)(unsafe.Pointer(&uint32Eface))._type + uint64Type *_type = (*eface)(unsafe.Pointer(&uint64Eface))._type + stringType *_type = (*eface)(unsafe.Pointer(&stringEface))._type + sliceType *_type = (*eface)(unsafe.Pointer(&sliceEface))._type +) + // The conv and assert functions below do very similar things. // The convXXX functions are guaranteed by the compiler to succeed. // The assertXXX functions may fail (either panicking or returning false, @@ -290,69 +318,54 @@ func convT2E(t *_type, elem unsafe.Pointer) (e eface) { return } -func convT2E16(t *_type, val uint16) (e eface) { - var x unsafe.Pointer +func convT16(val uint16) (x unsafe.Pointer) { if val == 0 { x = unsafe.Pointer(&zeroVal[0]) } else { - x = mallocgc(2, t, false) + x = mallocgc(2, uint16Type, false) *(*uint16)(x) = val } - e._type = t - e.data = x return } -func convT2E32(t *_type, val uint32) (e eface) { - var x unsafe.Pointer +func convT32(val uint32) (x unsafe.Pointer) { if val == 0 { x = unsafe.Pointer(&zeroVal[0]) } else { - x = mallocgc(4, t, false) + x = mallocgc(4, uint32Type, false) *(*uint32)(x) = val } - e._type = t - e.data = x return } -func convT2E64(t *_type, val uint64) (e eface) { - var x unsafe.Pointer +func convT64(val uint64) (x unsafe.Pointer) { if val == 0 { x = unsafe.Pointer(&zeroVal[0]) } else { - x = mallocgc(8, t, false) + x = mallocgc(8, uint64Type, false) *(*uint64)(x) = val } - e._type = t - e.data = x return } -func convT2Estring(t *_type, val string) (e eface) { - var x unsafe.Pointer +func convTstring(val string) (x unsafe.Pointer) { if val == "" { x = unsafe.Pointer(&zeroVal[0]) } else { - x = mallocgc(unsafe.Sizeof(val), t, true) + x = mallocgc(unsafe.Sizeof(val), stringType, true) *(*string)(x) = val } - e._type = t - e.data = x return } -func convT2Eslice(t *_type, val []byte) (e eface) { +func convTslice(val []byte) (x unsafe.Pointer) { // Note: this must work for any element type, not just byte. - var x unsafe.Pointer if (*slice)(unsafe.Pointer(&val)).array == nil { x = unsafe.Pointer(&zeroVal[0]) } else { - x = mallocgc(unsafe.Sizeof(val), t, true) + x = mallocgc(unsafe.Sizeof(val), sliceType, true) *(*[]byte)(x) = val } - e._type = t - e.data = x return } @@ -385,77 +398,6 @@ func convT2I(tab *itab, elem unsafe.Pointer) (i iface) { return } -func convT2I16(tab *itab, val uint16) (i iface) { - t := tab._type - var x unsafe.Pointer - if val == 0 { - x = unsafe.Pointer(&zeroVal[0]) - } else { - x = mallocgc(2, t, false) - *(*uint16)(x) = val - } - i.tab = tab - i.data = x - return -} - -func convT2I32(tab *itab, val uint32) (i iface) { - t := tab._type - var x unsafe.Pointer - if val == 0 { - x = unsafe.Pointer(&zeroVal[0]) - } else { - x = mallocgc(4, t, false) - *(*uint32)(x) = val - } - i.tab = tab - i.data = x - return -} - -func convT2I64(tab *itab, val uint64) (i iface) { - t := tab._type - var x unsafe.Pointer - if val == 0 { - x = unsafe.Pointer(&zeroVal[0]) - } else { - x = mallocgc(8, t, false) - *(*uint64)(x) = val - } - i.tab = tab - i.data = x - return -} - -func convT2Istring(tab *itab, val string) (i iface) { - t := tab._type - var x unsafe.Pointer - if val == "" { - x = unsafe.Pointer(&zeroVal[0]) - } else { - x = mallocgc(unsafe.Sizeof(val), t, true) - *(*string)(x) = val - } - i.tab = tab - i.data = x - return -} - -func convT2Islice(tab *itab, val []byte) (i iface) { - // Note: this must work for any element type, not just byte. - t := tab._type - var x unsafe.Pointer - if (*slice)(unsafe.Pointer(&val)).array == nil { - x = unsafe.Pointer(&zeroVal[0]) - } else { - x = mallocgc(unsafe.Sizeof(val), t, true) - *(*[]byte)(x) = val - } - i.tab = tab - i.data = x - return -} - func convT2Inoptr(tab *itab, elem unsafe.Pointer) (i iface) { t := tab._type if raceenabled { From 9fc22d29092933460fe00bdaccea179f29e9960d Mon Sep 17 00:00:00 2001 From: Michael Stapelberg Date: Fri, 2 Nov 2018 11:23:54 +0100 Subject: [PATCH 035/111] net: update zoneCache on cache misses to cover appearing interfaces performance differences are in measurement noise as per benchcmp: benchmark old ns/op new ns/op delta BenchmarkUDP6LinkLocalUnicast-12 5012 5009 -0.06% Fixes #28535 Change-Id: Id022e2ed089ce8388a2398e755848ec94e77e653 Reviewed-on: https://go-review.googlesource.com/c/146941 Run-TryBot: Mikio Hara Reviewed-by: Mikio Hara --- src/net/interface.go | 40 ++++++++++++++++++++++++--------- src/net/interface_bsd_test.go | 5 +++++ src/net/interface_linux_test.go | 25 +++++++++++++++++++++ src/net/interface_unix_test.go | 34 ++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/net/interface.go b/src/net/interface.go index 375a4568e3..46b0400f2f 100644 --- a/src/net/interface.go +++ b/src/net/interface.go @@ -102,7 +102,7 @@ func Interfaces() ([]Interface, error) { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} } if len(ift) != 0 { - zoneCache.update(ift) + zoneCache.update(ift, false) } return ift, nil } @@ -159,7 +159,7 @@ func InterfaceByName(name string) (*Interface, error) { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} } if len(ift) != 0 { - zoneCache.update(ift) + zoneCache.update(ift, false) } for _, ifi := range ift { if name == ifi.Name { @@ -187,18 +187,21 @@ var zoneCache = ipv6ZoneCache{ toName: make(map[int]string), } -func (zc *ipv6ZoneCache) update(ift []Interface) { +// update refreshes the network interface information if the cache was last +// updated more than 1 minute ago, or if force is set. It returns whether the +// cache was updated. +func (zc *ipv6ZoneCache) update(ift []Interface, force bool) (updated bool) { zc.Lock() defer zc.Unlock() now := time.Now() - if zc.lastFetched.After(now.Add(-60 * time.Second)) { - return + if !force && zc.lastFetched.After(now.Add(-60*time.Second)) { + return false } zc.lastFetched = now if len(ift) == 0 { var err error if ift, err = interfaceTable(0); err != nil { - return + return false } } zc.toIndex = make(map[string]int, len(ift)) @@ -209,16 +212,25 @@ func (zc *ipv6ZoneCache) update(ift []Interface) { zc.toName[ifi.Index] = ifi.Name } } + return true } func (zc *ipv6ZoneCache) name(index int) string { if index == 0 { return "" } - zoneCache.update(nil) + updated := zoneCache.update(nil, false) zoneCache.RLock() - defer zoneCache.RUnlock() name, ok := zoneCache.toName[index] + zoneCache.RUnlock() + if !ok { + if !updated { + zoneCache.update(nil, true) + zoneCache.RLock() + name, ok = zoneCache.toName[index] + zoneCache.RUnlock() + } + } if !ok { name = uitoa(uint(index)) } @@ -229,10 +241,18 @@ func (zc *ipv6ZoneCache) index(name string) int { if name == "" { return 0 } - zoneCache.update(nil) + updated := zoneCache.update(nil, false) zoneCache.RLock() - defer zoneCache.RUnlock() index, ok := zoneCache.toIndex[name] + zoneCache.RUnlock() + if !ok { + if !updated { + zoneCache.update(nil, true) + zoneCache.RLock() + index, ok = zoneCache.toIndex[name] + zoneCache.RUnlock() + } + } if !ok { index, _, _ = dtoi(name) } diff --git a/src/net/interface_bsd_test.go b/src/net/interface_bsd_test.go index 69b0fbcab3..947dde71e6 100644 --- a/src/net/interface_bsd_test.go +++ b/src/net/interface_bsd_test.go @@ -7,6 +7,7 @@ package net import ( + "errors" "fmt" "os/exec" "runtime" @@ -53,3 +54,7 @@ func (ti *testInterface) setPointToPoint(suffix int) error { }) return nil } + +func (ti *testInterface) setLinkLocal(suffix int) error { + return errors.New("not yet implemented for BSD") +} diff --git a/src/net/interface_linux_test.go b/src/net/interface_linux_test.go index 6959ddb3d9..0699fec636 100644 --- a/src/net/interface_linux_test.go +++ b/src/net/interface_linux_test.go @@ -35,6 +35,31 @@ func (ti *testInterface) setBroadcast(suffix int) error { return nil } +func (ti *testInterface) setLinkLocal(suffix int) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "add", ti.name, "type", "dummy"}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "add", ti.local, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "del", ti.local, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "delete", ti.name, "type", "dummy"}, + }) + return nil +} + func (ti *testInterface) setPointToPoint(suffix int) error { ti.name = fmt.Sprintf("gotest%d", suffix) xname, err := exec.LookPath("ip") diff --git a/src/net/interface_unix_test.go b/src/net/interface_unix_test.go index c3d981dc5c..20e75cd036 100644 --- a/src/net/interface_unix_test.go +++ b/src/net/interface_unix_test.go @@ -176,3 +176,37 @@ func TestInterfaceArrivalAndDeparture(t *testing.T) { } } } + +func TestInterfaceArrivalAndDepartureZoneCache(t *testing.T) { + if testing.Short() { + t.Skip("avoid external network") + } + if os.Getuid() != 0 { + t.Skip("must be root") + } + + // Ensure zoneCache is filled: + _, _ = Listen("tcp", "[fe80::1%nonexistant]:0") + + ti := &testInterface{local: "fe80::1"} + if err := ti.setLinkLocal(0); err != nil { + t.Skipf("test requires external command: %v", err) + } + if err := ti.setup(); err != nil { + t.Fatal(err) + } + defer ti.teardown() + + time.Sleep(3 * time.Millisecond) + + // If Listen fails (on Linux with “bind: invalid argumentâ€), zoneCache was + // not updated when encountering a nonexistant interface: + ln, err := Listen("tcp", "[fe80::1%"+ti.name+"]:0") + if err != nil { + t.Fatal(err) + } + ln.Close() + if err := ti.teardown(); err != nil { + t.Fatal(err) + } +} From c1a16b7dadfee27b03a2a70a20c3cf339a069a40 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Sun, 28 Oct 2018 11:19:33 -0700 Subject: [PATCH 036/111] cmd/compile: loop in disjoint OpOffPtr check We collapse OpOffPtrs during generic rewrites. However, we also use disjoint at the same time. Instead of waiting for all OpOffPtrs to be collapsed before the disjointness rules can kick in, burrow through all OpOffPtrs immediately. Change-Id: I60d0a70a9b4605b1817db7c4aab0c0d789651c90 Reviewed-on: https://go-review.googlesource.com/c/145206 Run-TryBot: Josh Bleecher Snyder Reviewed-by: Michael Munday Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/rewrite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index ed5bce861e..17d7cb3414 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -542,7 +542,7 @@ func disjoint(p1 *Value, n1 int64, p2 *Value, n2 int64) bool { } baseAndOffset := func(ptr *Value) (base *Value, offset int64) { base, offset = ptr, 0 - if base.Op == OpOffPtr { + for base.Op == OpOffPtr { offset += base.AuxInt base = base.Args[0] } From 510eea2dfcabbc8916c7c59aa37046269ad29497 Mon Sep 17 00:00:00 2001 From: Mikio Hara Date: Tue, 6 Nov 2018 12:48:17 +0900 Subject: [PATCH 037/111] net/http: update bundled SOCKS client Updates socks_bundle.go to git rev 26e67e7 for: - 26e67e7 internal/socks: fix socket descriptor leakage in Dialer.Dial Change-Id: I9ab27a85504d77f1ca2e97cb005f5e37fd3c3ff4 Reviewed-on: https://go-review.googlesource.com/c/147717 Run-TryBot: Mikio Hara Reviewed-by: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/net/http/socks_bundle.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/http/socks_bundle.go b/src/net/http/socks_bundle.go index e4314b4128..e6640dd404 100644 --- a/src/net/http/socks_bundle.go +++ b/src/net/http/socks_bundle.go @@ -380,6 +380,7 @@ func (d *socksDialer) Dial(network, address string) (net.Conn, error) { return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} } if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { + c.Close() return nil, err } return c, nil From 0e4a0b93d25f56eda3b6026a98bdee4cf6fc7b8f Mon Sep 17 00:00:00 2001 From: Raghavendra Nagaraj Date: Tue, 6 Nov 2018 09:02:03 +0000 Subject: [PATCH 038/111] reflect: fix StructOf panics from too many methods in embedded fields Previously we panicked if the number of methods present for an embedded field was >= 32. This change removes that limit and now StructOf dynamically calls itself to create space for the number of methods. Fixes #25402 Change-Id: I3b1deb119796d25f7e6eee1cdb126327b49a0b5e GitHub-Last-Rev: 16da71ad6b23563f3ed26f1914adf41e3d42de69 GitHub-Pull-Request: golang/go#26865 Reviewed-on: https://go-review.googlesource.com/c/128479 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/reflect/all_test.go | 11 ++++++ src/reflect/type.go | 79 ++++++++++------------------------------- 2 files changed, 30 insertions(+), 60 deletions(-) diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index c463b61c57..4b215f120c 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -5019,6 +5019,17 @@ func TestStructOfWithInterface(t *testing.T) { }) } +func TestStructOfTooManyFields(t *testing.T) { + // Bug Fix: #25402 - this should not panic + tt := StructOf([]StructField{ + {Name: "Time", Type: TypeOf(time.Time{}), Anonymous: true}, + }) + + if _, present := tt.MethodByName("After"); !present { + t.Errorf("Expected method `After` to be found") + } +} + func TestChanOf(t *testing.T) { // check construction and use of type not in binary type T string diff --git a/src/reflect/type.go b/src/reflect/type.go index a04234ca69..5bbab79fc0 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -1889,6 +1889,8 @@ func MapOf(key, elem Type) Type { return ti.(Type) } +// TODO(crawshaw): as these funcTypeFixedN structs have no methods, +// they could be defined at runtime using the StructOf function. type funcTypeFixed4 struct { funcType args [4]*rtype @@ -2278,42 +2280,6 @@ type structTypeUncommon struct { u uncommonType } -// A *rtype representing a struct is followed directly in memory by an -// array of method objects representing the methods attached to the -// struct. To get the same layout for a run time generated type, we -// need an array directly following the uncommonType memory. The types -// structTypeFixed4, ...structTypeFixedN are used to do this. -// -// A similar strategy is used for funcTypeFixed4, ...funcTypeFixedN. - -// TODO(crawshaw): as these structTypeFixedN and funcTypeFixedN structs -// have no methods, they could be defined at runtime using the StructOf -// function. - -type structTypeFixed4 struct { - structType - u uncommonType - m [4]method -} - -type structTypeFixed8 struct { - structType - u uncommonType - m [8]method -} - -type structTypeFixed16 struct { - structType - u uncommonType - m [16]method -} - -type structTypeFixed32 struct { - structType - u uncommonType - m [32]method -} - // isLetter reports whether a given 'rune' is classified as a Letter. func isLetter(ch rune) bool { return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) @@ -2571,33 +2537,26 @@ func StructOf(fields []StructField) Type { var typ *structType var ut *uncommonType - switch { - case len(methods) == 0: + if len(methods) == 0 { t := new(structTypeUncommon) typ = &t.structType ut = &t.u - case len(methods) <= 4: - t := new(structTypeFixed4) - typ = &t.structType - ut = &t.u - copy(t.m[:], methods) - case len(methods) <= 8: - t := new(structTypeFixed8) - typ = &t.structType - ut = &t.u - copy(t.m[:], methods) - case len(methods) <= 16: - t := new(structTypeFixed16) - typ = &t.structType - ut = &t.u - copy(t.m[:], methods) - case len(methods) <= 32: - t := new(structTypeFixed32) - typ = &t.structType - ut = &t.u - copy(t.m[:], methods) - default: - panic("reflect.StructOf: too many methods") + } else { + // A *rtype representing a struct is followed directly in memory by an + // array of method objects representing the methods attached to the + // struct. To get the same layout for a run time generated type, we + // need an array directly following the uncommonType memory. + // A similar strategy is used for funcTypeFixed4, ...funcTypeFixedN. + tt := New(StructOf([]StructField{ + {Name: "S", Type: TypeOf(structType{})}, + {Name: "U", Type: TypeOf(uncommonType{})}, + {Name: "M", Type: ArrayOf(len(methods), TypeOf(methods[0]))}, + })) + + typ = (*structType)(unsafe.Pointer(tt.Elem().Field(0).UnsafeAddr())) + ut = (*uncommonType)(unsafe.Pointer(tt.Elem().Field(1).UnsafeAddr())) + + copy(tt.Elem().Field(2).Slice(0, len(methods)).Interface().([]method), methods) } // TODO(sbinet): Once we allow embedding multiple types, // methods will need to be sorted like the compiler does. From e1978a2d7a6deac29aa778a17a1cbea25586abc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Chigot?= Date: Fri, 2 Nov 2018 13:14:07 +0100 Subject: [PATCH 039/111] cmd/compile/internal/gc: update cgo_import_dynamic for AIX On AIX, cmd/link needs two information in order to generate a dynamic import, the library and its object needed. Currently, cmd/link isn't able to retrieve this object only with the name of the library. Therefore, the library pattern in cgo_import_dynamic must be "lib.a/obj.o". Change-Id: Ib8b8aaa9807c9fa6af46ece4e312d58073ed6ec1 Reviewed-on: https://go-review.googlesource.com/c/146957 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/cgo/doc.go | 4 ++ src/cmd/compile/internal/gc/lex.go | 8 ++++ src/cmd/compile/internal/gc/lex_test.go | 56 ++++++++++++++++++++----- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index 157cd94d65..08d64130df 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -827,6 +827,10 @@ The directives are: possibly version in the dynamic library, and the optional "" names the specific library where the symbol should be found. + On AIX, the library pattern is slightly different. It must be + "lib.a/obj.o" with obj.o the member of this library exporting + this symbol. + In the , # or @ can be used to introduce a symbol version. Examples: diff --git a/src/cmd/compile/internal/gc/lex.go b/src/cmd/compile/internal/gc/lex.go index 3b302a5124..bd68ebffff 100644 --- a/src/cmd/compile/internal/gc/lex.go +++ b/src/cmd/compile/internal/gc/lex.go @@ -114,6 +114,14 @@ func (p *noder) pragcgo(pos syntax.Pos, text string) { case len(f) == 3 && !isQuoted(f[1]) && !isQuoted(f[2]): case len(f) == 4 && !isQuoted(f[1]) && !isQuoted(f[2]) && isQuoted(f[3]): f[3] = strings.Trim(f[3], `"`) + if objabi.GOOS == "aix" && f[3] != "" { + // On Aix, library pattern must be "lib.a/object.o" + n := strings.Split(f[3], "/") + if len(n) != 2 || !strings.HasSuffix(n[0], ".a") || !strings.HasSuffix(n[1], ".o") { + p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_import_dynamic local [remote ["lib.a/object.o"]]`}) + return + } + } default: p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_import_dynamic local [remote ["library"]]`}) return diff --git a/src/cmd/compile/internal/gc/lex_test.go b/src/cmd/compile/internal/gc/lex_test.go index fecf570fa1..e05726c9f3 100644 --- a/src/cmd/compile/internal/gc/lex_test.go +++ b/src/cmd/compile/internal/gc/lex_test.go @@ -7,6 +7,7 @@ package gc import ( "cmd/compile/internal/syntax" "reflect" + "runtime" "testing" ) @@ -49,10 +50,12 @@ func TestPragmaFields(t *testing.T) { } func TestPragcgo(t *testing.T) { - var tests = []struct { + type testStruct struct { in string want []string - }{ + } + + var tests = []testStruct{ {`go:cgo_export_dynamic local`, []string{`cgo_export_dynamic`, `local`}}, {`go:cgo_export_dynamic local remote`, []string{`cgo_export_dynamic`, `local`, `remote`}}, {`go:cgo_export_dynamic local' remote'`, []string{`cgo_export_dynamic`, `local'`, `remote'`}}, @@ -61,8 +64,6 @@ func TestPragcgo(t *testing.T) { {`go:cgo_export_static local' remote'`, []string{`cgo_export_static`, `local'`, `remote'`}}, {`go:cgo_import_dynamic local`, []string{`cgo_import_dynamic`, `local`}}, {`go:cgo_import_dynamic local remote`, []string{`cgo_import_dynamic`, `local`, `remote`}}, - {`go:cgo_import_dynamic local remote "library"`, []string{`cgo_import_dynamic`, `local`, `remote`, `library`}}, - {`go:cgo_import_dynamic local' remote' "lib rary"`, []string{`cgo_import_dynamic`, `local'`, `remote'`, `lib rary`}}, {`go:cgo_import_static local`, []string{`cgo_import_static`, `local`}}, {`go:cgo_import_static local'`, []string{`cgo_import_static`, `local'`}}, {`go:cgo_dynamic_linker "/path/"`, []string{`cgo_dynamic_linker`, `/path/`}}, @@ -71,17 +72,50 @@ func TestPragcgo(t *testing.T) { {`go:cgo_ldflag "a rg"`, []string{`cgo_ldflag`, `a rg`}}, } + if runtime.GOOS != "aix" { + tests = append(tests, []testStruct{ + {`go:cgo_import_dynamic local remote "library"`, []string{`cgo_import_dynamic`, `local`, `remote`, `library`}}, + {`go:cgo_import_dynamic local' remote' "lib rary"`, []string{`cgo_import_dynamic`, `local'`, `remote'`, `lib rary`}}, + }...) + } else { + // cgo_import_dynamic with a library is slightly different on AIX + // as the library field must follow the pattern [libc.a/object.o]. + tests = append(tests, []testStruct{ + {`go:cgo_import_dynamic local remote "lib.a/obj.o"`, []string{`cgo_import_dynamic`, `local`, `remote`, `lib.a/obj.o`}}, + // This test must fail. + {`go:cgo_import_dynamic local' remote' "library"`, []string{`: usage: //go:cgo_import_dynamic local [remote ["lib.a/object.o"]]`}}, + }...) + + } + var p noder var nopos syntax.Pos for _, tt := range tests { - p.pragcgobuf = nil - p.pragcgo(nopos, tt.in) - got := p.pragcgobuf - want := [][]string{tt.want} - if !reflect.DeepEqual(got, want) { - t.Errorf("pragcgo(%q) = %q; want %q", tt.in, got, want) - continue + p.err = make(chan syntax.Error) + gotch := make(chan [][]string) + go func() { + p.pragcgobuf = nil + p.pragcgo(nopos, tt.in) + if p.pragcgobuf != nil { + gotch <- p.pragcgobuf + } + }() + + select { + case e := <-p.err: + want := tt.want[0] + if e.Error() != want { + t.Errorf("pragcgo(%q) = %q; want %q", tt.in, e, want) + continue + } + case got := <-gotch: + want := [][]string{tt.want} + if !reflect.DeepEqual(got, want) { + t.Errorf("pragcgo(%q) = %q; want %q", tt.in, got, want) + continue + } } + } } From aa9bcea3907a74f45303b3bdb603b9952cc72b7b Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Fri, 5 Oct 2018 14:21:39 -0400 Subject: [PATCH 040/111] runtime: improve performance of memclr, memmove on ppc64x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This improves the asm implementations for memmove and memclr on ppc64x through use of vsx loads and stores when size is >= 32 bytes. For memclr, dcbz is used when the size is >= 512 and aligned to 128. Memclr/64 13.3ns ± 0% 10.7ns ± 0% -19.55% (p=0.000 n=8+7) Memclr/96 14.9ns ± 0% 11.4ns ± 0% -23.49% (p=0.000 n=8+8) Memclr/128 16.3ns ± 0% 12.3ns ± 0% -24.54% (p=0.000 n=8+8) Memclr/160 17.3ns ± 0% 13.0ns ± 0% -24.86% (p=0.000 n=8+8) Memclr/256 20.0ns ± 0% 15.3ns ± 0% -23.62% (p=0.000 n=8+8) Memclr/512 34.2ns ± 0% 10.2ns ± 0% -70.20% (p=0.000 n=8+8) Memclr/4096 178ns ± 0% 23ns ± 0% -87.13% (p=0.000 n=8+8) Memclr/65536 2.67µs ± 0% 0.30µs ± 0% -88.89% (p=0.000 n=7+8) Memclr/1M 43.2µs ± 0% 10.0µs ± 0% -76.85% (p=0.000 n=8+8) Memclr/4M 173µs ± 0% 40µs ± 0% -76.88% (p=0.000 n=8+8) Memclr/8M 349µs ± 0% 82µs ± 0% -76.58% (p=0.000 n=8+8) Memclr/16M 701µs ± 7% 672µs ± 0% -4.05% (p=0.040 n=8+7) Memclr/64M 2.70ms ± 0% 2.67ms ± 0% -0.96% (p=0.000 n=8+7) Memmove/32 6.59ns ± 0% 5.84ns ± 0% -11.34% (p=0.029 n=4+4) Memmove/64 7.91ns ± 0% 6.97ns ± 0% -11.92% (p=0.029 n=4+4) Memmove/128 10.5ns ± 0% 8.8ns ± 0% -16.24% (p=0.029 n=4+4) Memmove/256 21.0ns ± 0% 12.9ns ± 0% -38.57% (p=0.029 n=4+4) Memmove/512 28.4ns ± 0% 26.2ns ± 0% -7.75% (p=0.029 n=4+4) Memmove/1024 48.2ns ± 1% 39.4ns ± 0% -18.26% (p=0.029 n=4+4) Memmove/2048 85.4ns ± 0% 69.0ns ± 0% -19.20% (p=0.029 n=4+4) Memmove/4096 159ns ± 0% 128ns ± 0% -19.50% (p=0.029 n=4+4) Change-Id: I8c1adf88790845bf31444a15249456006eb5bf8b Reviewed-on: https://go-review.googlesource.com/c/141217 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Michael Munday --- src/runtime/memclr_ppc64x.s | 129 +++++++++++++++++++++++++++++++---- src/runtime/memmove_ppc64x.s | 51 +++++++++----- 2 files changed, 149 insertions(+), 31 deletions(-) diff --git a/src/runtime/memclr_ppc64x.s b/src/runtime/memclr_ppc64x.s index 3b23ce89d8..072963f756 100644 --- a/src/runtime/memclr_ppc64x.s +++ b/src/runtime/memclr_ppc64x.s @@ -14,34 +14,68 @@ TEXT runtime·memclrNoHeapPointers(SB), NOSPLIT|NOFRAME, $0-16 // Determine if there are doublewords to clear check: ANDCC $7, R4, R5 // R5: leftover bytes to clear - SRAD $3, R4, R6 // R6: double words to clear + SRD $3, R4, R6 // R6: double words to clear CMP R6, $0, CR1 // CR1[EQ] set if no double words - BC 12, 6, nozerolarge // only single bytes - MOVD R6, CTR // R6 = number of double words - SRADCC $2, R6, R7 // 32 byte chunks? - BNE zero32setup + BC 12, 6, nozerolarge // only single bytes + CMP R4, $512 + BLT under512 // special case for < 512 + ANDCC $127, R3, R8 // check for 128 alignment of address + BEQ zero512setup + + ANDCC $7, R3, R15 + BEQ zero512xsetup // at least 8 byte aligned + + // zero bytes up to 8 byte alignment + + ANDCC $1, R3, R15 // check for byte alignment + BEQ byte2 + MOVB R0, 0(R3) // zero 1 byte + ADD $1, R3 // bump ptr by 1 + ADD $-1, R4 + +byte2: + ANDCC $2, R3, R15 // check for 2 byte alignment + BEQ byte4 + MOVH R0, 0(R3) // zero 2 bytes + ADD $2, R3 // bump ptr by 2 + ADD $-2, R4 + +byte4: + ANDCC $4, R3, R15 // check for 4 byte alignment + BEQ zero512xsetup + MOVW R0, 0(R3) // zero 4 bytes + ADD $4, R3 // bump ptr by 4 + ADD $-4, R4 + BR zero512xsetup // ptr should now be 8 byte aligned + +under512: + MOVD R6, CTR // R6 = number of double words + SRDCC $2, R6, R7 // 32 byte chunks? + BNE zero32setup // Clear double words zero8: MOVD R0, 0(R3) // double word ADD $8, R3 + ADD $-8, R4 BC 16, 0, zero8 // dec ctr, br zero8 if ctr not 0 - BR nozerolarge // handle remainder + BR nozerolarge // handle leftovers // Prepare to clear 32 bytes at a time. zero32setup: - DCBTST (R3) // prepare data cache - MOVD R7, CTR // number of 32 byte chunks + DCBTST (R3) // prepare data cache + XXLXOR VS32, VS32, VS32 // clear VS32 (V0) + MOVD R7, CTR // number of 32 byte chunks + MOVD $16, R8 zero32: - MOVD R0, 0(R3) // clear 4 double words - MOVD R0, 8(R3) - MOVD R0, 16(R3) - MOVD R0, 24(R3) + STXVD2X VS32, (R3+R0) // store 16 bytes + STXVD2X VS32, (R3+R8) ADD $32, R3 + ADD $-32, R4 BC 16, 0, zero32 // dec ctr, br zero32 if ctr not 0 RLDCLCC $61, R4, $3, R6 // remaining doublewords BEQ nozerolarge @@ -49,8 +83,8 @@ zero32: BR zero8 nozerolarge: - CMP R5, $0 // any remaining bytes - BC 4, 1, LR // ble lr + ANDCC $7, R4, R5 // any remaining bytes + BC 4, 1, LR // ble lr zerotail: MOVD R5, CTR // set up to clear tail bytes @@ -60,3 +94,70 @@ zerotailloop: ADD $1, R3 BC 16, 0, zerotailloop // dec ctr, br zerotailloop if ctr not 0 RET + +zero512xsetup: // 512 chunk with extra needed + ANDCC $8, R3, R11 // 8 byte alignment? + BEQ zero512setup16 + MOVD R0, 0(R3) // clear 8 bytes + ADD $8, R3 // update ptr to next 8 + ADD $-8, R4 // dec count by 8 + +zero512setup16: + ANDCC $127, R3, R14 // < 128 byte alignment + BEQ zero512setup // handle 128 byte alignment + MOVD $128, R15 + SUB R14, R15, R14 // find increment to 128 alignment + SRD $4, R14, R15 // number of 16 byte chunks + +zero512presetup: + MOVD R15, CTR // loop counter of 16 bytes + XXLXOR VS32, VS32, VS32 // clear VS32 (V0) + +zero512preloop: // clear up to 128 alignment + STXVD2X VS32, (R3+R0) // clear 16 bytes + ADD $16, R3 // update ptr + ADD $-16, R4 // dec count + BC 16, 0, zero512preloop + +zero512setup: // setup for dcbz loop + CMP R4, $512 // check if at least 512 + BLT remain + SRD $9, R4, R8 // loop count for 512 chunks + MOVD R8, CTR // set up counter + MOVD $128, R9 // index regs for 128 bytes + MOVD $256, R10 + MOVD $384, R11 + +zero512: + DCBZ (R3+R0) // clear first chunk + DCBZ (R3+R9) // clear second chunk + DCBZ (R3+R10) // clear third chunk + DCBZ (R3+R11) // clear fourth chunk + ADD $512, R3 + ADD $-512, R4 + BC 16, 0, zero512 + +remain: + CMP R4, $128 // check if 128 byte chunks left + BLT smaller + DCBZ (R3+R0) // clear 128 + ADD $128, R3 + ADD $-128, R4 + BR remain + +smaller: + ANDCC $127, R4, R7 // find leftovers + BEQ done + CMP R7, $64 // more than 64, do 32 at a time + BLT zero8setup // less than 64, do 8 at a time + SRD $5, R7, R7 // set up counter for 32 + BR zero32setup + +zero8setup: + SRDCC $3, R7, R7 // less than 8 bytes + BEQ nozerolarge + MOVD R7, CTR + BR zero8 + +done: + RET diff --git a/src/runtime/memmove_ppc64x.s b/src/runtime/memmove_ppc64x.s index b79f76d388..60cbcc41ec 100644 --- a/src/runtime/memmove_ppc64x.s +++ b/src/runtime/memmove_ppc64x.s @@ -16,7 +16,7 @@ TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24 // copy so a more efficient move can be done check: ANDCC $7, R5, R7 // R7: bytes to copy - SRAD $3, R5, R6 // R6: double words to copy + SRD $3, R5, R6 // R6: double words to copy CMP R6, $0, CR1 // CR1[EQ] set if no double words to copy // Determine overlap by subtracting dest - src and comparing against the @@ -31,9 +31,9 @@ check: // Copying forward if no overlap. BC 12, 6, noforwardlarge // "BEQ CR1, noforwardlarge" - MOVD R6,CTR // R6 = number of double words - SRADCC $2,R6,R8 // 32 byte chunks? + SRDCC $2,R6,R8 // 32 byte chunks? BNE forward32setup // + MOVD R6,CTR // R6 = number of double words // Move double words @@ -51,17 +51,14 @@ forward32setup: DCBTST (R3) // prepare data cache DCBT (R4) MOVD R8, CTR // double work count + MOVD $16, R8 forward32: - MOVD 0(R4), R8 // load 4 double words - MOVD 8(R4), R9 - MOVD 16(R4), R14 - MOVD 24(R4), R15 - ADD $32,R4 - MOVD R8, 0(R3) // store those 4 - MOVD R9, 8(R3) - MOVD R14,16(R3) - MOVD R15,24(R3) + LXVD2X (R4+R0), VS32 // load 16 bytes + LXVD2X (R4+R8), VS33 + ADD $32, R4 + STXVD2X VS32, (R3+R0) // store 16 bytes + STXVD2X VS33, (R3+R8) ADD $32,R3 // bump up for next set BC 16, 0, forward32 // continue RLDCLCC $61,R5,$3,R6 // remaining doublewords @@ -71,7 +68,7 @@ forward32: noforwardlarge: CMP R7,$0 // any remaining bytes - BC 4, 1, LR + BC 4, 1, LR // ble lr forwardtail: MOVD R7, CTR // move tail bytes @@ -101,19 +98,39 @@ backwardtailloop: SUB $1,R4 MOVBZ R8, -1(R3) SUB $1,R3 - BC 16, 0, backwardtailloop + BC 16, 0, backwardtailloop // bndz nobackwardtail: - CMP R6,$0 - BC 4, 5, LR + BC 4, 5, LR // ble CR1 lr backwardlarge: MOVD R6, CTR + SUB R3, R4, R9 // Use vsx if moving + CMP R9, $32 // at least 32 byte chunks + BLT backwardlargeloop // and distance >= 32 + SRDCC $2,R6,R8 // 32 byte chunks + BNE backward32setup backwardlargeloop: MOVD -8(R4), R8 SUB $8,R4 MOVD R8, -8(R3) SUB $8,R3 - BC 16, 0, backwardlargeloop // + BC 16, 0, backwardlargeloop // bndz RET + +backward32setup: + MOVD R8, CTR // set up loop ctr + MOVD $16, R8 // 32 bytes at at time + +backward32loop: + SUB $32, R4 + SUB $32, R3 + LXVD2X (R4+R0), VS32 // load 16 bytes + LXVD2X (R4+R8), VS33 + STXVD2X VS32, (R3+R0) // store 16 bytes + STXVD2X VS33, (R3+R8) + BC 16, 0, backward32loop // bndz + BC 4, 5, LR // ble CR1 lr + MOVD R6, CTR + BR backwardlargeloop From 8b4692096bf85f04df53d4104cb82cc3c8095df7 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 5 Nov 2018 15:51:35 -0500 Subject: [PATCH 041/111] cmd/vendor: add x/tools/go/analysis/cmd/vet-lite + deps This change adds the vet-lite command (the future cmd/vet) and all its dependencies from x/tools, but not its tests and their dependencies. It was created with these commands: $ (cd $GOPATH/src/golang.org/x/tools && git checkout c76e1ad) $ cd GOROOT/src/cmd $ govendor add $(go list -deps golang.org/x/tools/go/analysis/cmd/vet-lite | grep golang.org/x/tools) $ rm -fr $(find vendor/golang.org/x/tools/ -name testdata) $ rm $(find vendor/golang.org/x/tools/ -name \*_test.go) I feel sure I am holding govendor wrong. Please advise. A followup CL will make cmd/vet behave like vet-lite, initially just for users that opt in, and soon after for all users, at which point cmd/vet will be replaced in its entirety by a copy of vet-lite's small main.go. In the meantime, anyone can try the new tool using these commands: $ go build cmd/vendor/golang.org/x/tools/go/analysis/cmd/vet-lite $ export GOVETTOOL=$(which vet-lite) $ go vet your/project/... Change-Id: Iea168111a32ce62f82f9fb706385ca0f368bc869 Reviewed-on: https://go-review.googlesource.com/c/147444 Reviewed-by: Russ Cox --- src/cmd/vendor/golang.org/x/tools/LICENSE | 27 + src/cmd/vendor/golang.org/x/tools/PATENTS | 22 + .../x/tools/go/analysis/analysis.go | 192 ++++ .../x/tools/go/analysis/cmd/vet-lite/main.go | 83 ++ .../golang.org/x/tools/go/analysis/doc.go | 328 ++++++ .../analysis/internal/analysisflags/flags.go | 223 ++++ .../tools/go/analysis/internal/facts/facts.go | 299 ++++++ .../go/analysis/internal/facts/imports.go | 88 ++ .../internal/unitchecker/unitchecker.go | 306 ++++++ .../go/analysis/passes/asmdecl/asmdecl.go | 759 ++++++++++++++ .../tools/go/analysis/passes/assign/assign.go | 68 ++ .../tools/go/analysis/passes/atomic/atomic.go | 96 ++ .../x/tools/go/analysis/passes/bools/bools.go | 214 ++++ .../go/analysis/passes/buildtag/buildtag.go | 159 +++ .../go/analysis/passes/cgocall/cgocall.go | 226 ++++ .../go/analysis/passes/composite/composite.go | 108 ++ .../go/analysis/passes/composite/whitelist.go | 33 + .../go/analysis/passes/copylock/copylock.go | 300 ++++++ .../go/analysis/passes/ctrlflow/ctrlflow.go | 225 ++++ .../passes/httpresponse/httpresponse.go | 177 ++++ .../go/analysis/passes/inspect/inspect.go | 45 + .../passes/internal/analysisutil/util.go | 106 ++ .../passes/loopclosure/loopclosure.go | 130 +++ .../analysis/passes/lostcancel/lostcancel.go | 304 ++++++ .../go/analysis/passes/nilfunc/nilfunc.go | 74 ++ .../go/analysis/passes/pkgfact/pkgfact.go | 127 +++ .../tools/go/analysis/passes/printf/printf.go | 964 ++++++++++++++++++ .../tools/go/analysis/passes/printf/types.go | 223 ++++ .../x/tools/go/analysis/passes/shift/dead.go | 101 ++ .../x/tools/go/analysis/passes/shift/shift.go | 128 +++ .../analysis/passes/stdmethods/stdmethods.go | 211 ++++ .../go/analysis/passes/structtag/structtag.go | 260 +++++ .../x/tools/go/analysis/passes/tests/tests.go | 175 ++++ .../passes/unreachable/unreachable.go | 314 ++++++ .../go/analysis/passes/unsafeptr/unsafeptr.go | 130 +++ .../passes/unusedresult/unusedresult.go | 131 +++ .../x/tools/go/analysis/validate.go | 104 ++ .../x/tools/go/ast/astutil/enclosing.go | 627 ++++++++++++ .../x/tools/go/ast/astutil/imports.go | 471 +++++++++ .../x/tools/go/ast/astutil/rewrite.go | 477 +++++++++ .../golang.org/x/tools/go/ast/astutil/util.go | 14 + .../x/tools/go/ast/inspector/inspector.go | 182 ++++ .../x/tools/go/ast/inspector/typeof.go | 216 ++++ .../golang.org/x/tools/go/cfg/builder.go | 510 +++++++++ .../vendor/golang.org/x/tools/go/cfg/cfg.go | 150 +++ .../x/tools/go/types/objectpath/objectpath.go | 523 ++++++++++ .../x/tools/go/types/typeutil/callee.go | 46 + .../x/tools/go/types/typeutil/imports.go | 31 + .../x/tools/go/types/typeutil/map.go | 313 ++++++ .../tools/go/types/typeutil/methodsetcache.go | 72 ++ .../x/tools/go/types/typeutil/ui.go | 52 + src/cmd/vendor/vendor.json | 204 ++++ 52 files changed, 11348 insertions(+) create mode 100644 src/cmd/vendor/golang.org/x/tools/LICENSE create mode 100644 src/cmd/vendor/golang.org/x/tools/PATENTS create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/analysis.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/cmd/vet-lite/main.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/doc.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/internal/analysisflags/flags.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/facts.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/imports.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/internal/unitchecker/unitchecker.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/assign/assign.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/atomic/atomic.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/bools/bools.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/buildtag/buildtag.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/cgocall/cgocall.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/composite.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/whitelist.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/copylock/copylock.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/ctrlflow/ctrlflow.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/httpresponse/httpresponse.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/inspect/inspect.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/internal/analysisutil/util.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/loopclosure/loopclosure.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/lostcancel/lostcancel.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/nilfunc/nilfunc.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/pkgfact/pkgfact.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/types.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/dead.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/shift.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/stdmethods/stdmethods.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/structtag/structtag.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/tests/tests.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unreachable/unreachable.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unsafeptr/unsafeptr.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unusedresult/unusedresult.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/analysis/validate.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/astutil/enclosing.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/astutil/imports.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/astutil/util.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/inspector/inspector.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/ast/inspector/typeof.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/cfg/builder.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/cfg/cfg.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/typeutil/callee.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/typeutil/imports.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/typeutil/map.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/typeutil/methodsetcache.go create mode 100644 src/cmd/vendor/golang.org/x/tools/go/types/typeutil/ui.go diff --git a/src/cmd/vendor/golang.org/x/tools/LICENSE b/src/cmd/vendor/golang.org/x/tools/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/cmd/vendor/golang.org/x/tools/PATENTS b/src/cmd/vendor/golang.org/x/tools/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/analysis.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/analysis.go new file mode 100644 index 0000000000..21baa02a8d --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/analysis.go @@ -0,0 +1,192 @@ +package analysis + +import ( + "flag" + "fmt" + "go/ast" + "go/token" + "go/types" + "reflect" +) + +// An Analyzer describes an analysis function and its options. +type Analyzer struct { + // The Name of the analyzer must be a valid Go identifier + // as it may appear in command-line flags, URLs, and so on. + Name string + + // Doc is the documentation for the analyzer. + // The part before the first "\n\n" is the title + // (no capital or period, max ~60 letters). + Doc string + + // Flags defines any flags accepted by the analyzer. + // The manner in which these flags are exposed to the user + // depends on the driver which runs the analyzer. + Flags flag.FlagSet + + // Run applies the analyzer to a package. + // It returns an error if the analyzer failed. + // + // On success, the Run function may return a result + // computed by the Analyzer; its type must match ResultType. + // The driver makes this result available as an input to + // another Analyzer that depends directly on this one (see + // Requires) when it analyzes the same package. + // + // To pass analysis results between packages (and thus + // potentially between address spaces), use Facts, which are + // serializable. + Run func(*Pass) (interface{}, error) + + // RunDespiteErrors allows the driver to invoke + // the Run method of this analyzer even on a + // package that contains parse or type errors. + RunDespiteErrors bool + + // Requires is a set of analyzers that must run successfully + // before this one on a given package. This analyzer may inspect + // the outputs produced by each analyzer in Requires. + // The graph over analyzers implied by Requires edges must be acyclic. + // + // Requires establishes a "horizontal" dependency between + // analysis passes (different analyzers, same package). + Requires []*Analyzer + + // ResultType is the type of the optional result of the Run function. + ResultType reflect.Type + + // FactTypes indicates that this analyzer imports and exports + // Facts of the specified concrete types. + // An analyzer that uses facts may assume that its import + // dependencies have been similarly analyzed before it runs. + // Facts must be pointers. + // + // FactTypes establishes a "vertical" dependency between + // analysis passes (same analyzer, different packages). + FactTypes []Fact +} + +func (a *Analyzer) String() string { return a.Name } + +// A Pass provides information to the Run function that +// applies a specific analyzer to a single Go package. +// +// It forms the interface between the analysis logic and the driver +// program, and has both input and an output components. +// +// As in a compiler, one pass may depend on the result computed by another. +// +// The Run function should not call any of the Pass functions concurrently. +type Pass struct { + Analyzer *Analyzer // the identity of the current analyzer + + // syntax and type information + Fset *token.FileSet // file position information + Files []*ast.File // the abstract syntax tree of each file + OtherFiles []string // names of non-Go files of this package + Pkg *types.Package // type information about the package + TypesInfo *types.Info // type information about the syntax trees + + // Report reports a Diagnostic, a finding about a specific location + // in the analyzed source code such as a potential mistake. + // It may be called by the Run function. + Report func(Diagnostic) + + // ResultOf provides the inputs to this analysis pass, which are + // the corresponding results of its prerequisite analyzers. + // The map keys are the elements of Analysis.Required, + // and the type of each corresponding value is the required + // analysis's ResultType. + ResultOf map[*Analyzer]interface{} + + // -- facts -- + + // ImportObjectFact retrieves a fact associated with obj. + // Given a value ptr of type *T, where *T satisfies Fact, + // ImportObjectFact copies the value to *ptr. + // + // ImportObjectFact panics if called after the pass is complete. + // ImportObjectFact is not concurrency-safe. + ImportObjectFact func(obj types.Object, fact Fact) bool + + // ImportPackageFact retrieves a fact associated with package pkg, + // which must be this package or one of its dependencies. + // See comments for ImportObjectFact. + ImportPackageFact func(pkg *types.Package, fact Fact) bool + + // ExportObjectFact associates a fact of type *T with the obj, + // replacing any previous fact of that type. + // + // ExportObjectFact panics if it is called after the pass is + // complete, or if obj does not belong to the package being analyzed. + // ExportObjectFact is not concurrency-safe. + ExportObjectFact func(obj types.Object, fact Fact) + + // ExportPackageFact associates a fact with the current package. + // See comments for ExportObjectFact. + ExportPackageFact func(fact Fact) + + /* Further fields may be added in future. */ + // For example, suggested or applied refactorings. +} + +// Reportf is a helper function that reports a Diagnostic using the +// specified position and formatted error message. +func (pass *Pass) Reportf(pos token.Pos, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + pass.Report(Diagnostic{Pos: pos, Message: msg}) +} + +func (pass *Pass) String() string { + return fmt.Sprintf("%s@%s", pass.Analyzer.Name, pass.Pkg.Path()) +} + +// A Fact is an intermediate fact produced during analysis. +// +// Each fact is associated with a named declaration (a types.Object) or +// with a package as a whole. A single object or package may have +// multiple associated facts, but only one of any particular fact type. +// +// A Fact represents a predicate such as "never returns", but does not +// represent the subject of the predicate such as "function F" or "package P". +// +// Facts may be produced in one analysis pass and consumed by another +// analysis pass even if these are in different address spaces. +// If package P imports Q, all facts about Q produced during +// analysis of that package will be available during later analysis of P. +// Facts are analogous to type export data in a build system: +// just as export data enables separate compilation of several passes, +// facts enable "separate analysis". +// +// Each pass (a, p) starts with the set of facts produced by the +// same analyzer a applied to the packages directly imported by p. +// The analysis may add facts to the set, and they may be exported in turn. +// An analysis's Run function may retrieve facts by calling +// Pass.Import{Object,Package}Fact and update them using +// Pass.Export{Object,Package}Fact. +// +// A fact is logically private to its Analysis. To pass values +// between different analyzers, use the results mechanism; +// see Analyzer.Requires, Analyzer.ResultType, and Pass.ResultOf. +// +// A Fact type must be a pointer. +// Facts are encoded and decoded using encoding/gob. +// A Fact may implement the GobEncoder/GobDecoder interfaces +// to customize its encoding. Fact encoding should not fail. +// +// A Fact should not be modified once exported. +type Fact interface { + AFact() // dummy method to avoid type errors +} + +// A Diagnostic is a message associated with a source location. +// +// An Analyzer may return a variety of diagnostics; the optional Category, +// which should be a constant, may be used to classify them. +// It is primarily intended to make it easy to look up documentation. +type Diagnostic struct { + Pos token.Pos + Category string // optional + Message string +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/cmd/vet-lite/main.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/cmd/vet-lite/main.go new file mode 100644 index 0000000000..ae66a7df28 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/cmd/vet-lite/main.go @@ -0,0 +1,83 @@ +// The vet-lite command is a driver for static checkers conforming to +// the golang.org/x/tools/go/analysis API. It must be run by go vet: +// +// $ GOVETTOOL=$(which vet-lite) go vet +// +// For a checker also capable of running standalone, use multichecker. +package main + +import ( + "flag" + "log" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/internal/analysisflags" + "golang.org/x/tools/go/analysis/internal/unitchecker" + + "golang.org/x/tools/go/analysis/passes/asmdecl" + "golang.org/x/tools/go/analysis/passes/assign" + "golang.org/x/tools/go/analysis/passes/atomic" + "golang.org/x/tools/go/analysis/passes/bools" + "golang.org/x/tools/go/analysis/passes/buildtag" + "golang.org/x/tools/go/analysis/passes/cgocall" + "golang.org/x/tools/go/analysis/passes/composite" + "golang.org/x/tools/go/analysis/passes/copylock" + "golang.org/x/tools/go/analysis/passes/httpresponse" + "golang.org/x/tools/go/analysis/passes/loopclosure" + "golang.org/x/tools/go/analysis/passes/lostcancel" + "golang.org/x/tools/go/analysis/passes/nilfunc" + "golang.org/x/tools/go/analysis/passes/pkgfact" + "golang.org/x/tools/go/analysis/passes/printf" + "golang.org/x/tools/go/analysis/passes/shift" + "golang.org/x/tools/go/analysis/passes/stdmethods" + "golang.org/x/tools/go/analysis/passes/structtag" + "golang.org/x/tools/go/analysis/passes/tests" + "golang.org/x/tools/go/analysis/passes/unreachable" + "golang.org/x/tools/go/analysis/passes/unsafeptr" + "golang.org/x/tools/go/analysis/passes/unusedresult" +) + +var analyzers = []*analysis.Analyzer{ + // For now, just the traditional vet suite: + asmdecl.Analyzer, + assign.Analyzer, + atomic.Analyzer, + bools.Analyzer, + buildtag.Analyzer, + cgocall.Analyzer, + composite.Analyzer, + copylock.Analyzer, + httpresponse.Analyzer, + loopclosure.Analyzer, + lostcancel.Analyzer, + nilfunc.Analyzer, + pkgfact.Analyzer, + printf.Analyzer, + // shadow.Analyzer, // experimental; not enabled by default + shift.Analyzer, + stdmethods.Analyzer, + structtag.Analyzer, + tests.Analyzer, + unreachable.Analyzer, + unsafeptr.Analyzer, + unusedresult.Analyzer, +} + +func main() { + log.SetFlags(0) + log.SetPrefix("vet: ") + + if err := analysis.Validate(analyzers); err != nil { + log.Fatal(err) + } + + analyzers = analysisflags.Parse(analyzers, true) + + args := flag.Args() + if len(args) != 1 || !strings.HasSuffix(args[0], ".cfg") { + log.Fatalf("invalid command: want .cfg file (this reduced version of vet is intended to be run only by the 'go vet' command)") + } + + unitchecker.Main(args[0], analyzers) +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/doc.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/doc.go new file mode 100644 index 0000000000..4223ab80fc --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/doc.go @@ -0,0 +1,328 @@ +/* + +The analysis package defines the interface between a modular static +analysis and an analysis driver program. + + +THIS INTERFACE IS EXPERIMENTAL AND SUBJECT TO CHANGE. +We aim to finalize it by November 2018. + +Background + +A static analysis is a function that inspects a package of Go code and +reports a set of diagnostics (typically mistakes in the code), and +perhaps produces other results as well, such as suggested refactorings +or other facts. An analysis that reports mistakes is informally called a +"checker". For example, the printf checker reports mistakes in +fmt.Printf format strings. + +A "modular" analysis is one that inspects one package at a time but can +save information from a lower-level package and use it when inspecting a +higher-level package, analogous to separate compilation in a toolchain. +The printf checker is modular: when it discovers that a function such as +log.Fatalf delegates to fmt.Printf, it records this fact, and checks +calls to that function too, including calls made from another package. + +By implementing a common interface, checkers from a variety of sources +can be easily selected, incorporated, and reused in a wide range of +driver programs including command-line tools (such as vet), text editors and +IDEs, build and test systems (such as go build, Bazel, or Buck), test +frameworks, code review tools, code-base indexers (such as SourceGraph), +documentation viewers (such as godoc), batch pipelines for large code +bases, and so on. + + +Analyzer + +The primary type in the API is Analyzer. An Analyzer statically +describes an analysis function: its name, documentation, flags, +relationship to other analyzers, and of course, its logic. + +To define an analysis, a user declares a (logically constant) variable +of type Analyzer. Here is a typical example from one of the analyzers in +the go/analysis/passes/ subdirectory: + + package unusedresult + + var Analyzer = &analysis.Analyzer{ + Name: "unusedresult", + Doc: "check for unused results of calls to some functions", + Run: run, + ... + } + + func run(pass *analysis.Pass) (interface{}, error) { + ... + } + + +An analysis driver is a program such as vet that runs a set of +analyses and prints the diagnostics that they report. +The driver program must import the list of Analyzers it needs. +Typically each Analyzer resides in a separate package. +To add a new Analyzer to an existing driver, add another item to the list: + + import ( "unusedresult"; "nilness"; "printf" ) + + var analyses = []*analysis.Analyzer{ + unusedresult.Analyzer, + nilness.Analyzer, + printf.Analyzer, + } + +A driver may use the name, flags, and documentation to provide on-line +help that describes the analyses its performs. +The vet command, shown below, is an example of a driver that runs +multiple analyzers. It is based on the multichecker package +(see the "Standalone commands" section for details). + + $ go build golang.org/x/tools/cmd/vet + $ ./vet help + vet is a tool for static analysis of Go programs. + + Usage: vet [-flag] [package] + + Registered analyzers: + + asmdecl report mismatches between assembly files and Go declarations + assign check for useless assignments + atomic check for common mistakes using the sync/atomic package + ... + unusedresult check for unused results of calls to some functions + + $ ./vet help unusedresult + unusedresult: check for unused results of calls to some functions + + Analyzer flags: + + -unusedresult.funcs value + comma-separated list of functions whose results must be used (default Error,String) + -unusedresult.stringmethods value + comma-separated list of names of methods of type func() string whose results must be used + + Some functions like fmt.Errorf return a result and have no side effects, + so it is always a mistake to discard the result. This analyzer reports + calls to certain functions in which the result of the call is ignored. + + The set of functions may be controlled using flags. + +The Analyzer type has more fields besides those shown above: + + type Analyzer struct { + Name string + Doc string + Flags flag.FlagSet + Run func(*Pass) (interface{}, error) + RunDespiteErrors bool + ResultType reflect.Type + Requires []*Analyzer + FactTypes []Fact + } + +The Flags field declares a set of named (global) flag variables that +control analysis behavior. Unlike vet, analysis flags are not declared +directly in the command line FlagSet; it is up to the driver to set the +flag variables. A driver for a single analysis, a, might expose its flag +f directly on the command line as -f, whereas a driver for multiple +analyses might prefix the flag name by the analysis name (-a.f) to avoid +ambiguity. An IDE might expose the flags through a graphical interface, +and a batch pipeline might configure them from a config file. +See the "findcall" analyzer for an example of flags in action. + +The RunDespiteErrors flag indicates whether the analysis is equipped to +handle ill-typed code. If not, the driver will skip the analysis if +there were parse or type errors. +The optional ResultType field specifies the type of the result value +computed by this analysis and made available to other analyses. +The Requires field specifies a list of analyses upon which +this one depends and whose results it may access, and it constrains the +order in which a driver may run analyses. +The FactTypes field is discussed in the section on Modularity. +The analysis package provides a Validate function to perform basic +sanity checks on an Analyzer, such as that its Requires graph is +acyclic, its fact and result types are unique, and so on. + +Finally, the Run field contains a function to be called by the driver to +execute the analysis on a single package. The driver passes it an +instance of the Pass type. + + +Pass + +A Pass describes a single unit of work: the application of a particular +Analyzer to a particular package of Go code. +The Pass provides information to the Analyzer's Run function about the +package being analyzed, and provides operations to the Run function for +reporting diagnostics and other information back to the driver. + + type Pass struct { + Fset *token.FileSet + Files []*ast.File + OtherFiles []string + Pkg *types.Package + TypesInfo *types.Info + ResultOf map[*Analyzer]interface{} + Report func(Diagnostic) + ... + } + +The Fset, Files, Pkg, and TypesInfo fields provide the syntax trees, +type information, and source positions for a single package of Go code. + +The OtherFiles field provides the names, but not the contents, of non-Go +files such as assembly that are part of this package. See the "asmdecl" +or "buildtags" analyzers for examples of loading non-Go files and report +diagnostics against them. + +The ResultOf field provides the results computed by the analyzers +required by this one, as expressed in its Analyzer.Requires field. The +driver runs the required analyzers first and makes their results +available in this map. Each Analyzer must return a value of the type +described in its Analyzer.ResultType field. +For example, the "ctrlflow" analyzer returns a *ctrlflow.CFGs, which +provides a control-flow graph for each function in the package (see +golang.org/x/tools/go/cfg); the "inspect" analyzer returns a value that +enables other Analyzers to traverse the syntax trees of the package more +efficiently; and the "buildssa" analyzer constructs an SSA-form +intermediate representation. +Each of these Analyzers extends the capabilities of later Analyzers +without adding a dependency to the core API, so an analysis tool pays +only for the extensions it needs. + +The Report function emits a diagnostic, a message associated with a +source position. For most analyses, diagnostics are their primary +result. +For convenience, Pass provides a helper method, Reportf, to report a new +diagnostic by formatting a string. +Diagnostic is defined as: + + type Diagnostic struct { + Pos token.Pos + Category string // optional + Message string + } + +The optional Category field is a short identifier that classifies the +kind of message when an analysis produces several kinds of diagnostic. + +Most Analyzers inspect typed Go syntax trees, but a few, such as asmdecl +and buildtag, inspect the raw text of Go source files or even non-Go +files such as assembly. To report a diagnostic against a line of a +raw text file, use the following sequence: + + content, err := ioutil.ReadFile(filename) + if err != nil { ... } + tf := fset.AddFile(filename, -1, len(content)) + tf.SetLinesForContent(content) + ... + pass.Reportf(tf.LineStart(line), "oops") + + +Modular analysis with Facts + +To improve efficiency and scalability, large programs are routinely +built using separate compilation: units of the program are compiled +separately, and recompiled only when one of their dependencies changes; +independent modules may be compiled in parallel. The same technique may +be applied to static analyses, for the same benefits. Such analyses are +described as "modular". + +A compiler’s type checker is an example of a modular static analysis. +Many other checkers we would like to apply to Go programs can be +understood as alternative or non-standard type systems. For example, +vet's printf checker infers whether a function has the "printf wrapper" +type, and it applies stricter checks to calls of such functions. In +addition, it records which functions are printf wrappers for use by +later analysis units to identify other printf wrappers by induction. +A result such as “f is a printf wrapper†that is not interesting by +itself but serves as a stepping stone to an interesting result (such as +a diagnostic) is called a "fact". + +The analysis API allows an analysis to define new types of facts, to +associate facts of these types with objects (named entities) declared +within the current package, or with the package as a whole, and to query +for an existing fact of a given type associated with an object or +package. + +An Analyzer that uses facts must declare their types: + + var Analyzer = &analysis.Analyzer{ + Name: "printf", + FactTypes: []reflect.Type{reflect.TypeOf(new(isWrapper))}, + ... + } + + type isWrapper struct{} // => *types.Func f “is a printf wrapper†+ +A driver program ensures that facts for a pass’s dependencies are +generated before analyzing the pass and are responsible for propagating +facts between from one pass to another, possibly across address spaces. +Consequently, Facts must be serializable. The API requires that drivers +use the gob encoding, an efficient, robust, self-describing binary +protocol. A fact type may implement the GobEncoder/GobDecoder interfaces +if the default encoding is unsuitable. Facts should be stateless. + +The Pass type has functions to import and export facts, +associated either with an object or with a package: + + type Pass struct { + ... + ExportObjectFact func(types.Object, Fact) + ImportObjectFact func(types.Object, Fact) bool + + ExportPackageFact func(fact Fact) + ImportPackageFact func(*types.Package, Fact) bool + } + +An Analyzer may only export facts associated with the current package or +its objects, though it may import facts from any package or object that +is an import dependency of the current package. + +Conceptually, ExportObjectFact(obj, fact) inserts fact into a hidden map keyed by +the pair (obj, TypeOf(fact)), and the ImportObjectFact function +retrieves the entry from this map and copies its value into the variable +pointed to by fact. This scheme assumes that the concrete type of fact +is a pointer; this assumption is checked by the Validate function. +See the "printf" analyzer for an example of object facts in action. + + +Testing an Analyzer + +The analysistest subpackage provides utilities for testing an Analyzer. +In a few lines of code, it is possible to run an analyzer on a package +of testdata files and check that it reported all the expected +diagnostics and facts (and no more). Expectations are expressed using +"// want ..." comments in the input code. + + +Standalone commands + +Analyzers are provided in the form of packages that a driver program is +expected to import. The vet command imports a set of several analyses, +but users may wish to define their own analysis commands that perform +additional checks. To simplify the task of creating an analysis command, +either for a single analyzer or for a whole suite, we provide the +singlechecker and multichecker subpackages. + +The singlechecker package provides the main function for a command that +runs one analysis. By convention, each analyzer such as +go/passes/findcall should be accompanied by a singlechecker-based +command such as go/analysis/passes/findcall/cmd/findcall, defined in its +entirety as: + + package main + + import ( + "golang.org/x/tools/go/analysis/passes/findcall" + "golang.org/x/tools/go/analysis/singlechecker" + ) + + func main() { singlechecker.Main(findcall.Analyzer) } + +A tool that provides multiple analyzers can use multichecker in a +similar way, giving it the list of Analyzers. + + + +*/ +package analysis diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/analysisflags/flags.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/analysisflags/flags.go new file mode 100644 index 0000000000..d6c13f2685 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/analysisflags/flags.go @@ -0,0 +1,223 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package analysisflags defines helpers for processing flags of +// analysis driver tools. +package analysisflags + +import ( + "crypto/sha256" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "strconv" + + "golang.org/x/tools/go/analysis" +) + +// Parse creates a flag for each of the analyzer's flags, +// including (in multi mode) an --analysis.enable flag, +// parses the flags, then filters and returns the list of +// analyzers enabled by flags. +func Parse(analyzers []*analysis.Analyzer, multi bool) []*analysis.Analyzer { + // Connect each analysis flag to the command line as -analysis.flag. + type analysisFlag struct { + Name string + Bool bool + Usage string + } + var analysisFlags []analysisFlag + + enabled := make(map[*analysis.Analyzer]*triState) + for _, a := range analyzers { + var prefix string + + // Add -analysis.enable flag. + if multi { + prefix = a.Name + "." + + enable := new(triState) + enableName := prefix + "enable" + enableUsage := "enable " + a.Name + " analysis" + flag.Var(enable, enableName, enableUsage) + enabled[a] = enable + analysisFlags = append(analysisFlags, analysisFlag{enableName, true, enableUsage}) + } + + a.Flags.VisitAll(func(f *flag.Flag) { + if !multi && flag.Lookup(f.Name) != nil { + log.Printf("%s flag -%s would conflict with driver; skipping", a.Name, f.Name) + return + } + + name := prefix + f.Name + flag.Var(f.Value, name, f.Usage) + + var isBool bool + if b, ok := f.Value.(interface{ IsBoolFlag() bool }); ok { + isBool = b.IsBoolFlag() + } + analysisFlags = append(analysisFlags, analysisFlag{name, isBool, f.Usage}) + }) + } + + // standard flags: -flags, -V. + printflags := flag.Bool("flags", false, "print analyzer flags in JSON") + addVersionFlag() + + flag.Parse() // (ExitOnError) + + // -flags: print flags so that go vet knows which ones are legitimate. + if *printflags { + data, err := json.MarshalIndent(analysisFlags, "", "\t") + if err != nil { + log.Fatal(err) + } + os.Stdout.Write(data) + os.Exit(0) + } + + // If any --foo.enable flag is true, run only those analyzers. Otherwise, + // if any --foo.enable flag is false, run all but those analyzers. + if multi { + var hasTrue, hasFalse bool + for _, ts := range enabled { + switch *ts { + case setTrue: + hasTrue = true + case setFalse: + hasFalse = true + } + } + + var keep []*analysis.Analyzer + if hasTrue { + for _, a := range analyzers { + if *enabled[a] == setTrue { + keep = append(keep, a) + } + } + analyzers = keep + } else if hasFalse { + for _, a := range analyzers { + if *enabled[a] != setFalse { + keep = append(keep, a) + } + } + analyzers = keep + } + } + + return analyzers +} + +// addVersionFlag registers a -V flag that, if set, +// prints the executable version and exits 0. +// +// It is a variable not a function to permit easy +// overriding in the copy vendored in $GOROOT/src/cmd/vet: +// +// func init() { addVersionFlag = objabi.AddVersionFlag } +var addVersionFlag = func() { + flag.Var(versionFlag{}, "V", "print version and exit") +} + +// versionFlag minimally complies with the -V protocol required by "go vet". +type versionFlag struct{} + +func (versionFlag) IsBoolFlag() bool { return true } +func (versionFlag) Get() interface{} { return nil } +func (versionFlag) String() string { return "" } +func (versionFlag) Set(s string) error { + if s != "full" { + log.Fatalf("unsupported flag value: -V=%s", s) + } + + // This replicates the miminal subset of + // cmd/internal/objabi.AddVersionFlag, which is private to the + // go tool yet forms part of our command-line interface. + // TODO(adonovan): clarify the contract. + + // Print the tool version so the build system can track changes. + // Formats: + // $progname version devel ... buildID=... + // $progname version go1.9.1 + progname := os.Args[0] + f, err := os.Open(progname) + if err != nil { + log.Fatal(err) + } + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + log.Fatal(err) + } + f.Close() + fmt.Printf("%s version devel comments-go-here buildID=%02x\n", + progname, string(h.Sum(nil))) + os.Exit(0) + return nil +} + +// A triState is a boolean that knows whether +// it has been set to either true or false. +// It is used to identify whether a flag appears; +// the standard boolean flag cannot +// distinguish missing from unset. +// It also satisfies flag.Value. +type triState int + +const ( + unset triState = iota + setTrue + setFalse +) + +func triStateFlag(name string, value triState, usage string) *triState { + flag.Var(&value, name, usage) + return &value +} + +// triState implements flag.Value, flag.Getter, and flag.boolFlag. +// They work like boolean flags: we can say vet -printf as well as vet -printf=true +func (ts *triState) Get() interface{} { + return *ts == setTrue +} + +func (ts triState) isTrue() bool { + return ts == setTrue +} + +func (ts *triState) Set(value string) error { + b, err := strconv.ParseBool(value) + if err != nil { + // This error message looks poor but package "flag" adds + // "invalid boolean value %q for -foo.enable: %s" + return fmt.Errorf("want true or false") + } + if b { + *ts = setTrue + } else { + *ts = setFalse + } + return nil +} + +func (ts *triState) String() string { + switch *ts { + case unset: + return "true" + case setTrue: + return "true" + case setFalse: + return "false" + } + panic("not reached") +} + +func (ts triState) IsBoolFlag() bool { + return true +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/facts.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/facts.go new file mode 100644 index 0000000000..468f148900 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/facts.go @@ -0,0 +1,299 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package facts defines a serializable set of analysis.Fact. +// +// It provides a partial implementation of the Fact-related parts of the +// analysis.Pass interface for use in analysis drivers such as "go vet" +// and other build systems. +// +// The serial format is unspecified and may change, so the same version +// of this package must be used for reading and writing serialized facts. +// +// The handling of facts in the analysis system parallels the handling +// of type information in the compiler: during compilation of package P, +// the compiler emits an export data file that describes the type of +// every object (named thing) defined in package P, plus every object +// indirectly reachable from one of those objects. Thus the downstream +// compiler of package Q need only load one export data file per direct +// import of Q, and it will learn everything about the API of package P +// and everything it needs to know about the API of P's dependencies. +// +// Similarly, analysis of package P emits a fact set containing facts +// about all objects exported from P, plus additional facts about only +// those objects of P's dependencies that are reachable from the API of +// package P; the downstream analysis of Q need only load one fact set +// per direct import of Q. +// +// The notion of "exportedness" that matters here is that of the +// compiler. According to the language spec, a method pkg.T.f is +// unexported simply because its name starts with lowercase. But the +// compiler must nonethless export f so that downstream compilations can +// accurately ascertain whether pkg.T implements an interface pkg.I +// defined as interface{f()}. Exported thus means "described in export +// data". +// +package facts + +import ( + "bytes" + "encoding/gob" + "fmt" + "go/types" + "io/ioutil" + "log" + "reflect" + "sort" + "sync" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/types/objectpath" +) + +const debug = false + +// A Set is a set of analysis.Facts. +// +// Decode creates a Set of facts by reading from the imports of a given +// package, and Encode writes out the set. Between these operation, +// the Import and Export methods will query and update the set. +// +// All of Set's methods except String are safe to call concurrently. +type Set struct { + pkg *types.Package + mu sync.Mutex + m map[key]analysis.Fact +} + +type key struct { + pkg *types.Package + obj types.Object // (object facts only) + t reflect.Type +} + +// ImportObjectFact implements analysis.Pass.ImportObjectFact. +func (s *Set) ImportObjectFact(obj types.Object, ptr analysis.Fact) bool { + if obj == nil { + panic("nil object") + } + key := key{pkg: obj.Pkg(), obj: obj, t: reflect.TypeOf(ptr)} + s.mu.Lock() + defer s.mu.Unlock() + if v, ok := s.m[key]; ok { + reflect.ValueOf(ptr).Elem().Set(reflect.ValueOf(v).Elem()) + return true + } + return false +} + +// ExportObjectFact implements analysis.Pass.ExportObjectFact. +func (s *Set) ExportObjectFact(obj types.Object, fact analysis.Fact) { + if obj.Pkg() != s.pkg { + log.Panicf("in package %s: ExportObjectFact(%s, %T): can't set fact on object belonging another package", + s.pkg, obj, fact) + } + key := key{pkg: obj.Pkg(), obj: obj, t: reflect.TypeOf(fact)} + s.mu.Lock() + s.m[key] = fact // clobber any existing entry + s.mu.Unlock() +} + +// ImportPackageFact implements analysis.Pass.ImportPackageFact. +func (s *Set) ImportPackageFact(pkg *types.Package, ptr analysis.Fact) bool { + if pkg == nil { + panic("nil package") + } + key := key{pkg: pkg, t: reflect.TypeOf(ptr)} + s.mu.Lock() + defer s.mu.Unlock() + if v, ok := s.m[key]; ok { + reflect.ValueOf(ptr).Elem().Set(reflect.ValueOf(v).Elem()) + return true + } + return false +} + +// ExportPackageFact implements analysis.Pass.ExportPackageFact. +func (s *Set) ExportPackageFact(fact analysis.Fact) { + key := key{pkg: s.pkg, t: reflect.TypeOf(fact)} + s.mu.Lock() + s.m[key] = fact // clobber any existing entry + s.mu.Unlock() +} + +// gobFact is the Gob declaration of a serialized fact. +type gobFact struct { + PkgPath string // path of package + Object objectpath.Path // optional path of object relative to package itself + Fact analysis.Fact // type and value of user-defined Fact +} + +// Decode decodes all the facts relevant to the analysis of package pkg. +// The read function reads serialized fact data from an external source +// for one of of pkg's direct imports. The empty file is a valid +// encoding of an empty fact set. +// +// It is the caller's responsibility to call gob.Register on all +// necessary fact types. +func Decode(pkg *types.Package, read func(packagePath string) ([]byte, error)) (*Set, error) { + // Compute the import map for this package. + // See the package doc comment. + packages := importMap(pkg.Imports()) + + // Read facts from imported packages. + // Facts may describe indirectly imported packages, or their objects. + m := make(map[key]analysis.Fact) // one big bucket + for _, imp := range pkg.Imports() { + logf := func(format string, args ...interface{}) { + if debug { + prefix := fmt.Sprintf("in %s, importing %s: ", + pkg.Path(), imp.Path()) + log.Print(prefix, fmt.Sprintf(format, args...)) + } + } + + // Read the gob-encoded facts. + data, err := read(imp.Path()) + if err != nil { + return nil, fmt.Errorf("in %s, can't import facts for package %q: %v", + pkg.Path(), imp.Path(), err) + } + if len(data) == 0 { + continue // no facts + } + var gobFacts []gobFact + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&gobFacts); err != nil { + return nil, fmt.Errorf("decoding facts for %q: %v", imp.Path(), err) + } + if debug { + logf("decoded %d facts: %v", len(gobFacts), gobFacts) + } + + // Parse each one into a key and a Fact. + for _, f := range gobFacts { + factPkg := packages[f.PkgPath] + if factPkg == nil { + // Fact relates to a dependency that was + // unused in this translation unit. Skip. + logf("no package %q; discarding %v", f.PkgPath, f.Fact) + continue + } + key := key{pkg: factPkg, t: reflect.TypeOf(f.Fact)} + if f.Object != "" { + // object fact + obj, err := objectpath.Object(factPkg, f.Object) + if err != nil { + // (most likely due to unexported object) + // TODO(adonovan): audit for other possibilities. + logf("no object for path: %v; discarding %s", err, f.Fact) + continue + } + key.obj = obj + logf("read %T fact %s for %v", f.Fact, f.Fact, key.obj) + } else { + // package fact + logf("read %T fact %s for %v", f.Fact, f.Fact, factPkg) + } + m[key] = f.Fact + } + } + + return &Set{pkg: pkg, m: m}, nil +} + +// Encode encodes a set of facts to a memory buffer. +// +// It may fail if one of the Facts could not be gob-encoded, but this is +// a sign of a bug in an Analyzer. +func (s *Set) Encode() []byte { + + // TODO(adonovan): opt: use a more efficient encoding + // that avoids repeating PkgPath for each fact. + + // Gather all facts, including those from imported packages. + var gobFacts []gobFact + + s.mu.Lock() + for k, fact := range s.m { + if debug { + log.Printf("%v => %s\n", k, fact) + } + var object objectpath.Path + if k.obj != nil { + path, err := objectpath.For(k.obj) + if err != nil { + if debug { + log.Printf("discarding fact %s about %s\n", fact, k.obj) + } + continue // object not accessible from package API; discard fact + } + object = path + } + gobFacts = append(gobFacts, gobFact{ + PkgPath: k.pkg.Path(), + Object: object, + Fact: fact, + }) + } + s.mu.Unlock() + + // Sort facts by (package, object, type) for determinism. + sort.Slice(gobFacts, func(i, j int) bool { + x, y := gobFacts[i], gobFacts[j] + if x.PkgPath != y.PkgPath { + return x.PkgPath < y.PkgPath + } + if x.Object != y.Object { + return x.Object < y.Object + } + tx := reflect.TypeOf(x.Fact) + ty := reflect.TypeOf(y.Fact) + if tx != ty { + return tx.String() < ty.String() + } + return false // equal + }) + + var buf bytes.Buffer + if len(gobFacts) > 0 { + if err := gob.NewEncoder(&buf).Encode(gobFacts); err != nil { + // Fact encoding should never fail. Identify the culprit. + for _, gf := range gobFacts { + if err := gob.NewEncoder(ioutil.Discard).Encode(gf); err != nil { + fact := gf.Fact + pkgpath := reflect.TypeOf(fact).Elem().PkgPath() + log.Panicf("internal error: gob encoding of analysis fact %s failed: %v; please report a bug against fact %T in package %q", + fact, err, fact, pkgpath) + } + } + } + } + + if debug { + log.Printf("package %q: encode %d facts, %d bytes\n", + s.pkg.Path(), len(gobFacts), buf.Len()) + } + + return buf.Bytes() +} + +// String is provided only for debugging, and must not be called +// concurrent with any Import/Export method. +func (s *Set) String() string { + var buf bytes.Buffer + buf.WriteString("{") + for k, f := range s.m { + if buf.Len() > 1 { + buf.WriteString(", ") + } + if k.obj != nil { + buf.WriteString(k.obj.String()) + } else { + buf.WriteString(k.pkg.Path()) + } + fmt.Fprintf(&buf, ": %v", f) + } + buf.WriteString("}") + return buf.String() +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/imports.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/imports.go new file mode 100644 index 0000000000..34740f48e0 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/facts/imports.go @@ -0,0 +1,88 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package facts + +import "go/types" + +// importMap computes the import map for a package by traversing the +// entire exported API each of its imports. +// +// This is a workaround for the fact that we cannot access the map used +// internally by the types.Importer returned by go/importer. The entries +// in this map are the packages and objects that may be relevant to the +// current analysis unit. +// +// Packages in the map that are only indirectly imported may be +// incomplete (!pkg.Complete()). +// +func importMap(imports []*types.Package) map[string]*types.Package { + objects := make(map[types.Object]bool) + packages := make(map[string]*types.Package) + + var addObj func(obj types.Object) bool + var addType func(T types.Type) + + addObj = func(obj types.Object) bool { + if !objects[obj] { + objects[obj] = true + addType(obj.Type()) + if pkg := obj.Pkg(); pkg != nil { + packages[pkg.Path()] = pkg + } + return true + } + return false + } + + addType = func(T types.Type) { + switch T := T.(type) { + case *types.Basic: + // nop + case *types.Named: + if addObj(T.Obj()) { + for i := 0; i < T.NumMethods(); i++ { + addObj(T.Method(i)) + } + } + case *types.Pointer: + addType(T.Elem()) + case *types.Slice: + addType(T.Elem()) + case *types.Array: + addType(T.Elem()) + case *types.Chan: + addType(T.Elem()) + case *types.Map: + addType(T.Key()) + addType(T.Elem()) + case *types.Signature: + addType(T.Params()) + addType(T.Results()) + case *types.Struct: + for i := 0; i < T.NumFields(); i++ { + addObj(T.Field(i)) + } + case *types.Tuple: + for i := 0; i < T.Len(); i++ { + addObj(T.At(i)) + } + case *types.Interface: + for i := 0; i < T.NumMethods(); i++ { + addObj(T.Method(i)) + } + } + } + + for _, imp := range imports { + packages[imp.Path()] = imp + + scope := imp.Scope() + for _, name := range scope.Names() { + addObj(scope.Lookup(name)) + } + } + + return packages +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/unitchecker/unitchecker.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/unitchecker/unitchecker.go new file mode 100644 index 0000000000..b67c943b4e --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/internal/unitchecker/unitchecker.go @@ -0,0 +1,306 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The unitchecker package defines the main function for an analysis +// driver that analyzes a single compilation unit during a build. +// It is invoked by a build system such as "go vet": +// +// $ GOVETTOOL=$(which vet) go vet +// +// It supports the following command-line protocol: +// +// -V=full describe executable (to the build tool) +// -flags describe flags (to the build tool) +// foo.cfg description of compilation unit (from the build tool) +// +// This package does not depend on go/packages. +// If you need a standalone tool, use multichecker, +// which supports this mode but can also load packages +// from source using go/packages. +package unitchecker + +// TODO(adonovan): +// - with gccgo, go build does not build standard library, +// so we will not get to analyze it. Yet we must in order +// to create base facts for, say, the fmt package for the +// printf checker. +// - support JSON output, factored with multichecker. + +import ( + "encoding/gob" + "encoding/json" + "fmt" + "go/ast" + "go/build" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "io/ioutil" + "log" + "os" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/internal/facts" +) + +// A Config describes a compilation unit to be analyzed. +// It is provided to the tool in a JSON-encoded file +// whose name ends with ".cfg". +type Config struct { + Compiler string + Dir string + ImportPath string + GoFiles []string + OtherFiles []string // TODO(adonovan): make go vet populate this (github.com/golang/go/issues/27665) + ImportMap map[string]string + PackageFile map[string]string + Standard map[string]bool + PackageVetx map[string]string + VetxOnly bool + VetxOutput string + SucceedOnTypecheckFailure bool +} + +// Main reads the *.cfg file, runs the analysis, +// and calls os.Exit with an appropriate error code. +func Main(configFile string, analyzers []*analysis.Analyzer) { + cfg, err := readConfig(configFile) + if err != nil { + log.Fatal(err) + } + + fset := token.NewFileSet() + diags, err := run(fset, cfg, analyzers) + if err != nil { + log.Fatal(err) + } + + if len(diags) > 0 { + for _, diag := range diags { + fmt.Fprintf(os.Stderr, "%s: %s\n", fset.Position(diag.Pos), diag.Message) + } + os.Exit(1) + } + + os.Exit(0) +} + +func readConfig(filename string) (*Config, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + cfg := new(Config) + if err := json.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("cannot decode JSON config file %s: %v", filename, err) + } + if len(cfg.GoFiles) == 0 { + // The go command disallows packages with no files. + // The only exception is unsafe, but the go command + // doesn't call vet on it. + return nil, fmt.Errorf("package has no files: %s", cfg.ImportPath) + } + return cfg, nil +} + +func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]analysis.Diagnostic, error) { + // Load, parse, typecheck. + var files []*ast.File + for _, name := range cfg.GoFiles { + f, err := parser.ParseFile(fset, name, nil, parser.ParseComments) + if err != nil { + if cfg.SucceedOnTypecheckFailure { + // Silently succeed; let the compiler + // report parse errors. + err = nil + } + return nil, err + } + files = append(files, f) + } + compilerImporter := importer.For(cfg.Compiler, func(path string) (io.ReadCloser, error) { + // path is a resolved package path, not an import path. + file, ok := cfg.PackageFile[path] + if !ok { + if cfg.Compiler == "gccgo" && cfg.Standard[path] { + return nil, nil // fall back to default gccgo lookup + } + return nil, fmt.Errorf("no package file for %q", path) + } + return os.Open(file) + }) + importer := importerFunc(func(importPath string) (*types.Package, error) { + path, ok := cfg.ImportMap[importPath] // resolve vendoring, etc + if !ok { + return nil, fmt.Errorf("can't resolve import %q", path) + } + return compilerImporter.Import(path) + }) + tc := &types.Config{ + Importer: importer, + Sizes: types.SizesFor("gc", build.Default.GOARCH), // assume gccgo ≡ gc? + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Scopes: make(map[ast.Node]*types.Scope), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + } + pkg, err := tc.Check(cfg.ImportPath, fset, files, info) + if err != nil { + if cfg.SucceedOnTypecheckFailure { + // Silently succeed; let the compiler + // report type errors. + err = nil + } + return nil, err + } + + // Register fact types with gob. + // In VetxOnly mode, analyzers are only for their facts, + // so we can skip any analysis that neither produces facts + // nor depends on any analysis that produces facts. + // Also build a map to hold working state and result. + type action struct { + once sync.Once + result interface{} + err error + usesFacts bool // (transitively uses) + diagnostics []analysis.Diagnostic + } + actions := make(map[*analysis.Analyzer]*action) + var registerFacts func(a *analysis.Analyzer) bool + registerFacts = func(a *analysis.Analyzer) bool { + act, ok := actions[a] + if !ok { + act = new(action) + var usesFacts bool + for _, f := range a.FactTypes { + usesFacts = true + gob.Register(f) + } + for _, req := range a.Requires { + if registerFacts(req) { + usesFacts = true + } + } + act.usesFacts = usesFacts + actions[a] = act + } + return act.usesFacts + } + var filtered []*analysis.Analyzer + for _, a := range analyzers { + if registerFacts(a) || !cfg.VetxOnly { + filtered = append(filtered, a) + } + } + analyzers = filtered + + // Read facts from imported packages. + read := func(path string) ([]byte, error) { + if vetx, ok := cfg.PackageVetx[path]; ok { + return ioutil.ReadFile(vetx) + } + return nil, nil // no .vetx file, no facts + } + facts, err := facts.Decode(pkg, read) + if err != nil { + return nil, err + } + + // In parallel, execute the DAG of analyzers. + var exec func(a *analysis.Analyzer) *action + var execAll func(analyzers []*analysis.Analyzer) + exec = func(a *analysis.Analyzer) *action { + act := actions[a] + act.once.Do(func() { + execAll(a.Requires) // prefetch dependencies in parallel + + // The inputs to this analysis are the + // results of its prerequisites. + inputs := make(map[*analysis.Analyzer]interface{}) + var failed []string + for _, req := range a.Requires { + reqact := exec(req) + if reqact.err != nil { + failed = append(failed, req.String()) + continue + } + inputs[req] = reqact.result + } + + // Report an error if any dependency failed. + if failed != nil { + sort.Strings(failed) + act.err = fmt.Errorf("failed prerequisites: %s", strings.Join(failed, ", ")) + return + } + + pass := &analysis.Pass{ + Analyzer: a, + Fset: fset, + Files: files, + OtherFiles: cfg.OtherFiles, + Pkg: pkg, + TypesInfo: info, + ResultOf: inputs, + Report: func(d analysis.Diagnostic) { act.diagnostics = append(act.diagnostics, d) }, + ImportObjectFact: facts.ImportObjectFact, + ExportObjectFact: facts.ExportObjectFact, + ImportPackageFact: facts.ImportPackageFact, + ExportPackageFact: facts.ExportPackageFact, + } + + t0 := time.Now() + act.result, act.err = a.Run(pass) + if false { + log.Printf("analysis %s = %s", pass, time.Since(t0)) + } + }) + return act + } + execAll = func(analyzers []*analysis.Analyzer) { + var wg sync.WaitGroup + for _, a := range analyzers { + wg.Add(1) + go func(a *analysis.Analyzer) { + _ = exec(a) + wg.Done() + }(a) + } + wg.Wait() + } + + execAll(analyzers) + + // Return diagnostics from root analyzers. + var diags []analysis.Diagnostic + for _, a := range analyzers { + act := actions[a] + if act.err != nil { + return nil, act.err // some analysis failed + } + diags = append(diags, act.diagnostics...) + } + + data := facts.Encode() + if err := ioutil.WriteFile(cfg.VetxOutput, data, 0666); err != nil { + return nil, fmt.Errorf("failed to write analysis facts: %v", err) + } + + return diags, nil +} + +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go new file mode 100644 index 0000000000..11dfbf6bc8 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/asmdecl/asmdecl.go @@ -0,0 +1,759 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package asmdecl defines an Analyzer that reports mismatches between +// assembly files and Go declarations. +package asmdecl + +import ( + "bytes" + "fmt" + "go/ast" + "go/build" + "go/token" + "go/types" + "log" + "regexp" + "strconv" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" +) + +var Analyzer = &analysis.Analyzer{ + Name: "asmdecl", + Doc: "report mismatches between assembly files and Go declarations", + Run: run, +} + +// 'kind' is a kind of assembly variable. +// The kinds 1, 2, 4, 8 stand for values of that size. +type asmKind int + +// These special kinds are not valid sizes. +const ( + asmString asmKind = 100 + iota + asmSlice + asmArray + asmInterface + asmEmptyInterface + asmStruct + asmComplex +) + +// An asmArch describes assembly parameters for an architecture +type asmArch struct { + name string + bigEndian bool + stack string + lr bool + // calculated during initialization + sizes types.Sizes + intSize int + ptrSize int + maxAlign int +} + +// An asmFunc describes the expected variables for a function on a given architecture. +type asmFunc struct { + arch *asmArch + size int // size of all arguments + vars map[string]*asmVar + varByOffset map[int]*asmVar +} + +// An asmVar describes a single assembly variable. +type asmVar struct { + name string + kind asmKind + typ string + off int + size int + inner []*asmVar +} + +var ( + asmArch386 = asmArch{name: "386", bigEndian: false, stack: "SP", lr: false} + asmArchArm = asmArch{name: "arm", bigEndian: false, stack: "R13", lr: true} + asmArchArm64 = asmArch{name: "arm64", bigEndian: false, stack: "RSP", lr: true} + asmArchAmd64 = asmArch{name: "amd64", bigEndian: false, stack: "SP", lr: false} + asmArchAmd64p32 = asmArch{name: "amd64p32", bigEndian: false, stack: "SP", lr: false} + asmArchMips = asmArch{name: "mips", bigEndian: true, stack: "R29", lr: true} + asmArchMipsLE = asmArch{name: "mipsle", bigEndian: false, stack: "R29", lr: true} + asmArchMips64 = asmArch{name: "mips64", bigEndian: true, stack: "R29", lr: true} + asmArchMips64LE = asmArch{name: "mips64le", bigEndian: false, stack: "R29", lr: true} + asmArchPpc64 = asmArch{name: "ppc64", bigEndian: true, stack: "R1", lr: true} + asmArchPpc64LE = asmArch{name: "ppc64le", bigEndian: false, stack: "R1", lr: true} + asmArchS390X = asmArch{name: "s390x", bigEndian: true, stack: "R15", lr: true} + asmArchWasm = asmArch{name: "wasm", bigEndian: false, stack: "SP", lr: false} + + arches = []*asmArch{ + &asmArch386, + &asmArchArm, + &asmArchArm64, + &asmArchAmd64, + &asmArchAmd64p32, + &asmArchMips, + &asmArchMipsLE, + &asmArchMips64, + &asmArchMips64LE, + &asmArchPpc64, + &asmArchPpc64LE, + &asmArchS390X, + &asmArchWasm, + } +) + +func init() { + for _, arch := range arches { + arch.sizes = types.SizesFor("gc", arch.name) + if arch.sizes == nil { + // TODO(adonovan): fix: now that asmdecl is not in the standard + // library we cannot assume types.SizesFor is consistent with arches. + // For now, assume 64-bit norms and print a warning. + // But this warning should really be deferred until we attempt to use + // arch, which is very unlikely. + arch.sizes = types.SizesFor("gc", "amd64") + log.Printf("unknown architecture %s", arch.name) + } + arch.intSize = int(arch.sizes.Sizeof(types.Typ[types.Int])) + arch.ptrSize = int(arch.sizes.Sizeof(types.Typ[types.UnsafePointer])) + arch.maxAlign = int(arch.sizes.Alignof(types.Typ[types.Int64])) + } +} + +var ( + re = regexp.MustCompile + asmPlusBuild = re(`//\s+\+build\s+([^\n]+)`) + asmTEXT = re(`\bTEXT\b(.*)·([^\(]+)\(SB\)(?:\s*,\s*([0-9A-Z|+()]+))?(?:\s*,\s*\$(-?[0-9]+)(?:-([0-9]+))?)?`) + asmDATA = re(`\b(DATA|GLOBL)\b`) + asmNamedFP = re(`([a-zA-Z0-9_\xFF-\x{10FFFF}]+)(?:\+([0-9]+))\(FP\)`) + asmUnnamedFP = re(`[^+\-0-9](([0-9]+)\(FP\))`) + asmSP = re(`[^+\-0-9](([0-9]+)\(([A-Z0-9]+)\))`) + asmOpcode = re(`^\s*(?:[A-Z0-9a-z_]+:)?\s*([A-Z]+)\s*([^,]*)(?:,\s*(.*))?`) + ppc64Suff = re(`([BHWD])(ZU|Z|U|BR)?$`) +) + +func run(pass *analysis.Pass) (interface{}, error) { + // No work if no assembly files. + var sfiles []string + for _, fname := range pass.OtherFiles { + if strings.HasSuffix(fname, ".s") { + sfiles = append(sfiles, fname) + } + } + if sfiles == nil { + return nil, nil + } + + // Gather declarations. knownFunc[name][arch] is func description. + knownFunc := make(map[string]map[string]*asmFunc) + + for _, f := range pass.Files { + for _, decl := range f.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok && decl.Body == nil { + knownFunc[decl.Name.Name] = asmParseDecl(pass, decl) + } + } + } + +Files: + for _, fname := range sfiles { + content, tf, err := analysisutil.ReadFile(pass.Fset, fname) + if err != nil { + return nil, err + } + + // Determine architecture from file name if possible. + var arch string + var archDef *asmArch + for _, a := range arches { + if strings.HasSuffix(fname, "_"+a.name+".s") { + arch = a.name + archDef = a + break + } + } + + lines := strings.SplitAfter(string(content), "\n") + var ( + fn *asmFunc + fnName string + localSize, argSize int + wroteSP bool + haveRetArg bool + retLine []int + ) + + flushRet := func() { + if fn != nil && fn.vars["ret"] != nil && !haveRetArg && len(retLine) > 0 { + v := fn.vars["ret"] + for _, line := range retLine { + pass.Reportf(analysisutil.LineStart(tf, line), "[%s] %s: RET without writing to %d-byte ret+%d(FP)", arch, fnName, v.size, v.off) + } + } + retLine = nil + } + for lineno, line := range lines { + lineno++ + + badf := func(format string, args ...interface{}) { + pass.Reportf(analysisutil.LineStart(tf, lineno), "[%s] %s: %s", arch, fnName, fmt.Sprintf(format, args...)) + } + + if arch == "" { + // Determine architecture from +build line if possible. + if m := asmPlusBuild.FindStringSubmatch(line); m != nil { + // There can be multiple architectures in a single +build line, + // so accumulate them all and then prefer the one that + // matches build.Default.GOARCH. + var archCandidates []*asmArch + for _, fld := range strings.Fields(m[1]) { + for _, a := range arches { + if a.name == fld { + archCandidates = append(archCandidates, a) + } + } + } + for _, a := range archCandidates { + if a.name == build.Default.GOARCH { + archCandidates = []*asmArch{a} + break + } + } + if len(archCandidates) > 0 { + arch = archCandidates[0].name + archDef = archCandidates[0] + } + } + } + + if m := asmTEXT.FindStringSubmatch(line); m != nil { + flushRet() + if arch == "" { + // Arch not specified by filename or build tags. + // Fall back to build.Default.GOARCH. + for _, a := range arches { + if a.name == build.Default.GOARCH { + arch = a.name + archDef = a + break + } + } + if arch == "" { + badf("%s: cannot determine architecture for assembly file") + continue Files + } + } + fnName = m[2] + if pkgName := strings.TrimSpace(m[1]); pkgName != "" { + pathParts := strings.Split(pkgName, "∕") + pkgName = pathParts[len(pathParts)-1] + if pkgName != pass.Pkg.Path() { + badf("[%s] cannot check cross-package assembly function: %s is in package %s", arch, fnName, pkgName) + fn = nil + fnName = "" + continue + } + } + flag := m[3] + fn = knownFunc[fnName][arch] + if fn != nil { + size, _ := strconv.Atoi(m[5]) + if size != fn.size && (flag != "7" && !strings.Contains(flag, "NOSPLIT") || size != 0) { + badf("wrong argument size %d; expected $...-%d", size, fn.size) + } + } + localSize, _ = strconv.Atoi(m[4]) + localSize += archDef.intSize + if archDef.lr && !strings.Contains(flag, "NOFRAME") { + // Account for caller's saved LR + localSize += archDef.intSize + } + argSize, _ = strconv.Atoi(m[5]) + if fn == nil && !strings.Contains(fnName, "<>") { + badf("function %s missing Go declaration", fnName) + } + wroteSP = false + haveRetArg = false + continue + } else if strings.Contains(line, "TEXT") && strings.Contains(line, "SB") { + // function, but not visible from Go (didn't match asmTEXT), so stop checking + flushRet() + fn = nil + fnName = "" + continue + } + + if strings.Contains(line, "RET") { + retLine = append(retLine, lineno) + } + + if fnName == "" { + continue + } + + if asmDATA.FindStringSubmatch(line) != nil { + fn = nil + } + + if archDef == nil { + continue + } + + if strings.Contains(line, ", "+archDef.stack) || strings.Contains(line, ",\t"+archDef.stack) { + wroteSP = true + continue + } + + for _, m := range asmSP.FindAllStringSubmatch(line, -1) { + if m[3] != archDef.stack || wroteSP { + continue + } + off := 0 + if m[1] != "" { + off, _ = strconv.Atoi(m[2]) + } + if off >= localSize { + if fn != nil { + v := fn.varByOffset[off-localSize] + if v != nil { + badf("%s should be %s+%d(FP)", m[1], v.name, off-localSize) + continue + } + } + if off >= localSize+argSize { + badf("use of %s points beyond argument frame", m[1]) + continue + } + badf("use of %s to access argument frame", m[1]) + } + } + + if fn == nil { + continue + } + + for _, m := range asmUnnamedFP.FindAllStringSubmatch(line, -1) { + off, _ := strconv.Atoi(m[2]) + v := fn.varByOffset[off] + if v != nil { + badf("use of unnamed argument %s; offset %d is %s+%d(FP)", m[1], off, v.name, v.off) + } else { + badf("use of unnamed argument %s", m[1]) + } + } + + for _, m := range asmNamedFP.FindAllStringSubmatch(line, -1) { + name := m[1] + off := 0 + if m[2] != "" { + off, _ = strconv.Atoi(m[2]) + } + if name == "ret" || strings.HasPrefix(name, "ret_") { + haveRetArg = true + } + v := fn.vars[name] + if v == nil { + // Allow argframe+0(FP). + if name == "argframe" && off == 0 { + continue + } + v = fn.varByOffset[off] + if v != nil { + badf("unknown variable %s; offset %d is %s+%d(FP)", name, off, v.name, v.off) + } else { + badf("unknown variable %s", name) + } + continue + } + asmCheckVar(badf, fn, line, m[0], off, v) + } + } + flushRet() + } + return nil, nil +} + +func asmKindForType(t types.Type, size int) asmKind { + switch t := t.Underlying().(type) { + case *types.Basic: + switch t.Kind() { + case types.String: + return asmString + case types.Complex64, types.Complex128: + return asmComplex + } + return asmKind(size) + case *types.Pointer, *types.Chan, *types.Map, *types.Signature: + return asmKind(size) + case *types.Struct: + return asmStruct + case *types.Interface: + if t.Empty() { + return asmEmptyInterface + } + return asmInterface + case *types.Array: + return asmArray + case *types.Slice: + return asmSlice + } + panic("unreachable") +} + +// A component is an assembly-addressable component of a composite type, +// or a composite type itself. +type component struct { + size int + offset int + kind asmKind + typ string + suffix string // Such as _base for string base, _0_lo for lo half of first element of [1]uint64 on 32 bit machine. + outer string // The suffix for immediately containing composite type. +} + +func newComponent(suffix string, kind asmKind, typ string, offset, size int, outer string) component { + return component{suffix: suffix, kind: kind, typ: typ, offset: offset, size: size, outer: outer} +} + +// componentsOfType generates a list of components of type t. +// For example, given string, the components are the string itself, the base, and the length. +func componentsOfType(arch *asmArch, t types.Type) []component { + return appendComponentsRecursive(arch, t, nil, "", 0) +} + +// appendComponentsRecursive implements componentsOfType. +// Recursion is required to correct handle structs and arrays, +// which can contain arbitrary other types. +func appendComponentsRecursive(arch *asmArch, t types.Type, cc []component, suffix string, off int) []component { + s := t.String() + size := int(arch.sizes.Sizeof(t)) + kind := asmKindForType(t, size) + cc = append(cc, newComponent(suffix, kind, s, off, size, suffix)) + + switch kind { + case 8: + if arch.ptrSize == 4 { + w1, w2 := "lo", "hi" + if arch.bigEndian { + w1, w2 = w2, w1 + } + cc = append(cc, newComponent(suffix+"_"+w1, 4, "half "+s, off, 4, suffix)) + cc = append(cc, newComponent(suffix+"_"+w2, 4, "half "+s, off+4, 4, suffix)) + } + + case asmEmptyInterface: + cc = append(cc, newComponent(suffix+"_type", asmKind(arch.ptrSize), "interface type", off, arch.ptrSize, suffix)) + cc = append(cc, newComponent(suffix+"_data", asmKind(arch.ptrSize), "interface data", off+arch.ptrSize, arch.ptrSize, suffix)) + + case asmInterface: + cc = append(cc, newComponent(suffix+"_itable", asmKind(arch.ptrSize), "interface itable", off, arch.ptrSize, suffix)) + cc = append(cc, newComponent(suffix+"_data", asmKind(arch.ptrSize), "interface data", off+arch.ptrSize, arch.ptrSize, suffix)) + + case asmSlice: + cc = append(cc, newComponent(suffix+"_base", asmKind(arch.ptrSize), "slice base", off, arch.ptrSize, suffix)) + cc = append(cc, newComponent(suffix+"_len", asmKind(arch.intSize), "slice len", off+arch.ptrSize, arch.intSize, suffix)) + cc = append(cc, newComponent(suffix+"_cap", asmKind(arch.intSize), "slice cap", off+arch.ptrSize+arch.intSize, arch.intSize, suffix)) + + case asmString: + cc = append(cc, newComponent(suffix+"_base", asmKind(arch.ptrSize), "string base", off, arch.ptrSize, suffix)) + cc = append(cc, newComponent(suffix+"_len", asmKind(arch.intSize), "string len", off+arch.ptrSize, arch.intSize, suffix)) + + case asmComplex: + fsize := size / 2 + cc = append(cc, newComponent(suffix+"_real", asmKind(fsize), fmt.Sprintf("real(complex%d)", size*8), off, fsize, suffix)) + cc = append(cc, newComponent(suffix+"_imag", asmKind(fsize), fmt.Sprintf("imag(complex%d)", size*8), off+fsize, fsize, suffix)) + + case asmStruct: + tu := t.Underlying().(*types.Struct) + fields := make([]*types.Var, tu.NumFields()) + for i := 0; i < tu.NumFields(); i++ { + fields[i] = tu.Field(i) + } + offsets := arch.sizes.Offsetsof(fields) + for i, f := range fields { + cc = appendComponentsRecursive(arch, f.Type(), cc, suffix+"_"+f.Name(), off+int(offsets[i])) + } + + case asmArray: + tu := t.Underlying().(*types.Array) + elem := tu.Elem() + // Calculate offset of each element array. + fields := []*types.Var{ + types.NewVar(token.NoPos, nil, "fake0", elem), + types.NewVar(token.NoPos, nil, "fake1", elem), + } + offsets := arch.sizes.Offsetsof(fields) + elemoff := int(offsets[1]) + for i := 0; i < int(tu.Len()); i++ { + cc = appendComponentsRecursive(arch, elem, cc, suffix+"_"+strconv.Itoa(i), i*elemoff) + } + } + + return cc +} + +// asmParseDecl parses a function decl for expected assembly variables. +func asmParseDecl(pass *analysis.Pass, decl *ast.FuncDecl) map[string]*asmFunc { + var ( + arch *asmArch + fn *asmFunc + offset int + ) + + // addParams adds asmVars for each of the parameters in list. + // isret indicates whether the list are the arguments or the return values. + // TODO(adonovan): simplify by passing (*types.Signature).{Params,Results} + // instead of list. + addParams := func(list []*ast.Field, isret bool) { + argnum := 0 + for _, fld := range list { + t := pass.TypesInfo.Types[fld.Type].Type + + // Work around github.com/golang/go/issues/28277. + if t == nil { + if ell, ok := fld.Type.(*ast.Ellipsis); ok { + t = types.NewSlice(pass.TypesInfo.Types[ell.Elt].Type) + } + } + + align := int(arch.sizes.Alignof(t)) + size := int(arch.sizes.Sizeof(t)) + offset += -offset & (align - 1) + cc := componentsOfType(arch, t) + + // names is the list of names with this type. + names := fld.Names + if len(names) == 0 { + // Anonymous args will be called arg, arg1, arg2, ... + // Similarly so for return values: ret, ret1, ret2, ... + name := "arg" + if isret { + name = "ret" + } + if argnum > 0 { + name += strconv.Itoa(argnum) + } + names = []*ast.Ident{ast.NewIdent(name)} + } + argnum += len(names) + + // Create variable for each name. + for _, id := range names { + name := id.Name + for _, c := range cc { + outer := name + c.outer + v := asmVar{ + name: name + c.suffix, + kind: c.kind, + typ: c.typ, + off: offset + c.offset, + size: c.size, + } + if vo := fn.vars[outer]; vo != nil { + vo.inner = append(vo.inner, &v) + } + fn.vars[v.name] = &v + for i := 0; i < v.size; i++ { + fn.varByOffset[v.off+i] = &v + } + } + offset += size + } + } + } + + m := make(map[string]*asmFunc) + for _, arch = range arches { + fn = &asmFunc{ + arch: arch, + vars: make(map[string]*asmVar), + varByOffset: make(map[int]*asmVar), + } + offset = 0 + addParams(decl.Type.Params.List, false) + if decl.Type.Results != nil && len(decl.Type.Results.List) > 0 { + offset += -offset & (arch.maxAlign - 1) + addParams(decl.Type.Results.List, true) + } + fn.size = offset + m[arch.name] = fn + } + + return m +} + +// asmCheckVar checks a single variable reference. +func asmCheckVar(badf func(string, ...interface{}), fn *asmFunc, line, expr string, off int, v *asmVar) { + m := asmOpcode.FindStringSubmatch(line) + if m == nil { + if !strings.HasPrefix(strings.TrimSpace(line), "//") { + badf("cannot find assembly opcode") + } + return + } + + // Determine operand sizes from instruction. + // Typically the suffix suffices, but there are exceptions. + var src, dst, kind asmKind + op := m[1] + switch fn.arch.name + "." + op { + case "386.FMOVLP": + src, dst = 8, 4 + case "arm.MOVD": + src = 8 + case "arm.MOVW": + src = 4 + case "arm.MOVH", "arm.MOVHU": + src = 2 + case "arm.MOVB", "arm.MOVBU": + src = 1 + // LEA* opcodes don't really read the second arg. + // They just take the address of it. + case "386.LEAL": + dst = 4 + case "amd64.LEAQ": + dst = 8 + case "amd64p32.LEAL": + dst = 4 + default: + switch fn.arch.name { + case "386", "amd64": + if strings.HasPrefix(op, "F") && (strings.HasSuffix(op, "D") || strings.HasSuffix(op, "DP")) { + // FMOVDP, FXCHD, etc + src = 8 + break + } + if strings.HasPrefix(op, "P") && strings.HasSuffix(op, "RD") { + // PINSRD, PEXTRD, etc + src = 4 + break + } + if strings.HasPrefix(op, "F") && (strings.HasSuffix(op, "F") || strings.HasSuffix(op, "FP")) { + // FMOVFP, FXCHF, etc + src = 4 + break + } + if strings.HasSuffix(op, "SD") { + // MOVSD, SQRTSD, etc + src = 8 + break + } + if strings.HasSuffix(op, "SS") { + // MOVSS, SQRTSS, etc + src = 4 + break + } + if strings.HasPrefix(op, "SET") { + // SETEQ, etc + src = 1 + break + } + switch op[len(op)-1] { + case 'B': + src = 1 + case 'W': + src = 2 + case 'L': + src = 4 + case 'D', 'Q': + src = 8 + } + case "ppc64", "ppc64le": + // Strip standard suffixes to reveal size letter. + m := ppc64Suff.FindStringSubmatch(op) + if m != nil { + switch m[1][0] { + case 'B': + src = 1 + case 'H': + src = 2 + case 'W': + src = 4 + case 'D': + src = 8 + } + } + case "mips", "mipsle", "mips64", "mips64le": + switch op { + case "MOVB", "MOVBU": + src = 1 + case "MOVH", "MOVHU": + src = 2 + case "MOVW", "MOVWU", "MOVF": + src = 4 + case "MOVV", "MOVD": + src = 8 + } + case "s390x": + switch op { + case "MOVB", "MOVBZ": + src = 1 + case "MOVH", "MOVHZ": + src = 2 + case "MOVW", "MOVWZ", "FMOVS": + src = 4 + case "MOVD", "FMOVD": + src = 8 + } + } + } + if dst == 0 { + dst = src + } + + // Determine whether the match we're holding + // is the first or second argument. + if strings.Index(line, expr) > strings.Index(line, ",") { + kind = dst + } else { + kind = src + } + + vk := v.kind + vs := v.size + vt := v.typ + switch vk { + case asmInterface, asmEmptyInterface, asmString, asmSlice: + // allow reference to first word (pointer) + vk = v.inner[0].kind + vs = v.inner[0].size + vt = v.inner[0].typ + } + + if off != v.off { + var inner bytes.Buffer + for i, vi := range v.inner { + if len(v.inner) > 1 { + fmt.Fprintf(&inner, ",") + } + fmt.Fprintf(&inner, " ") + if i == len(v.inner)-1 { + fmt.Fprintf(&inner, "or ") + } + fmt.Fprintf(&inner, "%s+%d(FP)", vi.name, vi.off) + } + badf("invalid offset %s; expected %s+%d(FP)%s", expr, v.name, v.off, inner.String()) + return + } + if kind != 0 && kind != vk { + var inner bytes.Buffer + if len(v.inner) > 0 { + fmt.Fprintf(&inner, " containing") + for i, vi := range v.inner { + if i > 0 && len(v.inner) > 2 { + fmt.Fprintf(&inner, ",") + } + fmt.Fprintf(&inner, " ") + if i > 0 && i == len(v.inner)-1 { + fmt.Fprintf(&inner, "and ") + } + fmt.Fprintf(&inner, "%s+%d(FP)", vi.name, vi.off) + } + } + badf("invalid %s of %s; %s is %d-byte value%s", op, expr, vt, vs, inner.String()) + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/assign/assign.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/assign/assign.go new file mode 100644 index 0000000000..4dff2908c3 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/assign/assign.go @@ -0,0 +1,68 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package assign defines an Analyzer that detects useless assignments. +package assign + +// TODO(adonovan): check also for assignments to struct fields inside +// methods that are on T instead of *T. + +import ( + "go/ast" + "go/token" + "reflect" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for useless assignments + +This checker reports assignments of the form x = x or a[i] = a[i]. +These are almost always useless, and even when they aren't they are +usually a mistake.` + +var Analyzer = &analysis.Analyzer{ + Name: "assign", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.AssignStmt)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + stmt := n.(*ast.AssignStmt) + if stmt.Tok != token.ASSIGN { + return // ignore := + } + if len(stmt.Lhs) != len(stmt.Rhs) { + // If LHS and RHS have different cardinality, they can't be the same. + return + } + for i, lhs := range stmt.Lhs { + rhs := stmt.Rhs[i] + if analysisutil.HasSideEffects(pass.TypesInfo, lhs) || + analysisutil.HasSideEffects(pass.TypesInfo, rhs) { + continue // expressions may not be equal + } + if reflect.TypeOf(lhs) != reflect.TypeOf(rhs) { + continue // short-circuit the heavy-weight gofmt check + } + le := analysisutil.Format(pass.Fset, lhs) + re := analysisutil.Format(pass.Fset, rhs) + if le == re { + pass.Reportf(stmt.Pos(), "self-assignment of %s to %s", re, le) + } + } + }) + + return nil, nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/atomic/atomic.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/atomic/atomic.go new file mode 100644 index 0000000000..45243d6f8c --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/atomic/atomic.go @@ -0,0 +1,96 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package atomic defines an Analyzer that checks for common mistakes +// using the sync/atomic package. +package atomic + +import ( + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for common mistakes using the sync/atomic package + +The atomic checker looks for assignment statements of the form: + + x = atomic.AddUint64(&x, 1) + +which are not atomic.` + +var Analyzer = &analysis.Analyzer{ + Name: "atomic", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.AssignStmt)(nil), + } + inspect.Preorder(nodeFilter, func(node ast.Node) { + n := node.(*ast.AssignStmt) + if len(n.Lhs) != len(n.Rhs) { + return + } + if len(n.Lhs) == 1 && n.Tok == token.DEFINE { + return + } + + for i, right := range n.Rhs { + call, ok := right.(*ast.CallExpr) + if !ok { + continue + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + continue + } + pkgIdent, _ := sel.X.(*ast.Ident) + pkgName, ok := pass.TypesInfo.Uses[pkgIdent].(*types.PkgName) + if !ok || pkgName.Imported().Path() != "sync/atomic" { + continue + } + + switch sel.Sel.Name { + case "AddInt32", "AddInt64", "AddUint32", "AddUint64", "AddUintptr": + checkAtomicAddAssignment(pass, n.Lhs[i], call) + } + } + }) + return nil, nil +} + +// checkAtomicAddAssignment walks the atomic.Add* method calls checking +// for assigning the return value to the same variable being used in the +// operation +func checkAtomicAddAssignment(pass *analysis.Pass, left ast.Expr, call *ast.CallExpr) { + if len(call.Args) != 2 { + return + } + arg := call.Args[0] + broken := false + + gofmt := func(e ast.Expr) string { return analysisutil.Format(pass.Fset, e) } + + if uarg, ok := arg.(*ast.UnaryExpr); ok && uarg.Op == token.AND { + broken = gofmt(left) == gofmt(uarg.X) + } else if star, ok := left.(*ast.StarExpr); ok { + broken = gofmt(star.X) == gofmt(arg) + } + + if broken { + pass.Reportf(left.Pos(), "direct assignment to atomic value") + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/bools/bools.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/bools/bools.go new file mode 100644 index 0000000000..0e6f2695f3 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/bools/bools.go @@ -0,0 +1,214 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bools defines an Analyzer that detects common mistakes +// involving boolean operators. +package bools + +import ( + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +var Analyzer = &analysis.Analyzer{ + Name: "bools", + Doc: "check for common mistakes involving boolean operators", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.BinaryExpr)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + e := n.(*ast.BinaryExpr) + + var op boolOp + switch e.Op { + case token.LOR: + op = or + case token.LAND: + op = and + default: + return + } + + // TODO(adonovan): this reports n(n-1)/2 errors for an + // expression e||...||e of depth n. Fix. + // See https://github.com/golang/go/issues/28086. + comm := op.commutativeSets(pass.TypesInfo, e) + for _, exprs := range comm { + op.checkRedundant(pass, exprs) + op.checkSuspect(pass, exprs) + } + }) + return nil, nil +} + +type boolOp struct { + name string + tok token.Token // token corresponding to this operator + badEq token.Token // token corresponding to the equality test that should not be used with this operator +} + +var ( + or = boolOp{"or", token.LOR, token.NEQ} + and = boolOp{"and", token.LAND, token.EQL} +) + +// commutativeSets returns all side effect free sets of +// expressions in e that are connected by op. +// For example, given 'a || b || f() || c || d' with the or op, +// commutativeSets returns {{b, a}, {d, c}}. +func (op boolOp) commutativeSets(info *types.Info, e *ast.BinaryExpr) [][]ast.Expr { + exprs := op.split(e) + + // Partition the slice of expressions into commutative sets. + i := 0 + var sets [][]ast.Expr + for j := 0; j <= len(exprs); j++ { + if j == len(exprs) || hasSideEffects(info, exprs[j]) { + if i < j { + sets = append(sets, exprs[i:j]) + } + i = j + 1 + } + } + + return sets +} + +// checkRedundant checks for expressions of the form +// e && e +// e || e +// Exprs must contain only side effect free expressions. +func (op boolOp) checkRedundant(pass *analysis.Pass, exprs []ast.Expr) { + seen := make(map[string]bool) + for _, e := range exprs { + efmt := analysisutil.Format(pass.Fset, e) + if seen[efmt] { + pass.Reportf(e.Pos(), "redundant %s: %s %s %s", op.name, efmt, op.tok, efmt) + } else { + seen[efmt] = true + } + } +} + +// checkSuspect checks for expressions of the form +// x != c1 || x != c2 +// x == c1 && x == c2 +// where c1 and c2 are constant expressions. +// If c1 and c2 are the same then it's redundant; +// if c1 and c2 are different then it's always true or always false. +// Exprs must contain only side effect free expressions. +func (op boolOp) checkSuspect(pass *analysis.Pass, exprs []ast.Expr) { + // seen maps from expressions 'x' to equality expressions 'x != c'. + seen := make(map[string]string) + + for _, e := range exprs { + bin, ok := e.(*ast.BinaryExpr) + if !ok || bin.Op != op.badEq { + continue + } + + // In order to avoid false positives, restrict to cases + // in which one of the operands is constant. We're then + // interested in the other operand. + // In the rare case in which both operands are constant + // (e.g. runtime.GOOS and "windows"), we'll only catch + // mistakes if the LHS is repeated, which is how most + // code is written. + var x ast.Expr + switch { + case pass.TypesInfo.Types[bin.Y].Value != nil: + x = bin.X + case pass.TypesInfo.Types[bin.X].Value != nil: + x = bin.Y + default: + continue + } + + // e is of the form 'x != c' or 'x == c'. + xfmt := analysisutil.Format(pass.Fset, x) + efmt := analysisutil.Format(pass.Fset, e) + if prev, found := seen[xfmt]; found { + // checkRedundant handles the case in which efmt == prev. + if efmt != prev { + pass.Reportf(e.Pos(), "suspect %s: %s %s %s", op.name, efmt, op.tok, prev) + } + } else { + seen[xfmt] = efmt + } + } +} + +// hasSideEffects reports whether evaluation of e has side effects. +func hasSideEffects(info *types.Info, e ast.Expr) bool { + safe := true + ast.Inspect(e, func(node ast.Node) bool { + switch n := node.(type) { + case *ast.CallExpr: + typVal := info.Types[n.Fun] + switch { + case typVal.IsType(): + // Type conversion, which is safe. + case typVal.IsBuiltin(): + // Builtin func, conservatively assumed to not + // be safe for now. + safe = false + return false + default: + // A non-builtin func or method call. + // Conservatively assume that all of them have + // side effects for now. + safe = false + return false + } + case *ast.UnaryExpr: + if n.Op == token.ARROW { + safe = false + return false + } + } + return true + }) + return !safe +} + +// split returns a slice of all subexpressions in e that are connected by op. +// For example, given 'a || (b || c) || d' with the or op, +// split returns []{d, c, b, a}. +func (op boolOp) split(e ast.Expr) (exprs []ast.Expr) { + for { + e = unparen(e) + if b, ok := e.(*ast.BinaryExpr); ok && b.Op == op.tok { + exprs = append(exprs, op.split(b.Y)...) + e = b.X + } else { + exprs = append(exprs, e) + break + } + } + return +} + +// unparen returns e with any enclosing parentheses stripped. +func unparen(e ast.Expr) ast.Expr { + for { + p, ok := e.(*ast.ParenExpr) + if !ok { + return e + } + e = p.X + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/buildtag/buildtag.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/buildtag/buildtag.go new file mode 100644 index 0000000000..5a441e609b --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/buildtag/buildtag.go @@ -0,0 +1,159 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package buildtag defines an Analyzer that checks build tags. +package buildtag + +import ( + "bytes" + "fmt" + "go/ast" + "strings" + "unicode" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" +) + +var Analyzer = &analysis.Analyzer{ + Name: "buildtag", + Doc: "check that +build tags are well-formed and correctly located", + Run: runBuildTag, +} + +func runBuildTag(pass *analysis.Pass) (interface{}, error) { + for _, f := range pass.Files { + checkGoFile(pass, f) + } + for _, name := range pass.OtherFiles { + if err := checkOtherFile(pass, name); err != nil { + return nil, err + } + } + return nil, nil +} + +func checkGoFile(pass *analysis.Pass, f *ast.File) { + pastCutoff := false + for _, group := range f.Comments { + // A +build comment is ignored after or adjoining the package declaration. + if group.End()+1 >= f.Package { + pastCutoff = true + } + + // "+build" is ignored within or after a /*...*/ comment. + if !strings.HasPrefix(group.List[0].Text, "//") { + pastCutoff = true + continue + } + + // Check each line of a //-comment. + for _, c := range group.List { + if !strings.Contains(c.Text, "+build") { + continue + } + if err := checkLine(c.Text, pastCutoff); err != nil { + pass.Reportf(c.Pos(), "%s", err) + } + } + } +} + +func checkOtherFile(pass *analysis.Pass, filename string) error { + content, tf, err := analysisutil.ReadFile(pass.Fset, filename) + if err != nil { + return err + } + + // We must look at the raw lines, as build tags may appear in non-Go + // files such as assembly files. + lines := bytes.SplitAfter(content, nl) + + // Determine cutpoint where +build comments are no longer valid. + // They are valid in leading // comments in the file followed by + // a blank line. + // + // This must be done as a separate pass because of the + // requirement that the comment be followed by a blank line. + var cutoff int + for i, line := range lines { + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, slashSlash) { + if len(line) > 0 { + break + } + cutoff = i + } + } + + for i, line := range lines { + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, slashSlash) { + continue + } + if !bytes.Contains(line, []byte("+build")) { + continue + } + if err := checkLine(string(line), i >= cutoff); err != nil { + pass.Reportf(analysisutil.LineStart(tf, i+1), "%s", err) + continue + } + } + return nil +} + +// checkLine checks a line that starts with "//" and contains "+build". +func checkLine(line string, pastCutoff bool) error { + line = strings.TrimPrefix(line, "//") + line = strings.TrimSpace(line) + + if strings.HasPrefix(line, "+build") { + fields := strings.Fields(line) + if fields[0] != "+build" { + // Comment is something like +buildasdf not +build. + return fmt.Errorf("possible malformed +build comment") + } + if pastCutoff { + return fmt.Errorf("+build comment must appear before package clause and be followed by a blank line") + } + if err := checkArguments(fields); err != nil { + return err + } + } else { + // Comment with +build but not at beginning. + if !pastCutoff { + return fmt.Errorf("possible malformed +build comment") + } + } + return nil +} + +func checkArguments(fields []string) error { + // The original version of this checker in vet could examine + // files with malformed build tags that would cause the file to + // be always ignored by "go build". However, drivers for the new + // analysis API will analyze only the files selected to form a + // package, so these checks will never fire. + // TODO(adonovan): rethink this. + + for _, arg := range fields[1:] { + for _, elem := range strings.Split(arg, ",") { + if strings.HasPrefix(elem, "!!") { + return fmt.Errorf("invalid double negative in build constraint: %s", arg) + } + elem = strings.TrimPrefix(elem, "!") + for _, c := range elem { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { + return fmt.Errorf("invalid non-alphanumeric build constraint: %s", arg) + } + } + } + } + return nil +} + +var ( + nl = []byte("\n") + slashSlash = []byte("//") +) diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/cgocall/cgocall.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/cgocall/cgocall.go new file mode 100644 index 0000000000..7eb24a4a91 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/cgocall/cgocall.go @@ -0,0 +1,226 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cgocall defines an Analyzer that detects some violations of +// the cgo pointer passing rules. +package cgocall + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "log" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `detect some violations of the cgo pointer passing rules + +Check for invalid cgo pointer passing. +This looks for code that uses cgo to call C code passing values +whose types are almost always invalid according to the cgo pointer +sharing rules. +Specifically, it warns about attempts to pass a Go chan, map, func, +or slice to C, either directly, or via a pointer, array, or struct.` + +var Analyzer = &analysis.Analyzer{ + Name: "cgocall", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.CallExpr)(nil), + } + inspect.WithStack(nodeFilter, func(n ast.Node, push bool, stack []ast.Node) bool { + if !push { + return true + } + call, name := findCall(pass.Fset, stack) + if call == nil { + return true // not a call we need to check + } + + // A call to C.CBytes passes a pointer but is always safe. + if name == "CBytes" { + return true + } + + if false { + fmt.Printf("%s: inner call to C.%s\n", pass.Fset.Position(n.Pos()), name) + fmt.Printf("%s: outer call to C.%s\n", pass.Fset.Position(call.Lparen), name) + } + + for _, arg := range call.Args { + if !typeOKForCgoCall(cgoBaseType(pass.TypesInfo, arg), make(map[types.Type]bool)) { + pass.Reportf(arg.Pos(), "possibly passing Go type with embedded pointer to C") + break + } + + // Check for passing the address of a bad type. + if conv, ok := arg.(*ast.CallExpr); ok && len(conv.Args) == 1 && + isUnsafePointer(pass.TypesInfo, conv.Fun) { + arg = conv.Args[0] + } + if u, ok := arg.(*ast.UnaryExpr); ok && u.Op == token.AND { + if !typeOKForCgoCall(cgoBaseType(pass.TypesInfo, u.X), make(map[types.Type]bool)) { + pass.Reportf(arg.Pos(), "possibly passing Go type with embedded pointer to C") + break + } + } + } + return true + }) + return nil, nil +} + +// findCall returns the CallExpr that we need to check, which may not be +// the same as the one we're currently visiting, due to code generation. +// It also returns the name of the function, such as "f" for C.f(...). +// +// This checker was initially written in vet to inpect unprocessed cgo +// source files using partial type information. However, Analyzers in +// the new analysis API are presented with the type-checked, processed +// Go ASTs resulting from cgo processing files, so we must choose +// between: +// +// a) locating the cgo file (e.g. from //line directives) +// and working with that, or +// b) working with the file generated by cgo. +// +// We cannot use (a) because it does not provide type information, which +// the analyzer needs, and it is infeasible for the analyzer to run the +// type checker on this file. Thus we choose (b), which is fragile, +// because the checker may need to change each time the cgo processor +// changes. +// +// Consider a cgo source file containing this header: +// +// /* void f(void *x, *y); */ +// import "C" +// +// The cgo tool expands a call such as: +// +// C.f(x, y) +// +// to this: +// +// 1 func(param0, param1 unsafe.Pointer) { +// 2 ... various checks on params ... +// 3 (_Cfunc_f)(param0, param1) +// 4 }(x, y) +// +// We first locate the _Cfunc_f call on line 3, then +// walk up the stack of enclosing nodes until we find +// the call on line 4. +// +func findCall(fset *token.FileSet, stack []ast.Node) (*ast.CallExpr, string) { + last := len(stack) - 1 + call := stack[last].(*ast.CallExpr) + if id, ok := analysisutil.Unparen(call.Fun).(*ast.Ident); ok { + if name := strings.TrimPrefix(id.Name, "_Cfunc_"); name != id.Name { + // Find the outer call with the arguments (x, y) we want to check. + for i := last - 1; i >= 0; i-- { + if outer, ok := stack[i].(*ast.CallExpr); ok { + return outer, name + } + } + // This shouldn't happen. + // Perhaps the code generator has changed? + log.Printf("%s: can't find outer call for C.%s(...)", + fset.Position(call.Lparen), name) + } + } + return nil, "" +} + +// cgoBaseType tries to look through type conversions involving +// unsafe.Pointer to find the real type. It converts: +// unsafe.Pointer(x) => x +// *(*unsafe.Pointer)(unsafe.Pointer(&x)) => x +func cgoBaseType(info *types.Info, arg ast.Expr) types.Type { + switch arg := arg.(type) { + case *ast.CallExpr: + if len(arg.Args) == 1 && isUnsafePointer(info, arg.Fun) { + return cgoBaseType(info, arg.Args[0]) + } + case *ast.StarExpr: + call, ok := arg.X.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + break + } + // Here arg is *f(v). + t := info.Types[call.Fun].Type + if t == nil { + break + } + ptr, ok := t.Underlying().(*types.Pointer) + if !ok { + break + } + // Here arg is *(*p)(v) + elem, ok := ptr.Elem().Underlying().(*types.Basic) + if !ok || elem.Kind() != types.UnsafePointer { + break + } + // Here arg is *(*unsafe.Pointer)(v) + call, ok = call.Args[0].(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + break + } + // Here arg is *(*unsafe.Pointer)(f(v)) + if !isUnsafePointer(info, call.Fun) { + break + } + // Here arg is *(*unsafe.Pointer)(unsafe.Pointer(v)) + u, ok := call.Args[0].(*ast.UnaryExpr) + if !ok || u.Op != token.AND { + break + } + // Here arg is *(*unsafe.Pointer)(unsafe.Pointer(&v)) + return cgoBaseType(info, u.X) + } + + return info.Types[arg].Type +} + +// typeOKForCgoCall reports whether the type of arg is OK to pass to a +// C function using cgo. This is not true for Go types with embedded +// pointers. m is used to avoid infinite recursion on recursive types. +func typeOKForCgoCall(t types.Type, m map[types.Type]bool) bool { + if t == nil || m[t] { + return true + } + m[t] = true + switch t := t.Underlying().(type) { + case *types.Chan, *types.Map, *types.Signature, *types.Slice: + return false + case *types.Pointer: + return typeOKForCgoCall(t.Elem(), m) + case *types.Array: + return typeOKForCgoCall(t.Elem(), m) + case *types.Struct: + for i := 0; i < t.NumFields(); i++ { + if !typeOKForCgoCall(t.Field(i).Type(), m) { + return false + } + } + } + return true +} + +func isUnsafePointer(info *types.Info, e ast.Expr) bool { + t := info.Types[e].Type + return t != nil && t.Underlying() == types.Typ[types.UnsafePointer] +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/composite.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/composite.go new file mode 100644 index 0000000000..b7cfe8a95d --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/composite.go @@ -0,0 +1,108 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package composite defines an Analyzer that checks for unkeyed +// composite literals. +package composite + +import ( + "go/ast" + "go/types" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `checked for unkeyed composite literals + +This analyzer reports a diagnostic for composite literals of struct +types imported from another package that do not use the field-keyed +syntax. Such literals are fragile because the addition of a new field +(even if unexported) to the struct will cause compilation to fail.` + +var Analyzer = &analysis.Analyzer{ + Name: "composites", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +var whitelist = true + +func init() { + Analyzer.Flags.BoolVar(&whitelist, "whitelist", whitelist, "use composite white list; for testing only") +} + +// runUnkeyedLiteral checks if a composite literal is a struct literal with +// unkeyed fields. +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.CompositeLit)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + cl := n.(*ast.CompositeLit) + + typ := pass.TypesInfo.Types[cl].Type + if typ == nil { + // cannot determine composite literals' type, skip it + return + } + typeName := typ.String() + if whitelist && unkeyedLiteral[typeName] { + // skip whitelisted types + return + } + under := typ.Underlying() + for { + ptr, ok := under.(*types.Pointer) + if !ok { + break + } + under = ptr.Elem().Underlying() + } + if _, ok := under.(*types.Struct); !ok { + // skip non-struct composite literals + return + } + if isLocalType(pass, typ) { + // allow unkeyed locally defined composite literal + return + } + + // check if the CompositeLit contains an unkeyed field + allKeyValue := true + for _, e := range cl.Elts { + if _, ok := e.(*ast.KeyValueExpr); !ok { + allKeyValue = false + break + } + } + if allKeyValue { + // all the composite literal fields are keyed + return + } + + pass.Reportf(cl.Pos(), "%s composite literal uses unkeyed fields", typeName) + }) + return nil, nil +} + +func isLocalType(pass *analysis.Pass, typ types.Type) bool { + switch x := typ.(type) { + case *types.Struct: + // struct literals are local types + return true + case *types.Pointer: + return isLocalType(pass, x.Elem()) + case *types.Named: + // names in package foo are local to foo_test too + return strings.TrimSuffix(x.Obj().Pkg().Path(), "_test") == strings.TrimSuffix(pass.Pkg.Path(), "_test") + } + return false +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/whitelist.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/whitelist.go new file mode 100644 index 0000000000..ab609f279b --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/composite/whitelist.go @@ -0,0 +1,33 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package composite + +// unkeyedLiteral is a white list of types in the standard packages +// that are used with unkeyed literals we deem to be acceptable. +var unkeyedLiteral = map[string]bool{ + // These image and image/color struct types are frozen. We will never add fields to them. + "image/color.Alpha16": true, + "image/color.Alpha": true, + "image/color.CMYK": true, + "image/color.Gray16": true, + "image/color.Gray": true, + "image/color.NRGBA64": true, + "image/color.NRGBA": true, + "image/color.NYCbCrA": true, + "image/color.RGBA64": true, + "image/color.RGBA": true, + "image/color.YCbCr": true, + "image.Point": true, + "image.Rectangle": true, + "image.Uniform": true, + + "unicode.Range16": true, + + // These three structs are used in generated test main files, + // but the generator can be trusted. + "testing.InternalBenchmark": true, + "testing.InternalExample": true, + "testing.InternalTest": true, +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/copylock/copylock.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/copylock/copylock.go new file mode 100644 index 0000000000..067aed57df --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/copylock/copylock.go @@ -0,0 +1,300 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package copylock defines an Analyzer that checks for locks +// erroneously passed by value. +package copylock + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for locks erroneously passed by value + +Inadvertently copying a value containing a lock, such as sync.Mutex or +sync.WaitGroup, may cause both copies to malfunction. Generally such +values should be referred to through a pointer.` + +var Analyzer = &analysis.Analyzer{ + Name: "copylocks", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.AssignStmt)(nil), + (*ast.CallExpr)(nil), + (*ast.CompositeLit)(nil), + (*ast.FuncDecl)(nil), + (*ast.FuncLit)(nil), + (*ast.GenDecl)(nil), + (*ast.RangeStmt)(nil), + (*ast.ReturnStmt)(nil), + } + inspect.Preorder(nodeFilter, func(node ast.Node) { + switch node := node.(type) { + case *ast.RangeStmt: + checkCopyLocksRange(pass, node) + case *ast.FuncDecl: + checkCopyLocksFunc(pass, node.Name.Name, node.Recv, node.Type) + case *ast.FuncLit: + checkCopyLocksFunc(pass, "func", nil, node.Type) + case *ast.CallExpr: + checkCopyLocksCallExpr(pass, node) + case *ast.AssignStmt: + checkCopyLocksAssign(pass, node) + case *ast.GenDecl: + checkCopyLocksGenDecl(pass, node) + case *ast.CompositeLit: + checkCopyLocksCompositeLit(pass, node) + case *ast.ReturnStmt: + checkCopyLocksReturnStmt(pass, node) + } + }) + return nil, nil +} + +// checkCopyLocksAssign checks whether an assignment +// copies a lock. +func checkCopyLocksAssign(pass *analysis.Pass, as *ast.AssignStmt) { + for i, x := range as.Rhs { + if path := lockPathRhs(pass, x); path != nil { + pass.Reportf(x.Pos(), "assignment copies lock value to %v: %v", analysisutil.Format(pass.Fset, as.Lhs[i]), path) + } + } +} + +// checkCopyLocksGenDecl checks whether lock is copied +// in variable declaration. +func checkCopyLocksGenDecl(pass *analysis.Pass, gd *ast.GenDecl) { + if gd.Tok != token.VAR { + return + } + for _, spec := range gd.Specs { + valueSpec := spec.(*ast.ValueSpec) + for i, x := range valueSpec.Values { + if path := lockPathRhs(pass, x); path != nil { + pass.Reportf(x.Pos(), "variable declaration copies lock value to %v: %v", valueSpec.Names[i].Name, path) + } + } + } +} + +// checkCopyLocksCompositeLit detects lock copy inside a composite literal +func checkCopyLocksCompositeLit(pass *analysis.Pass, cl *ast.CompositeLit) { + for _, x := range cl.Elts { + if node, ok := x.(*ast.KeyValueExpr); ok { + x = node.Value + } + if path := lockPathRhs(pass, x); path != nil { + pass.Reportf(x.Pos(), "literal copies lock value from %v: %v", analysisutil.Format(pass.Fset, x), path) + } + } +} + +// checkCopyLocksReturnStmt detects lock copy in return statement +func checkCopyLocksReturnStmt(pass *analysis.Pass, rs *ast.ReturnStmt) { + for _, x := range rs.Results { + if path := lockPathRhs(pass, x); path != nil { + pass.Reportf(x.Pos(), "return copies lock value: %v", path) + } + } +} + +// checkCopyLocksCallExpr detects lock copy in the arguments to a function call +func checkCopyLocksCallExpr(pass *analysis.Pass, ce *ast.CallExpr) { + var id *ast.Ident + switch fun := ce.Fun.(type) { + case *ast.Ident: + id = fun + case *ast.SelectorExpr: + id = fun.Sel + } + if fun, ok := pass.TypesInfo.Uses[id].(*types.Builtin); ok { + switch fun.Name() { + case "new", "len", "cap", "Sizeof": + return + } + } + for _, x := range ce.Args { + if path := lockPathRhs(pass, x); path != nil { + pass.Reportf(x.Pos(), "call of %s copies lock value: %v", analysisutil.Format(pass.Fset, ce.Fun), path) + } + } +} + +// checkCopyLocksFunc checks whether a function might +// inadvertently copy a lock, by checking whether +// its receiver, parameters, or return values +// are locks. +func checkCopyLocksFunc(pass *analysis.Pass, name string, recv *ast.FieldList, typ *ast.FuncType) { + if recv != nil && len(recv.List) > 0 { + expr := recv.List[0].Type + if path := lockPath(pass.Pkg, pass.TypesInfo.Types[expr].Type); path != nil { + pass.Reportf(expr.Pos(), "%s passes lock by value: %v", name, path) + } + } + + if typ.Params != nil { + for _, field := range typ.Params.List { + expr := field.Type + if path := lockPath(pass.Pkg, pass.TypesInfo.Types[expr].Type); path != nil { + pass.Reportf(expr.Pos(), "%s passes lock by value: %v", name, path) + } + } + } + + // Don't check typ.Results. If T has a Lock field it's OK to write + // return T{} + // because that is returning the zero value. Leave result checking + // to the return statement. +} + +// checkCopyLocksRange checks whether a range statement +// might inadvertently copy a lock by checking whether +// any of the range variables are locks. +func checkCopyLocksRange(pass *analysis.Pass, r *ast.RangeStmt) { + checkCopyLocksRangeVar(pass, r.Tok, r.Key) + checkCopyLocksRangeVar(pass, r.Tok, r.Value) +} + +func checkCopyLocksRangeVar(pass *analysis.Pass, rtok token.Token, e ast.Expr) { + if e == nil { + return + } + id, isId := e.(*ast.Ident) + if isId && id.Name == "_" { + return + } + + var typ types.Type + if rtok == token.DEFINE { + if !isId { + return + } + obj := pass.TypesInfo.Defs[id] + if obj == nil { + return + } + typ = obj.Type() + } else { + typ = pass.TypesInfo.Types[e].Type + } + + if typ == nil { + return + } + if path := lockPath(pass.Pkg, typ); path != nil { + pass.Reportf(e.Pos(), "range var %s copies lock: %v", analysisutil.Format(pass.Fset, e), path) + } +} + +type typePath []types.Type + +// String pretty-prints a typePath. +func (path typePath) String() string { + n := len(path) + var buf bytes.Buffer + for i := range path { + if i > 0 { + fmt.Fprint(&buf, " contains ") + } + // The human-readable path is in reverse order, outermost to innermost. + fmt.Fprint(&buf, path[n-i-1].String()) + } + return buf.String() +} + +func lockPathRhs(pass *analysis.Pass, x ast.Expr) typePath { + if _, ok := x.(*ast.CompositeLit); ok { + return nil + } + if _, ok := x.(*ast.CallExpr); ok { + // A call may return a zero value. + return nil + } + if star, ok := x.(*ast.StarExpr); ok { + if _, ok := star.X.(*ast.CallExpr); ok { + // A call may return a pointer to a zero value. + return nil + } + } + return lockPath(pass.Pkg, pass.TypesInfo.Types[x].Type) +} + +// lockPath returns a typePath describing the location of a lock value +// contained in typ. If there is no contained lock, it returns nil. +func lockPath(tpkg *types.Package, typ types.Type) typePath { + if typ == nil { + return nil + } + + for { + atyp, ok := typ.Underlying().(*types.Array) + if !ok { + break + } + typ = atyp.Elem() + } + + // We're only interested in the case in which the underlying + // type is a struct. (Interfaces and pointers are safe to copy.) + styp, ok := typ.Underlying().(*types.Struct) + if !ok { + return nil + } + + // We're looking for cases in which a pointer to this type + // is a sync.Locker, but a value is not. This differentiates + // embedded interfaces from embedded values. + if types.Implements(types.NewPointer(typ), lockerType) && !types.Implements(typ, lockerType) { + return []types.Type{typ} + } + + // In go1.10, sync.noCopy did not implement Locker. + // (The Unlock method was added only in CL 121876.) + // TODO(adonovan): remove workaround when we drop go1.10. + if named, ok := typ.(*types.Named); ok && + named.Obj().Name() == "noCopy" && + named.Obj().Pkg().Path() == "sync" { + return []types.Type{typ} + } + + nfields := styp.NumFields() + for i := 0; i < nfields; i++ { + ftyp := styp.Field(i).Type() + subpath := lockPath(tpkg, ftyp) + if subpath != nil { + return append(subpath, typ) + } + } + + return nil +} + +var lockerType *types.Interface + +// Construct a sync.Locker interface type. +func init() { + nullary := types.NewSignature(nil, nil, nil, false) // func() + methods := []*types.Func{ + types.NewFunc(token.NoPos, nil, "Lock", nullary), + types.NewFunc(token.NoPos, nil, "Unlock", nullary), + } + lockerType = types.NewInterface(methods, nil).Complete() +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/ctrlflow/ctrlflow.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/ctrlflow/ctrlflow.go new file mode 100644 index 0000000000..75655c5bad --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/ctrlflow/ctrlflow.go @@ -0,0 +1,225 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ctrlflow is an analysis that provides a syntactic +// control-flow graph (CFG) for the body of a function. +// It records whether a function cannot return. +// By itself, it does not report any diagnostics. +package ctrlflow + +import ( + "go/ast" + "go/types" + "log" + "reflect" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" + "golang.org/x/tools/go/cfg" + "golang.org/x/tools/go/types/typeutil" +) + +var Analyzer = &analysis.Analyzer{ + Name: "ctrlflow", + Doc: "build a control-flow graph", + Run: run, + ResultType: reflect.TypeOf(new(CFGs)), + FactTypes: []analysis.Fact{new(noReturn)}, + Requires: []*analysis.Analyzer{inspect.Analyzer}, +} + +// noReturn is a fact indicating that a function does not return. +type noReturn struct{} + +func (*noReturn) AFact() {} + +func (*noReturn) String() string { return "noReturn" } + +// A CFGs holds the control-flow graphs +// for all the functions of the current package. +type CFGs struct { + defs map[*ast.Ident]types.Object // from Pass.TypesInfo.Defs + funcDecls map[*types.Func]*declInfo + funcLits map[*ast.FuncLit]*litInfo + pass *analysis.Pass // transient; nil after construction +} + +// CFGs has two maps: funcDecls for named functions and funcLits for +// unnamed ones. Unlike funcLits, the funcDecls map is not keyed by its +// syntax node, *ast.FuncDecl, because callMayReturn needs to do a +// look-up by *types.Func, and you can get from an *ast.FuncDecl to a +// *types.Func but not the other way. + +type declInfo struct { + decl *ast.FuncDecl + cfg *cfg.CFG // iff decl.Body != nil + started bool // to break cycles + noReturn bool +} + +type litInfo struct { + cfg *cfg.CFG + noReturn bool +} + +// FuncDecl returns the control-flow graph for a named function. +// It returns nil if decl.Body==nil. +func (c *CFGs) FuncDecl(decl *ast.FuncDecl) *cfg.CFG { + if decl.Body == nil { + return nil + } + fn := c.defs[decl.Name].(*types.Func) + return c.funcDecls[fn].cfg +} + +// FuncLit returns the control-flow graph for a literal function. +func (c *CFGs) FuncLit(lit *ast.FuncLit) *cfg.CFG { + return c.funcLits[lit].cfg +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + // Because CFG construction consumes and produces noReturn + // facts, CFGs for exported FuncDecls must be built before 'run' + // returns; we cannot construct them lazily. + // (We could build CFGs for FuncLits lazily, + // but the benefit is marginal.) + + // Pass 1. Map types.Funcs to ast.FuncDecls in this package. + funcDecls := make(map[*types.Func]*declInfo) // functions and methods + funcLits := make(map[*ast.FuncLit]*litInfo) + + var decls []*types.Func // keys(funcDecls), in order + var lits []*ast.FuncLit // keys(funcLits), in order + + nodeFilter := []ast.Node{ + (*ast.FuncDecl)(nil), + (*ast.FuncLit)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + switch n := n.(type) { + case *ast.FuncDecl: + fn := pass.TypesInfo.Defs[n.Name].(*types.Func) + funcDecls[fn] = &declInfo{decl: n} + decls = append(decls, fn) + + case *ast.FuncLit: + funcLits[n] = new(litInfo) + lits = append(lits, n) + } + }) + + c := &CFGs{ + defs: pass.TypesInfo.Defs, + funcDecls: funcDecls, + funcLits: funcLits, + pass: pass, + } + + // Pass 2. Build CFGs. + + // Build CFGs for named functions. + // Cycles in the static call graph are broken + // arbitrarily but deterministically. + // We create noReturn facts as discovered. + for _, fn := range decls { + c.buildDecl(fn, funcDecls[fn]) + } + + // Build CFGs for literal functions. + // These aren't relevant to facts (since they aren't named) + // but are required for the CFGs.FuncLit API. + for _, lit := range lits { + li := funcLits[lit] + if li.cfg == nil { + li.cfg = cfg.New(lit.Body, c.callMayReturn) + if !hasReachableReturn(li.cfg) { + li.noReturn = true + } + } + } + + // All CFGs are now built. + c.pass = nil + + return c, nil +} + +// di.cfg may be nil on return. +func (c *CFGs) buildDecl(fn *types.Func, di *declInfo) { + // buildDecl may call itself recursively for the same function, + // because cfg.New is passed the callMayReturn method, which + // builds the CFG of the callee, leading to recursion. + // The buildDecl call tree thus resembles the static call graph. + // We mark each node when we start working on it to break cycles. + + if !di.started { // break cycle + di.started = true + + if isIntrinsicNoReturn(fn) { + di.noReturn = true + } + if di.decl.Body != nil { + di.cfg = cfg.New(di.decl.Body, c.callMayReturn) + if !hasReachableReturn(di.cfg) { + di.noReturn = true + } + } + if di.noReturn { + c.pass.ExportObjectFact(fn, new(noReturn)) + } + + // debugging + if false { + log.Printf("CFG for %s:\n%s (noreturn=%t)\n", fn, di.cfg.Format(c.pass.Fset), di.noReturn) + } + } +} + +// callMayReturn reports whether the called function may return. +// It is passed to the CFG builder. +func (c *CFGs) callMayReturn(call *ast.CallExpr) (r bool) { + if id, ok := call.Fun.(*ast.Ident); ok && c.pass.TypesInfo.Uses[id] == panicBuiltin { + return false // panic never returns + } + + // Is this a static call? + fn := typeutil.StaticCallee(c.pass.TypesInfo, call) + if fn == nil { + return true // callee not statically known; be conservative + } + + // Function or method declared in this package? + if di, ok := c.funcDecls[fn]; ok { + c.buildDecl(fn, di) + return !di.noReturn + } + + // Not declared in this package. + // Is there a fact from another package? + return !c.pass.ImportObjectFact(fn, new(noReturn)) +} + +var panicBuiltin = types.Universe.Lookup("panic").(*types.Builtin) + +func hasReachableReturn(g *cfg.CFG) bool { + for _, b := range g.Blocks { + if b.Live && b.Return() != nil { + return true + } + } + return false +} + +// isIntrinsicNoReturn reports whether a function intrinsically never +// returns because it stops execution of the calling thread. +// It is the base case in the recursion. +func isIntrinsicNoReturn(fn *types.Func) bool { + // Add functions here as the need arises, but don't allocate memory. + path, name := fn.Pkg().Path(), fn.Name() + return path == "syscall" && (name == "Exit" || name == "ExitProcess" || name == "ExitThread") || + path == "runtime" && name == "Goexit" +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/httpresponse/httpresponse.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/httpresponse/httpresponse.go new file mode 100644 index 0000000000..0cf21b8cd1 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/httpresponse/httpresponse.go @@ -0,0 +1,177 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpresponse defines an Analyzer that checks for mistakes +// using HTTP responses. +package httpresponse + +import ( + "go/ast" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for mistakes using HTTP responses + +A common mistake when using the net/http package is to defer a function +call to close the http.Response Body before checking the error that +determines whether the response is valid: + + resp, err := http.Head(url) + defer resp.Body.Close() + if err != nil { + log.Fatal(err) + } + // (defer statement belongs here) + +This checker helps uncover latent nil dereference bugs by reporting a +diagnostic for such mistakes.` + +var Analyzer = &analysis.Analyzer{ + Name: "httpresponse", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + // Fast path: if the package doesn't import net/http, + // skip the traversal. + if !imports(pass.Pkg, "net/http") { + return nil, nil + } + + nodeFilter := []ast.Node{ + (*ast.CallExpr)(nil), + } + inspect.WithStack(nodeFilter, func(n ast.Node, push bool, stack []ast.Node) bool { + if !push { + return true + } + call := n.(*ast.CallExpr) + if !isHTTPFuncOrMethodOnClient(pass.TypesInfo, call) { + return true // the function call is not related to this check. + } + + // Find the innermost containing block, and get the list + // of statements starting with the one containing call. + stmts := restOfBlock(stack) + if len(stmts) < 2 { + return true // the call to the http function is the last statement of the block. + } + + asg, ok := stmts[0].(*ast.AssignStmt) + if !ok { + return true // the first statement is not assignment. + } + resp := rootIdent(asg.Lhs[0]) + if resp == nil { + return true // could not find the http.Response in the assignment. + } + + def, ok := stmts[1].(*ast.DeferStmt) + if !ok { + return true // the following statement is not a defer. + } + root := rootIdent(def.Call.Fun) + if root == nil { + return true // could not find the receiver of the defer call. + } + + if resp.Obj == root.Obj { + pass.Reportf(root.Pos(), "using %s before checking for errors", resp.Name) + } + return true + }) + return nil, nil +} + +// isHTTPFuncOrMethodOnClient checks whether the given call expression is on +// either a function of the net/http package or a method of http.Client that +// returns (*http.Response, error). +func isHTTPFuncOrMethodOnClient(info *types.Info, expr *ast.CallExpr) bool { + fun, _ := expr.Fun.(*ast.SelectorExpr) + sig, _ := info.Types[fun].Type.(*types.Signature) + if sig == nil { + return false // the call is not of the form x.f() + } + + res := sig.Results() + if res.Len() != 2 { + return false // the function called does not return two values. + } + if ptr, ok := res.At(0).Type().(*types.Pointer); !ok || !isNamedType(ptr.Elem(), "net/http", "Response") { + return false // the first return type is not *http.Response. + } + + errorType := types.Universe.Lookup("error").Type() + if !types.Identical(res.At(1).Type(), errorType) { + return false // the second return type is not error + } + + typ := info.Types[fun.X].Type + if typ == nil { + id, ok := fun.X.(*ast.Ident) + return ok && id.Name == "http" // function in net/http package. + } + + if isNamedType(typ, "net/http", "Client") { + return true // method on http.Client. + } + ptr, ok := typ.(*types.Pointer) + return ok && isNamedType(ptr.Elem(), "net/http", "Client") // method on *http.Client. +} + +// restOfBlock, given a traversal stack, finds the innermost containing +// block and returns the suffix of its statements starting with the +// current node (the last element of stack). +func restOfBlock(stack []ast.Node) []ast.Stmt { + for i := len(stack) - 1; i >= 0; i-- { + if b, ok := stack[i].(*ast.BlockStmt); ok { + for j, v := range b.List { + if v == stack[i+1] { + return b.List[j:] + } + } + break + } + } + return nil +} + +// rootIdent finds the root identifier x in a chain of selections x.y.z, or nil if not found. +func rootIdent(n ast.Node) *ast.Ident { + switch n := n.(type) { + case *ast.SelectorExpr: + return rootIdent(n.X) + case *ast.Ident: + return n + default: + return nil + } +} + +// isNamedType reports whether t is the named type path.name. +func isNamedType(t types.Type, path, name string) bool { + n, ok := t.(*types.Named) + if !ok { + return false + } + obj := n.Obj() + return obj.Name() == name && obj.Pkg() != nil && obj.Pkg().Path() == path +} + +func imports(pkg *types.Package, path string) bool { + for _, imp := range pkg.Imports() { + if imp.Path() == path { + return true + } + } + return false +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/inspect/inspect.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/inspect/inspect.go new file mode 100644 index 0000000000..bd06549984 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/inspect/inspect.go @@ -0,0 +1,45 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package inspect defines an Analyzer that provides an AST inspector +// (golang.org/x/tools/go/ast/inspect.Inspect) for the syntax trees of a +// package. It is only a building block for other analyzers. +// +// Example of use in another analysis: +// +// import "golang.org/x/tools/go/analysis/passes/inspect" +// +// var Analyzer = &analysis.Analyzer{ +// ... +// Requires: reflect.TypeOf(new(inspect.Analyzer)), +// } +// +// func run(pass *analysis.Pass) (interface{}, error) { +// inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) +// inspect.Preorder(nil, func(n ast.Node) { +// ... +// }) +// return nil +// } +// +package inspect + +import ( + "reflect" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/ast/inspector" +) + +var Analyzer = &analysis.Analyzer{ + Name: "inspect", + Doc: "optimize AST traversal for later passes", + Run: run, + RunDespiteErrors: true, + ResultType: reflect.TypeOf(new(inspector.Inspector)), +} + +func run(pass *analysis.Pass) (interface{}, error) { + return inspector.New(pass.Files), nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/internal/analysisutil/util.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/internal/analysisutil/util.go new file mode 100644 index 0000000000..13a458d9d6 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/internal/analysisutil/util.go @@ -0,0 +1,106 @@ +// Package analysisutil defines various helper functions +// used by two or more packages beneath go/analysis. +package analysisutil + +import ( + "bytes" + "go/ast" + "go/printer" + "go/token" + "go/types" + "io/ioutil" +) + +// Format returns a string representation of the expression. +func Format(fset *token.FileSet, x ast.Expr) string { + var b bytes.Buffer + printer.Fprint(&b, fset, x) + return b.String() +} + +// HasSideEffects reports whether evaluation of e has side effects. +func HasSideEffects(info *types.Info, e ast.Expr) bool { + safe := true + ast.Inspect(e, func(node ast.Node) bool { + switch n := node.(type) { + case *ast.CallExpr: + typVal := info.Types[n.Fun] + switch { + case typVal.IsType(): + // Type conversion, which is safe. + case typVal.IsBuiltin(): + // Builtin func, conservatively assumed to not + // be safe for now. + safe = false + return false + default: + // A non-builtin func or method call. + // Conservatively assume that all of them have + // side effects for now. + safe = false + return false + } + case *ast.UnaryExpr: + if n.Op == token.ARROW { + safe = false + return false + } + } + return true + }) + return !safe +} + +// Unparen returns e with any enclosing parentheses stripped. +func Unparen(e ast.Expr) ast.Expr { + for { + p, ok := e.(*ast.ParenExpr) + if !ok { + return e + } + e = p.X + } +} + +// ReadFile reads a file and adds it to the FileSet +// so that we can report errors against it using lineStart. +func ReadFile(fset *token.FileSet, filename string) ([]byte, *token.File, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, nil, err + } + tf := fset.AddFile(filename, -1, len(content)) + tf.SetLinesForContent(content) + return content, tf, nil +} + +// LineStart returns the position of the start of the specified line +// within file f, or NoPos if there is no line of that number. +func LineStart(f *token.File, line int) token.Pos { + // Use binary search to find the start offset of this line. + // + // TODO(adonovan): eventually replace this function with the + // simpler and more efficient (*go/token.File).LineStart, added + // in go1.12. + + min := 0 // inclusive + max := f.Size() // exclusive + for { + offset := (min + max) / 2 + pos := f.Pos(offset) + posn := f.Position(pos) + if posn.Line == line { + return pos - (token.Pos(posn.Column) - 1) + } + + if min+1 >= max { + return token.NoPos + } + + if posn.Line < line { + min = offset + } else { + max = offset + } + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/loopclosure/loopclosure.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/loopclosure/loopclosure.go new file mode 100644 index 0000000000..da0714069f --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/loopclosure/loopclosure.go @@ -0,0 +1,130 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package loopclosure defines an Analyzer that checks for references to +// enclosing loop variables from within nested functions. +package loopclosure + +import ( + "go/ast" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +// TODO(adonovan): also report an error for the following structure, +// which is often used to ensure that deferred calls do not accumulate +// in a loop: +// +// for i, x := range c { +// func() { +// ...reference to i or x... +// }() +// } + +const Doc = `check references to loop variables from within nested functions + +This analyzer checks for references to loop variables from within a +function literal inside the loop body. It checks only instances where +the function literal is called in a defer or go statement that is the +last statement in the loop body, as otherwise we would need whole +program analysis. + +For example: + + for i, v := range s { + go func() { + println(i, v) // not what you might expect + }() + } + +See: https://golang.org/doc/go_faq.html#closures_and_goroutines` + +var Analyzer = &analysis.Analyzer{ + Name: "loopclosure", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.RangeStmt)(nil), + (*ast.ForStmt)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + // Find the variables updated by the loop statement. + var vars []*ast.Ident + addVar := func(expr ast.Expr) { + if id, ok := expr.(*ast.Ident); ok { + vars = append(vars, id) + } + } + var body *ast.BlockStmt + switch n := n.(type) { + case *ast.RangeStmt: + body = n.Body + addVar(n.Key) + addVar(n.Value) + case *ast.ForStmt: + body = n.Body + switch post := n.Post.(type) { + case *ast.AssignStmt: + // e.g. for p = head; p != nil; p = p.next + for _, lhs := range post.Lhs { + addVar(lhs) + } + case *ast.IncDecStmt: + // e.g. for i := 0; i < n; i++ + addVar(post.X) + } + } + if vars == nil { + return + } + + // Inspect a go or defer statement + // if it's the last one in the loop body. + // (We give up if there are following statements, + // because it's hard to prove go isn't followed by wait, + // or defer by return.) + if len(body.List) == 0 { + return + } + var last *ast.CallExpr + switch s := body.List[len(body.List)-1].(type) { + case *ast.GoStmt: + last = s.Call + case *ast.DeferStmt: + last = s.Call + default: + return + } + lit, ok := last.Fun.(*ast.FuncLit) + if !ok { + return + } + ast.Inspect(lit.Body, func(n ast.Node) bool { + id, ok := n.(*ast.Ident) + if !ok || id.Obj == nil { + return true + } + if pass.TypesInfo.Types[id].Type == nil { + // Not referring to a variable (e.g. struct field name) + return true + } + for _, v := range vars { + if v.Obj == id.Obj { + pass.Reportf(id.Pos(), "loop variable %s captured by func literal", + id.Name) + } + } + return true + }) + }) + return nil, nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/lostcancel/lostcancel.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/lostcancel/lostcancel.go new file mode 100644 index 0000000000..fcf9f553a9 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/lostcancel/lostcancel.go @@ -0,0 +1,304 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lostcancel defines an Analyzer that checks for failure to +// call a context cancelation function. +package lostcancel + +import ( + "fmt" + "go/ast" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/ctrlflow" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" + "golang.org/x/tools/go/cfg" +) + +const Doc = `check cancel func returned by context.WithCancel is called + +The cancelation function returned by context.WithCancel, WithTimeout, +and WithDeadline must be called or the new context will remain live +until its parent context is cancelled. +(The background context is never cancelled.)` + +var Analyzer = &analysis.Analyzer{ + Name: "lostcancel", + Doc: Doc, + Run: run, + Requires: []*analysis.Analyzer{ + inspect.Analyzer, + ctrlflow.Analyzer, + }, +} + +const debug = false + +var contextPackage = "context" + +// checkLostCancel reports a failure to the call the cancel function +// returned by context.WithCancel, either because the variable was +// assigned to the blank identifier, or because there exists a +// control-flow path from the call to a return statement and that path +// does not "use" the cancel function. Any reference to the variable +// counts as a use, even within a nested function literal. +// +// checkLostCancel analyzes a single named or literal function. +func run(pass *analysis.Pass) (interface{}, error) { + // Fast path: bypass check if file doesn't use context.WithCancel. + if !hasImport(pass.Pkg, contextPackage) { + return nil, nil + } + + // Call runFunc for each Func{Decl,Lit}. + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + nodeTypes := []ast.Node{ + (*ast.FuncLit)(nil), + (*ast.FuncDecl)(nil), + } + inspect.Preorder(nodeTypes, func(n ast.Node) { + runFunc(pass, n) + }) + return nil, nil +} + +func runFunc(pass *analysis.Pass, node ast.Node) { + // Maps each cancel variable to its defining ValueSpec/AssignStmt. + cancelvars := make(map[*types.Var]ast.Node) + + // TODO(adonovan): opt: refactor to make a single pass + // over the AST using inspect.WithStack and node types + // {FuncDecl,FuncLit,CallExpr,SelectorExpr}. + + // Find the set of cancel vars to analyze. + stack := make([]ast.Node, 0, 32) + ast.Inspect(node, func(n ast.Node) bool { + switch n.(type) { + case *ast.FuncLit: + if len(stack) > 0 { + return false // don't stray into nested functions + } + case nil: + stack = stack[:len(stack)-1] // pop + return true + } + stack = append(stack, n) // push + + // Look for [{AssignStmt,ValueSpec} CallExpr SelectorExpr]: + // + // ctx, cancel := context.WithCancel(...) + // ctx, cancel = context.WithCancel(...) + // var ctx, cancel = context.WithCancel(...) + // + if isContextWithCancel(pass.TypesInfo, n) && isCall(stack[len(stack)-2]) { + var id *ast.Ident // id of cancel var + stmt := stack[len(stack)-3] + switch stmt := stmt.(type) { + case *ast.ValueSpec: + if len(stmt.Names) > 1 { + id = stmt.Names[1] + } + case *ast.AssignStmt: + if len(stmt.Lhs) > 1 { + id, _ = stmt.Lhs[1].(*ast.Ident) + } + } + if id != nil { + if id.Name == "_" { + pass.Reportf(id.Pos(), + "the cancel function returned by context.%s should be called, not discarded, to avoid a context leak", + n.(*ast.SelectorExpr).Sel.Name) + } else if v, ok := pass.TypesInfo.Uses[id].(*types.Var); ok { + cancelvars[v] = stmt + } else if v, ok := pass.TypesInfo.Defs[id].(*types.Var); ok { + cancelvars[v] = stmt + } + } + } + + return true + }) + + if len(cancelvars) == 0 { + return // no need to inspect CFG + } + + // Obtain the CFG. + cfgs := pass.ResultOf[ctrlflow.Analyzer].(*ctrlflow.CFGs) + var g *cfg.CFG + var sig *types.Signature + switch node := node.(type) { + case *ast.FuncDecl: + g = cfgs.FuncDecl(node) + sig, _ = pass.TypesInfo.Defs[node.Name].Type().(*types.Signature) + case *ast.FuncLit: + g = cfgs.FuncLit(node) + sig, _ = pass.TypesInfo.Types[node.Type].Type.(*types.Signature) + } + if sig == nil { + return // missing type information + } + + // Print CFG. + if debug { + fmt.Println(g.Format(pass.Fset)) + } + + // Examine the CFG for each variable in turn. + // (It would be more efficient to analyze all cancelvars in a + // single pass over the AST, but seldom is there more than one.) + for v, stmt := range cancelvars { + if ret := lostCancelPath(pass, g, v, stmt, sig); ret != nil { + lineno := pass.Fset.Position(stmt.Pos()).Line + pass.Reportf(stmt.Pos(), "the %s function is not used on all paths (possible context leak)", v.Name()) + pass.Reportf(ret.Pos(), "this return statement may be reached without using the %s var defined on line %d", v.Name(), lineno) + } + } +} + +func isCall(n ast.Node) bool { _, ok := n.(*ast.CallExpr); return ok } + +func hasImport(pkg *types.Package, path string) bool { + for _, imp := range pkg.Imports() { + if imp.Path() == path { + return true + } + } + return false +} + +// isContextWithCancel reports whether n is one of the qualified identifiers +// context.With{Cancel,Timeout,Deadline}. +func isContextWithCancel(info *types.Info, n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + switch sel.Sel.Name { + case "WithCancel", "WithTimeout", "WithDeadline": + if x, ok := sel.X.(*ast.Ident); ok { + if pkgname, ok := info.Uses[x].(*types.PkgName); ok { + return pkgname.Imported().Path() == contextPackage + } + // Import failed, so we can't check package path. + // Just check the local package name (heuristic). + return x.Name == "context" + } + } + } + return false +} + +// lostCancelPath finds a path through the CFG, from stmt (which defines +// the 'cancel' variable v) to a return statement, that doesn't "use" v. +// If it finds one, it returns the return statement (which may be synthetic). +// sig is the function's type, if known. +func lostCancelPath(pass *analysis.Pass, g *cfg.CFG, v *types.Var, stmt ast.Node, sig *types.Signature) *ast.ReturnStmt { + vIsNamedResult := sig != nil && tupleContains(sig.Results(), v) + + // uses reports whether stmts contain a "use" of variable v. + uses := func(pass *analysis.Pass, v *types.Var, stmts []ast.Node) bool { + found := false + for _, stmt := range stmts { + ast.Inspect(stmt, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.Ident: + if pass.TypesInfo.Uses[n] == v { + found = true + } + case *ast.ReturnStmt: + // A naked return statement counts as a use + // of the named result variables. + if n.Results == nil && vIsNamedResult { + found = true + } + } + return !found + }) + } + return found + } + + // blockUses computes "uses" for each block, caching the result. + memo := make(map[*cfg.Block]bool) + blockUses := func(pass *analysis.Pass, v *types.Var, b *cfg.Block) bool { + res, ok := memo[b] + if !ok { + res = uses(pass, v, b.Nodes) + memo[b] = res + } + return res + } + + // Find the var's defining block in the CFG, + // plus the rest of the statements of that block. + var defblock *cfg.Block + var rest []ast.Node +outer: + for _, b := range g.Blocks { + for i, n := range b.Nodes { + if n == stmt { + defblock = b + rest = b.Nodes[i+1:] + break outer + } + } + } + if defblock == nil { + panic("internal error: can't find defining block for cancel var") + } + + // Is v "used" in the remainder of its defining block? + if uses(pass, v, rest) { + return nil + } + + // Does the defining block return without using v? + if ret := defblock.Return(); ret != nil { + return ret + } + + // Search the CFG depth-first for a path, from defblock to a + // return block, in which v is never "used". + seen := make(map[*cfg.Block]bool) + var search func(blocks []*cfg.Block) *ast.ReturnStmt + search = func(blocks []*cfg.Block) *ast.ReturnStmt { + for _, b := range blocks { + if !seen[b] { + seen[b] = true + + // Prune the search if the block uses v. + if blockUses(pass, v, b) { + continue + } + + // Found path to return statement? + if ret := b.Return(); ret != nil { + if debug { + fmt.Printf("found path to return in block %s\n", b) + } + return ret // found + } + + // Recur + if ret := search(b.Succs); ret != nil { + if debug { + fmt.Printf(" from block %s\n", b) + } + return ret + } + } + } + return nil + } + return search(defblock.Succs) +} + +func tupleContains(tuple *types.Tuple, v *types.Var) bool { + for i := 0; i < tuple.Len(); i++ { + if tuple.At(i) == v { + return true + } + } + return false +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/nilfunc/nilfunc.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/nilfunc/nilfunc.go new file mode 100644 index 0000000000..9c2d4df20a --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/nilfunc/nilfunc.go @@ -0,0 +1,74 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package nilfunc defines an Analyzer that checks for useless +// comparisons against nil. +package nilfunc + +import ( + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for useless comparisons between functions and nil + +A useless comparison is one like f == nil as opposed to f() == nil.` + +var Analyzer = &analysis.Analyzer{ + Name: "nilfunc", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.BinaryExpr)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + e := n.(*ast.BinaryExpr) + + // Only want == or != comparisons. + if e.Op != token.EQL && e.Op != token.NEQ { + return + } + + // Only want comparisons with a nil identifier on one side. + var e2 ast.Expr + switch { + case pass.TypesInfo.Types[e.X].IsNil(): + e2 = e.Y + case pass.TypesInfo.Types[e.Y].IsNil(): + e2 = e.X + default: + return + } + + // Only want identifiers or selector expressions. + var obj types.Object + switch v := e2.(type) { + case *ast.Ident: + obj = pass.TypesInfo.Uses[v] + case *ast.SelectorExpr: + obj = pass.TypesInfo.Uses[v.Sel] + default: + return + } + + // Only want functions. + if _, ok := obj.(*types.Func); !ok { + return + } + + pass.Reportf(e.Pos(), "comparison of function %v %v nil is always %v", obj.Name(), e.Op, e.Op == token.NEQ) + }) + return nil, nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/pkgfact/pkgfact.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/pkgfact/pkgfact.go new file mode 100644 index 0000000000..e053086732 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/pkgfact/pkgfact.go @@ -0,0 +1,127 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The pkgfact package is a demonstration and test of the package fact +// mechanism. +// +// The output of the pkgfact analysis is a set of key/values pairs +// gathered from the analyzed package and its imported dependencies. +// Each key/value pair comes from a top-level constant declaration +// whose name starts and ends with "_". For example: +// +// package p +// +// const _greeting_ = "hello" +// const _audience_ = "world" +// +// the pkgfact analysis output for package p would be: +// +// {"greeting": "hello", "audience": "world"}. +// +// In addition, the analysis reports a diagnostic at each import +// showing which key/value pairs it contributes. +package pkgfact + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "reflect" + "sort" + "strings" + + "golang.org/x/tools/go/analysis" +) + +var Analyzer = &analysis.Analyzer{ + Name: "pkgfact", + Doc: "gather name/value pairs from constant declarations", + Run: run, + FactTypes: []analysis.Fact{new(pairsFact)}, + ResultType: reflect.TypeOf(map[string]string{}), +} + +// A pairsFact is a package-level fact that records +// an set of key=value strings accumulated from constant +// declarations in this package and its dependencies. +// Elements are ordered by keys, which are unique. +type pairsFact []string + +func (f *pairsFact) AFact() {} +func (f *pairsFact) String() string { return "pairs(" + strings.Join(*f, ", ") + ")" } + +func run(pass *analysis.Pass) (interface{}, error) { + result := make(map[string]string) + + // At each import, print the fact from the imported + // package and accumulate its information into the result. + // (Warning: accumulation leads to quadratic growth of work.) + doImport := func(spec *ast.ImportSpec) { + pkg := imported(pass.TypesInfo, spec) + var fact pairsFact + if pass.ImportPackageFact(pkg, &fact) { + for _, pair := range fact { + eq := strings.IndexByte(pair, '=') + result[pair[:eq]] = pair[1+eq:] + } + pass.Reportf(spec.Pos(), "%s", strings.Join(fact, " ")) + } + } + + // At each "const _name_ = value", add a fact into env. + doConst := func(spec *ast.ValueSpec) { + if len(spec.Names) == len(spec.Values) { + for i := range spec.Names { + name := spec.Names[i].Name + if strings.HasPrefix(name, "_") && strings.HasSuffix(name, "_") { + + if key := strings.Trim(name, "_"); key != "" { + value := pass.TypesInfo.Types[spec.Values[i]].Value.String() + result[key] = value + } + } + } + } + } + + for _, f := range pass.Files { + for _, decl := range f.Decls { + if decl, ok := decl.(*ast.GenDecl); ok { + for _, spec := range decl.Specs { + switch decl.Tok { + case token.IMPORT: + doImport(spec.(*ast.ImportSpec)) + case token.CONST: + doConst(spec.(*ast.ValueSpec)) + } + } + } + } + } + + // Sort/deduplicate the result and save it as a package fact. + keys := make([]string, 0, len(result)) + for key := range result { + keys = append(keys, key) + } + sort.Strings(keys) + var fact pairsFact + for _, key := range keys { + fact = append(fact, fmt.Sprintf("%s=%s", key, result[key])) + } + if len(fact) > 0 { + pass.ExportPackageFact(&fact) + } + + return result, nil +} + +func imported(info *types.Info, spec *ast.ImportSpec) *types.Package { + obj, ok := info.Implicits[spec] + if !ok { + obj = info.Defs[spec.Name] // renaming import + } + return obj.(*types.PkgName).Imported() +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go new file mode 100644 index 0000000000..23f634fd98 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go @@ -0,0 +1,964 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the printf-checker. + +package printf + +import ( + "bytes" + "fmt" + "go/ast" + "go/constant" + "go/token" + "go/types" + "regexp" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" + "golang.org/x/tools/go/types/typeutil" +) + +func init() { + Analyzer.Flags.Var(isPrint, "funcs", "comma-separated list of print function names to check") +} + +var Analyzer = &analysis.Analyzer{ + Name: "printf", + Doc: "check printf-like invocations", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, + FactTypes: []analysis.Fact{new(isWrapper)}, +} + +const doc = `check consistency of Printf format strings and arguments + +The check applies to known functions (for example, those in package fmt) +as well as any detected wrappers of known functions. + +A function that wants to avail itself of printf checking but does not +get found by this analyzer's heuristics (for example, due to use of +dynamic calls) can insert a bogus call: + + if false { + fmt.Sprintf(format, args...) // enable printf checking + } + +The -funcs flag specifies a comma-separated list of names of additional +known formatting functions or methods. If the name contains a period, +it must denote a specific function using one of the following forms: + + dir/pkg.Function + dir/pkg.Type.Method + (*dir/pkg.Type).Method + +Otherwise the name is interpreted as a case-insensitive unqualified +identifier such as "errorf". Either way, if a listed name ends in f, the +function is assumed to be Printf-like, taking a format string before the +argument list. Otherwise it is assumed to be Print-like, taking a list +of arguments with no format string. +` + +// isWrapper is a fact indicating that a function is a print or printf wrapper. +type isWrapper struct{ Printf bool } + +func (f *isWrapper) AFact() {} + +func (f *isWrapper) String() string { + if f.Printf { + return "printfWrapper" + } else { + return "printWrapper" + } +} + +func run(pass *analysis.Pass) (interface{}, error) { + findPrintfLike(pass) + checkCall(pass) + return nil, nil +} + +type printfWrapper struct { + obj *types.Func + fdecl *ast.FuncDecl + format *types.Var + args *types.Var + callers []printfCaller + failed bool // if true, not a printf wrapper +} + +type printfCaller struct { + w *printfWrapper + call *ast.CallExpr +} + +// maybePrintfWrapper decides whether decl (a declared function) may be a wrapper +// around a fmt.Printf or fmt.Print function. If so it returns a printfWrapper +// function describing the declaration. Later processing will analyze the +// graph of potential printf wrappers to pick out the ones that are true wrappers. +// A function may be a Printf or Print wrapper if its last argument is ...interface{}. +// If the next-to-last argument is a string, then this may be a Printf wrapper. +// Otherwise it may be a Print wrapper. +func maybePrintfWrapper(info *types.Info, decl ast.Decl) *printfWrapper { + // Look for functions with final argument type ...interface{}. + fdecl, ok := decl.(*ast.FuncDecl) + if !ok || fdecl.Body == nil { + return nil + } + fn := info.Defs[fdecl.Name].(*types.Func) + + sig := fn.Type().(*types.Signature) + if !sig.Variadic() { + return nil // not variadic + } + + params := sig.Params() + nparams := params.Len() // variadic => nonzero + + args := params.At(nparams - 1) + iface, ok := args.Type().(*types.Slice).Elem().(*types.Interface) + if !ok || !iface.Empty() { + return nil // final (args) param is not ...interface{} + } + + // Is second last param 'format string'? + var format *types.Var + if nparams >= 2 { + if p := params.At(nparams - 2); p.Type() == types.Typ[types.String] { + format = p + } + } + + return &printfWrapper{ + obj: fn, + fdecl: fdecl, + format: format, + args: args, + } +} + +// findPrintfLike scans the entire package to find printf-like functions. +func findPrintfLike(pass *analysis.Pass) (interface{}, error) { + // Gather potential wrappers and call graph between them. + byObj := make(map[*types.Func]*printfWrapper) + var wrappers []*printfWrapper + for _, file := range pass.Files { + for _, decl := range file.Decls { + w := maybePrintfWrapper(pass.TypesInfo, decl) + if w == nil { + continue + } + byObj[w.obj] = w + wrappers = append(wrappers, w) + } + } + + // Walk the graph to figure out which are really printf wrappers. + for _, w := range wrappers { + // Scan function for calls that could be to other printf-like functions. + ast.Inspect(w.fdecl.Body, func(n ast.Node) bool { + if w.failed { + return false + } + + // TODO: Relax these checks; issue 26555. + if assign, ok := n.(*ast.AssignStmt); ok { + for _, lhs := range assign.Lhs { + if match(pass.TypesInfo, lhs, w.format) || + match(pass.TypesInfo, lhs, w.args) { + // Modifies the format + // string or args in + // some way, so not a + // simple wrapper. + w.failed = true + return false + } + } + } + if un, ok := n.(*ast.UnaryExpr); ok && un.Op == token.AND { + if match(pass.TypesInfo, un.X, w.format) || + match(pass.TypesInfo, un.X, w.args) { + // Taking the address of the + // format string or args, + // so not a simple wrapper. + w.failed = true + return false + } + } + + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) == 0 || !match(pass.TypesInfo, call.Args[len(call.Args)-1], w.args) { + return true + } + + fn, kind := printfNameAndKind(pass, call) + if kind != 0 { + checkPrintfFwd(pass, w, call, kind) + return true + } + + // If the call is to another function in this package, + // maybe we will find out it is printf-like later. + // Remember this call for later checking. + if fn != nil && fn.Pkg() == pass.Pkg && byObj[fn] != nil { + callee := byObj[fn] + callee.callers = append(callee.callers, printfCaller{w, call}) + } + + return true + }) + } + return nil, nil +} + +func match(info *types.Info, arg ast.Expr, param *types.Var) bool { + id, ok := arg.(*ast.Ident) + return ok && info.ObjectOf(id) == param +} + +const ( + kindPrintf = 1 + kindPrint = 2 +) + +// checkPrintfFwd checks that a printf-forwarding wrapper is forwarding correctly. +// It diagnoses writing fmt.Printf(format, args) instead of fmt.Printf(format, args...). +func checkPrintfFwd(pass *analysis.Pass, w *printfWrapper, call *ast.CallExpr, kind int) { + matched := kind == kindPrint || + kind == kindPrintf && len(call.Args) >= 2 && match(pass.TypesInfo, call.Args[len(call.Args)-2], w.format) + if !matched { + return + } + + if !call.Ellipsis.IsValid() { + typ, ok := pass.TypesInfo.Types[call.Fun].Type.(*types.Signature) + if !ok { + return + } + if len(call.Args) > typ.Params().Len() { + // If we're passing more arguments than what the + // print/printf function can take, adding an ellipsis + // would break the program. For example: + // + // func foo(arg1 string, arg2 ...interface{} { + // fmt.Printf("%s %v", arg1, arg2) + // } + return + } + desc := "printf" + if kind == kindPrint { + desc = "print" + } + pass.Reportf(call.Pos(), "missing ... in args forwarded to %s-like function", desc) + return + } + fn := w.obj + var fact isWrapper + if !pass.ImportObjectFact(fn, &fact) { + fact.Printf = kind == kindPrintf + pass.ExportObjectFact(fn, &fact) + for _, caller := range w.callers { + checkPrintfFwd(pass, caller.w, caller.call, kind) + } + } +} + +// isPrint records the print functions. +// If a key ends in 'f' then it is assumed to be a formatted print. +// +// Keys are either values returned by (*types.Func).FullName, +// or case-insensitive identifiers such as "errorf". +// +// The -funcs flag adds to this set. +var isPrint = stringSet{ + "fmt.Errorf": true, + "fmt.Fprint": true, + "fmt.Fprintf": true, + "fmt.Fprintln": true, + "fmt.Print": true, // technically these three + "fmt.Printf": true, // are redundant because they + "fmt.Println": true, // forward to Fprint{,f,ln} + "fmt.Sprint": true, + "fmt.Sprintf": true, + "fmt.Sprintln": true, + + // *testing.T and B are detected by induction, but testing.TB is + // an interface and the inference can't follow dynamic calls. + "(testing.TB).Error": true, + "(testing.TB).Errorf": true, + "(testing.TB).Fatal": true, + "(testing.TB).Fatalf": true, + "(testing.TB).Log": true, + "(testing.TB).Logf": true, + "(testing.TB).Skip": true, + "(testing.TB).Skipf": true, +} + +// formatString returns the format string argument and its index within +// the given printf-like call expression. +// +// The last parameter before variadic arguments is assumed to be +// a format string. +// +// The first string literal or string constant is assumed to be a format string +// if the call's signature cannot be determined. +// +// If it cannot find any format string parameter, it returns ("", -1). +func formatString(pass *analysis.Pass, call *ast.CallExpr) (format string, idx int) { + typ := pass.TypesInfo.Types[call.Fun].Type + if typ != nil { + if sig, ok := typ.(*types.Signature); ok { + if !sig.Variadic() { + // Skip checking non-variadic functions. + return "", -1 + } + idx := sig.Params().Len() - 2 + if idx < 0 { + // Skip checking variadic functions without + // fixed arguments. + return "", -1 + } + s, ok := stringConstantArg(pass, call, idx) + if !ok { + // The last argument before variadic args isn't a string. + return "", -1 + } + return s, idx + } + } + + // Cannot determine call's signature. Fall back to scanning for the first + // string constant in the call. + for idx := range call.Args { + if s, ok := stringConstantArg(pass, call, idx); ok { + return s, idx + } + if pass.TypesInfo.Types[call.Args[idx]].Type == types.Typ[types.String] { + // Skip checking a call with a non-constant format + // string argument, since its contents are unavailable + // for validation. + return "", -1 + } + } + return "", -1 +} + +// stringConstantArg returns call's string constant argument at the index idx. +// +// ("", false) is returned if call's argument at the index idx isn't a string +// constant. +func stringConstantArg(pass *analysis.Pass, call *ast.CallExpr, idx int) (string, bool) { + if idx >= len(call.Args) { + return "", false + } + arg := call.Args[idx] + lit := pass.TypesInfo.Types[arg].Value + if lit != nil && lit.Kind() == constant.String { + return constant.StringVal(lit), true + } + return "", false +} + +// checkCall triggers the print-specific checks if the call invokes a print function. +func checkCall(pass *analysis.Pass) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + nodeFilter := []ast.Node{ + (*ast.CallExpr)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + call := n.(*ast.CallExpr) + fn, kind := printfNameAndKind(pass, call) + switch kind { + case kindPrintf: + checkPrintf(pass, call, fn) + case kindPrint: + checkPrint(pass, call, fn) + } + }) +} + +func printfNameAndKind(pass *analysis.Pass, call *ast.CallExpr) (fn *types.Func, kind int) { + fn, _ = typeutil.Callee(pass.TypesInfo, call).(*types.Func) + if fn == nil { + return nil, 0 + } + + var fact isWrapper + if pass.ImportObjectFact(fn, &fact) { + if fact.Printf { + return fn, kindPrintf + } else { + return fn, kindPrint + } + } + + _, ok := isPrint[fn.FullName()] + if !ok { + // Next look up just "printf", for use with -printf.funcs. + _, ok = isPrint[strings.ToLower(fn.Name())] + } + if ok { + if strings.HasSuffix(fn.Name(), "f") { + kind = kindPrintf + } else { + kind = kindPrint + } + } + return fn, kind +} + +// isFormatter reports whether t satisfies fmt.Formatter. +// Unlike fmt.Stringer, it's impossible to satisfy fmt.Formatter without importing fmt. +func isFormatter(pass *analysis.Pass, t types.Type) bool { + for _, imp := range pass.Pkg.Imports() { + if imp.Path() == "fmt" { + formatter := imp.Scope().Lookup("Formatter").Type().Underlying().(*types.Interface) + return types.Implements(t, formatter) + } + } + return false +} + +// formatState holds the parsed representation of a printf directive such as "%3.*[4]d". +// It is constructed by parsePrintfVerb. +type formatState struct { + verb rune // the format verb: 'd' for "%d" + format string // the full format directive from % through verb, "%.3d". + name string // Printf, Sprintf etc. + flags []byte // the list of # + etc. + argNums []int // the successive argument numbers that are consumed, adjusted to refer to actual arg in call + firstArg int // Index of first argument after the format in the Printf call. + // Used only during parse. + pass *analysis.Pass + call *ast.CallExpr + argNum int // Which argument we're expecting to format now. + hasIndex bool // Whether the argument is indexed. + indexPending bool // Whether we have an indexed argument that has not resolved. + nbytes int // number of bytes of the format string consumed. +} + +// checkPrintf checks a call to a formatted print routine such as Printf. +func checkPrintf(pass *analysis.Pass, call *ast.CallExpr, fn *types.Func) { + format, idx := formatString(pass, call) + if idx < 0 { + if false { + pass.Reportf(call.Lparen, "can't check non-constant format in call to %s", fn.Name()) + } + return + } + + firstArg := idx + 1 // Arguments are immediately after format string. + if !strings.Contains(format, "%") { + if len(call.Args) > firstArg { + pass.Reportf(call.Lparen, "%s call has arguments but no formatting directives", fn.Name()) + } + return + } + // Hard part: check formats against args. + argNum := firstArg + maxArgNum := firstArg + anyIndex := false + for i, w := 0, 0; i < len(format); i += w { + w = 1 + if format[i] != '%' { + continue + } + state := parsePrintfVerb(pass, call, fn.Name(), format[i:], firstArg, argNum) + if state == nil { + return + } + w = len(state.format) + if !okPrintfArg(pass, call, state) { // One error per format is enough. + return + } + if state.hasIndex { + anyIndex = true + } + if len(state.argNums) > 0 { + // Continue with the next sequential argument. + argNum = state.argNums[len(state.argNums)-1] + 1 + } + for _, n := range state.argNums { + if n >= maxArgNum { + maxArgNum = n + 1 + } + } + } + // Dotdotdot is hard. + if call.Ellipsis.IsValid() && maxArgNum >= len(call.Args)-1 { + return + } + // If any formats are indexed, extra arguments are ignored. + if anyIndex { + return + } + // There should be no leftover arguments. + if maxArgNum != len(call.Args) { + expect := maxArgNum - firstArg + numArgs := len(call.Args) - firstArg + pass.Reportf(call.Pos(), "%s call needs %v but has %v", fn.Name(), count(expect, "arg"), count(numArgs, "arg")) + } +} + +// parseFlags accepts any printf flags. +func (s *formatState) parseFlags() { + for s.nbytes < len(s.format) { + switch c := s.format[s.nbytes]; c { + case '#', '0', '+', '-', ' ': + s.flags = append(s.flags, c) + s.nbytes++ + default: + return + } + } +} + +// scanNum advances through a decimal number if present. +func (s *formatState) scanNum() { + for ; s.nbytes < len(s.format); s.nbytes++ { + c := s.format[s.nbytes] + if c < '0' || '9' < c { + return + } + } +} + +// parseIndex scans an index expression. It returns false if there is a syntax error. +func (s *formatState) parseIndex() bool { + if s.nbytes == len(s.format) || s.format[s.nbytes] != '[' { + return true + } + // Argument index present. + s.nbytes++ // skip '[' + start := s.nbytes + s.scanNum() + ok := true + if s.nbytes == len(s.format) || s.nbytes == start || s.format[s.nbytes] != ']' { + ok = false + s.nbytes = strings.Index(s.format, "]") + if s.nbytes < 0 { + s.pass.Reportf(s.call.Pos(), "%s format %s is missing closing ]", s.name, s.format) + return false + } + } + arg32, err := strconv.ParseInt(s.format[start:s.nbytes], 10, 32) + if err != nil || !ok || arg32 <= 0 || arg32 > int64(len(s.call.Args)-s.firstArg) { + s.pass.Reportf(s.call.Pos(), "%s format has invalid argument index [%s]", s.name, s.format[start:s.nbytes]) + return false + } + s.nbytes++ // skip ']' + arg := int(arg32) + arg += s.firstArg - 1 // We want to zero-index the actual arguments. + s.argNum = arg + s.hasIndex = true + s.indexPending = true + return true +} + +// parseNum scans a width or precision (or *). It returns false if there's a bad index expression. +func (s *formatState) parseNum() bool { + if s.nbytes < len(s.format) && s.format[s.nbytes] == '*' { + if s.indexPending { // Absorb it. + s.indexPending = false + } + s.nbytes++ + s.argNums = append(s.argNums, s.argNum) + s.argNum++ + } else { + s.scanNum() + } + return true +} + +// parsePrecision scans for a precision. It returns false if there's a bad index expression. +func (s *formatState) parsePrecision() bool { + // If there's a period, there may be a precision. + if s.nbytes < len(s.format) && s.format[s.nbytes] == '.' { + s.flags = append(s.flags, '.') // Treat precision as a flag. + s.nbytes++ + if !s.parseIndex() { + return false + } + if !s.parseNum() { + return false + } + } + return true +} + +// parsePrintfVerb looks the formatting directive that begins the format string +// and returns a formatState that encodes what the directive wants, without looking +// at the actual arguments present in the call. The result is nil if there is an error. +func parsePrintfVerb(pass *analysis.Pass, call *ast.CallExpr, name, format string, firstArg, argNum int) *formatState { + state := &formatState{ + format: format, + name: name, + flags: make([]byte, 0, 5), + argNum: argNum, + argNums: make([]int, 0, 1), + nbytes: 1, // There's guaranteed to be a percent sign. + firstArg: firstArg, + pass: pass, + call: call, + } + // There may be flags. + state.parseFlags() + // There may be an index. + if !state.parseIndex() { + return nil + } + // There may be a width. + if !state.parseNum() { + return nil + } + // There may be a precision. + if !state.parsePrecision() { + return nil + } + // Now a verb, possibly prefixed by an index (which we may already have). + if !state.indexPending && !state.parseIndex() { + return nil + } + if state.nbytes == len(state.format) { + pass.Reportf(call.Pos(), "%s format %s is missing verb at end of string", name, state.format) + return nil + } + verb, w := utf8.DecodeRuneInString(state.format[state.nbytes:]) + state.verb = verb + state.nbytes += w + if verb != '%' { + state.argNums = append(state.argNums, state.argNum) + } + state.format = state.format[:state.nbytes] + return state +} + +// printfArgType encodes the types of expressions a printf verb accepts. It is a bitmask. +type printfArgType int + +const ( + argBool printfArgType = 1 << iota + argInt + argRune + argString + argFloat + argComplex + argPointer + anyType printfArgType = ^0 +) + +type printVerb struct { + verb rune // User may provide verb through Formatter; could be a rune. + flags string // known flags are all ASCII + typ printfArgType +} + +// Common flag sets for printf verbs. +const ( + noFlag = "" + numFlag = " -+.0" + sharpNumFlag = " -+.0#" + allFlags = " -+.0#" +) + +// printVerbs identifies which flags are known to printf for each verb. +var printVerbs = []printVerb{ + // '-' is a width modifier, always valid. + // '.' is a precision for float, max width for strings. + // '+' is required sign for numbers, Go format for %v. + // '#' is alternate format for several verbs. + // ' ' is spacer for numbers + {'%', noFlag, 0}, + {'b', numFlag, argInt | argFloat | argComplex}, + {'c', "-", argRune | argInt}, + {'d', numFlag, argInt | argPointer}, + {'e', sharpNumFlag, argFloat | argComplex}, + {'E', sharpNumFlag, argFloat | argComplex}, + {'f', sharpNumFlag, argFloat | argComplex}, + {'F', sharpNumFlag, argFloat | argComplex}, + {'g', sharpNumFlag, argFloat | argComplex}, + {'G', sharpNumFlag, argFloat | argComplex}, + {'o', sharpNumFlag, argInt}, + {'p', "-#", argPointer}, + {'q', " -+.0#", argRune | argInt | argString}, + {'s', " -+.0", argString}, + {'t', "-", argBool}, + {'T', "-", anyType}, + {'U', "-#", argRune | argInt}, + {'v', allFlags, anyType}, + {'x', sharpNumFlag, argRune | argInt | argString | argPointer}, + {'X', sharpNumFlag, argRune | argInt | argString | argPointer}, +} + +// okPrintfArg compares the formatState to the arguments actually present, +// reporting any discrepancies it can discern. If the final argument is ellipsissed, +// there's little it can do for that. +func okPrintfArg(pass *analysis.Pass, call *ast.CallExpr, state *formatState) (ok bool) { + var v printVerb + found := false + // Linear scan is fast enough for a small list. + for _, v = range printVerbs { + if v.verb == state.verb { + found = true + break + } + } + + // Does current arg implement fmt.Formatter? + formatter := false + if state.argNum < len(call.Args) { + if tv, ok := pass.TypesInfo.Types[call.Args[state.argNum]]; ok { + formatter = isFormatter(pass, tv.Type) + } + } + + if !formatter { + if !found { + pass.Reportf(call.Pos(), "%s format %s has unknown verb %c", state.name, state.format, state.verb) + return false + } + for _, flag := range state.flags { + // TODO: Disable complaint about '0' for Go 1.10. To be fixed properly in 1.11. + // See issues 23598 and 23605. + if flag == '0' { + continue + } + if !strings.ContainsRune(v.flags, rune(flag)) { + pass.Reportf(call.Pos(), "%s format %s has unrecognized flag %c", state.name, state.format, flag) + return false + } + } + } + // Verb is good. If len(state.argNums)>trueArgs, we have something like %.*s and all + // but the final arg must be an integer. + trueArgs := 1 + if state.verb == '%' { + trueArgs = 0 + } + nargs := len(state.argNums) + for i := 0; i < nargs-trueArgs; i++ { + argNum := state.argNums[i] + if !argCanBeChecked(pass, call, i, state) { + return + } + arg := call.Args[argNum] + if !matchArgType(pass, argInt, nil, arg) { + pass.Reportf(call.Pos(), "%s format %s uses non-int %s as argument of *", state.name, state.format, analysisutil.Format(pass.Fset, arg)) + return false + } + } + + if state.verb == '%' || formatter { + return true + } + argNum := state.argNums[len(state.argNums)-1] + if !argCanBeChecked(pass, call, len(state.argNums)-1, state) { + return false + } + arg := call.Args[argNum] + if isFunctionValue(pass, arg) && state.verb != 'p' && state.verb != 'T' { + pass.Reportf(call.Pos(), "%s format %s arg %s is a func value, not called", state.name, state.format, analysisutil.Format(pass.Fset, arg)) + return false + } + if !matchArgType(pass, v.typ, nil, arg) { + typeString := "" + if typ := pass.TypesInfo.Types[arg].Type; typ != nil { + typeString = typ.String() + } + pass.Reportf(call.Pos(), "%s format %s has arg %s of wrong type %s", state.name, state.format, analysisutil.Format(pass.Fset, arg), typeString) + return false + } + if v.typ&argString != 0 && v.verb != 'T' && !bytes.Contains(state.flags, []byte{'#'}) && recursiveStringer(pass, arg) { + pass.Reportf(call.Pos(), "%s format %s with arg %s causes recursive String method call", state.name, state.format, analysisutil.Format(pass.Fset, arg)) + return false + } + return true +} + +// recursiveStringer reports whether the argument e is a potential +// recursive call to stringer, such as t and &t in these examples: +// +// func (t *T) String() string { printf("%s", t) } +// func (t T) String() string { printf("%s", t) } +// func (t T) String() string { printf("%s", &t) } +// +func recursiveStringer(pass *analysis.Pass, e ast.Expr) bool { + typ := pass.TypesInfo.Types[e].Type + + // It's unlikely to be a recursive stringer if it has a Format method. + if isFormatter(pass, typ) { + return false + } + + // Does e allow e.String()? + obj, _, _ := types.LookupFieldOrMethod(typ, false, pass.Pkg, "String") + stringMethod, ok := obj.(*types.Func) + if !ok { + return false + } + + // Is the expression e within the body of that String method? + return stringMethod.Pkg() == pass.Pkg && stringMethod.Scope().Contains(e.Pos()) +} + +// isFunctionValue reports whether the expression is a function as opposed to a function call. +// It is almost always a mistake to print a function value. +func isFunctionValue(pass *analysis.Pass, e ast.Expr) bool { + if typ := pass.TypesInfo.Types[e].Type; typ != nil { + _, ok := typ.(*types.Signature) + return ok + } + return false +} + +// argCanBeChecked reports whether the specified argument is statically present; +// it may be beyond the list of arguments or in a terminal slice... argument, which +// means we can't see it. +func argCanBeChecked(pass *analysis.Pass, call *ast.CallExpr, formatArg int, state *formatState) bool { + argNum := state.argNums[formatArg] + if argNum <= 0 { + // Shouldn't happen, so catch it with prejudice. + panic("negative arg num") + } + if argNum < len(call.Args)-1 { + return true // Always OK. + } + if call.Ellipsis.IsValid() { + return false // We just can't tell; there could be many more arguments. + } + if argNum < len(call.Args) { + return true + } + // There are bad indexes in the format or there are fewer arguments than the format needs. + // This is the argument number relative to the format: Printf("%s", "hi") will give 1 for the "hi". + arg := argNum - state.firstArg + 1 // People think of arguments as 1-indexed. + pass.Reportf(call.Pos(), "%s format %s reads arg #%d, but call has %v", state.name, state.format, arg, count(len(call.Args)-state.firstArg, "arg")) + return false +} + +// printFormatRE is the regexp we match and report as a possible format string +// in the first argument to unformatted prints like fmt.Print. +// We exclude the space flag, so that printing a string like "x % y" is not reported as a format. +var printFormatRE = regexp.MustCompile(`%` + flagsRE + numOptRE + `\.?` + numOptRE + indexOptRE + verbRE) + +const ( + flagsRE = `[+\-#]*` + indexOptRE = `(\[[0-9]+\])?` + numOptRE = `([0-9]+|` + indexOptRE + `\*)?` + verbRE = `[bcdefgopqstvxEFGTUX]` +) + +// checkPrint checks a call to an unformatted print routine such as Println. +func checkPrint(pass *analysis.Pass, call *ast.CallExpr, fn *types.Func) { + firstArg := 0 + typ := pass.TypesInfo.Types[call.Fun].Type + if typ == nil { + // Skip checking functions with unknown type. + return + } + if sig, ok := typ.(*types.Signature); ok { + if !sig.Variadic() { + // Skip checking non-variadic functions. + return + } + params := sig.Params() + firstArg = params.Len() - 1 + + typ := params.At(firstArg).Type() + typ = typ.(*types.Slice).Elem() + it, ok := typ.(*types.Interface) + if !ok || !it.Empty() { + // Skip variadic functions accepting non-interface{} args. + return + } + } + args := call.Args + if len(args) <= firstArg { + // Skip calls without variadic args. + return + } + args = args[firstArg:] + + if firstArg == 0 { + if sel, ok := call.Args[0].(*ast.SelectorExpr); ok { + if x, ok := sel.X.(*ast.Ident); ok { + if x.Name == "os" && strings.HasPrefix(sel.Sel.Name, "Std") { + pass.Reportf(call.Pos(), "%s does not take io.Writer but has first arg %s", fn.Name(), analysisutil.Format(pass.Fset, call.Args[0])) + } + } + } + } + + arg := args[0] + if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { + // Ignore trailing % character in lit.Value. + // The % in "abc 0.0%" couldn't be a formatting directive. + s := strings.TrimSuffix(lit.Value, `%"`) + if strings.Contains(s, "%") { + m := printFormatRE.FindStringSubmatch(s) + if m != nil { + pass.Reportf(call.Pos(), "%s call has possible formatting directive %s", fn.Name(), m[0]) + } + } + } + if strings.HasSuffix(fn.Name(), "ln") { + // The last item, if a string, should not have a newline. + arg = args[len(args)-1] + if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { + str, _ := strconv.Unquote(lit.Value) + if strings.HasSuffix(str, "\n") { + pass.Reportf(call.Pos(), "%s arg list ends with redundant newline", fn.Name()) + } + } + } + for _, arg := range args { + if isFunctionValue(pass, arg) { + pass.Reportf(call.Pos(), "%s arg %s is a func value, not called", fn.Name(), analysisutil.Format(pass.Fset, arg)) + } + if recursiveStringer(pass, arg) { + pass.Reportf(call.Pos(), "%s arg %s causes recursive call to String method", fn.Name(), analysisutil.Format(pass.Fset, arg)) + } + } +} + +// count(n, what) returns "1 what" or "N whats" +// (assuming the plural of what is whats). +func count(n int, what string) string { + if n == 1 { + return "1 " + what + } + return fmt.Sprintf("%d %ss", n, what) +} + +// stringSet is a set-of-nonempty-strings-valued flag. +// Note: elements without a '.' get lower-cased. +type stringSet map[string]bool + +func (ss stringSet) String() string { + var list []string + for name := range ss { + list = append(list, name) + } + sort.Strings(list) + return strings.Join(list, ",") +} + +func (ss stringSet) Set(flag string) error { + for _, name := range strings.Split(flag, ",") { + if len(name) == 0 { + return fmt.Errorf("empty string") + } + if !strings.Contains(name, ".") { + name = strings.ToLower(name) + } + ss[name] = true + } + return nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/types.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/types.go new file mode 100644 index 0000000000..701d08bea2 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/types.go @@ -0,0 +1,223 @@ +package printf + +import ( + "go/ast" + "go/build" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" +) + +var errorType = types.Universe.Lookup("error").Type().Underlying().(*types.Interface) + +// matchArgType reports an error if printf verb t is not appropriate +// for operand arg. +// +// typ is used only for recursive calls; external callers must supply nil. +// +// (Recursion arises from the compound types {map,chan,slice} which +// may be printed with %d etc. if that is appropriate for their element +// types.) +func matchArgType(pass *analysis.Pass, t printfArgType, typ types.Type, arg ast.Expr) bool { + return matchArgTypeInternal(pass, t, typ, arg, make(map[types.Type]bool)) +} + +// matchArgTypeInternal is the internal version of matchArgType. It carries a map +// remembering what types are in progress so we don't recur when faced with recursive +// types or mutually recursive types. +func matchArgTypeInternal(pass *analysis.Pass, t printfArgType, typ types.Type, arg ast.Expr, inProgress map[types.Type]bool) bool { + // %v, %T accept any argument type. + if t == anyType { + return true + } + if typ == nil { + // external call + typ = pass.TypesInfo.Types[arg].Type + if typ == nil { + return true // probably a type check problem + } + } + // If the type implements fmt.Formatter, we have nothing to check. + if isFormatter(pass, typ) { + return true + } + // If we can use a string, might arg (dynamically) implement the Stringer or Error interface? + if t&argString != 0 && isConvertibleToString(pass, typ) { + return true + } + + typ = typ.Underlying() + if inProgress[typ] { + // We're already looking at this type. The call that started it will take care of it. + return true + } + inProgress[typ] = true + + switch typ := typ.(type) { + case *types.Signature: + return t&argPointer != 0 + + case *types.Map: + // Recur: map[int]int matches %d. + return t&argPointer != 0 || + (matchArgTypeInternal(pass, t, typ.Key(), arg, inProgress) && matchArgTypeInternal(pass, t, typ.Elem(), arg, inProgress)) + + case *types.Chan: + return t&argPointer != 0 + + case *types.Array: + // Same as slice. + if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 { + return true // %s matches []byte + } + // Recur: []int matches %d. + return t&argPointer != 0 || matchArgTypeInternal(pass, t, typ.Elem(), arg, inProgress) + + case *types.Slice: + // Same as array. + if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 { + return true // %s matches []byte + } + // Recur: []int matches %d. But watch out for + // type T []T + // If the element is a pointer type (type T[]*T), it's handled fine by the Pointer case below. + return t&argPointer != 0 || matchArgTypeInternal(pass, t, typ.Elem(), arg, inProgress) + + case *types.Pointer: + // Ugly, but dealing with an edge case: a known pointer to an invalid type, + // probably something from a failed import. + if typ.Elem().String() == "invalid type" { + if false { + pass.Reportf(arg.Pos(), "printf argument %v is pointer to invalid or unknown type", analysisutil.Format(pass.Fset, arg)) + } + return true // special case + } + // If it's actually a pointer with %p, it prints as one. + if t == argPointer { + return true + } + // If it's pointer to struct, that's equivalent in our analysis to whether we can print the struct. + if str, ok := typ.Elem().Underlying().(*types.Struct); ok { + return matchStructArgType(pass, t, str, arg, inProgress) + } + // Check whether the rest can print pointers. + return t&argPointer != 0 + + case *types.Struct: + return matchStructArgType(pass, t, typ, arg, inProgress) + + case *types.Interface: + // There's little we can do. + // Whether any particular verb is valid depends on the argument. + // The user may have reasonable prior knowledge of the contents of the interface. + return true + + case *types.Basic: + switch typ.Kind() { + case types.UntypedBool, + types.Bool: + return t&argBool != 0 + + case types.UntypedInt, + types.Int, + types.Int8, + types.Int16, + types.Int32, + types.Int64, + types.Uint, + types.Uint8, + types.Uint16, + types.Uint32, + types.Uint64, + types.Uintptr: + return t&argInt != 0 + + case types.UntypedFloat, + types.Float32, + types.Float64: + return t&argFloat != 0 + + case types.UntypedComplex, + types.Complex64, + types.Complex128: + return t&argComplex != 0 + + case types.UntypedString, + types.String: + return t&argString != 0 + + case types.UnsafePointer: + return t&(argPointer|argInt) != 0 + + case types.UntypedRune: + return t&(argInt|argRune) != 0 + + case types.UntypedNil: + return false + + case types.Invalid: + if false { + pass.Reportf(arg.Pos(), "printf argument %v has invalid or unknown type", analysisutil.Format(pass.Fset, arg)) + } + return true // Probably a type check problem. + } + panic("unreachable") + } + + return false +} + +func isConvertibleToString(pass *analysis.Pass, typ types.Type) bool { + if bt, ok := typ.(*types.Basic); ok && bt.Kind() == types.UntypedNil { + // We explicitly don't want untyped nil, which is + // convertible to both of the interfaces below, as it + // would just panic anyway. + return false + } + if types.ConvertibleTo(typ, errorType) { + return true // via .Error() + } + + // Does it implement fmt.Stringer? + if obj, _, _ := types.LookupFieldOrMethod(typ, false, nil, "String"); obj != nil { + if fn, ok := obj.(*types.Func); ok { + sig := fn.Type().(*types.Signature) + if sig.Params().Len() == 0 && + sig.Results().Len() == 1 && + sig.Results().At(0).Type() == types.Typ[types.String] { + return true + } + } + } + + return false +} + +// hasBasicType reports whether x's type is a types.Basic with the given kind. +func hasBasicType(pass *analysis.Pass, x ast.Expr, kind types.BasicKind) bool { + t := pass.TypesInfo.Types[x].Type + if t != nil { + t = t.Underlying() + } + b, ok := t.(*types.Basic) + return ok && b.Kind() == kind +} + +// matchStructArgType reports whether all the elements of the struct match the expected +// type. For instance, with "%d" all the elements must be printable with the "%d" format. +func matchStructArgType(pass *analysis.Pass, t printfArgType, typ *types.Struct, arg ast.Expr, inProgress map[types.Type]bool) bool { + for i := 0; i < typ.NumFields(); i++ { + typf := typ.Field(i) + if !matchArgTypeInternal(pass, t, typf.Type(), arg, inProgress) { + return false + } + if t&argString != 0 && !typf.Exported() && isConvertibleToString(pass, typf.Type()) { + // Issue #17798: unexported Stringer or error cannot be properly fomatted. + return false + } + } + return true +} + +var archSizes = types.SizesFor("gc", build.Default.GOARCH) diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/dead.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/dead.go new file mode 100644 index 0000000000..43415a98d6 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/dead.go @@ -0,0 +1,101 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package shift + +// Simplified dead code detector. +// Used for skipping shift checks on unreachable arch-specific code. + +import ( + "go/ast" + "go/constant" + "go/types" +) + +// updateDead puts unreachable "if" and "case" nodes into dead. +func updateDead(info *types.Info, dead map[ast.Node]bool, node ast.Node) { + if dead[node] { + // The node is already marked as dead. + return + } + + // setDead marks the node and all the children as dead. + setDead := func(n ast.Node) { + ast.Inspect(n, func(node ast.Node) bool { + if node != nil { + dead[node] = true + } + return true + }) + } + + switch stmt := node.(type) { + case *ast.IfStmt: + // "if" branch is dead if its condition evaluates + // to constant false. + v := info.Types[stmt.Cond].Value + if v == nil { + return + } + if !constant.BoolVal(v) { + setDead(stmt.Body) + return + } + if stmt.Else != nil { + setDead(stmt.Else) + } + case *ast.SwitchStmt: + // Case clause with empty switch tag is dead if it evaluates + // to constant false. + if stmt.Tag == nil { + BodyLoopBool: + for _, stmt := range stmt.Body.List { + cc := stmt.(*ast.CaseClause) + if cc.List == nil { + // Skip default case. + continue + } + for _, expr := range cc.List { + v := info.Types[expr].Value + if v == nil || v.Kind() != constant.Bool || constant.BoolVal(v) { + continue BodyLoopBool + } + } + setDead(cc) + } + return + } + + // Case clause is dead if its constant value doesn't match + // the constant value from the switch tag. + // TODO: This handles integer comparisons only. + v := info.Types[stmt.Tag].Value + if v == nil || v.Kind() != constant.Int { + return + } + tagN, ok := constant.Uint64Val(v) + if !ok { + return + } + BodyLoopInt: + for _, x := range stmt.Body.List { + cc := x.(*ast.CaseClause) + if cc.List == nil { + // Skip default case. + continue + } + for _, expr := range cc.List { + v := info.Types[expr].Value + if v == nil { + continue BodyLoopInt + } + n, ok := constant.Uint64Val(v) + if !ok || tagN == n { + continue BodyLoopInt + } + } + setDead(cc) + } + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/shift.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/shift.go new file mode 100644 index 0000000000..56b150b2b1 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/shift/shift.go @@ -0,0 +1,128 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package shift defines an Analyzer that checks for shifts that exceed +// the width of an integer. +package shift + +// TODO(adonovan): integrate with ctrflow (CFG-based) dead code analysis. May +// have impedance mismatch due to its (non-)treatment of constant +// expressions (such as runtime.GOARCH=="386"). + +import ( + "go/ast" + "go/build" + "go/constant" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +var Analyzer = &analysis.Analyzer{ + Name: "shift", + Doc: "check for shifts that equal or exceed the width of the integer", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + // Do a complete pass to compute dead nodes. + dead := make(map[ast.Node]bool) + nodeFilter := []ast.Node{ + (*ast.IfStmt)(nil), + (*ast.SwitchStmt)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + // TODO(adonovan): move updateDead into this file. + updateDead(pass.TypesInfo, dead, n) + }) + + nodeFilter = []ast.Node{ + (*ast.AssignStmt)(nil), + (*ast.BinaryExpr)(nil), + } + inspect.Preorder(nodeFilter, func(node ast.Node) { + if dead[node] { + // Skip shift checks on unreachable nodes. + return + } + + switch node := node.(type) { + case *ast.BinaryExpr: + if node.Op == token.SHL || node.Op == token.SHR { + checkLongShift(pass, node, node.X, node.Y) + } + case *ast.AssignStmt: + if len(node.Lhs) != 1 || len(node.Rhs) != 1 { + return + } + if node.Tok == token.SHL_ASSIGN || node.Tok == token.SHR_ASSIGN { + checkLongShift(pass, node, node.Lhs[0], node.Rhs[0]) + } + } + }) + return nil, nil +} + +// checkLongShift checks if shift or shift-assign operations shift by more than +// the length of the underlying variable. +func checkLongShift(pass *analysis.Pass, node ast.Node, x, y ast.Expr) { + if pass.TypesInfo.Types[x].Value != nil { + // Ignore shifts of constants. + // These are frequently used for bit-twiddling tricks + // like ^uint(0) >> 63 for 32/64 bit detection and compatibility. + return + } + + v := pass.TypesInfo.Types[y].Value + if v == nil { + return + } + amt, ok := constant.Int64Val(v) + if !ok { + return + } + t := pass.TypesInfo.Types[x].Type + if t == nil { + return + } + b, ok := t.Underlying().(*types.Basic) + if !ok { + return + } + var size int64 + switch b.Kind() { + case types.Uint8, types.Int8: + size = 8 + case types.Uint16, types.Int16: + size = 16 + case types.Uint32, types.Int32: + size = 32 + case types.Uint64, types.Int64: + size = 64 + case types.Int, types.Uint: + size = uintBitSize + case types.Uintptr: + size = uintptrBitSize + default: + return + } + if amt >= size { + ident := analysisutil.Format(pass.Fset, x) + pass.Reportf(node.Pos(), "%s (%d bits) too small for shift of %d", ident, size, amt) + } +} + +var ( + uintBitSize = 8 * archSizes.Sizeof(types.Typ[types.Uint]) + uintptrBitSize = 8 * archSizes.Sizeof(types.Typ[types.Uintptr]) +) + +var archSizes = types.SizesFor("gc", build.Default.GOARCH) diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/stdmethods/stdmethods.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/stdmethods/stdmethods.go new file mode 100644 index 0000000000..eead289e4c --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/stdmethods/stdmethods.go @@ -0,0 +1,211 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package stdmethods defines an Analyzer that checks for misspellings +// in the signatures of methods similar to well-known interfaces. +package stdmethods + +import ( + "bytes" + "fmt" + "go/ast" + "go/printer" + "go/token" + "go/types" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check signature of methods of well-known interfaces + +Sometimes a type may be intended to satisfy an interface but may fail to +do so because of a mistake in its method signature. +For example, the result of this WriteTo method should be (int64, error), +not error, to satisfy io.WriterTo: + + type myWriterTo struct{...} + func (myWriterTo) WriteTo(w io.Writer) error { ... } + +This check ensures that each method whose name matches one of several +well-known interface methods from the standard library has the correct +signature for that interface. + +Checked method names include: + Format GobEncode GobDecode MarshalJSON MarshalXML + Peek ReadByte ReadFrom ReadRune Scan Seek + UnmarshalJSON UnreadByte UnreadRune WriteByte + WriteTo +` + +var Analyzer = &analysis.Analyzer{ + Name: "stdmethods", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +// canonicalMethods lists the input and output types for Go methods +// that are checked using dynamic interface checks. Because the +// checks are dynamic, such methods would not cause a compile error +// if they have the wrong signature: instead the dynamic check would +// fail, sometimes mysteriously. If a method is found with a name listed +// here but not the input/output types listed here, vet complains. +// +// A few of the canonical methods have very common names. +// For example, a type might implement a Scan method that +// has nothing to do with fmt.Scanner, but we still want to check +// the methods that are intended to implement fmt.Scanner. +// To do that, the arguments that have a = prefix are treated as +// signals that the canonical meaning is intended: if a Scan +// method doesn't have a fmt.ScanState as its first argument, +// we let it go. But if it does have a fmt.ScanState, then the +// rest has to match. +var canonicalMethods = map[string]struct{ args, results []string }{ + // "Flush": {{}, {"error"}}, // http.Flusher and jpeg.writer conflict + "Format": {[]string{"=fmt.State", "rune"}, []string{}}, // fmt.Formatter + "GobDecode": {[]string{"[]byte"}, []string{"error"}}, // gob.GobDecoder + "GobEncode": {[]string{}, []string{"[]byte", "error"}}, // gob.GobEncoder + "MarshalJSON": {[]string{}, []string{"[]byte", "error"}}, // json.Marshaler + "MarshalXML": {[]string{"*xml.Encoder", "xml.StartElement"}, []string{"error"}}, // xml.Marshaler + "ReadByte": {[]string{}, []string{"byte", "error"}}, // io.ByteReader + "ReadFrom": {[]string{"=io.Reader"}, []string{"int64", "error"}}, // io.ReaderFrom + "ReadRune": {[]string{}, []string{"rune", "int", "error"}}, // io.RuneReader + "Scan": {[]string{"=fmt.ScanState", "rune"}, []string{"error"}}, // fmt.Scanner + "Seek": {[]string{"=int64", "int"}, []string{"int64", "error"}}, // io.Seeker + "UnmarshalJSON": {[]string{"[]byte"}, []string{"error"}}, // json.Unmarshaler + "UnmarshalXML": {[]string{"*xml.Decoder", "xml.StartElement"}, []string{"error"}}, // xml.Unmarshaler + "UnreadByte": {[]string{}, []string{"error"}}, + "UnreadRune": {[]string{}, []string{"error"}}, + "WriteByte": {[]string{"byte"}, []string{"error"}}, // jpeg.writer (matching bufio.Writer) + "WriteTo": {[]string{"=io.Writer"}, []string{"int64", "error"}}, // io.WriterTo +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.FuncDecl)(nil), + (*ast.InterfaceType)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + switch n := n.(type) { + case *ast.FuncDecl: + if n.Recv != nil { + canonicalMethod(pass, n.Name, n.Type) + } + case *ast.InterfaceType: + for _, field := range n.Methods.List { + for _, id := range field.Names { + canonicalMethod(pass, id, field.Type.(*ast.FuncType)) + } + } + } + }) + return nil, nil +} + +func canonicalMethod(pass *analysis.Pass, id *ast.Ident, t *ast.FuncType) { + // Expected input/output. + expect, ok := canonicalMethods[id.Name] + if !ok { + return + } + + // Actual input/output + args := typeFlatten(t.Params.List) + var results []ast.Expr + if t.Results != nil { + results = typeFlatten(t.Results.List) + } + + // Do the =s (if any) all match? + if !matchParams(pass, expect.args, args, "=") || !matchParams(pass, expect.results, results, "=") { + return + } + + // Everything must match. + if !matchParams(pass, expect.args, args, "") || !matchParams(pass, expect.results, results, "") { + expectFmt := id.Name + "(" + argjoin(expect.args) + ")" + if len(expect.results) == 1 { + expectFmt += " " + argjoin(expect.results) + } else if len(expect.results) > 1 { + expectFmt += " (" + argjoin(expect.results) + ")" + } + + var buf bytes.Buffer + if err := printer.Fprint(&buf, pass.Fset, t); err != nil { + fmt.Fprintf(&buf, "<%s>", err) + } + actual := buf.String() + actual = strings.TrimPrefix(actual, "func") + actual = id.Name + actual + + pass.Reportf(id.Pos(), "method %s should have signature %s", actual, expectFmt) + } +} + +func argjoin(x []string) string { + y := make([]string, len(x)) + for i, s := range x { + if s[0] == '=' { + s = s[1:] + } + y[i] = s + } + return strings.Join(y, ", ") +} + +// Turn parameter list into slice of types +// (in the ast, types are Exprs). +// Have to handle f(int, bool) and f(x, y, z int) +// so not a simple 1-to-1 conversion. +func typeFlatten(l []*ast.Field) []ast.Expr { + var t []ast.Expr + for _, f := range l { + if len(f.Names) == 0 { + t = append(t, f.Type) + continue + } + for range f.Names { + t = append(t, f.Type) + } + } + return t +} + +// Does each type in expect with the given prefix match the corresponding type in actual? +func matchParams(pass *analysis.Pass, expect []string, actual []ast.Expr, prefix string) bool { + for i, x := range expect { + if !strings.HasPrefix(x, prefix) { + continue + } + if i >= len(actual) { + return false + } + if !matchParamType(pass.Fset, pass.Pkg, x, actual[i]) { + return false + } + } + if prefix == "" && len(actual) > len(expect) { + return false + } + return true +} + +// Does this one type match? +func matchParamType(fset *token.FileSet, pkg *types.Package, expect string, actual ast.Expr) bool { + expect = strings.TrimPrefix(expect, "=") + // Strip package name if we're in that package. + if n := len(pkg.Name()); len(expect) > n && expect[:n] == pkg.Name() && expect[n] == '.' { + expect = expect[n+1:] + } + + // Overkill but easy. + var buf bytes.Buffer + printer.Fprint(&buf, fset, actual) + return buf.String() == expect +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/structtag/structtag.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/structtag/structtag.go new file mode 100644 index 0000000000..78133fe6f3 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/structtag/structtag.go @@ -0,0 +1,260 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package structtag defines an Analyzer that checks struct field tags +// are well formed. +package structtag + +import ( + "errors" + "go/ast" + "go/token" + "go/types" + "path/filepath" + "reflect" + "strconv" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check that struct field tags conform to reflect.StructTag.Get + +Also report certain struct tags (json, xml) used with unexported fields.` + +var Analyzer = &analysis.Analyzer{ + Name: "structtag", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.StructType)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + styp := pass.TypesInfo.Types[n.(*ast.StructType)].Type.(*types.Struct) + var seen map[[2]string]token.Pos + for i := 0; i < styp.NumFields(); i++ { + field := styp.Field(i) + tag := styp.Tag(i) + checkCanonicalFieldTag(pass, field, tag, &seen) + } + }) + return nil, nil +} + +var checkTagDups = []string{"json", "xml"} +var checkTagSpaces = map[string]bool{"json": true, "xml": true, "asn1": true} + +// checkCanonicalFieldTag checks a single struct field tag. +func checkCanonicalFieldTag(pass *analysis.Pass, field *types.Var, tag string, seen *map[[2]string]token.Pos) { + for _, key := range checkTagDups { + checkTagDuplicates(pass, tag, key, field, field, seen) + } + + if err := validateStructTag(tag); err != nil { + pass.Reportf(field.Pos(), "struct field tag %#q not compatible with reflect.StructTag.Get: %s", tag, err) + } + + // Check for use of json or xml tags with unexported fields. + + // Embedded struct. Nothing to do for now, but that + // may change, depending on what happens with issue 7363. + // TODO(adonovan): investigate, now that that issue is fixed. + if field.Anonymous() { + return + } + + if field.Exported() { + return + } + + for _, enc := range [...]string{"json", "xml"} { + if reflect.StructTag(tag).Get(enc) != "" { + pass.Reportf(field.Pos(), "struct field %s has %s tag but is not exported", field.Name(), enc) + return + } + } +} + +// checkTagDuplicates checks a single struct field tag to see if any tags are +// duplicated. nearest is the field that's closest to the field being checked, +// while still being part of the top-level struct type. +func checkTagDuplicates(pass *analysis.Pass, tag, key string, nearest, field *types.Var, seen *map[[2]string]token.Pos) { + val := reflect.StructTag(tag).Get(key) + if val == "-" { + // Ignored, even if the field is anonymous. + return + } + if val == "" || val[0] == ',' { + if field.Anonymous() { + typ, ok := field.Type().Underlying().(*types.Struct) + if !ok { + return + } + for i := 0; i < typ.NumFields(); i++ { + field := typ.Field(i) + if !field.Exported() { + continue + } + tag := typ.Tag(i) + checkTagDuplicates(pass, tag, key, nearest, field, seen) + } + } + // Ignored if the field isn't anonymous. + return + } + if key == "xml" && field.Name() == "XMLName" { + // XMLName defines the XML element name of the struct being + // checked. That name cannot collide with element or attribute + // names defined on other fields of the struct. Vet does not have a + // check for untagged fields of type struct defining their own name + // by containing a field named XMLName; see issue 18256. + return + } + if i := strings.Index(val, ","); i >= 0 { + if key == "xml" { + // Use a separate namespace for XML attributes. + for _, opt := range strings.Split(val[i:], ",") { + if opt == "attr" { + key += " attribute" // Key is part of the error message. + break + } + } + } + val = val[:i] + } + if *seen == nil { + *seen = map[[2]string]token.Pos{} + } + if pos, ok := (*seen)[[2]string{key, val}]; ok { + posn := pass.Fset.Position(pos) + posn.Filename = filepath.Base(posn.Filename) + posn.Column = 0 + pass.Reportf(nearest.Pos(), "struct field %s repeats %s tag %q also at %s", field.Name(), key, val, posn) + } else { + (*seen)[[2]string{key, val}] = field.Pos() + } +} + +var ( + errTagSyntax = errors.New("bad syntax for struct tag pair") + errTagKeySyntax = errors.New("bad syntax for struct tag key") + errTagValueSyntax = errors.New("bad syntax for struct tag value") + errTagValueSpace = errors.New("suspicious space in struct tag value") + errTagSpace = errors.New("key:\"value\" pairs not separated by spaces") +) + +// validateStructTag parses the struct tag and returns an error if it is not +// in the canonical format, which is a space-separated list of key:"value" +// settings. The value may contain spaces. +func validateStructTag(tag string) error { + // This code is based on the StructTag.Get code in package reflect. + + n := 0 + for ; tag != ""; n++ { + if n > 0 && tag != "" && tag[0] != ' ' { + // More restrictive than reflect, but catches likely mistakes + // like `x:"foo",y:"bar"`, which parses as `x:"foo" ,y:"bar"` with second key ",y". + return errTagSpace + } + // Skip leading space. + i := 0 + for i < len(tag) && tag[i] == ' ' { + i++ + } + tag = tag[i:] + if tag == "" { + break + } + + // Scan to colon. A space, a quote or a control character is a syntax error. + // Strictly speaking, control chars include the range [0x7f, 0x9f], not just + // [0x00, 0x1f], but in practice, we ignore the multi-byte control characters + // as it is simpler to inspect the tag's bytes than the tag's runes. + i = 0 + for i < len(tag) && tag[i] > ' ' && tag[i] != ':' && tag[i] != '"' && tag[i] != 0x7f { + i++ + } + if i == 0 { + return errTagKeySyntax + } + if i+1 >= len(tag) || tag[i] != ':' { + return errTagSyntax + } + if tag[i+1] != '"' { + return errTagValueSyntax + } + key := tag[:i] + tag = tag[i+1:] + + // Scan quoted string to find value. + i = 1 + for i < len(tag) && tag[i] != '"' { + if tag[i] == '\\' { + i++ + } + i++ + } + if i >= len(tag) { + return errTagValueSyntax + } + qvalue := tag[:i+1] + tag = tag[i+1:] + + value, err := strconv.Unquote(qvalue) + if err != nil { + return errTagValueSyntax + } + + if !checkTagSpaces[key] { + continue + } + + switch key { + case "xml": + // If the first or last character in the XML tag is a space, it is + // suspicious. + if strings.Trim(value, " ") != value { + return errTagValueSpace + } + + // If there are multiple spaces, they are suspicious. + if strings.Count(value, " ") > 1 { + return errTagValueSpace + } + + // If there is no comma, skip the rest of the checks. + comma := strings.IndexRune(value, ',') + if comma < 0 { + continue + } + + // If the character before a comma is a space, this is suspicious. + if comma > 0 && value[comma-1] == ' ' { + return errTagValueSpace + } + value = value[comma+1:] + case "json": + // JSON allows using spaces in the name, so skip it. + comma := strings.IndexRune(value, ',') + if comma < 0 { + continue + } + value = value[comma+1:] + } + + if strings.IndexByte(value, ' ') >= 0 { + return errTagValueSpace + } + } + return nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/tests/tests.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/tests/tests.go new file mode 100644 index 0000000000..35b0a3e7cc --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/tests/tests.go @@ -0,0 +1,175 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tests defines an Analyzer that checks for common mistaken +// usages of tests and examples. +package tests + +import ( + "go/ast" + "go/types" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/tools/go/analysis" +) + +const Doc = `check for common mistaken usages of tests and examples + +The tests checker walks Test, Benchmark and Example functions checking +malformed names, wrong signatures and examples documenting non-existent +identifiers.` + +var Analyzer = &analysis.Analyzer{ + Name: "tests", + Doc: Doc, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + for _, f := range pass.Files { + if !strings.HasSuffix(pass.Fset.File(f.Pos()).Name(), "_test.go") { + continue + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv != nil { + // Ignore non-functions or functions with receivers. + continue + } + + switch { + case strings.HasPrefix(fn.Name.Name, "Example"): + checkExample(pass, fn) + case strings.HasPrefix(fn.Name.Name, "Test"): + checkTest(pass, fn, "Test") + case strings.HasPrefix(fn.Name.Name, "Benchmark"): + checkTest(pass, fn, "Benchmark") + } + } + } + return nil, nil +} + +func isExampleSuffix(s string) bool { + r, size := utf8.DecodeRuneInString(s) + return size > 0 && unicode.IsLower(r) +} + +func isTestSuffix(name string) bool { + if len(name) == 0 { + // "Test" is ok. + return true + } + r, _ := utf8.DecodeRuneInString(name) + return !unicode.IsLower(r) +} + +func isTestParam(typ ast.Expr, wantType string) bool { + ptr, ok := typ.(*ast.StarExpr) + if !ok { + // Not a pointer. + return false + } + // No easy way of making sure it's a *testing.T or *testing.B: + // ensure the name of the type matches. + if name, ok := ptr.X.(*ast.Ident); ok { + return name.Name == wantType + } + if sel, ok := ptr.X.(*ast.SelectorExpr); ok { + return sel.Sel.Name == wantType + } + return false +} + +func lookup(pkg *types.Package, name string) types.Object { + if o := pkg.Scope().Lookup(name); o != nil { + return o + } + + // If this package is ".../foo_test" and it imports a package + // ".../foo", try looking in the latter package. + // This heuristic should work even on build systems that do not + // record any special link between the packages. + if basePath := strings.TrimSuffix(pkg.Path(), "_test"); basePath != pkg.Path() { + for _, imp := range pkg.Imports() { + if imp.Path() == basePath { + return imp.Scope().Lookup(name) + } + } + } + return nil +} + +func checkExample(pass *analysis.Pass, fn *ast.FuncDecl) { + fnName := fn.Name.Name + if params := fn.Type.Params; len(params.List) != 0 { + pass.Reportf(fn.Pos(), "%s should be niladic", fnName) + } + if results := fn.Type.Results; results != nil && len(results.List) != 0 { + pass.Reportf(fn.Pos(), "%s should return nothing", fnName) + } + + if fnName == "Example" { + // Nothing more to do. + return + } + + var ( + exName = strings.TrimPrefix(fnName, "Example") + elems = strings.SplitN(exName, "_", 3) + ident = elems[0] + obj = lookup(pass.Pkg, ident) + ) + if ident != "" && obj == nil { + // Check ExampleFoo and ExampleBadFoo. + pass.Reportf(fn.Pos(), "%s refers to unknown identifier: %s", fnName, ident) + // Abort since obj is absent and no subsequent checks can be performed. + return + } + if len(elems) < 2 { + // Nothing more to do. + return + } + + if ident == "" { + // Check Example_suffix and Example_BadSuffix. + if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) { + pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, residual) + } + return + } + + mmbr := elems[1] + if !isExampleSuffix(mmbr) { + // Check ExampleFoo_Method and ExampleFoo_BadMethod. + if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj == nil { + pass.Reportf(fn.Pos(), "%s refers to unknown field or method: %s.%s", fnName, ident, mmbr) + } + } + if len(elems) == 3 && !isExampleSuffix(elems[2]) { + // Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix. + pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, elems[2]) + } +} + +func checkTest(pass *analysis.Pass, fn *ast.FuncDecl, prefix string) { + // Want functions with 0 results and 1 parameter. + if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 || + fn.Type.Params == nil || + len(fn.Type.Params.List) != 1 || + len(fn.Type.Params.List[0].Names) > 1 { + return + } + + // The param must look like a *testing.T or *testing.B. + if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) { + return + } + + if !isTestSuffix(fn.Name.Name[len(prefix):]) { + pass.Reportf(fn.Pos(), "%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix) + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unreachable/unreachable.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unreachable/unreachable.go new file mode 100644 index 0000000000..19bc9c2db9 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unreachable/unreachable.go @@ -0,0 +1,314 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package unreachable defines an Analyzer that checks for unreachable code. +package unreachable + +// TODO(adonovan): use the new cfg package, which is more precise. + +import ( + "go/ast" + "go/token" + "log" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for unreachable code + +The unreachable analyzer finds statements that execution can never reach +because they are preceded by an return statement, a call to panic, an +infinite loop, or similar constructs.` + +var Analyzer = &analysis.Analyzer{ + Name: "unreachable", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + RunDespiteErrors: true, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.FuncDecl)(nil), + (*ast.FuncLit)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + var body *ast.BlockStmt + switch n := n.(type) { + case *ast.FuncDecl: + body = n.Body + case *ast.FuncLit: + body = n.Body + } + if body == nil { + return + } + d := &deadState{ + pass: pass, + hasBreak: make(map[ast.Stmt]bool), + hasGoto: make(map[string]bool), + labels: make(map[string]ast.Stmt), + } + d.findLabels(body) + d.reachable = true + d.findDead(body) + }) + return nil, nil +} + +type deadState struct { + pass *analysis.Pass + hasBreak map[ast.Stmt]bool + hasGoto map[string]bool + labels map[string]ast.Stmt + breakTarget ast.Stmt + + reachable bool +} + +// findLabels gathers information about the labels defined and used by stmt +// and about which statements break, whether a label is involved or not. +func (d *deadState) findLabels(stmt ast.Stmt) { + switch x := stmt.(type) { + default: + log.Fatalf("%s: internal error in findLabels: unexpected statement %T", d.pass.Fset.Position(x.Pos()), x) + + case *ast.AssignStmt, + *ast.BadStmt, + *ast.DeclStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.ExprStmt, + *ast.GoStmt, + *ast.IncDecStmt, + *ast.ReturnStmt, + *ast.SendStmt: + // no statements inside + + case *ast.BlockStmt: + for _, stmt := range x.List { + d.findLabels(stmt) + } + + case *ast.BranchStmt: + switch x.Tok { + case token.GOTO: + if x.Label != nil { + d.hasGoto[x.Label.Name] = true + } + + case token.BREAK: + stmt := d.breakTarget + if x.Label != nil { + stmt = d.labels[x.Label.Name] + } + if stmt != nil { + d.hasBreak[stmt] = true + } + } + + case *ast.IfStmt: + d.findLabels(x.Body) + if x.Else != nil { + d.findLabels(x.Else) + } + + case *ast.LabeledStmt: + d.labels[x.Label.Name] = x.Stmt + d.findLabels(x.Stmt) + + // These cases are all the same, but the x.Body only works + // when the specific type of x is known, so the cases cannot + // be merged. + case *ast.ForStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.RangeStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.SelectStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.SwitchStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.TypeSwitchStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.CommClause: + for _, stmt := range x.Body { + d.findLabels(stmt) + } + + case *ast.CaseClause: + for _, stmt := range x.Body { + d.findLabels(stmt) + } + } +} + +// findDead walks the statement looking for dead code. +// If d.reachable is false on entry, stmt itself is dead. +// When findDead returns, d.reachable tells whether the +// statement following stmt is reachable. +func (d *deadState) findDead(stmt ast.Stmt) { + // Is this a labeled goto target? + // If so, assume it is reachable due to the goto. + // This is slightly conservative, in that we don't + // check that the goto is reachable, so + // L: goto L + // will not provoke a warning. + // But it's good enough. + if x, isLabel := stmt.(*ast.LabeledStmt); isLabel && d.hasGoto[x.Label.Name] { + d.reachable = true + } + + if !d.reachable { + switch stmt.(type) { + case *ast.EmptyStmt: + // do not warn about unreachable empty statements + default: + d.pass.Reportf(stmt.Pos(), "unreachable code") + d.reachable = true // silence error about next statement + } + } + + switch x := stmt.(type) { + default: + log.Fatalf("%s: internal error in findDead: unexpected statement %T", d.pass.Fset.Position(x.Pos()), x) + + case *ast.AssignStmt, + *ast.BadStmt, + *ast.DeclStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.GoStmt, + *ast.IncDecStmt, + *ast.SendStmt: + // no control flow + + case *ast.BlockStmt: + for _, stmt := range x.List { + d.findDead(stmt) + } + + case *ast.BranchStmt: + switch x.Tok { + case token.BREAK, token.GOTO, token.FALLTHROUGH: + d.reachable = false + case token.CONTINUE: + // NOTE: We accept "continue" statements as terminating. + // They are not necessary in the spec definition of terminating, + // because a continue statement cannot be the final statement + // before a return. But for the more general problem of syntactically + // identifying dead code, continue redirects control flow just + // like the other terminating statements. + d.reachable = false + } + + case *ast.ExprStmt: + // Call to panic? + call, ok := x.X.(*ast.CallExpr) + if ok { + name, ok := call.Fun.(*ast.Ident) + if ok && name.Name == "panic" && name.Obj == nil { + d.reachable = false + } + } + + case *ast.ForStmt: + d.findDead(x.Body) + d.reachable = x.Cond != nil || d.hasBreak[x] + + case *ast.IfStmt: + d.findDead(x.Body) + if x.Else != nil { + r := d.reachable + d.reachable = true + d.findDead(x.Else) + d.reachable = d.reachable || r + } else { + // might not have executed if statement + d.reachable = true + } + + case *ast.LabeledStmt: + d.findDead(x.Stmt) + + case *ast.RangeStmt: + d.findDead(x.Body) + d.reachable = true + + case *ast.ReturnStmt: + d.reachable = false + + case *ast.SelectStmt: + // NOTE: Unlike switch and type switch below, we don't care + // whether a select has a default, because a select without a + // default blocks until one of the cases can run. That's different + // from a switch without a default, which behaves like it has + // a default with an empty body. + anyReachable := false + for _, comm := range x.Body.List { + d.reachable = true + for _, stmt := range comm.(*ast.CommClause).Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] + + case *ast.SwitchStmt: + anyReachable := false + hasDefault := false + for _, cas := range x.Body.List { + cc := cas.(*ast.CaseClause) + if cc.List == nil { + hasDefault = true + } + d.reachable = true + for _, stmt := range cc.Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] || !hasDefault + + case *ast.TypeSwitchStmt: + anyReachable := false + hasDefault := false + for _, cas := range x.Body.List { + cc := cas.(*ast.CaseClause) + if cc.List == nil { + hasDefault = true + } + d.reachable = true + for _, stmt := range cc.Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] || !hasDefault + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unsafeptr/unsafeptr.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unsafeptr/unsafeptr.go new file mode 100644 index 0000000000..116d622b36 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unsafeptr/unsafeptr.go @@ -0,0 +1,130 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package unsafeptr defines an Analyzer that checks for invalid +// conversions of uintptr to unsafe.Pointer. +package unsafeptr + +import ( + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +const Doc = `check for invalid conversions of uintptr to unsafe.Pointer + +The unsafeptr analyzer reports likely incorrect uses of unsafe.Pointer +to convert integers to pointers. A conversion from uintptr to +unsafe.Pointer is invalid if it implies that there is a uintptr-typed +word in memory that holds a pointer value, because that word will be +invisible to stack copying and to the garbage collector.` + +var Analyzer = &analysis.Analyzer{ + Name: "unsafeptr", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.CallExpr)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + x := n.(*ast.CallExpr) + if len(x.Args) != 1 { + return + } + if hasBasicType(pass.TypesInfo, x.Fun, types.UnsafePointer) && + hasBasicType(pass.TypesInfo, x.Args[0], types.Uintptr) && + !isSafeUintptr(pass.TypesInfo, x.Args[0]) { + pass.Reportf(x.Pos(), "possible misuse of unsafe.Pointer") + } + }) + return nil, nil +} + +// isSafeUintptr reports whether x - already known to be a uintptr - +// is safe to convert to unsafe.Pointer. It is safe if x is itself derived +// directly from an unsafe.Pointer via conversion and pointer arithmetic +// or if x is the result of reflect.Value.Pointer or reflect.Value.UnsafeAddr +// or obtained from the Data field of a *reflect.SliceHeader or *reflect.StringHeader. +func isSafeUintptr(info *types.Info, x ast.Expr) bool { + switch x := x.(type) { + case *ast.ParenExpr: + return isSafeUintptr(info, x.X) + + case *ast.SelectorExpr: + switch x.Sel.Name { + case "Data": + // reflect.SliceHeader and reflect.StringHeader are okay, + // but only if they are pointing at a real slice or string. + // It's not okay to do: + // var x SliceHeader + // x.Data = uintptr(unsafe.Pointer(...)) + // ... use x ... + // p := unsafe.Pointer(x.Data) + // because in the middle the garbage collector doesn't + // see x.Data as a pointer and so x.Data may be dangling + // by the time we get to the conversion at the end. + // For now approximate by saying that *Header is okay + // but Header is not. + pt, ok := info.Types[x.X].Type.(*types.Pointer) + if ok { + t, ok := pt.Elem().(*types.Named) + if ok && t.Obj().Pkg().Path() == "reflect" { + switch t.Obj().Name() { + case "StringHeader", "SliceHeader": + return true + } + } + } + } + + case *ast.CallExpr: + switch len(x.Args) { + case 0: + // maybe call to reflect.Value.Pointer or reflect.Value.UnsafeAddr. + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok { + break + } + switch sel.Sel.Name { + case "Pointer", "UnsafeAddr": + t, ok := info.Types[sel.X].Type.(*types.Named) + if ok && t.Obj().Pkg().Path() == "reflect" && t.Obj().Name() == "Value" { + return true + } + } + + case 1: + // maybe conversion of uintptr to unsafe.Pointer + return hasBasicType(info, x.Fun, types.Uintptr) && + hasBasicType(info, x.Args[0], types.UnsafePointer) + } + + case *ast.BinaryExpr: + switch x.Op { + case token.ADD, token.SUB, token.AND_NOT: + return isSafeUintptr(info, x.X) && !isSafeUintptr(info, x.Y) + } + } + return false +} + +// hasBasicType reports whether x's type is a types.Basic with the given kind. +func hasBasicType(info *types.Info, x ast.Expr, kind types.BasicKind) bool { + t := info.Types[x].Type + if t != nil { + t = t.Underlying() + } + b, ok := t.(*types.Basic) + return ok && b.Kind() == kind +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unusedresult/unusedresult.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unusedresult/unusedresult.go new file mode 100644 index 0000000000..76d4ab2382 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/unusedresult/unusedresult.go @@ -0,0 +1,131 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package unusedresult defines an analyzer that checks for unused +// results of calls to certain pure functions. +package unusedresult + +import ( + "go/ast" + "go/token" + "go/types" + "sort" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +// TODO(adonovan): make this analysis modular: export a mustUseResult +// fact for each function that tail-calls one of the functions that we +// check, and check those functions too. + +const Doc = `check for unused results of calls to some functions + +Some functions like fmt.Errorf return a result and have no side effects, +so it is always a mistake to discard the result. This analyzer reports +calls to certain functions in which the result of the call is ignored. + +The set of functions may be controlled using flags.` + +var Analyzer = &analysis.Analyzer{ + Name: "unusedresult", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +// flags +var funcs, stringMethods stringSetFlag + +func init() { + // TODO(adonovan): provide a comment syntax to allow users to + // add their functions to this set using facts. + funcs.Set("errors.New,fmt.Errorf,fmt.Sprintf,fmt.Sprint,sort.Reverse") + Analyzer.Flags.Var(&funcs, "funcs", + "comma-separated list of functions whose results must be used") + + stringMethods.Set("Error,String") + Analyzer.Flags.Var(&stringMethods, "stringmethods", + "comma-separated list of names of methods of type func() string whose results must be used") +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.ExprStmt)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + call, ok := analysisutil.Unparen(n.(*ast.ExprStmt).X).(*ast.CallExpr) + if !ok { + return // not a call statement + } + fun := analysisutil.Unparen(call.Fun) + + if pass.TypesInfo.Types[fun].IsType() { + return // a conversion, not a call + } + + selector, ok := fun.(*ast.SelectorExpr) + if !ok { + return // neither a method call nor a qualified ident + } + + sel, ok := pass.TypesInfo.Selections[selector] + if ok && sel.Kind() == types.MethodVal { + // method (e.g. foo.String()) + obj := sel.Obj().(*types.Func) + sig := sel.Type().(*types.Signature) + if types.Identical(sig, sigNoArgsStringResult) { + if stringMethods[obj.Name()] { + pass.Reportf(call.Lparen, "result of (%s).%s call not used", + sig.Recv().Type(), obj.Name()) + } + } + } else if !ok { + // package-qualified function (e.g. fmt.Errorf) + obj := pass.TypesInfo.Uses[selector.Sel] + if obj, ok := obj.(*types.Func); ok { + qname := obj.Pkg().Path() + "." + obj.Name() + if funcs[qname] { + pass.Reportf(call.Lparen, "result of %v call not used", qname) + } + } + } + }) + return nil, nil +} + +// func() string +var sigNoArgsStringResult = types.NewSignature(nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String])), + false) + +type stringSetFlag map[string]bool + +func (ss *stringSetFlag) String() string { + var items []string + for item := range *ss { + items = append(items, item) + } + sort.Strings(items) + return strings.Join(items, ",") +} + +func (ss *stringSetFlag) Set(s string) error { + m := make(map[string]bool) // clobber previous value + if s != "" { + for _, name := range strings.Split(s, ",") { + if name == "" { + continue // TODO: report error? proceed? + } + m[name] = true + } + } + *ss = m + return nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/validate.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/validate.go new file mode 100644 index 0000000000..6e6cf4984f --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/validate.go @@ -0,0 +1,104 @@ +package analysis + +import ( + "fmt" + "reflect" + "unicode" +) + +// Validate reports an error if any of the analyzers are misconfigured. +// Checks include: +// that the name is a valid identifier; +// that analyzer names are unique; +// that the Requires graph is acylic; +// that analyzer fact types are unique; +// that each fact type is a pointer. +func Validate(analyzers []*Analyzer) error { + names := make(map[string]bool) + + // Map each fact type to its sole generating analyzer. + factTypes := make(map[reflect.Type]*Analyzer) + + // Traverse the Requires graph, depth first. + const ( + white = iota + grey + black + finished + ) + color := make(map[*Analyzer]uint8) + var visit func(a *Analyzer) error + visit = func(a *Analyzer) error { + if a == nil { + return fmt.Errorf("nil *Analyzer") + } + if color[a] == white { + color[a] = grey + + // names + if !validIdent(a.Name) { + return fmt.Errorf("invalid analyzer name %q", a) + } + if names[a.Name] { + return fmt.Errorf("duplicate analyzer name %q", a) + } + names[a.Name] = true + + if a.Doc == "" { + return fmt.Errorf("analyzer %q is undocumented", a) + } + + // fact types + for _, f := range a.FactTypes { + if f == nil { + return fmt.Errorf("analyzer %s has nil FactType", a) + } + t := reflect.TypeOf(f) + if prev := factTypes[t]; prev != nil { + return fmt.Errorf("fact type %s registered by two analyzers: %v, %v", + t, a, prev) + } + if t.Kind() != reflect.Ptr { + return fmt.Errorf("%s: fact type %s is not a pointer", a, t) + } + factTypes[t] = a + } + + // recursion + for i, req := range a.Requires { + if err := visit(req); err != nil { + return fmt.Errorf("%s.Requires[%d]: %v", a.Name, i, err) + } + } + color[a] = black + } + + return nil + } + for _, a := range analyzers { + if err := visit(a); err != nil { + return err + } + } + + // Reject duplicates among analyzers. + // Precondition: color[a] == black. + // Postcondition: color[a] == finished. + for _, a := range analyzers { + if color[a] == finished { + return fmt.Errorf("duplicate analyzer: %s", a.Name) + } + color[a] = finished + } + + return nil +} + +func validIdent(name string) bool { + for i, r := range name { + if !(r == '_' || unicode.IsLetter(r) || i > 0 && unicode.IsDigit(r)) { + return false + } + } + return name != "" +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/enclosing.go b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/enclosing.go new file mode 100644 index 0000000000..6b7052b892 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/enclosing.go @@ -0,0 +1,627 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package astutil + +// This file defines utilities for working with source positions. + +import ( + "fmt" + "go/ast" + "go/token" + "sort" +) + +// PathEnclosingInterval returns the node that encloses the source +// interval [start, end), and all its ancestors up to the AST root. +// +// The definition of "enclosing" used by this function considers +// additional whitespace abutting a node to be enclosed by it. +// In this example: +// +// z := x + y // add them +// <-A-> +// <----B-----> +// +// the ast.BinaryExpr(+) node is considered to enclose interval B +// even though its [Pos()..End()) is actually only interval A. +// This behaviour makes user interfaces more tolerant of imperfect +// input. +// +// This function treats tokens as nodes, though they are not included +// in the result. e.g. PathEnclosingInterval("+") returns the +// enclosing ast.BinaryExpr("x + y"). +// +// If start==end, the 1-char interval following start is used instead. +// +// The 'exact' result is true if the interval contains only path[0] +// and perhaps some adjacent whitespace. It is false if the interval +// overlaps multiple children of path[0], or if it contains only +// interior whitespace of path[0]. +// In this example: +// +// z := x + y // add them +// <--C--> <---E--> +// ^ +// D +// +// intervals C, D and E are inexact. C is contained by the +// z-assignment statement, because it spans three of its children (:=, +// x, +). So too is the 1-char interval D, because it contains only +// interior whitespace of the assignment. E is considered interior +// whitespace of the BlockStmt containing the assignment. +// +// Precondition: [start, end) both lie within the same file as root. +// TODO(adonovan): return (nil, false) in this case and remove precond. +// Requires FileSet; see loader.tokenFileContainsPos. +// +// Postcondition: path is never nil; it always contains at least 'root'. +// +func PathEnclosingInterval(root *ast.File, start, end token.Pos) (path []ast.Node, exact bool) { + // fmt.Printf("EnclosingInterval %d %d\n", start, end) // debugging + + // Precondition: node.[Pos..End) and adjoining whitespace contain [start, end). + var visit func(node ast.Node) bool + visit = func(node ast.Node) bool { + path = append(path, node) + + nodePos := node.Pos() + nodeEnd := node.End() + + // fmt.Printf("visit(%T, %d, %d)\n", node, nodePos, nodeEnd) // debugging + + // Intersect [start, end) with interval of node. + if start < nodePos { + start = nodePos + } + if end > nodeEnd { + end = nodeEnd + } + + // Find sole child that contains [start, end). + children := childrenOf(node) + l := len(children) + for i, child := range children { + // [childPos, childEnd) is unaugmented interval of child. + childPos := child.Pos() + childEnd := child.End() + + // [augPos, augEnd) is whitespace-augmented interval of child. + augPos := childPos + augEnd := childEnd + if i > 0 { + augPos = children[i-1].End() // start of preceding whitespace + } + if i < l-1 { + nextChildPos := children[i+1].Pos() + // Does [start, end) lie between child and next child? + if start >= augEnd && end <= nextChildPos { + return false // inexact match + } + augEnd = nextChildPos // end of following whitespace + } + + // fmt.Printf("\tchild %d: [%d..%d)\tcontains interval [%d..%d)?\n", + // i, augPos, augEnd, start, end) // debugging + + // Does augmented child strictly contain [start, end)? + if augPos <= start && end <= augEnd { + _, isToken := child.(tokenNode) + return isToken || visit(child) + } + + // Does [start, end) overlap multiple children? + // i.e. left-augmented child contains start + // but LR-augmented child does not contain end. + if start < childEnd && end > augEnd { + break + } + } + + // No single child contained [start, end), + // so node is the result. Is it exact? + + // (It's tempting to put this condition before the + // child loop, but it gives the wrong result in the + // case where a node (e.g. ExprStmt) and its sole + // child have equal intervals.) + if start == nodePos && end == nodeEnd { + return true // exact match + } + + return false // inexact: overlaps multiple children + } + + if start > end { + start, end = end, start + } + + if start < root.End() && end > root.Pos() { + if start == end { + end = start + 1 // empty interval => interval of size 1 + } + exact = visit(root) + + // Reverse the path: + for i, l := 0, len(path); i < l/2; i++ { + path[i], path[l-1-i] = path[l-1-i], path[i] + } + } else { + // Selection lies within whitespace preceding the + // first (or following the last) declaration in the file. + // The result nonetheless always includes the ast.File. + path = append(path, root) + } + + return +} + +// tokenNode is a dummy implementation of ast.Node for a single token. +// They are used transiently by PathEnclosingInterval but never escape +// this package. +// +type tokenNode struct { + pos token.Pos + end token.Pos +} + +func (n tokenNode) Pos() token.Pos { + return n.pos +} + +func (n tokenNode) End() token.Pos { + return n.end +} + +func tok(pos token.Pos, len int) ast.Node { + return tokenNode{pos, pos + token.Pos(len)} +} + +// childrenOf returns the direct non-nil children of ast.Node n. +// It may include fake ast.Node implementations for bare tokens. +// it is not safe to call (e.g.) ast.Walk on such nodes. +// +func childrenOf(n ast.Node) []ast.Node { + var children []ast.Node + + // First add nodes for all true subtrees. + ast.Inspect(n, func(node ast.Node) bool { + if node == n { // push n + return true // recur + } + if node != nil { // push child + children = append(children, node) + } + return false // no recursion + }) + + // Then add fake Nodes for bare tokens. + switch n := n.(type) { + case *ast.ArrayType: + children = append(children, + tok(n.Lbrack, len("[")), + tok(n.Elt.End(), len("]"))) + + case *ast.AssignStmt: + children = append(children, + tok(n.TokPos, len(n.Tok.String()))) + + case *ast.BasicLit: + children = append(children, + tok(n.ValuePos, len(n.Value))) + + case *ast.BinaryExpr: + children = append(children, tok(n.OpPos, len(n.Op.String()))) + + case *ast.BlockStmt: + children = append(children, + tok(n.Lbrace, len("{")), + tok(n.Rbrace, len("}"))) + + case *ast.BranchStmt: + children = append(children, + tok(n.TokPos, len(n.Tok.String()))) + + case *ast.CallExpr: + children = append(children, + tok(n.Lparen, len("(")), + tok(n.Rparen, len(")"))) + if n.Ellipsis != 0 { + children = append(children, tok(n.Ellipsis, len("..."))) + } + + case *ast.CaseClause: + if n.List == nil { + children = append(children, + tok(n.Case, len("default"))) + } else { + children = append(children, + tok(n.Case, len("case"))) + } + children = append(children, tok(n.Colon, len(":"))) + + case *ast.ChanType: + switch n.Dir { + case ast.RECV: + children = append(children, tok(n.Begin, len("<-chan"))) + case ast.SEND: + children = append(children, tok(n.Begin, len("chan<-"))) + case ast.RECV | ast.SEND: + children = append(children, tok(n.Begin, len("chan"))) + } + + case *ast.CommClause: + if n.Comm == nil { + children = append(children, + tok(n.Case, len("default"))) + } else { + children = append(children, + tok(n.Case, len("case"))) + } + children = append(children, tok(n.Colon, len(":"))) + + case *ast.Comment: + // nop + + case *ast.CommentGroup: + // nop + + case *ast.CompositeLit: + children = append(children, + tok(n.Lbrace, len("{")), + tok(n.Rbrace, len("{"))) + + case *ast.DeclStmt: + // nop + + case *ast.DeferStmt: + children = append(children, + tok(n.Defer, len("defer"))) + + case *ast.Ellipsis: + children = append(children, + tok(n.Ellipsis, len("..."))) + + case *ast.EmptyStmt: + // nop + + case *ast.ExprStmt: + // nop + + case *ast.Field: + // TODO(adonovan): Field.{Doc,Comment,Tag}? + + case *ast.FieldList: + children = append(children, + tok(n.Opening, len("(")), + tok(n.Closing, len(")"))) + + case *ast.File: + // TODO test: Doc + children = append(children, + tok(n.Package, len("package"))) + + case *ast.ForStmt: + children = append(children, + tok(n.For, len("for"))) + + case *ast.FuncDecl: + // TODO(adonovan): FuncDecl.Comment? + + // Uniquely, FuncDecl breaks the invariant that + // preorder traversal yields tokens in lexical order: + // in fact, FuncDecl.Recv precedes FuncDecl.Type.Func. + // + // As a workaround, we inline the case for FuncType + // here and order things correctly. + // + children = nil // discard ast.Walk(FuncDecl) info subtrees + children = append(children, tok(n.Type.Func, len("func"))) + if n.Recv != nil { + children = append(children, n.Recv) + } + children = append(children, n.Name) + if n.Type.Params != nil { + children = append(children, n.Type.Params) + } + if n.Type.Results != nil { + children = append(children, n.Type.Results) + } + if n.Body != nil { + children = append(children, n.Body) + } + + case *ast.FuncLit: + // nop + + case *ast.FuncType: + if n.Func != 0 { + children = append(children, + tok(n.Func, len("func"))) + } + + case *ast.GenDecl: + children = append(children, + tok(n.TokPos, len(n.Tok.String()))) + if n.Lparen != 0 { + children = append(children, + tok(n.Lparen, len("(")), + tok(n.Rparen, len(")"))) + } + + case *ast.GoStmt: + children = append(children, + tok(n.Go, len("go"))) + + case *ast.Ident: + children = append(children, + tok(n.NamePos, len(n.Name))) + + case *ast.IfStmt: + children = append(children, + tok(n.If, len("if"))) + + case *ast.ImportSpec: + // TODO(adonovan): ImportSpec.{Doc,EndPos}? + + case *ast.IncDecStmt: + children = append(children, + tok(n.TokPos, len(n.Tok.String()))) + + case *ast.IndexExpr: + children = append(children, + tok(n.Lbrack, len("{")), + tok(n.Rbrack, len("}"))) + + case *ast.InterfaceType: + children = append(children, + tok(n.Interface, len("interface"))) + + case *ast.KeyValueExpr: + children = append(children, + tok(n.Colon, len(":"))) + + case *ast.LabeledStmt: + children = append(children, + tok(n.Colon, len(":"))) + + case *ast.MapType: + children = append(children, + tok(n.Map, len("map"))) + + case *ast.ParenExpr: + children = append(children, + tok(n.Lparen, len("(")), + tok(n.Rparen, len(")"))) + + case *ast.RangeStmt: + children = append(children, + tok(n.For, len("for")), + tok(n.TokPos, len(n.Tok.String()))) + + case *ast.ReturnStmt: + children = append(children, + tok(n.Return, len("return"))) + + case *ast.SelectStmt: + children = append(children, + tok(n.Select, len("select"))) + + case *ast.SelectorExpr: + // nop + + case *ast.SendStmt: + children = append(children, + tok(n.Arrow, len("<-"))) + + case *ast.SliceExpr: + children = append(children, + tok(n.Lbrack, len("[")), + tok(n.Rbrack, len("]"))) + + case *ast.StarExpr: + children = append(children, tok(n.Star, len("*"))) + + case *ast.StructType: + children = append(children, tok(n.Struct, len("struct"))) + + case *ast.SwitchStmt: + children = append(children, tok(n.Switch, len("switch"))) + + case *ast.TypeAssertExpr: + children = append(children, + tok(n.Lparen-1, len(".")), + tok(n.Lparen, len("(")), + tok(n.Rparen, len(")"))) + + case *ast.TypeSpec: + // TODO(adonovan): TypeSpec.{Doc,Comment}? + + case *ast.TypeSwitchStmt: + children = append(children, tok(n.Switch, len("switch"))) + + case *ast.UnaryExpr: + children = append(children, tok(n.OpPos, len(n.Op.String()))) + + case *ast.ValueSpec: + // TODO(adonovan): ValueSpec.{Doc,Comment}? + + case *ast.BadDecl, *ast.BadExpr, *ast.BadStmt: + // nop + } + + // TODO(adonovan): opt: merge the logic of ast.Inspect() into + // the switch above so we can make interleaved callbacks for + // both Nodes and Tokens in the right order and avoid the need + // to sort. + sort.Sort(byPos(children)) + + return children +} + +type byPos []ast.Node + +func (sl byPos) Len() int { + return len(sl) +} +func (sl byPos) Less(i, j int) bool { + return sl[i].Pos() < sl[j].Pos() +} +func (sl byPos) Swap(i, j int) { + sl[i], sl[j] = sl[j], sl[i] +} + +// NodeDescription returns a description of the concrete type of n suitable +// for a user interface. +// +// TODO(adonovan): in some cases (e.g. Field, FieldList, Ident, +// StarExpr) we could be much more specific given the path to the AST +// root. Perhaps we should do that. +// +func NodeDescription(n ast.Node) string { + switch n := n.(type) { + case *ast.ArrayType: + return "array type" + case *ast.AssignStmt: + return "assignment" + case *ast.BadDecl: + return "bad declaration" + case *ast.BadExpr: + return "bad expression" + case *ast.BadStmt: + return "bad statement" + case *ast.BasicLit: + return "basic literal" + case *ast.BinaryExpr: + return fmt.Sprintf("binary %s operation", n.Op) + case *ast.BlockStmt: + return "block" + case *ast.BranchStmt: + switch n.Tok { + case token.BREAK: + return "break statement" + case token.CONTINUE: + return "continue statement" + case token.GOTO: + return "goto statement" + case token.FALLTHROUGH: + return "fall-through statement" + } + case *ast.CallExpr: + if len(n.Args) == 1 && !n.Ellipsis.IsValid() { + return "function call (or conversion)" + } + return "function call" + case *ast.CaseClause: + return "case clause" + case *ast.ChanType: + return "channel type" + case *ast.CommClause: + return "communication clause" + case *ast.Comment: + return "comment" + case *ast.CommentGroup: + return "comment group" + case *ast.CompositeLit: + return "composite literal" + case *ast.DeclStmt: + return NodeDescription(n.Decl) + " statement" + case *ast.DeferStmt: + return "defer statement" + case *ast.Ellipsis: + return "ellipsis" + case *ast.EmptyStmt: + return "empty statement" + case *ast.ExprStmt: + return "expression statement" + case *ast.Field: + // Can be any of these: + // struct {x, y int} -- struct field(s) + // struct {T} -- anon struct field + // interface {I} -- interface embedding + // interface {f()} -- interface method + // func (A) func(B) C -- receiver, param(s), result(s) + return "field/method/parameter" + case *ast.FieldList: + return "field/method/parameter list" + case *ast.File: + return "source file" + case *ast.ForStmt: + return "for loop" + case *ast.FuncDecl: + return "function declaration" + case *ast.FuncLit: + return "function literal" + case *ast.FuncType: + return "function type" + case *ast.GenDecl: + switch n.Tok { + case token.IMPORT: + return "import declaration" + case token.CONST: + return "constant declaration" + case token.TYPE: + return "type declaration" + case token.VAR: + return "variable declaration" + } + case *ast.GoStmt: + return "go statement" + case *ast.Ident: + return "identifier" + case *ast.IfStmt: + return "if statement" + case *ast.ImportSpec: + return "import specification" + case *ast.IncDecStmt: + if n.Tok == token.INC { + return "increment statement" + } + return "decrement statement" + case *ast.IndexExpr: + return "index expression" + case *ast.InterfaceType: + return "interface type" + case *ast.KeyValueExpr: + return "key/value association" + case *ast.LabeledStmt: + return "statement label" + case *ast.MapType: + return "map type" + case *ast.Package: + return "package" + case *ast.ParenExpr: + return "parenthesized " + NodeDescription(n.X) + case *ast.RangeStmt: + return "range loop" + case *ast.ReturnStmt: + return "return statement" + case *ast.SelectStmt: + return "select statement" + case *ast.SelectorExpr: + return "selector" + case *ast.SendStmt: + return "channel send" + case *ast.SliceExpr: + return "slice expression" + case *ast.StarExpr: + return "*-operation" // load/store expr or pointer type + case *ast.StructType: + return "struct type" + case *ast.SwitchStmt: + return "switch statement" + case *ast.TypeAssertExpr: + return "type assertion" + case *ast.TypeSpec: + return "type specification" + case *ast.TypeSwitchStmt: + return "type switch" + case *ast.UnaryExpr: + return fmt.Sprintf("unary %s operation", n.Op) + case *ast.ValueSpec: + return "value specification" + + } + panic(fmt.Sprintf("unexpected node type: %T", n)) +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/imports.go b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/imports.go new file mode 100644 index 0000000000..04ad6795d9 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/imports.go @@ -0,0 +1,471 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package astutil contains common utilities for working with the Go AST. +package astutil // import "golang.org/x/tools/go/ast/astutil" + +import ( + "fmt" + "go/ast" + "go/token" + "strconv" + "strings" +) + +// AddImport adds the import path to the file f, if absent. +func AddImport(fset *token.FileSet, f *ast.File, ipath string) (added bool) { + return AddNamedImport(fset, f, "", ipath) +} + +// AddNamedImport adds the import path to the file f, if absent. +// If name is not empty, it is used to rename the import. +// +// For example, calling +// AddNamedImport(fset, f, "pathpkg", "path") +// adds +// import pathpkg "path" +func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added bool) { + if imports(f, ipath) { + return false + } + + newImport := &ast.ImportSpec{ + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(ipath), + }, + } + if name != "" { + newImport.Name = &ast.Ident{Name: name} + } + + // Find an import decl to add to. + // The goal is to find an existing import + // whose import path has the longest shared + // prefix with ipath. + var ( + bestMatch = -1 // length of longest shared prefix + lastImport = -1 // index in f.Decls of the file's final import decl + impDecl *ast.GenDecl // import decl containing the best match + impIndex = -1 // spec index in impDecl containing the best match + + isThirdPartyPath = isThirdParty(ipath) + ) + for i, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + if ok && gen.Tok == token.IMPORT { + lastImport = i + // Do not add to import "C", to avoid disrupting the + // association with its doc comment, breaking cgo. + if declImports(gen, "C") { + continue + } + + // Match an empty import decl if that's all that is available. + if len(gen.Specs) == 0 && bestMatch == -1 { + impDecl = gen + } + + // Compute longest shared prefix with imports in this group and find best + // matched import spec. + // 1. Always prefer import spec with longest shared prefix. + // 2. While match length is 0, + // - for stdlib package: prefer first import spec. + // - for third party package: prefer first third party import spec. + // We cannot use last import spec as best match for third party package + // because grouped imports are usually placed last by goimports -local + // flag. + // See issue #19190. + seenAnyThirdParty := false + for j, spec := range gen.Specs { + impspec := spec.(*ast.ImportSpec) + p := importPath(impspec) + n := matchLen(p, ipath) + if n > bestMatch || (bestMatch == 0 && !seenAnyThirdParty && isThirdPartyPath) { + bestMatch = n + impDecl = gen + impIndex = j + } + seenAnyThirdParty = seenAnyThirdParty || isThirdParty(p) + } + } + } + + // If no import decl found, add one after the last import. + if impDecl == nil { + impDecl = &ast.GenDecl{ + Tok: token.IMPORT, + } + if lastImport >= 0 { + impDecl.TokPos = f.Decls[lastImport].End() + } else { + // There are no existing imports. + // Our new import, preceded by a blank line, goes after the package declaration + // and after the comment, if any, that starts on the same line as the + // package declaration. + impDecl.TokPos = f.Package + + file := fset.File(f.Package) + pkgLine := file.Line(f.Package) + for _, c := range f.Comments { + if file.Line(c.Pos()) > pkgLine { + break + } + // +2 for a blank line + impDecl.TokPos = c.End() + 2 + } + } + f.Decls = append(f.Decls, nil) + copy(f.Decls[lastImport+2:], f.Decls[lastImport+1:]) + f.Decls[lastImport+1] = impDecl + } + + // Insert new import at insertAt. + insertAt := 0 + if impIndex >= 0 { + // insert after the found import + insertAt = impIndex + 1 + } + impDecl.Specs = append(impDecl.Specs, nil) + copy(impDecl.Specs[insertAt+1:], impDecl.Specs[insertAt:]) + impDecl.Specs[insertAt] = newImport + pos := impDecl.Pos() + if insertAt > 0 { + // If there is a comment after an existing import, preserve the comment + // position by adding the new import after the comment. + if spec, ok := impDecl.Specs[insertAt-1].(*ast.ImportSpec); ok && spec.Comment != nil { + pos = spec.Comment.End() + } else { + // Assign same position as the previous import, + // so that the sorter sees it as being in the same block. + pos = impDecl.Specs[insertAt-1].Pos() + } + } + if newImport.Name != nil { + newImport.Name.NamePos = pos + } + newImport.Path.ValuePos = pos + newImport.EndPos = pos + + // Clean up parens. impDecl contains at least one spec. + if len(impDecl.Specs) == 1 { + // Remove unneeded parens. + impDecl.Lparen = token.NoPos + } else if !impDecl.Lparen.IsValid() { + // impDecl needs parens added. + impDecl.Lparen = impDecl.Specs[0].Pos() + } + + f.Imports = append(f.Imports, newImport) + + if len(f.Decls) <= 1 { + return true + } + + // Merge all the import declarations into the first one. + var first *ast.GenDecl + for i := 0; i < len(f.Decls); i++ { + decl := f.Decls[i] + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.IMPORT || declImports(gen, "C") { + continue + } + if first == nil { + first = gen + continue // Don't touch the first one. + } + // We now know there is more than one package in this import + // declaration. Ensure that it ends up parenthesized. + first.Lparen = first.Pos() + // Move the imports of the other import declaration to the first one. + for _, spec := range gen.Specs { + spec.(*ast.ImportSpec).Path.ValuePos = first.Pos() + first.Specs = append(first.Specs, spec) + } + f.Decls = append(f.Decls[:i], f.Decls[i+1:]...) + i-- + } + + return true +} + +func isThirdParty(importPath string) bool { + // Third party package import path usually contains "." (".com", ".org", ...) + // This logic is taken from golang.org/x/tools/imports package. + return strings.Contains(importPath, ".") +} + +// DeleteImport deletes the import path from the file f, if present. +func DeleteImport(fset *token.FileSet, f *ast.File, path string) (deleted bool) { + return DeleteNamedImport(fset, f, "", path) +} + +// DeleteNamedImport deletes the import with the given name and path from the file f, if present. +func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (deleted bool) { + var delspecs []*ast.ImportSpec + var delcomments []*ast.CommentGroup + + // Find the import nodes that import path, if any. + for i := 0; i < len(f.Decls); i++ { + decl := f.Decls[i] + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.IMPORT { + continue + } + for j := 0; j < len(gen.Specs); j++ { + spec := gen.Specs[j] + impspec := spec.(*ast.ImportSpec) + if impspec.Name == nil && name != "" { + continue + } + if impspec.Name != nil && impspec.Name.Name != name { + continue + } + if importPath(impspec) != path { + continue + } + + // We found an import spec that imports path. + // Delete it. + delspecs = append(delspecs, impspec) + deleted = true + copy(gen.Specs[j:], gen.Specs[j+1:]) + gen.Specs = gen.Specs[:len(gen.Specs)-1] + + // If this was the last import spec in this decl, + // delete the decl, too. + if len(gen.Specs) == 0 { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + i-- + break + } else if len(gen.Specs) == 1 { + if impspec.Doc != nil { + delcomments = append(delcomments, impspec.Doc) + } + if impspec.Comment != nil { + delcomments = append(delcomments, impspec.Comment) + } + for _, cg := range f.Comments { + // Found comment on the same line as the import spec. + if cg.End() < impspec.Pos() && fset.Position(cg.End()).Line == fset.Position(impspec.Pos()).Line { + delcomments = append(delcomments, cg) + break + } + } + + spec := gen.Specs[0].(*ast.ImportSpec) + + // Move the documentation right after the import decl. + if spec.Doc != nil { + for fset.Position(gen.TokPos).Line+1 < fset.Position(spec.Doc.Pos()).Line { + fset.File(gen.TokPos).MergeLine(fset.Position(gen.TokPos).Line) + } + } + for _, cg := range f.Comments { + if cg.End() < spec.Pos() && fset.Position(cg.End()).Line == fset.Position(spec.Pos()).Line { + for fset.Position(gen.TokPos).Line+1 < fset.Position(spec.Pos()).Line { + fset.File(gen.TokPos).MergeLine(fset.Position(gen.TokPos).Line) + } + break + } + } + } + if j > 0 { + lastImpspec := gen.Specs[j-1].(*ast.ImportSpec) + lastLine := fset.Position(lastImpspec.Path.ValuePos).Line + line := fset.Position(impspec.Path.ValuePos).Line + + // We deleted an entry but now there may be + // a blank line-sized hole where the import was. + if line-lastLine > 1 { + // There was a blank line immediately preceding the deleted import, + // so there's no need to close the hole. + // Do nothing. + } else if line != fset.File(gen.Rparen).LineCount() { + // There was no blank line. Close the hole. + fset.File(gen.Rparen).MergeLine(line) + } + } + j-- + } + } + + // Delete imports from f.Imports. + for i := 0; i < len(f.Imports); i++ { + imp := f.Imports[i] + for j, del := range delspecs { + if imp == del { + copy(f.Imports[i:], f.Imports[i+1:]) + f.Imports = f.Imports[:len(f.Imports)-1] + copy(delspecs[j:], delspecs[j+1:]) + delspecs = delspecs[:len(delspecs)-1] + i-- + break + } + } + } + + // Delete comments from f.Comments. + for i := 0; i < len(f.Comments); i++ { + cg := f.Comments[i] + for j, del := range delcomments { + if cg == del { + copy(f.Comments[i:], f.Comments[i+1:]) + f.Comments = f.Comments[:len(f.Comments)-1] + copy(delcomments[j:], delcomments[j+1:]) + delcomments = delcomments[:len(delcomments)-1] + i-- + break + } + } + } + + if len(delspecs) > 0 { + panic(fmt.Sprintf("deleted specs from Decls but not Imports: %v", delspecs)) + } + + return +} + +// RewriteImport rewrites any import of path oldPath to path newPath. +func RewriteImport(fset *token.FileSet, f *ast.File, oldPath, newPath string) (rewrote bool) { + for _, imp := range f.Imports { + if importPath(imp) == oldPath { + rewrote = true + // record old End, because the default is to compute + // it using the length of imp.Path.Value. + imp.EndPos = imp.End() + imp.Path.Value = strconv.Quote(newPath) + } + } + return +} + +// UsesImport reports whether a given import is used. +func UsesImport(f *ast.File, path string) (used bool) { + spec := importSpec(f, path) + if spec == nil { + return + } + + name := spec.Name.String() + switch name { + case "": + // If the package name is not explicitly specified, + // make an educated guess. This is not guaranteed to be correct. + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 { + name = path + } else { + name = path[lastSlash+1:] + } + case "_", ".": + // Not sure if this import is used - err on the side of caution. + return true + } + + ast.Walk(visitFn(func(n ast.Node) { + sel, ok := n.(*ast.SelectorExpr) + if ok && isTopName(sel.X, name) { + used = true + } + }), f) + + return +} + +type visitFn func(node ast.Node) + +func (fn visitFn) Visit(node ast.Node) ast.Visitor { + fn(node) + return fn +} + +// imports returns true if f imports path. +func imports(f *ast.File, path string) bool { + return importSpec(f, path) != nil +} + +// importSpec returns the import spec if f imports path, +// or nil otherwise. +func importSpec(f *ast.File, path string) *ast.ImportSpec { + for _, s := range f.Imports { + if importPath(s) == path { + return s + } + } + return nil +} + +// importPath returns the unquoted import path of s, +// or "" if the path is not properly quoted. +func importPath(s *ast.ImportSpec) string { + t, err := strconv.Unquote(s.Path.Value) + if err == nil { + return t + } + return "" +} + +// declImports reports whether gen contains an import of path. +func declImports(gen *ast.GenDecl, path string) bool { + if gen.Tok != token.IMPORT { + return false + } + for _, spec := range gen.Specs { + impspec := spec.(*ast.ImportSpec) + if importPath(impspec) == path { + return true + } + } + return false +} + +// matchLen returns the length of the longest path segment prefix shared by x and y. +func matchLen(x, y string) int { + n := 0 + for i := 0; i < len(x) && i < len(y) && x[i] == y[i]; i++ { + if x[i] == '/' { + n++ + } + } + return n +} + +// isTopName returns true if n is a top-level unresolved identifier with the given name. +func isTopName(n ast.Expr, name string) bool { + id, ok := n.(*ast.Ident) + return ok && id.Name == name && id.Obj == nil +} + +// Imports returns the file imports grouped by paragraph. +func Imports(fset *token.FileSet, f *ast.File) [][]*ast.ImportSpec { + var groups [][]*ast.ImportSpec + + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.IMPORT { + break + } + + group := []*ast.ImportSpec{} + + var lastLine int + for _, spec := range genDecl.Specs { + importSpec := spec.(*ast.ImportSpec) + pos := importSpec.Path.ValuePos + line := fset.Position(pos).Line + if lastLine > 0 && pos > 0 && line-lastLine > 1 { + groups = append(groups, group) + group = []*ast.ImportSpec{} + } + group = append(group, importSpec) + lastLine = line + } + groups = append(groups, group) + } + + return groups +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go new file mode 100644 index 0000000000..cf72ea990b --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go @@ -0,0 +1,477 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package astutil + +import ( + "fmt" + "go/ast" + "reflect" + "sort" +) + +// An ApplyFunc is invoked by Apply for each node n, even if n is nil, +// before and/or after the node's children, using a Cursor describing +// the current node and providing operations on it. +// +// The return value of ApplyFunc controls the syntax tree traversal. +// See Apply for details. +type ApplyFunc func(*Cursor) bool + +// Apply traverses a syntax tree recursively, starting with root, +// and calling pre and post for each node as described below. +// Apply returns the syntax tree, possibly modified. +// +// If pre is not nil, it is called for each node before the node's +// children are traversed (pre-order). If pre returns false, no +// children are traversed, and post is not called for that node. +// +// If post is not nil, and a prior call of pre didn't return false, +// post is called for each node after its children are traversed +// (post-order). If post returns false, traversal is terminated and +// Apply returns immediately. +// +// Only fields that refer to AST nodes are considered children; +// i.e., token.Pos, Scopes, Objects, and fields of basic types +// (strings, etc.) are ignored. +// +// Children are traversed in the order in which they appear in the +// respective node's struct definition. A package's files are +// traversed in the filenames' alphabetical order. +// +func Apply(root ast.Node, pre, post ApplyFunc) (result ast.Node) { + parent := &struct{ ast.Node }{root} + defer func() { + if r := recover(); r != nil && r != abort { + panic(r) + } + result = parent.Node + }() + a := &application{pre: pre, post: post} + a.apply(parent, "Node", nil, root) + return +} + +var abort = new(int) // singleton, to signal termination of Apply + +// A Cursor describes a node encountered during Apply. +// Information about the node and its parent is available +// from the Node, Parent, Name, and Index methods. +// +// If p is a variable of type and value of the current parent node +// c.Parent(), and f is the field identifier with name c.Name(), +// the following invariants hold: +// +// p.f == c.Node() if c.Index() < 0 +// p.f[c.Index()] == c.Node() if c.Index() >= 0 +// +// The methods Replace, Delete, InsertBefore, and InsertAfter +// can be used to change the AST without disrupting Apply. +type Cursor struct { + parent ast.Node + name string + iter *iterator // valid if non-nil + node ast.Node +} + +// Node returns the current Node. +func (c *Cursor) Node() ast.Node { return c.node } + +// Parent returns the parent of the current Node. +func (c *Cursor) Parent() ast.Node { return c.parent } + +// Name returns the name of the parent Node field that contains the current Node. +// If the parent is a *ast.Package and the current Node is a *ast.File, Name returns +// the filename for the current Node. +func (c *Cursor) Name() string { return c.name } + +// Index reports the index >= 0 of the current Node in the slice of Nodes that +// contains it, or a value < 0 if the current Node is not part of a slice. +// The index of the current node changes if InsertBefore is called while +// processing the current node. +func (c *Cursor) Index() int { + if c.iter != nil { + return c.iter.index + } + return -1 +} + +// field returns the current node's parent field value. +func (c *Cursor) field() reflect.Value { + return reflect.Indirect(reflect.ValueOf(c.parent)).FieldByName(c.name) +} + +// Replace replaces the current Node with n. +// The replacement node is not walked by Apply. +func (c *Cursor) Replace(n ast.Node) { + if _, ok := c.node.(*ast.File); ok { + file, ok := n.(*ast.File) + if !ok { + panic("attempt to replace *ast.File with non-*ast.File") + } + c.parent.(*ast.Package).Files[c.name] = file + return + } + + v := c.field() + if i := c.Index(); i >= 0 { + v = v.Index(i) + } + v.Set(reflect.ValueOf(n)) +} + +// Delete deletes the current Node from its containing slice. +// If the current Node is not part of a slice, Delete panics. +// As a special case, if the current node is a package file, +// Delete removes it from the package's Files map. +func (c *Cursor) Delete() { + if _, ok := c.node.(*ast.File); ok { + delete(c.parent.(*ast.Package).Files, c.name) + return + } + + i := c.Index() + if i < 0 { + panic("Delete node not contained in slice") + } + v := c.field() + l := v.Len() + reflect.Copy(v.Slice(i, l), v.Slice(i+1, l)) + v.Index(l - 1).Set(reflect.Zero(v.Type().Elem())) + v.SetLen(l - 1) + c.iter.step-- +} + +// InsertAfter inserts n after the current Node in its containing slice. +// If the current Node is not part of a slice, InsertAfter panics. +// Apply does not walk n. +func (c *Cursor) InsertAfter(n ast.Node) { + i := c.Index() + if i < 0 { + panic("InsertAfter node not contained in slice") + } + v := c.field() + v.Set(reflect.Append(v, reflect.Zero(v.Type().Elem()))) + l := v.Len() + reflect.Copy(v.Slice(i+2, l), v.Slice(i+1, l)) + v.Index(i + 1).Set(reflect.ValueOf(n)) + c.iter.step++ +} + +// InsertBefore inserts n before the current Node in its containing slice. +// If the current Node is not part of a slice, InsertBefore panics. +// Apply will not walk n. +func (c *Cursor) InsertBefore(n ast.Node) { + i := c.Index() + if i < 0 { + panic("InsertBefore node not contained in slice") + } + v := c.field() + v.Set(reflect.Append(v, reflect.Zero(v.Type().Elem()))) + l := v.Len() + reflect.Copy(v.Slice(i+1, l), v.Slice(i, l)) + v.Index(i).Set(reflect.ValueOf(n)) + c.iter.index++ +} + +// application carries all the shared data so we can pass it around cheaply. +type application struct { + pre, post ApplyFunc + cursor Cursor + iter iterator +} + +func (a *application) apply(parent ast.Node, name string, iter *iterator, n ast.Node) { + // convert typed nil into untyped nil + if v := reflect.ValueOf(n); v.Kind() == reflect.Ptr && v.IsNil() { + n = nil + } + + // avoid heap-allocating a new cursor for each apply call; reuse a.cursor instead + saved := a.cursor + a.cursor.parent = parent + a.cursor.name = name + a.cursor.iter = iter + a.cursor.node = n + + if a.pre != nil && !a.pre(&a.cursor) { + a.cursor = saved + return + } + + // walk children + // (the order of the cases matches the order of the corresponding node types in go/ast) + switch n := n.(type) { + case nil: + // nothing to do + + // Comments and fields + case *ast.Comment: + // nothing to do + + case *ast.CommentGroup: + if n != nil { + a.applyList(n, "List") + } + + case *ast.Field: + a.apply(n, "Doc", nil, n.Doc) + a.applyList(n, "Names") + a.apply(n, "Type", nil, n.Type) + a.apply(n, "Tag", nil, n.Tag) + a.apply(n, "Comment", nil, n.Comment) + + case *ast.FieldList: + a.applyList(n, "List") + + // Expressions + case *ast.BadExpr, *ast.Ident, *ast.BasicLit: + // nothing to do + + case *ast.Ellipsis: + a.apply(n, "Elt", nil, n.Elt) + + case *ast.FuncLit: + a.apply(n, "Type", nil, n.Type) + a.apply(n, "Body", nil, n.Body) + + case *ast.CompositeLit: + a.apply(n, "Type", nil, n.Type) + a.applyList(n, "Elts") + + case *ast.ParenExpr: + a.apply(n, "X", nil, n.X) + + case *ast.SelectorExpr: + a.apply(n, "X", nil, n.X) + a.apply(n, "Sel", nil, n.Sel) + + case *ast.IndexExpr: + a.apply(n, "X", nil, n.X) + a.apply(n, "Index", nil, n.Index) + + case *ast.SliceExpr: + a.apply(n, "X", nil, n.X) + a.apply(n, "Low", nil, n.Low) + a.apply(n, "High", nil, n.High) + a.apply(n, "Max", nil, n.Max) + + case *ast.TypeAssertExpr: + a.apply(n, "X", nil, n.X) + a.apply(n, "Type", nil, n.Type) + + case *ast.CallExpr: + a.apply(n, "Fun", nil, n.Fun) + a.applyList(n, "Args") + + case *ast.StarExpr: + a.apply(n, "X", nil, n.X) + + case *ast.UnaryExpr: + a.apply(n, "X", nil, n.X) + + case *ast.BinaryExpr: + a.apply(n, "X", nil, n.X) + a.apply(n, "Y", nil, n.Y) + + case *ast.KeyValueExpr: + a.apply(n, "Key", nil, n.Key) + a.apply(n, "Value", nil, n.Value) + + // Types + case *ast.ArrayType: + a.apply(n, "Len", nil, n.Len) + a.apply(n, "Elt", nil, n.Elt) + + case *ast.StructType: + a.apply(n, "Fields", nil, n.Fields) + + case *ast.FuncType: + a.apply(n, "Params", nil, n.Params) + a.apply(n, "Results", nil, n.Results) + + case *ast.InterfaceType: + a.apply(n, "Methods", nil, n.Methods) + + case *ast.MapType: + a.apply(n, "Key", nil, n.Key) + a.apply(n, "Value", nil, n.Value) + + case *ast.ChanType: + a.apply(n, "Value", nil, n.Value) + + // Statements + case *ast.BadStmt: + // nothing to do + + case *ast.DeclStmt: + a.apply(n, "Decl", nil, n.Decl) + + case *ast.EmptyStmt: + // nothing to do + + case *ast.LabeledStmt: + a.apply(n, "Label", nil, n.Label) + a.apply(n, "Stmt", nil, n.Stmt) + + case *ast.ExprStmt: + a.apply(n, "X", nil, n.X) + + case *ast.SendStmt: + a.apply(n, "Chan", nil, n.Chan) + a.apply(n, "Value", nil, n.Value) + + case *ast.IncDecStmt: + a.apply(n, "X", nil, n.X) + + case *ast.AssignStmt: + a.applyList(n, "Lhs") + a.applyList(n, "Rhs") + + case *ast.GoStmt: + a.apply(n, "Call", nil, n.Call) + + case *ast.DeferStmt: + a.apply(n, "Call", nil, n.Call) + + case *ast.ReturnStmt: + a.applyList(n, "Results") + + case *ast.BranchStmt: + a.apply(n, "Label", nil, n.Label) + + case *ast.BlockStmt: + a.applyList(n, "List") + + case *ast.IfStmt: + a.apply(n, "Init", nil, n.Init) + a.apply(n, "Cond", nil, n.Cond) + a.apply(n, "Body", nil, n.Body) + a.apply(n, "Else", nil, n.Else) + + case *ast.CaseClause: + a.applyList(n, "List") + a.applyList(n, "Body") + + case *ast.SwitchStmt: + a.apply(n, "Init", nil, n.Init) + a.apply(n, "Tag", nil, n.Tag) + a.apply(n, "Body", nil, n.Body) + + case *ast.TypeSwitchStmt: + a.apply(n, "Init", nil, n.Init) + a.apply(n, "Assign", nil, n.Assign) + a.apply(n, "Body", nil, n.Body) + + case *ast.CommClause: + a.apply(n, "Comm", nil, n.Comm) + a.applyList(n, "Body") + + case *ast.SelectStmt: + a.apply(n, "Body", nil, n.Body) + + case *ast.ForStmt: + a.apply(n, "Init", nil, n.Init) + a.apply(n, "Cond", nil, n.Cond) + a.apply(n, "Post", nil, n.Post) + a.apply(n, "Body", nil, n.Body) + + case *ast.RangeStmt: + a.apply(n, "Key", nil, n.Key) + a.apply(n, "Value", nil, n.Value) + a.apply(n, "X", nil, n.X) + a.apply(n, "Body", nil, n.Body) + + // Declarations + case *ast.ImportSpec: + a.apply(n, "Doc", nil, n.Doc) + a.apply(n, "Name", nil, n.Name) + a.apply(n, "Path", nil, n.Path) + a.apply(n, "Comment", nil, n.Comment) + + case *ast.ValueSpec: + a.apply(n, "Doc", nil, n.Doc) + a.applyList(n, "Names") + a.apply(n, "Type", nil, n.Type) + a.applyList(n, "Values") + a.apply(n, "Comment", nil, n.Comment) + + case *ast.TypeSpec: + a.apply(n, "Doc", nil, n.Doc) + a.apply(n, "Name", nil, n.Name) + a.apply(n, "Type", nil, n.Type) + a.apply(n, "Comment", nil, n.Comment) + + case *ast.BadDecl: + // nothing to do + + case *ast.GenDecl: + a.apply(n, "Doc", nil, n.Doc) + a.applyList(n, "Specs") + + case *ast.FuncDecl: + a.apply(n, "Doc", nil, n.Doc) + a.apply(n, "Recv", nil, n.Recv) + a.apply(n, "Name", nil, n.Name) + a.apply(n, "Type", nil, n.Type) + a.apply(n, "Body", nil, n.Body) + + // Files and packages + case *ast.File: + a.apply(n, "Doc", nil, n.Doc) + a.apply(n, "Name", nil, n.Name) + a.applyList(n, "Decls") + // Don't walk n.Comments; they have either been walked already if + // they are Doc comments, or they can be easily walked explicitly. + + case *ast.Package: + // collect and sort names for reproducible behavior + var names []string + for name := range n.Files { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + a.apply(n, name, nil, n.Files[name]) + } + + default: + panic(fmt.Sprintf("Apply: unexpected node type %T", n)) + } + + if a.post != nil && !a.post(&a.cursor) { + panic(abort) + } + + a.cursor = saved +} + +// An iterator controls iteration over a slice of nodes. +type iterator struct { + index, step int +} + +func (a *application) applyList(parent ast.Node, name string) { + // avoid heap-allocating a new iterator for each applyList call; reuse a.iter instead + saved := a.iter + a.iter.index = 0 + for { + // must reload parent.name each time, since cursor modifications might change it + v := reflect.Indirect(reflect.ValueOf(parent)).FieldByName(name) + if a.iter.index >= v.Len() { + break + } + + // element x may be nil in a bad AST - be cautious + var x ast.Node + if e := v.Index(a.iter.index); e.IsValid() { + x = e.Interface().(ast.Node) + } + + a.iter.step = 1 + a.apply(parent, name, &a.iter, x) + a.iter.index += a.iter.step + } + a.iter = saved +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/util.go b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/util.go new file mode 100644 index 0000000000..7630629824 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/util.go @@ -0,0 +1,14 @@ +package astutil + +import "go/ast" + +// Unparen returns e with any enclosing parentheses stripped. +func Unparen(e ast.Expr) ast.Expr { + for { + p, ok := e.(*ast.ParenExpr) + if !ok { + return e + } + e = p.X + } +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/inspector.go b/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/inspector.go new file mode 100644 index 0000000000..db88a95109 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/inspector.go @@ -0,0 +1,182 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package inspector provides helper functions for traversal over the +// syntax trees of a package, including node filtering by type, and +// materialization of the traversal stack. +// +// During construction, the inspector does a complete traversal and +// builds a list of push/pop events and their node type. Subsequent +// method calls that request a traversal scan this list, rather than walk +// the AST, and perform type filtering using efficient bit sets. +// +// Experiments suggest the inspector's traversals are about 2.5x faster +// than ast.Inspect, but it may take around 5 traversals for this +// benefit to amortize the inspector's construction cost. +// If efficiency is the primary concern, do not use use Inspector for +// one-off traversals. +package inspector + +// There are four orthogonal features in a traversal: +// 1 type filtering +// 2 pruning +// 3 postorder calls to f +// 4 stack +// Rather than offer all of them in the API, +// only a few combinations are exposed: +// - Preorder is the fastest and has fewest features, +// but is the most commonly needed traversal. +// - Nodes and WithStack both provide pruning and postorder calls, +// even though few clients need it, because supporting two versions +// is not justified. +// More combinations could be supported by expressing them as +// wrappers around a more generic traversal, but this was measured +// and found to degrade performance significantly (30%). + +import ( + "go/ast" +) + +// An Inspector provides methods for inspecting +// (traversing) the syntax trees of a package. +type Inspector struct { + events []event +} + +// New returns an Inspector for the specified syntax trees. +func New(files []*ast.File) *Inspector { + return &Inspector{traverse(files)} +} + +// An event represents a push or a pop +// of an ast.Node during a traversal. +type event struct { + node ast.Node + typ uint64 // typeOf(node) + index int // 1 + index of corresponding pop event, or 0 if this is a pop +} + +// Preorder visits all the nodes of the files supplied to New in +// depth-first order. It calls f(n) for each node n before it visits +// n's children. +// +// The types argument, if non-empty, enables type-based filtering of +// events. The function f if is called only for nodes whose type +// matches an element of the types slice. +func (in *Inspector) Preorder(types []ast.Node, f func(ast.Node)) { + // Because it avoids postorder calls to f, and the pruning + // check, Preorder is almost twice as fast as Nodes. The two + // features seem to contribute similar slowdowns (~1.4x each). + + mask := maskOf(types) + for i := 0; i < len(in.events); { + ev := in.events[i] + if ev.typ&mask != 0 { + if ev.index > 0 { + f(ev.node) + } + } + i++ + } +} + +// Nodes visits the nodes of the files supplied to New in depth-first +// order. It calls f(n, true) for each node n before it visits n's +// children. If f returns true, Nodes invokes f recursively for each +// of the non-nil children of the node, followed by a call of +// f(n, false). +// +// The types argument, if non-empty, enables type-based filtering of +// events. The function f if is called only for nodes whose type +// matches an element of the types slice. +func (in *Inspector) Nodes(types []ast.Node, f func(n ast.Node, push bool) (prune bool)) { + mask := maskOf(types) + for i := 0; i < len(in.events); { + ev := in.events[i] + if ev.typ&mask != 0 { + if ev.index > 0 { + // push + if !f(ev.node, true) { + i = ev.index // jump to corresponding pop + 1 + continue + } + } else { + // pop + f(ev.node, false) + } + } + i++ + } +} + +// WithStack visits nodes in a similar manner to Nodes, but it +// supplies each call to f an additional argument, the current +// traversal stack. The stack's first element is the outermost node, +// an *ast.File; its last is the innermost, n. +func (in *Inspector) WithStack(types []ast.Node, f func(n ast.Node, push bool, stack []ast.Node) (prune bool)) { + mask := maskOf(types) + var stack []ast.Node + for i := 0; i < len(in.events); { + ev := in.events[i] + if ev.index > 0 { + // push + stack = append(stack, ev.node) + if ev.typ&mask != 0 { + if !f(ev.node, true, stack) { + i = ev.index + stack = stack[:len(stack)-1] + continue + } + } + } else { + // pop + if ev.typ&mask != 0 { + f(ev.node, false, stack) + } + stack = stack[:len(stack)-1] + } + i++ + } +} + +// traverse builds the table of events representing a traversal. +func traverse(files []*ast.File) []event { + // Preallocate approximate number of events + // based on source file extent. + // This makes traverse faster by 4x (!). + var extent int + for _, f := range files { + extent += int(f.End() - f.Pos()) + } + // This estimate is based on the net/http package. + events := make([]event, 0, extent*33/100) + + var stack []event + for _, f := range files { + ast.Inspect(f, func(n ast.Node) bool { + if n != nil { + // push + ev := event{ + node: n, + typ: typeOf(n), + index: len(events), // push event temporarily holds own index + } + stack = append(stack, ev) + events = append(events, ev) + } else { + // pop + ev := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + events[ev.index].index = len(events) + 1 // make push refer to pop + + ev.index = 0 // turn ev into a pop event + events = append(events, ev) + } + return true + }) + } + + return events +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/typeof.go b/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/typeof.go new file mode 100644 index 0000000000..d61301b133 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/inspector/typeof.go @@ -0,0 +1,216 @@ +package inspector + +// This file defines func typeOf(ast.Node) uint64. +// +// The initial map-based implementation was too slow; +// see https://go-review.googlesource.com/c/tools/+/135655/1/go/ast/inspector/inspector.go#196 + +import "go/ast" + +const ( + nArrayType = iota + nAssignStmt + nBadDecl + nBadExpr + nBadStmt + nBasicLit + nBinaryExpr + nBlockStmt + nBranchStmt + nCallExpr + nCaseClause + nChanType + nCommClause + nComment + nCommentGroup + nCompositeLit + nDeclStmt + nDeferStmt + nEllipsis + nEmptyStmt + nExprStmt + nField + nFieldList + nFile + nForStmt + nFuncDecl + nFuncLit + nFuncType + nGenDecl + nGoStmt + nIdent + nIfStmt + nImportSpec + nIncDecStmt + nIndexExpr + nInterfaceType + nKeyValueExpr + nLabeledStmt + nMapType + nPackage + nParenExpr + nRangeStmt + nReturnStmt + nSelectStmt + nSelectorExpr + nSendStmt + nSliceExpr + nStarExpr + nStructType + nSwitchStmt + nTypeAssertExpr + nTypeSpec + nTypeSwitchStmt + nUnaryExpr + nValueSpec +) + +// typeOf returns a distinct single-bit value that represents the type of n. +// +// Various implementations were benchmarked with BenchmarkNewInspector: +// GOGC=off +// - type switch 4.9-5.5ms 2.1ms +// - binary search over a sorted list of types 5.5-5.9ms 2.5ms +// - linear scan, frequency-ordered list 5.9-6.1ms 2.7ms +// - linear scan, unordered list 6.4ms 2.7ms +// - hash table 6.5ms 3.1ms +// A perfect hash seemed like overkill. +// +// The compiler's switch statement is the clear winner +// as it produces a binary tree in code, +// with constant conditions and good branch prediction. +// (Sadly it is the most verbose in source code.) +// Binary search suffered from poor branch prediction. +// +func typeOf(n ast.Node) uint64 { + // Fast path: nearly half of all nodes are identifiers. + if _, ok := n.(*ast.Ident); ok { + return 1 << nIdent + } + + // These cases include all nodes encountered by ast.Inspect. + switch n.(type) { + case *ast.ArrayType: + return 1 << nArrayType + case *ast.AssignStmt: + return 1 << nAssignStmt + case *ast.BadDecl: + return 1 << nBadDecl + case *ast.BadExpr: + return 1 << nBadExpr + case *ast.BadStmt: + return 1 << nBadStmt + case *ast.BasicLit: + return 1 << nBasicLit + case *ast.BinaryExpr: + return 1 << nBinaryExpr + case *ast.BlockStmt: + return 1 << nBlockStmt + case *ast.BranchStmt: + return 1 << nBranchStmt + case *ast.CallExpr: + return 1 << nCallExpr + case *ast.CaseClause: + return 1 << nCaseClause + case *ast.ChanType: + return 1 << nChanType + case *ast.CommClause: + return 1 << nCommClause + case *ast.Comment: + return 1 << nComment + case *ast.CommentGroup: + return 1 << nCommentGroup + case *ast.CompositeLit: + return 1 << nCompositeLit + case *ast.DeclStmt: + return 1 << nDeclStmt + case *ast.DeferStmt: + return 1 << nDeferStmt + case *ast.Ellipsis: + return 1 << nEllipsis + case *ast.EmptyStmt: + return 1 << nEmptyStmt + case *ast.ExprStmt: + return 1 << nExprStmt + case *ast.Field: + return 1 << nField + case *ast.FieldList: + return 1 << nFieldList + case *ast.File: + return 1 << nFile + case *ast.ForStmt: + return 1 << nForStmt + case *ast.FuncDecl: + return 1 << nFuncDecl + case *ast.FuncLit: + return 1 << nFuncLit + case *ast.FuncType: + return 1 << nFuncType + case *ast.GenDecl: + return 1 << nGenDecl + case *ast.GoStmt: + return 1 << nGoStmt + case *ast.Ident: + return 1 << nIdent + case *ast.IfStmt: + return 1 << nIfStmt + case *ast.ImportSpec: + return 1 << nImportSpec + case *ast.IncDecStmt: + return 1 << nIncDecStmt + case *ast.IndexExpr: + return 1 << nIndexExpr + case *ast.InterfaceType: + return 1 << nInterfaceType + case *ast.KeyValueExpr: + return 1 << nKeyValueExpr + case *ast.LabeledStmt: + return 1 << nLabeledStmt + case *ast.MapType: + return 1 << nMapType + case *ast.Package: + return 1 << nPackage + case *ast.ParenExpr: + return 1 << nParenExpr + case *ast.RangeStmt: + return 1 << nRangeStmt + case *ast.ReturnStmt: + return 1 << nReturnStmt + case *ast.SelectStmt: + return 1 << nSelectStmt + case *ast.SelectorExpr: + return 1 << nSelectorExpr + case *ast.SendStmt: + return 1 << nSendStmt + case *ast.SliceExpr: + return 1 << nSliceExpr + case *ast.StarExpr: + return 1 << nStarExpr + case *ast.StructType: + return 1 << nStructType + case *ast.SwitchStmt: + return 1 << nSwitchStmt + case *ast.TypeAssertExpr: + return 1 << nTypeAssertExpr + case *ast.TypeSpec: + return 1 << nTypeSpec + case *ast.TypeSwitchStmt: + return 1 << nTypeSwitchStmt + case *ast.UnaryExpr: + return 1 << nUnaryExpr + case *ast.ValueSpec: + return 1 << nValueSpec + } + return 0 +} + +func maskOf(nodes []ast.Node) uint64 { + if nodes == nil { + return 1<<64 - 1 // match all node types + } + var mask uint64 + for _, n := range nodes { + mask |= typeOf(n) + } + return mask +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/cfg/builder.go b/src/cmd/vendor/golang.org/x/tools/go/cfg/builder.go new file mode 100644 index 0000000000..24e1aba033 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/cfg/builder.go @@ -0,0 +1,510 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cfg + +// This file implements the CFG construction pass. + +import ( + "fmt" + "go/ast" + "go/token" +) + +type builder struct { + cfg *CFG + mayReturn func(*ast.CallExpr) bool + current *Block + lblocks map[*ast.Object]*lblock // labeled blocks + targets *targets // linked stack of branch targets +} + +func (b *builder) stmt(_s ast.Stmt) { + // The label of the current statement. If non-nil, its _goto + // target is always set; its _break and _continue are set only + // within the body of switch/typeswitch/select/for/range. + // It is effectively an additional default-nil parameter of stmt(). + var label *lblock +start: + switch s := _s.(type) { + case *ast.BadStmt, + *ast.SendStmt, + *ast.IncDecStmt, + *ast.GoStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.AssignStmt: + // No effect on control flow. + b.add(s) + + case *ast.ExprStmt: + b.add(s) + if call, ok := s.X.(*ast.CallExpr); ok && !b.mayReturn(call) { + // Calls to panic, os.Exit, etc, never return. + b.current = b.newBlock("unreachable.call") + } + + case *ast.DeclStmt: + // Treat each var ValueSpec as a separate statement. + d := s.Decl.(*ast.GenDecl) + if d.Tok == token.VAR { + for _, spec := range d.Specs { + if spec, ok := spec.(*ast.ValueSpec); ok { + b.add(spec) + } + } + } + + case *ast.LabeledStmt: + label = b.labeledBlock(s.Label) + b.jump(label._goto) + b.current = label._goto + _s = s.Stmt + goto start // effectively: tailcall stmt(g, s.Stmt, label) + + case *ast.ReturnStmt: + b.add(s) + b.current = b.newBlock("unreachable.return") + + case *ast.BranchStmt: + b.branchStmt(s) + + case *ast.BlockStmt: + b.stmtList(s.List) + + case *ast.IfStmt: + if s.Init != nil { + b.stmt(s.Init) + } + then := b.newBlock("if.then") + done := b.newBlock("if.done") + _else := done + if s.Else != nil { + _else = b.newBlock("if.else") + } + b.add(s.Cond) + b.ifelse(then, _else) + b.current = then + b.stmt(s.Body) + b.jump(done) + + if s.Else != nil { + b.current = _else + b.stmt(s.Else) + b.jump(done) + } + + b.current = done + + case *ast.SwitchStmt: + b.switchStmt(s, label) + + case *ast.TypeSwitchStmt: + b.typeSwitchStmt(s, label) + + case *ast.SelectStmt: + b.selectStmt(s, label) + + case *ast.ForStmt: + b.forStmt(s, label) + + case *ast.RangeStmt: + b.rangeStmt(s, label) + + default: + panic(fmt.Sprintf("unexpected statement kind: %T", s)) + } +} + +func (b *builder) stmtList(list []ast.Stmt) { + for _, s := range list { + b.stmt(s) + } +} + +func (b *builder) branchStmt(s *ast.BranchStmt) { + var block *Block + switch s.Tok { + case token.BREAK: + if s.Label != nil { + if lb := b.labeledBlock(s.Label); lb != nil { + block = lb._break + } + } else { + for t := b.targets; t != nil && block == nil; t = t.tail { + block = t._break + } + } + + case token.CONTINUE: + if s.Label != nil { + if lb := b.labeledBlock(s.Label); lb != nil { + block = lb._continue + } + } else { + for t := b.targets; t != nil && block == nil; t = t.tail { + block = t._continue + } + } + + case token.FALLTHROUGH: + for t := b.targets; t != nil; t = t.tail { + block = t._fallthrough + } + + case token.GOTO: + if s.Label != nil { + block = b.labeledBlock(s.Label)._goto + } + } + if block == nil { + block = b.newBlock("undefined.branch") + } + b.jump(block) + b.current = b.newBlock("unreachable.branch") +} + +func (b *builder) switchStmt(s *ast.SwitchStmt, label *lblock) { + if s.Init != nil { + b.stmt(s.Init) + } + if s.Tag != nil { + b.add(s.Tag) + } + done := b.newBlock("switch.done") + if label != nil { + label._break = done + } + // We pull the default case (if present) down to the end. + // But each fallthrough label must point to the next + // body block in source order, so we preallocate a + // body block (fallthru) for the next case. + // Unfortunately this makes for a confusing block order. + var defaultBody *[]ast.Stmt + var defaultFallthrough *Block + var fallthru, defaultBlock *Block + ncases := len(s.Body.List) + for i, clause := range s.Body.List { + body := fallthru + if body == nil { + body = b.newBlock("switch.body") // first case only + } + + // Preallocate body block for the next case. + fallthru = done + if i+1 < ncases { + fallthru = b.newBlock("switch.body") + } + + cc := clause.(*ast.CaseClause) + if cc.List == nil { + // Default case. + defaultBody = &cc.Body + defaultFallthrough = fallthru + defaultBlock = body + continue + } + + var nextCond *Block + for _, cond := range cc.List { + nextCond = b.newBlock("switch.next") + b.add(cond) // one half of the tag==cond condition + b.ifelse(body, nextCond) + b.current = nextCond + } + b.current = body + b.targets = &targets{ + tail: b.targets, + _break: done, + _fallthrough: fallthru, + } + b.stmtList(cc.Body) + b.targets = b.targets.tail + b.jump(done) + b.current = nextCond + } + if defaultBlock != nil { + b.jump(defaultBlock) + b.current = defaultBlock + b.targets = &targets{ + tail: b.targets, + _break: done, + _fallthrough: defaultFallthrough, + } + b.stmtList(*defaultBody) + b.targets = b.targets.tail + } + b.jump(done) + b.current = done +} + +func (b *builder) typeSwitchStmt(s *ast.TypeSwitchStmt, label *lblock) { + if s.Init != nil { + b.stmt(s.Init) + } + if s.Assign != nil { + b.add(s.Assign) + } + + done := b.newBlock("typeswitch.done") + if label != nil { + label._break = done + } + var default_ *ast.CaseClause + for _, clause := range s.Body.List { + cc := clause.(*ast.CaseClause) + if cc.List == nil { + default_ = cc + continue + } + body := b.newBlock("typeswitch.body") + var next *Block + for _, casetype := range cc.List { + next = b.newBlock("typeswitch.next") + // casetype is a type, so don't call b.add(casetype). + // This block logically contains a type assertion, + // x.(casetype), but it's unclear how to represent x. + _ = casetype + b.ifelse(body, next) + b.current = next + } + b.current = body + b.typeCaseBody(cc, done) + b.current = next + } + if default_ != nil { + b.typeCaseBody(default_, done) + } else { + b.jump(done) + } + b.current = done +} + +func (b *builder) typeCaseBody(cc *ast.CaseClause, done *Block) { + b.targets = &targets{ + tail: b.targets, + _break: done, + } + b.stmtList(cc.Body) + b.targets = b.targets.tail + b.jump(done) +} + +func (b *builder) selectStmt(s *ast.SelectStmt, label *lblock) { + // First evaluate channel expressions. + // TODO(adonovan): fix: evaluate only channel exprs here. + for _, clause := range s.Body.List { + if comm := clause.(*ast.CommClause).Comm; comm != nil { + b.stmt(comm) + } + } + + done := b.newBlock("select.done") + if label != nil { + label._break = done + } + + var defaultBody *[]ast.Stmt + for _, cc := range s.Body.List { + clause := cc.(*ast.CommClause) + if clause.Comm == nil { + defaultBody = &clause.Body + continue + } + body := b.newBlock("select.body") + next := b.newBlock("select.next") + b.ifelse(body, next) + b.current = body + b.targets = &targets{ + tail: b.targets, + _break: done, + } + switch comm := clause.Comm.(type) { + case *ast.ExprStmt: // <-ch + // nop + case *ast.AssignStmt: // x := <-states[state].Chan + b.add(comm.Lhs[0]) + } + b.stmtList(clause.Body) + b.targets = b.targets.tail + b.jump(done) + b.current = next + } + if defaultBody != nil { + b.targets = &targets{ + tail: b.targets, + _break: done, + } + b.stmtList(*defaultBody) + b.targets = b.targets.tail + b.jump(done) + } + b.current = done +} + +func (b *builder) forStmt(s *ast.ForStmt, label *lblock) { + // ...init... + // jump loop + // loop: + // if cond goto body else done + // body: + // ...body... + // jump post + // post: (target of continue) + // ...post... + // jump loop + // done: (target of break) + if s.Init != nil { + b.stmt(s.Init) + } + body := b.newBlock("for.body") + done := b.newBlock("for.done") // target of 'break' + loop := body // target of back-edge + if s.Cond != nil { + loop = b.newBlock("for.loop") + } + cont := loop // target of 'continue' + if s.Post != nil { + cont = b.newBlock("for.post") + } + if label != nil { + label._break = done + label._continue = cont + } + b.jump(loop) + b.current = loop + if loop != body { + b.add(s.Cond) + b.ifelse(body, done) + b.current = body + } + b.targets = &targets{ + tail: b.targets, + _break: done, + _continue: cont, + } + b.stmt(s.Body) + b.targets = b.targets.tail + b.jump(cont) + + if s.Post != nil { + b.current = cont + b.stmt(s.Post) + b.jump(loop) // back-edge + } + b.current = done +} + +func (b *builder) rangeStmt(s *ast.RangeStmt, label *lblock) { + b.add(s.X) + + if s.Key != nil { + b.add(s.Key) + } + if s.Value != nil { + b.add(s.Value) + } + + // ... + // loop: (target of continue) + // if ... goto body else done + // body: + // ... + // jump loop + // done: (target of break) + + loop := b.newBlock("range.loop") + b.jump(loop) + b.current = loop + + body := b.newBlock("range.body") + done := b.newBlock("range.done") + b.ifelse(body, done) + b.current = body + + if label != nil { + label._break = done + label._continue = loop + } + b.targets = &targets{ + tail: b.targets, + _break: done, + _continue: loop, + } + b.stmt(s.Body) + b.targets = b.targets.tail + b.jump(loop) // back-edge + b.current = done +} + +// -------- helpers -------- + +// Destinations associated with unlabeled for/switch/select stmts. +// We push/pop one of these as we enter/leave each construct and for +// each BranchStmt we scan for the innermost target of the right type. +// +type targets struct { + tail *targets // rest of stack + _break *Block + _continue *Block + _fallthrough *Block +} + +// Destinations associated with a labeled block. +// We populate these as labels are encountered in forward gotos or +// labeled statements. +// +type lblock struct { + _goto *Block + _break *Block + _continue *Block +} + +// labeledBlock returns the branch target associated with the +// specified label, creating it if needed. +// +func (b *builder) labeledBlock(label *ast.Ident) *lblock { + lb := b.lblocks[label.Obj] + if lb == nil { + lb = &lblock{_goto: b.newBlock(label.Name)} + if b.lblocks == nil { + b.lblocks = make(map[*ast.Object]*lblock) + } + b.lblocks[label.Obj] = lb + } + return lb +} + +// newBlock appends a new unconnected basic block to b.cfg's block +// slice and returns it. +// It does not automatically become the current block. +// comment is an optional string for more readable debugging output. +func (b *builder) newBlock(comment string) *Block { + g := b.cfg + block := &Block{ + Index: int32(len(g.Blocks)), + comment: comment, + } + block.Succs = block.succs2[:0] + g.Blocks = append(g.Blocks, block) + return block +} + +func (b *builder) add(n ast.Node) { + b.current.Nodes = append(b.current.Nodes, n) +} + +// jump adds an edge from the current block to the target block, +// and sets b.current to nil. +func (b *builder) jump(target *Block) { + b.current.Succs = append(b.current.Succs, target) + b.current = nil +} + +// ifelse emits edges from the current block to the t and f blocks, +// and sets b.current to nil. +func (b *builder) ifelse(t, f *Block) { + b.current.Succs = append(b.current.Succs, t, f) + b.current = nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/cfg/cfg.go b/src/cmd/vendor/golang.org/x/tools/go/cfg/cfg.go new file mode 100644 index 0000000000..b075034bb4 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/cfg/cfg.go @@ -0,0 +1,150 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package constructs a simple control-flow graph (CFG) of the +// statements and expressions within a single function. +// +// Use cfg.New to construct the CFG for a function body. +// +// The blocks of the CFG contain all the function's non-control +// statements. The CFG does not contain control statements such as If, +// Switch, Select, and Branch, but does contain their subexpressions. +// For example, this source code: +// +// if x := f(); x != nil { +// T() +// } else { +// F() +// } +// +// produces this CFG: +// +// 1: x := f() +// x != nil +// succs: 2, 3 +// 2: T() +// succs: 4 +// 3: F() +// succs: 4 +// 4: +// +// The CFG does contain Return statements; even implicit returns are +// materialized (at the position of the function's closing brace). +// +// The CFG does not record conditions associated with conditional branch +// edges, nor the short-circuit semantics of the && and || operators, +// nor abnormal control flow caused by panic. If you need this +// information, use golang.org/x/tools/go/ssa instead. +// +package cfg + +import ( + "bytes" + "fmt" + "go/ast" + "go/format" + "go/token" +) + +// A CFG represents the control-flow graph of a single function. +// +// The entry point is Blocks[0]; there may be multiple return blocks. +type CFG struct { + Blocks []*Block // block[0] is entry; order otherwise undefined +} + +// A Block represents a basic block: a list of statements and +// expressions that are always evaluated sequentially. +// +// A block may have 0-2 successors: zero for a return block or a block +// that calls a function such as panic that never returns; one for a +// normal (jump) block; and two for a conditional (if) block. +type Block struct { + Nodes []ast.Node // statements, expressions, and ValueSpecs + Succs []*Block // successor nodes in the graph + Index int32 // index within CFG.Blocks + Live bool // block is reachable from entry + + comment string // for debugging + succs2 [2]*Block // underlying array for Succs +} + +// New returns a new control-flow graph for the specified function body, +// which must be non-nil. +// +// The CFG builder calls mayReturn to determine whether a given function +// call may return. For example, calls to panic, os.Exit, and log.Fatal +// do not return, so the builder can remove infeasible graph edges +// following such calls. The builder calls mayReturn only for a +// CallExpr beneath an ExprStmt. +func New(body *ast.BlockStmt, mayReturn func(*ast.CallExpr) bool) *CFG { + b := builder{ + mayReturn: mayReturn, + cfg: new(CFG), + } + b.current = b.newBlock("entry") + b.stmt(body) + + // Compute liveness (reachability from entry point), breadth-first. + q := make([]*Block, 0, len(b.cfg.Blocks)) + q = append(q, b.cfg.Blocks[0]) // entry point + for len(q) > 0 { + b := q[len(q)-1] + q = q[:len(q)-1] + + if !b.Live { + b.Live = true + q = append(q, b.Succs...) + } + } + + // Does control fall off the end of the function's body? + // Make implicit return explicit. + if b.current != nil && b.current.Live { + b.add(&ast.ReturnStmt{ + Return: body.End() - 1, + }) + } + + return b.cfg +} + +func (b *Block) String() string { + return fmt.Sprintf("block %d (%s)", b.Index, b.comment) +} + +// Return returns the return statement at the end of this block if present, nil otherwise. +func (b *Block) Return() (ret *ast.ReturnStmt) { + if len(b.Nodes) > 0 { + ret, _ = b.Nodes[len(b.Nodes)-1].(*ast.ReturnStmt) + } + return +} + +// Format formats the control-flow graph for ease of debugging. +func (g *CFG) Format(fset *token.FileSet) string { + var buf bytes.Buffer + for _, b := range g.Blocks { + fmt.Fprintf(&buf, ".%d: # %s\n", b.Index, b.comment) + for _, n := range b.Nodes { + fmt.Fprintf(&buf, "\t%s\n", formatNode(fset, n)) + } + if len(b.Succs) > 0 { + fmt.Fprintf(&buf, "\tsuccs:") + for _, succ := range b.Succs { + fmt.Fprintf(&buf, " %d", succ.Index) + } + buf.WriteByte('\n') + } + buf.WriteByte('\n') + } + return buf.String() +} + +func formatNode(fset *token.FileSet, n ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, fset, n) + // Indent secondary lines by a tab. + return string(bytes.Replace(buf.Bytes(), []byte("\n"), []byte("\n\t"), -1)) +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go b/src/cmd/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go new file mode 100644 index 0000000000..0d85488efb --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go @@ -0,0 +1,523 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package objectpath defines a naming scheme for types.Objects +// (that is, named entities in Go programs) relative to their enclosing +// package. +// +// Type-checker objects are canonical, so they are usually identified by +// their address in memory (a pointer), but a pointer has meaning only +// within one address space. By contrast, objectpath names allow the +// identity of an object to be sent from one program to another, +// establishing a correspondence between types.Object variables that are +// distinct but logically equivalent. +// +// A single object may have multiple paths. In this example, +// type A struct{ X int } +// type B A +// the field X has two paths due to its membership of both A and B. +// The For(obj) function always returns one of these paths, arbitrarily +// but consistently. +package objectpath + +import ( + "fmt" + "strconv" + "strings" + + "go/types" +) + +// A Path is an opaque name that identifies a types.Object +// relative to its package. Conceptually, the name consists of a +// sequence of destructuring operations applied to the package scope +// to obtain the original object. +// The name does not include the package itself. +type Path string + +// Encoding +// +// An object path is a textual and (with training) human-readable encoding +// of a sequence of destructuring operators, starting from a types.Package. +// The sequences represent a path through the package/object/type graph. +// We classify these operators by their type: +// +// PO package->object Package.Scope.Lookup +// OT object->type Object.Type +// TT type->type Type.{Elem,Key,Params,Results,Underlying} [EKPRU] +// TO type->object Type.{At,Field,Method,Obj} [AFMO] +// +// All valid paths start with a package and end at an object +// and thus may be defined by the regular language: +// +// objectpath = PO (OT TT* TO)* +// +// The concrete encoding follows directly: +// - The only PO operator is Package.Scope.Lookup, which requires an identifier. +// - The only OT operator is Object.Type, +// which we encode as '.' because dot cannot appear in an identifier. +// - The TT operators are encoded as [EKPRU]. +// - The OT operators are encoded as [AFMO]; +// three of these (At,Field,Method) require an integer operand, +// which is encoded as a string of decimal digits. +// These indices are stable across different representations +// of the same package, even source and export data. +// +// In the example below, +// +// package p +// +// type T interface { +// f() (a string, b struct{ X int }) +// } +// +// field X has the path "T.UM0.RA1.F0", +// representing the following sequence of operations: +// +// p.Lookup("T") T +// .Type().Underlying().Method(0). f +// .Type().Results().At(1) b +// .Type().Field(0) X +// +// The encoding is not maximally compact---every R or P is +// followed by an A, for example---but this simplifies the +// encoder and decoder. +// +const ( + // object->type operators + opType = '.' // .Type() (Object) + + // type->type operators + opElem = 'E' // .Elem() (Pointer, Slice, Array, Chan, Map) + opKey = 'K' // .Key() (Map) + opParams = 'P' // .Params() (Signature) + opResults = 'R' // .Results() (Signature) + opUnderlying = 'U' // .Underlying() (Named) + + // type->object operators + opAt = 'A' // .At(i) (Tuple) + opField = 'F' // .Field(i) (Struct) + opMethod = 'M' // .Method(i) (Named or Interface; not Struct: "promoted" names are ignored) + opObj = 'O' // .Obj() (Named) +) + +// The For function returns the path to an object relative to its package, +// or an error if the object is not accessible from the package's Scope. +// +// The For function guarantees to return a path only for the following objects: +// - package-level types +// - exported package-level non-types +// - methods +// - parameter and result variables +// - struct fields +// These objects are sufficient to define the API of their package. +// The objects described by a package's export data are drawn from this set. +// +// For does not return a path for predeclared names, imported package +// names, local names, and unexported package-level names (except +// types). +// +// Example: given this definition, +// +// package p +// +// type T interface { +// f() (a string, b struct{ X int }) +// } +// +// For(X) would return a path that denotes the following sequence of operations: +// +// p.Scope().Lookup("T") (TypeName T) +// .Type().Underlying().Method(0). (method Func f) +// .Type().Results().At(1) (field Var b) +// .Type().Field(0) (field Var X) +// +// where p is the package (*types.Package) to which X belongs. +func For(obj types.Object) (Path, error) { + pkg := obj.Pkg() + + // This table lists the cases of interest. + // + // Object Action + // ------ ------ + // nil reject + // builtin reject + // pkgname reject + // label reject + // var + // package-level accept + // func param/result accept + // local reject + // struct field accept + // const + // package-level accept + // local reject + // func + // package-level accept + // init functions reject + // concrete method accept + // interface method accept + // type + // package-level accept + // local reject + // + // The only accessible package-level objects are members of pkg itself. + // + // The cases are handled in four steps: + // + // 1. reject nil and builtin + // 2. accept package-level objects + // 3. reject obviously invalid objects + // 4. search the API for the path to the param/result/field/method. + + // 1. reference to nil or builtin? + if pkg == nil { + return "", fmt.Errorf("predeclared %s has no path", obj) + } + scope := pkg.Scope() + + // 2. package-level object? + if scope.Lookup(obj.Name()) == obj { + // Only exported objects (and non-exported types) have a path. + // Non-exported types may be referenced by other objects. + if _, ok := obj.(*types.TypeName); !ok && !obj.Exported() { + return "", fmt.Errorf("no path for non-exported %v", obj) + } + return Path(obj.Name()), nil + } + + // 3. Not a package-level object. + // Reject obviously non-viable cases. + switch obj := obj.(type) { + case *types.Const, // Only package-level constants have a path. + *types.TypeName, // Only package-level types have a path. + *types.Label, // Labels are function-local. + *types.PkgName: // PkgNames are file-local. + return "", fmt.Errorf("no path for %v", obj) + + case *types.Var: + // Could be: + // - a field (obj.IsField()) + // - a func parameter or result + // - a local var. + // Sadly there is no way to distinguish + // a param/result from a local + // so we must proceed to the find. + + case *types.Func: + // A func, if not package-level, must be a method. + if recv := obj.Type().(*types.Signature).Recv(); recv == nil { + return "", fmt.Errorf("func is not a method: %v", obj) + } + // TODO(adonovan): opt: if the method is concrete, + // do a specialized version of the rest of this function so + // that it's O(1) not O(|scope|). Basically 'find' is needed + // only for struct fields and interface methods. + + default: + panic(obj) + } + + // 4. Search the API for the path to the var (field/param/result) or method. + + // First inspect package-level named types. + // In the presence of path aliases, these give + // the best paths because non-types may + // refer to types, but not the reverse. + empty := make([]byte, 0, 48) // initial space + for _, name := range scope.Names() { + o := scope.Lookup(name) + tname, ok := o.(*types.TypeName) + if !ok { + continue // handle non-types in second pass + } + + path := append(empty, name...) + path = append(path, opType) + + T := o.Type() + + if tname.IsAlias() { + // type alias + if r := find(obj, T, path); r != nil { + return Path(r), nil + } + } else { + // defined (named) type + if r := find(obj, T.Underlying(), append(path, opUnderlying)); r != nil { + return Path(r), nil + } + } + } + + // Then inspect everything else: + // non-types, and declared methods of defined types. + for _, name := range scope.Names() { + o := scope.Lookup(name) + path := append(empty, name...) + if _, ok := o.(*types.TypeName); !ok { + if o.Exported() { + // exported non-type (const, var, func) + if r := find(obj, o.Type(), append(path, opType)); r != nil { + return Path(r), nil + } + } + continue + } + + // Inspect declared methods of defined types. + if T, ok := o.Type().(*types.Named); ok { + path = append(path, opType) + for i := 0; i < T.NumMethods(); i++ { + m := T.Method(i) + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return Path(path2), nil // found declared method + } + if r := find(obj, m.Type(), append(path2, opType)); r != nil { + return Path(r), nil + } + } + } + } + + return "", fmt.Errorf("can't find path for %v in %s", obj, pkg.Path()) +} + +func appendOpArg(path []byte, op byte, arg int) []byte { + path = append(path, op) + path = strconv.AppendInt(path, int64(arg), 10) + return path +} + +// find finds obj within type T, returning the path to it, or nil if not found. +func find(obj types.Object, T types.Type, path []byte) []byte { + switch T := T.(type) { + case *types.Basic, *types.Named: + // Named types belonging to pkg were handled already, + // so T must belong to another package. No path. + return nil + case *types.Pointer: + return find(obj, T.Elem(), append(path, opElem)) + case *types.Slice: + return find(obj, T.Elem(), append(path, opElem)) + case *types.Array: + return find(obj, T.Elem(), append(path, opElem)) + case *types.Chan: + return find(obj, T.Elem(), append(path, opElem)) + case *types.Map: + if r := find(obj, T.Key(), append(path, opKey)); r != nil { + return r + } + return find(obj, T.Elem(), append(path, opElem)) + case *types.Signature: + if r := find(obj, T.Params(), append(path, opParams)); r != nil { + return r + } + return find(obj, T.Results(), append(path, opResults)) + case *types.Struct: + for i := 0; i < T.NumFields(); i++ { + f := T.Field(i) + path2 := appendOpArg(path, opField, i) + if f == obj { + return path2 // found field var + } + if r := find(obj, f.Type(), append(path2, opType)); r != nil { + return r + } + } + return nil + case *types.Tuple: + for i := 0; i < T.Len(); i++ { + v := T.At(i) + path2 := appendOpArg(path, opAt, i) + if v == obj { + return path2 // found param/result var + } + if r := find(obj, v.Type(), append(path2, opType)); r != nil { + return r + } + } + return nil + case *types.Interface: + for i := 0; i < T.NumMethods(); i++ { + m := T.Method(i) + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return path2 // found interface method + } + if r := find(obj, m.Type(), append(path2, opType)); r != nil { + return r + } + } + return nil + } + panic(T) +} + +// Object returns the object denoted by path p within the package pkg. +func Object(pkg *types.Package, p Path) (types.Object, error) { + if p == "" { + return nil, fmt.Errorf("empty path") + } + + pathstr := string(p) + var pkgobj, suffix string + if dot := strings.IndexByte(pathstr, opType); dot < 0 { + pkgobj = pathstr + } else { + pkgobj = pathstr[:dot] + suffix = pathstr[dot:] // suffix starts with "." + } + + obj := pkg.Scope().Lookup(pkgobj) + if obj == nil { + return nil, fmt.Errorf("package %s does not contain %q", pkg.Path(), pkgobj) + } + + // abtraction of *types.{Pointer,Slice,Array,Chan,Map} + type hasElem interface { + Elem() types.Type + } + // abstraction of *types.{Interface,Named} + type hasMethods interface { + Method(int) *types.Func + NumMethods() int + } + + // The loop state is the pair (t, obj), + // exactly one of which is non-nil, initially obj. + // All suffixes start with '.' (the only object->type operation), + // followed by optional type->type operations, + // then a type->object operation. + // The cycle then repeats. + var t types.Type + for suffix != "" { + code := suffix[0] + suffix = suffix[1:] + + // Codes [AFM] have an integer operand. + var index int + switch code { + case opAt, opField, opMethod: + rest := strings.TrimLeft(suffix, "0123456789") + numerals := suffix[:len(suffix)-len(rest)] + suffix = rest + i, err := strconv.Atoi(numerals) + if err != nil { + return nil, fmt.Errorf("invalid path: bad numeric operand %q for code %q", numerals, code) + } + index = int(i) + case opObj: + // no operand + default: + // The suffix must end with a type->object operation. + if suffix == "" { + return nil, fmt.Errorf("invalid path: ends with %q, want [AFMO]", code) + } + } + + if code == opType { + if t != nil { + return nil, fmt.Errorf("invalid path: unexpected %q in type context", opType) + } + t = obj.Type() + obj = nil + continue + } + + if t == nil { + return nil, fmt.Errorf("invalid path: code %q in object context", code) + } + + // Inv: t != nil, obj == nil + + switch code { + case opElem: + hasElem, ok := t.(hasElem) // Pointer, Slice, Array, Chan, Map + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want pointer, slice, array, chan or map)", code, t, t) + } + t = hasElem.Elem() + + case opKey: + mapType, ok := t.(*types.Map) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want map)", code, t, t) + } + t = mapType.Key() + + case opParams: + sig, ok := t.(*types.Signature) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want signature)", code, t, t) + } + t = sig.Params() + + case opResults: + sig, ok := t.(*types.Signature) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want signature)", code, t, t) + } + t = sig.Results() + + case opUnderlying: + named, ok := t.(*types.Named) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %s, want named)", code, t, t) + } + t = named.Underlying() + + case opAt: + tuple, ok := t.(*types.Tuple) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %s, want tuple)", code, t, t) + } + if n := tuple.Len(); index >= n { + return nil, fmt.Errorf("tuple index %d out of range [0-%d)", index, n) + } + obj = tuple.At(index) + t = nil + + case opField: + structType, ok := t.(*types.Struct) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %T, want struct)", code, t, t) + } + if n := structType.NumFields(); index >= n { + return nil, fmt.Errorf("field index %d out of range [0-%d)", index, n) + } + obj = structType.Field(index) + t = nil + + case opMethod: + hasMethods, ok := t.(hasMethods) // Interface or Named + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %s, want interface or named)", code, t, t) + } + if n := hasMethods.NumMethods(); index >= n { + return nil, fmt.Errorf("method index %d out of range [0-%d)", index, n) + } + obj = hasMethods.Method(index) + t = nil + + case opObj: + named, ok := t.(*types.Named) + if !ok { + return nil, fmt.Errorf("cannot apply %q to %s (got %s, want named)", code, t, t) + } + obj = named.Obj() + t = nil + + default: + return nil, fmt.Errorf("invalid path: unknown code %q", code) + } + } + + if obj.Pkg() != pkg { + return nil, fmt.Errorf("path denotes %s, which belongs to a different package", obj) + } + + return obj, nil // success +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/callee.go b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/callee.go new file mode 100644 index 0000000000..38f596daf9 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/callee.go @@ -0,0 +1,46 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typeutil + +import ( + "go/ast" + "go/types" + + "golang.org/x/tools/go/ast/astutil" +) + +// Callee returns the named target of a function call, if any: +// a function, method, builtin, or variable. +func Callee(info *types.Info, call *ast.CallExpr) types.Object { + var obj types.Object + switch fun := astutil.Unparen(call.Fun).(type) { + case *ast.Ident: + obj = info.Uses[fun] // type, var, builtin, or declared func + case *ast.SelectorExpr: + if sel, ok := info.Selections[fun]; ok { + obj = sel.Obj() // method or field + } else { + obj = info.Uses[fun.Sel] // qualified identifier? + } + } + if _, ok := obj.(*types.TypeName); ok { + return nil // T(x) is a conversion, not a call + } + return obj +} + +// StaticCallee returns the target (function or method) of a static +// function call, if any. It returns nil for calls to builtins. +func StaticCallee(info *types.Info, call *ast.CallExpr) *types.Func { + if f, ok := Callee(info, call).(*types.Func); ok && !interfaceMethod(f) { + return f + } + return nil +} + +func interfaceMethod(f *types.Func) bool { + recv := f.Type().(*types.Signature).Recv() + return recv != nil && types.IsInterface(recv.Type()) +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/imports.go b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/imports.go new file mode 100644 index 0000000000..9c441dba9c --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/imports.go @@ -0,0 +1,31 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typeutil + +import "go/types" + +// Dependencies returns all dependencies of the specified packages. +// +// Dependent packages appear in topological order: if package P imports +// package Q, Q appears earlier than P in the result. +// The algorithm follows import statements in the order they +// appear in the source code, so the result is a total order. +// +func Dependencies(pkgs ...*types.Package) []*types.Package { + var result []*types.Package + seen := make(map[*types.Package]bool) + var visit func(pkgs []*types.Package) + visit = func(pkgs []*types.Package) { + for _, p := range pkgs { + if !seen[p] { + seen[p] = true + visit(p.Imports()) + result = append(result, p) + } + } + } + visit(pkgs) + return result +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/map.go b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/map.go new file mode 100644 index 0000000000..c7f7545006 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/map.go @@ -0,0 +1,313 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package typeutil defines various utilities for types, such as Map, +// a mapping from types.Type to interface{} values. +package typeutil // import "golang.org/x/tools/go/types/typeutil" + +import ( + "bytes" + "fmt" + "go/types" + "reflect" +) + +// Map is a hash-table-based mapping from types (types.Type) to +// arbitrary interface{} values. The concrete types that implement +// the Type interface are pointers. Since they are not canonicalized, +// == cannot be used to check for equivalence, and thus we cannot +// simply use a Go map. +// +// Just as with map[K]V, a nil *Map is a valid empty map. +// +// Not thread-safe. +// +type Map struct { + hasher Hasher // shared by many Maps + table map[uint32][]entry // maps hash to bucket; entry.key==nil means unused + length int // number of map entries +} + +// entry is an entry (key/value association) in a hash bucket. +type entry struct { + key types.Type + value interface{} +} + +// SetHasher sets the hasher used by Map. +// +// All Hashers are functionally equivalent but contain internal state +// used to cache the results of hashing previously seen types. +// +// A single Hasher created by MakeHasher() may be shared among many +// Maps. This is recommended if the instances have many keys in +// common, as it will amortize the cost of hash computation. +// +// A Hasher may grow without bound as new types are seen. Even when a +// type is deleted from the map, the Hasher never shrinks, since other +// types in the map may reference the deleted type indirectly. +// +// Hashers are not thread-safe, and read-only operations such as +// Map.Lookup require updates to the hasher, so a full Mutex lock (not a +// read-lock) is require around all Map operations if a shared +// hasher is accessed from multiple threads. +// +// If SetHasher is not called, the Map will create a private hasher at +// the first call to Insert. +// +func (m *Map) SetHasher(hasher Hasher) { + m.hasher = hasher +} + +// Delete removes the entry with the given key, if any. +// It returns true if the entry was found. +// +func (m *Map) Delete(key types.Type) bool { + if m != nil && m.table != nil { + hash := m.hasher.Hash(key) + bucket := m.table[hash] + for i, e := range bucket { + if e.key != nil && types.Identical(key, e.key) { + // We can't compact the bucket as it + // would disturb iterators. + bucket[i] = entry{} + m.length-- + return true + } + } + } + return false +} + +// At returns the map entry for the given key. +// The result is nil if the entry is not present. +// +func (m *Map) At(key types.Type) interface{} { + if m != nil && m.table != nil { + for _, e := range m.table[m.hasher.Hash(key)] { + if e.key != nil && types.Identical(key, e.key) { + return e.value + } + } + } + return nil +} + +// Set sets the map entry for key to val, +// and returns the previous entry, if any. +func (m *Map) Set(key types.Type, value interface{}) (prev interface{}) { + if m.table != nil { + hash := m.hasher.Hash(key) + bucket := m.table[hash] + var hole *entry + for i, e := range bucket { + if e.key == nil { + hole = &bucket[i] + } else if types.Identical(key, e.key) { + prev = e.value + bucket[i].value = value + return + } + } + + if hole != nil { + *hole = entry{key, value} // overwrite deleted entry + } else { + m.table[hash] = append(bucket, entry{key, value}) + } + } else { + if m.hasher.memo == nil { + m.hasher = MakeHasher() + } + hash := m.hasher.Hash(key) + m.table = map[uint32][]entry{hash: {entry{key, value}}} + } + + m.length++ + return +} + +// Len returns the number of map entries. +func (m *Map) Len() int { + if m != nil { + return m.length + } + return 0 +} + +// Iterate calls function f on each entry in the map in unspecified order. +// +// If f should mutate the map, Iterate provides the same guarantees as +// Go maps: if f deletes a map entry that Iterate has not yet reached, +// f will not be invoked for it, but if f inserts a map entry that +// Iterate has not yet reached, whether or not f will be invoked for +// it is unspecified. +// +func (m *Map) Iterate(f func(key types.Type, value interface{})) { + if m != nil { + for _, bucket := range m.table { + for _, e := range bucket { + if e.key != nil { + f(e.key, e.value) + } + } + } + } +} + +// Keys returns a new slice containing the set of map keys. +// The order is unspecified. +func (m *Map) Keys() []types.Type { + keys := make([]types.Type, 0, m.Len()) + m.Iterate(func(key types.Type, _ interface{}) { + keys = append(keys, key) + }) + return keys +} + +func (m *Map) toString(values bool) string { + if m == nil { + return "{}" + } + var buf bytes.Buffer + fmt.Fprint(&buf, "{") + sep := "" + m.Iterate(func(key types.Type, value interface{}) { + fmt.Fprint(&buf, sep) + sep = ", " + fmt.Fprint(&buf, key) + if values { + fmt.Fprintf(&buf, ": %q", value) + } + }) + fmt.Fprint(&buf, "}") + return buf.String() +} + +// String returns a string representation of the map's entries. +// Values are printed using fmt.Sprintf("%v", v). +// Order is unspecified. +// +func (m *Map) String() string { + return m.toString(true) +} + +// KeysString returns a string representation of the map's key set. +// Order is unspecified. +// +func (m *Map) KeysString() string { + return m.toString(false) +} + +//////////////////////////////////////////////////////////////////////// +// Hasher + +// A Hasher maps each type to its hash value. +// For efficiency, a hasher uses memoization; thus its memory +// footprint grows monotonically over time. +// Hashers are not thread-safe. +// Hashers have reference semantics. +// Call MakeHasher to create a Hasher. +type Hasher struct { + memo map[types.Type]uint32 +} + +// MakeHasher returns a new Hasher instance. +func MakeHasher() Hasher { + return Hasher{make(map[types.Type]uint32)} +} + +// Hash computes a hash value for the given type t such that +// Identical(t, t') => Hash(t) == Hash(t'). +func (h Hasher) Hash(t types.Type) uint32 { + hash, ok := h.memo[t] + if !ok { + hash = h.hashFor(t) + h.memo[t] = hash + } + return hash +} + +// hashString computes the Fowler–Noll–Vo hash of s. +func hashString(s string) uint32 { + var h uint32 + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +// hashFor computes the hash of t. +func (h Hasher) hashFor(t types.Type) uint32 { + // See Identical for rationale. + switch t := t.(type) { + case *types.Basic: + return uint32(t.Kind()) + + case *types.Array: + return 9043 + 2*uint32(t.Len()) + 3*h.Hash(t.Elem()) + + case *types.Slice: + return 9049 + 2*h.Hash(t.Elem()) + + case *types.Struct: + var hash uint32 = 9059 + for i, n := 0, t.NumFields(); i < n; i++ { + f := t.Field(i) + if f.Anonymous() { + hash += 8861 + } + hash += hashString(t.Tag(i)) + hash += hashString(f.Name()) // (ignore f.Pkg) + hash += h.Hash(f.Type()) + } + return hash + + case *types.Pointer: + return 9067 + 2*h.Hash(t.Elem()) + + case *types.Signature: + var hash uint32 = 9091 + if t.Variadic() { + hash *= 8863 + } + return hash + 3*h.hashTuple(t.Params()) + 5*h.hashTuple(t.Results()) + + case *types.Interface: + var hash uint32 = 9103 + for i, n := 0, t.NumMethods(); i < n; i++ { + // See go/types.identicalMethods for rationale. + // Method order is not significant. + // Ignore m.Pkg(). + m := t.Method(i) + hash += 3*hashString(m.Name()) + 5*h.Hash(m.Type()) + } + return hash + + case *types.Map: + return 9109 + 2*h.Hash(t.Key()) + 3*h.Hash(t.Elem()) + + case *types.Chan: + return 9127 + 2*uint32(t.Dir()) + 3*h.Hash(t.Elem()) + + case *types.Named: + // Not safe with a copying GC; objects may move. + return uint32(reflect.ValueOf(t.Obj()).Pointer()) + + case *types.Tuple: + return h.hashTuple(t) + } + panic(t) +} + +func (h Hasher) hashTuple(tuple *types.Tuple) uint32 { + // See go/types.identicalTypes for rationale. + n := tuple.Len() + var hash uint32 = 9137 + 2*uint32(n) + for i := 0; i < n; i++ { + hash += 3 * h.Hash(tuple.At(i).Type()) + } + return hash +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/methodsetcache.go b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/methodsetcache.go new file mode 100644 index 0000000000..32084610f4 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/methodsetcache.go @@ -0,0 +1,72 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a cache of method sets. + +package typeutil + +import ( + "go/types" + "sync" +) + +// A MethodSetCache records the method set of each type T for which +// MethodSet(T) is called so that repeat queries are fast. +// The zero value is a ready-to-use cache instance. +type MethodSetCache struct { + mu sync.Mutex + named map[*types.Named]struct{ value, pointer *types.MethodSet } // method sets for named N and *N + others map[types.Type]*types.MethodSet // all other types +} + +// MethodSet returns the method set of type T. It is thread-safe. +// +// If cache is nil, this function is equivalent to types.NewMethodSet(T). +// Utility functions can thus expose an optional *MethodSetCache +// parameter to clients that care about performance. +// +func (cache *MethodSetCache) MethodSet(T types.Type) *types.MethodSet { + if cache == nil { + return types.NewMethodSet(T) + } + cache.mu.Lock() + defer cache.mu.Unlock() + + switch T := T.(type) { + case *types.Named: + return cache.lookupNamed(T).value + + case *types.Pointer: + if N, ok := T.Elem().(*types.Named); ok { + return cache.lookupNamed(N).pointer + } + } + + // all other types + // (The map uses pointer equivalence, not type identity.) + mset := cache.others[T] + if mset == nil { + mset = types.NewMethodSet(T) + if cache.others == nil { + cache.others = make(map[types.Type]*types.MethodSet) + } + cache.others[T] = mset + } + return mset +} + +func (cache *MethodSetCache) lookupNamed(named *types.Named) struct{ value, pointer *types.MethodSet } { + if cache.named == nil { + cache.named = make(map[*types.Named]struct{ value, pointer *types.MethodSet }) + } + // Avoid recomputing mset(*T) for each distinct Pointer + // instance whose underlying type is a named type. + msets, ok := cache.named[named] + if !ok { + msets.value = types.NewMethodSet(named) + msets.pointer = types.NewMethodSet(types.NewPointer(named)) + cache.named[named] = msets + } + return msets +} diff --git a/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/ui.go b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/ui.go new file mode 100644 index 0000000000..9849c24cef --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/go/types/typeutil/ui.go @@ -0,0 +1,52 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typeutil + +// This file defines utilities for user interfaces that display types. + +import "go/types" + +// IntuitiveMethodSet returns the intuitive method set of a type T, +// which is the set of methods you can call on an addressable value of +// that type. +// +// The result always contains MethodSet(T), and is exactly MethodSet(T) +// for interface types and for pointer-to-concrete types. +// For all other concrete types T, the result additionally +// contains each method belonging to *T if there is no identically +// named method on T itself. +// +// This corresponds to user intuition about method sets; +// this function is intended only for user interfaces. +// +// The order of the result is as for types.MethodSet(T). +// +func IntuitiveMethodSet(T types.Type, msets *MethodSetCache) []*types.Selection { + isPointerToConcrete := func(T types.Type) bool { + ptr, ok := T.(*types.Pointer) + return ok && !types.IsInterface(ptr.Elem()) + } + + var result []*types.Selection + mset := msets.MethodSet(T) + if types.IsInterface(T) || isPointerToConcrete(T) { + for i, n := 0, mset.Len(); i < n; i++ { + result = append(result, mset.At(i)) + } + } else { + // T is some other concrete type. + // Report methods of T and *T, preferring those of T. + pmset := msets.MethodSet(types.NewPointer(T)) + for i, n := 0, pmset.Len(); i < n; i++ { + meth := pmset.At(i) + if m := mset.Lookup(meth.Obj().Pkg(), meth.Obj().Name()); m != nil { + meth = m + } + result = append(result, meth) + } + + } + return result +} diff --git a/src/cmd/vendor/vendor.json b/src/cmd/vendor/vendor.json index 6e077e4ae1..b952b93c08 100644 --- a/src/cmd/vendor/vendor.json +++ b/src/cmd/vendor/vendor.json @@ -165,6 +165,210 @@ "path": "golang.org/x/sys/windows/svc/mgr", "revision": "90868a75fefd03942536221d7c0e2f84ec62a668", "revisionTime": "2018-08-01T20:46:00Z" + }, + { + "checksumSHA1": "witNkDO7koGO7+oxpBMZBvoxz3c=", + "path": "golang.org/x/tools/go/analysis", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "NPcubwbqmr2yGfGztLqizwbXrwM=", + "path": "golang.org/x/tools/go/analysis/cmd/vet-lite", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "kWG+JiD2mA+2pnSeYJrKLHHgT+s=", + "path": "golang.org/x/tools/go/analysis/internal/analysisflags", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "c4FY3+yRC2GHON66hIU254nQxA8=", + "path": "golang.org/x/tools/go/analysis/internal/facts", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "Zuz7FbEMWtUNCKTA+ofVkkDl1Ic=", + "path": "golang.org/x/tools/go/analysis/internal/unitchecker", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "fxi2KL0typcqGp87Qa9CxSp89Sk=", + "path": "golang.org/x/tools/go/analysis/passes/asmdecl", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "AK5vKjJmQD1u/6v/s107upAF03w=", + "path": "golang.org/x/tools/go/analysis/passes/assign", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "qRQNlOhRmTPecqsjJMf3Rxd7M1g=", + "path": "golang.org/x/tools/go/analysis/passes/atomic", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "zhnbma06ExmGYTu5QaGAi5+QciY=", + "path": "golang.org/x/tools/go/analysis/passes/bools", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "HWcvlzqG20E9BaG4/j3u9tnUyZ4=", + "path": "golang.org/x/tools/go/analysis/passes/buildtag", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "ekVwfAw224CT/eBihMCzAOzIHiE=", + "path": "golang.org/x/tools/go/analysis/passes/cgocall", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "dwtQdPi0Jb9BYVr0Gynh5NpCSz8=", + "path": "golang.org/x/tools/go/analysis/passes/composite", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "6M8xb//gcLk3dSpRq6/fb/8Wvqk=", + "path": "golang.org/x/tools/go/analysis/passes/copylock", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "DPQnIktTEV7cNBNDRIpg0OK6v9Q=", + "path": "golang.org/x/tools/go/analysis/passes/ctrlflow", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "GJjZZhXYqMoGmym/2DpExqHP+Cw=", + "path": "golang.org/x/tools/go/analysis/passes/httpresponse", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "Q76YV1xYtBCBsZk7uKXqih7iHL4=", + "path": "golang.org/x/tools/go/analysis/passes/inspect", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "Y7NBmaqiGnVWf3yn16cwbWmgUhI=", + "path": "golang.org/x/tools/go/analysis/passes/internal/analysisutil", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "Fjj6sV+qmJwvxGt/i8fLIma9Lzs=", + "path": "golang.org/x/tools/go/analysis/passes/loopclosure", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "VZE2qx/m2esvfEreS0RCaVoWYhc=", + "path": "golang.org/x/tools/go/analysis/passes/lostcancel", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "xf9nMwSbFWJXDC9W+Gnus+uU0Nw=", + "path": "golang.org/x/tools/go/analysis/passes/nilfunc", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "PKByrfYKilYhkhAE01z5Om0Tr+w=", + "path": "golang.org/x/tools/go/analysis/passes/pkgfact", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "BrlVsK8u6SPMyvoWdkwS4IAXVRI=", + "path": "golang.org/x/tools/go/analysis/passes/printf", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "3w9Q99Mxrf8qEU+FH7lSyy5hwc4=", + "path": "golang.org/x/tools/go/analysis/passes/shift", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "oeR6BB6OmfoYReHyNLEX9BbF1cI=", + "path": "golang.org/x/tools/go/analysis/passes/stdmethods", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "rPSH7/W3vsomdmSIgdEDrzaCQyk=", + "path": "golang.org/x/tools/go/analysis/passes/structtag", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "yHrglPUc3Ia12nwO0l/I0ArT3to=", + "path": "golang.org/x/tools/go/analysis/passes/tests", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "en0VsP2OoNX40F/bNfbO6geSgi4=", + "path": "golang.org/x/tools/go/analysis/passes/unreachable", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "641akvyeQUx5MqoHiyKwRps4vEg=", + "path": "golang.org/x/tools/go/analysis/passes/unsafeptr", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "cm27h0jINv4jlgiHMn7q572FXTY=", + "path": "golang.org/x/tools/go/analysis/passes/unusedresult", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "/bQnex6L/nyDuZCIIRbM6Is/IRY=", + "path": "golang.org/x/tools/go/ast/astutil", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "qnZLWirp4hAxafiKvH+nnmgGf8Q=", + "path": "golang.org/x/tools/go/ast/inspector", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "+g97ZSLGNNbqfBzpYje8fA5PvXs=", + "path": "golang.org/x/tools/go/cfg", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "JWIR0GVqbDYhTW9mh4zpY/ve6Ro=", + "path": "golang.org/x/tools/go/types/objectpath", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" + }, + { + "checksumSHA1": "kyVWOWK3PkDKCtXRJffE60MrfOo=", + "path": "golang.org/x/tools/go/types/typeutil", + "revision": "c76e1ad98a635a7c069d7ab43d31fcf38381facc", + "revisionTime": "2018-11-05T19:48:08Z" } ], "rootPath": "/cmd" From 95a4f793c077ab7b13fdb7505b65ff19a97a07f9 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 6 Nov 2018 10:16:17 -0800 Subject: [PATCH 042/111] cmd/compile: don't deadcode eliminate labels Dead-code eliminating labels is tricky because there might be gotos that can still reach them. Bug probably introduced with CL 91056 Fixes #28616 Change-Id: I6680465134e3486dcb658896f5172606cc51b104 Reviewed-on: https://go-review.googlesource.com/c/147817 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick Reviewed-by: Iskander Sharipov --- src/cmd/compile/internal/gc/typecheck.go | 12 +++++++++++- test/fixedbugs/issue28616.go | 25 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue28616.go diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index 06dd176b37..8ec60cbbba 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -4084,6 +4084,12 @@ func deadcode(fn *Node) { } func deadcodeslice(nn Nodes) { + var lastLabel = -1 + for i, n := range nn.Slice() { + if n != nil && n.Op == OLABEL { + lastLabel = i + } + } for i, n := range nn.Slice() { // Cut is set to true when all nodes after i'th position // should be removed. @@ -4106,10 +4112,14 @@ func deadcodeslice(nn Nodes) { // If "then" or "else" branch ends with panic or return statement, // it is safe to remove all statements after this node. // isterminating is not used to avoid goto-related complications. + // We must be careful not to deadcode-remove labels, as they + // might be the target of a goto. See issue 28616. if body := body.Slice(); len(body) != 0 { switch body[(len(body) - 1)].Op { case ORETURN, ORETJMP, OPANIC: - cut = true + if i > lastLabel { + cut = true + } } } } diff --git a/test/fixedbugs/issue28616.go b/test/fixedbugs/issue28616.go new file mode 100644 index 0000000000..f1ba974797 --- /dev/null +++ b/test/fixedbugs/issue28616.go @@ -0,0 +1,25 @@ +// compile + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure we don't dead code eliminate a label. + +package p + +var i int + +func f() { + + if true { + + if i == 1 { + goto label + } + + return + } + +label: +} From 35c05542938416cde6a366505c24568ea5ccd98e Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 2 Nov 2018 16:51:14 -0400 Subject: [PATCH 043/111] cmd/asm: rename R18 to R18_PLATFORM on ARM64 In ARM64 ABI, R18 is the "platform register", the use of which is OS specific. The OS could choose to reserve this register. In practice, it seems fine to use R18 on Linux but not on darwin (iOS). Rename R18 to R18_PLATFORM to prevent accidental use. There is no R18 usage within the standard library (besides tests, which are updated). Fixes #26110 Change-Id: Icef7b9549e2049db1df307a0180a3c90a12d7a84 Reviewed-on: https://go-review.googlesource.com/c/147218 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- doc/asm.html | 1 + src/cmd/asm/internal/arch/arch.go | 3 ++ src/cmd/asm/internal/asm/operand_test.go | 1 + src/cmd/asm/internal/asm/testdata/arm64.s | 8 +-- src/cmd/asm/internal/asm/testdata/arm64enc.s | 54 ++++++++++---------- src/cmd/internal/obj/arm64/doc.go | 10 ++-- 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/doc/asm.html b/doc/asm.html index f2f8fad576..debb1e2fc6 100644 --- a/doc/asm.html +++ b/doc/asm.html @@ -740,6 +740,7 @@ The ARM64 port is in an experimental state.

R18 is the "platform register", reserved on the Apple platform. +To prevent accidental misuse, the register is named R18_PLATFORM. R27 and R28 are reserved by the compiler and linker. R29 is the frame pointer. R30 is the link register. diff --git a/src/cmd/asm/internal/arch/arch.go b/src/cmd/asm/internal/arch/arch.go index ecea6ba97d..eaa5cb8958 100644 --- a/src/cmd/asm/internal/arch/arch.go +++ b/src/cmd/asm/internal/arch/arch.go @@ -258,6 +258,9 @@ func archArm64() *Arch { for i := arm64.REG_R0; i <= arm64.REG_R31; i++ { register[obj.Rconv(i)] = int16(i) } + // Rename R18 to R18_PLATFORM to avoid accidental use. + register["R18_PLATFORM"] = register["R18"] + delete(register, "R18") for i := arm64.REG_F0; i <= arm64.REG_F31; i++ { register[obj.Rconv(i)] = int16(i) } diff --git a/src/cmd/asm/internal/asm/operand_test.go b/src/cmd/asm/internal/asm/operand_test.go index df60b71ebd..69393b6b20 100644 --- a/src/cmd/asm/internal/asm/operand_test.go +++ b/src/cmd/asm/internal/asm/operand_test.go @@ -607,6 +607,7 @@ var arm64OperandTests = []operandTest{ {"R0", "R0"}, {"R10", "R10"}, {"R11", "R11"}, + {"R18_PLATFORM", "R18"}, {"$4503601774854144.0", "$(4503601774854144.0)"}, {"$runtime·badsystemstack(SB)", "$runtime.badsystemstack(SB)"}, {"ZR", "ZR"}, diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s index b851ba411e..a577c4da9d 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64.s +++ b/src/cmd/asm/internal/asm/testdata/arm64.s @@ -47,8 +47,8 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 ADD R2.SXTX<<1, RSP, RSP // ffe7228b ADD ZR.SXTX<<1, R2, R3 // 43e43f8b ADDW R2.SXTW, R10, R12 // 4cc1220b - ADD R18.UXTX, R14, R17 // d161328b - ADDSW R18.UXTW, R14, R17 // d141322b + ADD R19.UXTX, R14, R17 // d161338b + ADDSW R19.UXTW, R14, R17 // d141332b ADDS R12.SXTX, R3, R1 // 61e02cab SUB R19.UXTH<<4, R2, R21 // 553033cb SUBW R1.UXTX<<1, R3, R2 // 6264214b @@ -144,7 +144,7 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 MOVD (R2)(R6.SXTW), R4 // 44c866f8 MOVD (R3)(R6), R5 // MOVD (R3)(R6*1), R5 // 656866f8 MOVD (R2)(R6), R4 // MOVD (R2)(R6*1), R4 // 446866f8 - MOVWU (R19)(R18<<2), R18 // 727a72b8 + MOVWU (R19)(R20<<2), R20 // 747a74b8 MOVD (R2)(R6<<3), R4 // 447866f8 MOVD (R3)(R7.SXTX<<3), R8 // 68f867f8 MOVWU (R5)(R4.UXTW), R10 // aa4864b8 @@ -154,7 +154,7 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 MOVHU (R1)(R2<<1), R5 // 25786278 MOVB (R9)(R3.UXTW), R6 // 2649a338 MOVB (R10)(R6), R15 // MOVB (R10)(R6*1), R15 // 4f69a638 - MOVH (R5)(R7.SXTX<<1), R18 // b2f8a778 + MOVH (R5)(R7.SXTX<<1), R19 // b3f8a778 MOVH (R8)(R4<<1), R10 // 0a79a478 MOVW (R9)(R8.SXTW<<2), R19 // 33d9a8b8 MOVW (R1)(R4.SXTX), R11 // 2be8a4b8 diff --git a/src/cmd/asm/internal/asm/testdata/arm64enc.s b/src/cmd/asm/internal/asm/testdata/arm64enc.s index 432ab74493..a2850e2e46 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64enc.s +++ b/src/cmd/asm/internal/asm/testdata/arm64enc.s @@ -56,7 +56,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 BFXILW $3, R27, $23, R14 // 6e670333 BFXIL $26, R8, $16, R20 // 14a55ab3 BICW R7@>15, R5, R16 // b03ce70a - BIC R12@>13, R12, R18 // 9235ec8a + BIC R12@>13, R12, R19 // 9335ec8a BICSW R25->20, R3, R20 // 7450b96a BICS R19->12, R1, R23 // 3730b3ea BICS R19, R1, R23 // 370033ea @@ -76,7 +76,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 CCMN LE, R30, R12, $6 // c6d34cba CCMPW VS, R29, $15, $7 // a76b4f7a CCMP LE, R7, $19, $3 // e3d853fa - CCMPW HS, R18, R6, $0 // 4022467a + CCMPW HS, R19, R6, $0 // 6022467a CCMP LT, R30, R6, $7 // c7b346fa CCMN MI, ZR, R1, $4 // e44341ba CSINCW HS, ZR, R27, R14 // ee279b1a @@ -118,7 +118,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 CRC32H R3, R21, R27 // bb46c31a CRC32W R22, R30, R9 // c94bd61a CRC32X R20, R4, R15 // 8f4cd49a - CRC32CB R18, R27, R22 // 7653d21a + CRC32CB R19, R27, R22 // 7653d31a CRC32CH R21, R0, R20 // 1454d51a CRC32CW R9, R3, R21 // 7558c91a CRC32CX R11, R0, R24 // 185ccb9a @@ -133,7 +133,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 CSINVW AL, R23, R21, R5 // e5e2955a CSINV LO, R2, R11, R14 // 4e308bda CSNEGW HS, R16, R29, R10 // 0a269d5a - CSNEG NE, R21, R18, R11 // ab1692da + CSNEG NE, R21, R19, R11 // ab1693da //TODO DC DCPS1 $11378 // 418ea5d4 DCPS2 $10699 // 6239a5d4 @@ -185,23 +185,23 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 MOVBU.P 42(R2), R12 // 4ca44238 MOVBU.W -27(R2), R14 // 4e5c5e38 MOVBU 2916(R24), R3 // 03936d39 - MOVBU (R18)(R14<<0), R23 // 577a6e38 + MOVBU (R19)(R14<<0), R23 // 777a6e38 MOVBU (R2)(R8.SXTX), R19 // 53e86838 MOVBU (R27)(R23), R14 // MOVBU (R27)(R23*1), R14 // 6e6b7738 MOVHU.P 107(R14), R13 // cdb54678 MOVHU.W 192(R3), R2 // 620c4c78 - MOVHU 6844(R4), R18 // 92787579 + MOVHU 6844(R4), R19 // 93787579 MOVHU (R5)(R25.SXTW), R15 // afc87978 - //TODO MOVBW.P 77(R18), R11 // 4bd6c438 + //TODO MOVBW.P 77(R19), R11 // 6bd6c438 MOVB.P 36(RSP), R27 // fb478238 - //TODO MOVBW.W -57(R18), R13 // 4d7edc38 + //TODO MOVBW.W -57(R19), R13 // 6d7edc38 MOVB.W -178(R16), R24 // 18ee9438 //TODO MOVBW 430(R8), R22 // 16b9c639 MOVB 997(R9), R23 // 37958f39 //TODO MOVBW (R2<<1)(R21), R15 // af7ae238 //TODO MOVBW (R26)(R0), R21 // 1568fa38 MOVB (R5)(R15), R16 // MOVB (R5)(R15*1), R16 // b068af38 - MOVB (R18)(R26.SXTW), R19 // 53caba38 + MOVB (R19)(R26.SXTW), R19 // 73caba38 MOVB (R29)(R30), R14 // MOVB (R29)(R30*1), R14 // ae6bbe38 //TODO MOVHW.P 218(R22), R25 // d9a6cd78 MOVH.P 179(R23), R5 // e5368b78 @@ -212,7 +212,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 //TODO MOVHW (R22)(R24.SXTX), R4 // c4eaf878 MOVH (R26)(R30.UXTW<<1), ZR // 5f5bbe78 MOVW.P -58(R16), R2 // 02669cb8 - MOVW.W -216(R18), R8 // 488e92b8 + MOVW.W -216(R19), R8 // 688e92b8 MOVW 4764(R23), R10 // ea9e92b9 MOVW (R8)(R3.UXTW), R17 // 1149a3b8 //TODO LDTR -0x1e(R3), R4 // 64285eb8 @@ -297,7 +297,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 RET // c0035fd6 REVW R8, R10 // 0a09c05a REV R1, R2 // 220cc0da - REV16W R21, R18 // b206c05a + REV16W R21, R19 // b306c05a REV16 R25, R4 // 2407c0da REV32 R27, R21 // 750bc0da EXTRW $27, R4, R25, R19 // 336f8413 @@ -308,7 +308,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 ROR R0, R23, R2 // e22ec09a SBCW R4, R8, R24 // 1801045a SBC R25, R10, R26 // 5a0119da - SBCSW R27, R18, R18 // 52021b7a + SBCSW R27, R19, R19 // 73021b7a SBCS R5, R9, R5 // 250105fa SBFIZW $9, R10, $18, R22 // 56451713 SBFIZ $6, R11, $15, R20 // 74397a93 @@ -337,7 +337,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 //TODO STNPW 44(R1), R3, R10 // 2a8c0528 //TODO STNP 0x108(R3), ZR, R7 // 67fc10a8 LDP.P -384(R3), (R22, R26) // 7668e8a8 - LDP.W 280(R8), (R18, R11) // 12add1a9 + LDP.W 280(R8), (R19, R11) // 13add1a9 STP.P (R22, R27), 352(R0) // 166c96a8 STP.W (R17, R11), 96(R8) // 112d86a9 MOVW.P R20, -28(R1) // 34441eb8 @@ -360,22 +360,22 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 MOVB R2, (R29)(R26) // MOVB R2, (R29)(R26*1) // a26b3a38 MOVH R11, -80(R23) // eb021b78 MOVH R11, (R27)(R14.SXTW<<1) // 6bdb2e78 - MOVB R18, (R0)(R4) // MOVB R18, (R0)(R4*1) // 12682438 + MOVB R19, (R0)(R4) // MOVB R19, (R0)(R4*1) // 13682438 MOVB R1, (R6)(R4) // MOVB R1, (R6)(R4*1) // c1682438 MOVH R3, (R11)(R13<<1) // 63792d78 //TODO STTR 55(R4), R29 // 9d7803b8 //TODO STTR 124(R5), R25 // b9c807f8 //TODO STTRB -28(R23), R16 // f04a1e38 - //TODO STTRH 9(R10), R18 // 52990078 + //TODO STTRH 9(R10), R19 // 53990078 STXP (R1, R2), (R3), R10 // 61082ac8 STXP (R1, R2), (RSP), R10 // e10b2ac8 STXPW (R1, R2), (R3), R10 // 61082a88 STXPW (R1, R2), (RSP), R10 // e10b2a88 - STXRW R2, (R19), R18 // 627e1288 + STXRW R2, (R19), R20 // 627e1488 STXR R15, (R21), R13 // af7e0dc8 STXRB R7, (R9), R24 // 277d1808 STXRH R12, (R3), R8 // 6c7c0848 - SUBW R20.UXTW<<2, R23, R18 // f24a344b + SUBW R20.UXTW<<2, R23, R19 // f34a344b SUB R5.SXTW<<2, R1, R26 // 3ac825cb SUB $(1923<<12), R4, R27 // SUB $7876608, R4, R27 // 9b0c5ed1 SUBW $(1923<<12), R4, R27 // SUBW $7876608, R4, R27 // 9b0c5e51 @@ -410,12 +410,12 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 UBFXW $3, R7, $20, R15 // ef580353 UBFX $33, R17, $25, R5 // 25e661d3 UDIVW R8, R21, R15 // af0ac81a - UDIV R2, R18, R21 // 550ac29a + UDIV R2, R19, R21 // 750ac29a UMADDL R0, R20, R17, R17 // 3152a09b UMSUBL R22, R4, R3, R7 // 6790b69b - UMNEGL R3, R18, R1 // 41fea39b + UMNEGL R3, R19, R1 // 61fea39b UMULH R24, R20, R24 // 987ed89b - UMULL R18, R22, R19 // d37eb29b + UMULL R19, R22, R19 // d37eb39b UXTBW R2, R6 // 461c0053 UXTHW R7, R20 // f43c0053 VCNT V0.B8, V0.B8 // 0058200e @@ -471,7 +471,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 //TODO FCVTAS F27, R7 // 6703241e //TODO FCVTAS F19, R26 // 7a02249e //TODO FCVTAS F4, R0 // 8000641e - //TODO FCVTAS F3, R18 // 7200649e + //TODO FCVTAS F3, R19 // 7300649e //TODO FCVTAU F18, F28 // 5cca217e //TODO VFCVTAU V30.S4, V27.S4 // dbcb216e //TODO FCVTAU F0, R2 // 0200251e @@ -482,16 +482,16 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 //TODO VFCVTL2 V15.H8, V25.S4 // f979214e //TODO FCVTMS F21, F28 // bcba215e //TODO VFCVTMS V5.D2, V2.D2 // a2b8614e - //TODO FCVTMS F31, R18 // f203301e + //TODO FCVTMS F31, R19 // f303301e //TODO FCVTMS F23, R16 // f002309e //TODO FCVTMS F16, R22 // 1602701e //TODO FCVTMS F14, R19 // d301709e //TODO FCVTMU F14, F8 // c8b9217e //TODO VFCVTMU V7.D2, V1.D2 // e1b8616e //TODO FCVTMU F2, R0 // 4000311e - //TODO FCVTMU F23, R18 // f202319e + //TODO FCVTMU F23, R19 // f302319e //TODO FCVTMU F16, R17 // 1102711e - //TODO FCVTMU F12, R18 // 9201719e + //TODO FCVTMU F12, R19 // 9301719e //TODO VFCVTN V23.D2, V26.S2 // fa6a610e //TODO VFCVTN2 V2.D2, V31.S4 // 5f68614e //TODO FCVTNS F3, F27 // 7ba8215e @@ -540,7 +540,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 //TODO FCVTZU $14, F24, R20 // 14cb191e //TODO FCVTZU $6, F25, R17 // 31eb199e //TODO FCVTZU $5, F17, R10 // 2aee591e - //TODO FCVTZU $6, F7, R18 // f2e8599e + //TODO FCVTZU $6, F7, R19 // f3e8599e FCVTZUSW F2, R9 // 4900391e FCVTZUS F12, R29 // 9d01399e FCVTZUDW F27, R22 // 7603791e @@ -682,11 +682,11 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8 VLD1.P (R19)(R4), [V24.B8, V25.B8] // VLD1.P (R19)(R4*1), [V24.B8, V25.B8] // 78a2c40c VLD1.P (R20)(R8), [V7.H8, V8.H8, V9.H8] // VLD1.P (R20)(R8*1), [V7.H8, V8.H8, V9.H8] // 8766c84c VLD1.P 32(R30), [V5.B8, V6.B8, V7.B8, V8.B8] // c523df0c - VLD1 (R18), V14.B[15] // 4e1e404d + VLD1 (R19), V14.B[15] // 6e1e404d VLD1 (R29), V0.H[1] // a04b400d VLD1 (R27), V2.S[0] // 6283400d VLD1 (R21), V5.D[1] // a586404d - VLD1.P 1(R18), V10.B[14] // 4a1adf4d + VLD1.P 1(R19), V10.B[14] // 6a1adf4d VLD1.P (R3)(R14), V16.B[11] // VLD1.P (R3)(R14*1), V16.B[11] // 700cce4d VLD1.P 2(R1), V28.H[2] // 3c50df0d VLD1.P (R13)(R20), V9.H[2] // VLD1.P (R13)(R20*1), V9.H[2] // a951d40d diff --git a/src/cmd/internal/obj/arm64/doc.go b/src/cmd/internal/obj/arm64/doc.go index 845fb22817..7fb129989b 100644 --- a/src/cmd/internal/obj/arm64/doc.go +++ b/src/cmd/internal/obj/arm64/doc.go @@ -89,7 +89,7 @@ such as str, stur, strb, sturb, strh, sturh stlr, stlrb. stlrh, st1. Examples: MOVD R29, 384(R19) <=> str x29, [x19,#384] MOVB.P R30, 30(R4) <=> strb w30, [x4],#30 - STLRH R21, (R18) <=> stlrh w21, [x18] + STLRH R21, (R19) <=> stlrh w21, [x19] (2) MADD, MADDW, MSUB, MSUBW, SMADDL, SMSUBL, UMADDL, UMSUBL , , , @@ -127,7 +127,7 @@ such as str, stur, strb, sturb, strh, sturh stlr, stlrb. stlrh, st1. Examples: CCMN VS, R13, R22, $10 <=> ccmn x13, x22, #0xa, vs - CCMPW HS, R18, R14, $11 <=> ccmp w18, w14, #0xb, cs + CCMPW HS, R19, R14, $11 <=> ccmp w19, w14, #0xb, cs (9) CSEL, CSELW, CSNEG, CSNEGW, CSINC, CSINCW , , , ; FCSELD, FCSELS , , , @@ -144,12 +144,12 @@ FCSELD, FCSELS , , , Examples: STLXR ZR, (R15), R16 <=> stlxr w16, xzr, [x15] - STXRB R9, (R21), R18 <=> stxrb w18, w9, [x21] + STXRB R9, (R21), R19 <=> stxrb w19, w9, [x21] (12) STLXP, STLXPW, STXP, STXPW (, ), (), Examples: - STLXP (R17, R18), (R4), R5 <=> stlxp w5, x17, x18, [x4] + STLXP (R17, R19), (R4), R5 <=> stlxp w5, x17, x19, [x4] STXPW (R30, R25), (R22), R13 <=> stxp w13, w30, w25, [x22] 2. Expressions for special arguments. @@ -173,7 +173,7 @@ Extended registers are written as {.{<<}}. can be UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW or SXTX. Examples: - ADDS R18.UXTB<<4, R9, R26 <=> adds x26, x9, w18, uxtb #4 + ADDS R19.UXTB<<4, R9, R26 <=> adds x26, x9, w19, uxtb #4 ADDSW R14.SXTX, R14, R6 <=> adds w6, w14, w14, sxtx Memory references: [{,#0}] is written as (Rn|RSP), a base register and an immediate From 9c772522ea365be6a916d428a981969befedad7f Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Tue, 6 Nov 2018 19:43:55 +0100 Subject: [PATCH 044/111] cmd/compile: add new format to known_formats This change fixes a TestFormat failure in fmt_test by adding a recently introduced new known format (%q for syntax.Error). Fixes #28621 Change-Id: I026ec88c334549a957a692c1652a860c57e23dae Reviewed-on: https://go-review.googlesource.com/c/147837 Reviewed-by: Ian Lance Taylor --- src/cmd/compile/fmt_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go index eaa2aa8dbd..05d13b58a5 100644 --- a/src/cmd/compile/fmt_test.go +++ b/src/cmd/compile/fmt_test.go @@ -660,6 +660,7 @@ var knownFormats = map[string]string{ "cmd/compile/internal/ssa.rbrank %d": "", "cmd/compile/internal/ssa.regMask %d": "", "cmd/compile/internal/ssa.register %d": "", + "cmd/compile/internal/syntax.Error %q": "", "cmd/compile/internal/syntax.Expr %#v": "", "cmd/compile/internal/syntax.Node %T": "", "cmd/compile/internal/syntax.Operator %s": "", From ca33f33b1420cf333c59c6458bb9bc8910c91ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 25 Oct 2018 13:42:46 +0100 Subject: [PATCH 045/111] cmd/go: make 'go test -h' print two lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Like every other command's -h flag. To achieve this, pass the command's usage function to the cmdflag package, since that package is used by multiple commands and cannot directly access *base.Command. This also lets us get rid of testFlag1 and testFlag2, and instead have contiguous raw strings for the test and testflag help docs. Fixes #26999. Change-Id: I2ebd66835ee61fa83270816a01fa312425224bb3 Reviewed-on: https://go-review.googlesource.com/c/144558 Run-TryBot: Daniel Martí TryBot-Result: Gobot Gobot Reviewed-by: Alan Donovan --- src/cmd/go/internal/cmdflag/flag.go | 6 ++--- src/cmd/go/internal/test/test.go | 34 ++++++++-------------------- src/cmd/go/internal/test/testflag.go | 4 ++-- src/cmd/go/internal/vet/vet.go | 2 +- src/cmd/go/internal/vet/vetflag.go | 4 ++-- src/cmd/go/main.go | 11 --------- src/cmd/go/testdata/script/help.txt | 8 ++++++- 7 files changed, 24 insertions(+), 45 deletions(-) diff --git a/src/cmd/go/internal/cmdflag/flag.go b/src/cmd/go/internal/cmdflag/flag.go index b2a67e6f74..7f2c53def8 100644 --- a/src/cmd/go/internal/cmdflag/flag.go +++ b/src/cmd/go/internal/cmdflag/flag.go @@ -79,15 +79,15 @@ func AddKnownFlags(cmd string, defns []*Defn) { // Parse sees if argument i is present in the definitions and if so, // returns its definition, value, and whether it consumed an extra word. -// If the flag begins (cmd+".") it is ignored for the purpose of this function. -func Parse(cmd string, defns []*Defn, args []string, i int) (f *Defn, value string, extra bool) { +// If the flag begins (cmd.Name()+".") it is ignored for the purpose of this function. +func Parse(cmd string, usage func(), defns []*Defn, args []string, i int) (f *Defn, value string, extra bool) { arg := args[i] if strings.HasPrefix(arg, "--") { // reduce two minuses to one arg = arg[1:] } switch arg { case "-?", "-h", "-help": - base.Usage() + usage() } if arg == "" || arg[0] != '-' { return diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 750b515e41..b38eb4c41d 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -124,16 +124,6 @@ A cached test result is treated as executing in no time at all, so a successful package test result will be cached and reused regardless of -timeout setting. -` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details. - -For more about build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go vet. -`, -} - -const testFlag1 = ` In addition to the build flags, the flags handled by 'go test' itself are: -args @@ -164,15 +154,13 @@ In addition to the build flags, the flags handled by 'go test' itself are: The test still runs (unless -c or -i is specified). The test binary also accepts flags that control execution of the test; these -flags are also accessible by 'go test'. -` +flags are also accessible by 'go test'. See 'go help testflag' for details. -// Usage prints the usage message for 'go test -h' and exits. -func Usage() { - os.Stderr.WriteString("usage: " + testUsage + "\n\n" + - strings.TrimSpace(testFlag1) + "\n\n\t" + - strings.TrimSpace(testFlag2) + "\n") - os.Exit(2) +For more about build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go vet. +`, } var HelpTestflag = &base.Command{ @@ -190,11 +178,6 @@ options of pprof control how the information is presented. The following flags are recognized by the 'go test' command and control the execution of any test: - ` + strings.TrimSpace(testFlag2) + ` -`, -} - -const testFlag2 = ` -bench regexp Run only those benchmarks matching a regular expression. By default, no benchmarks are run. @@ -414,7 +397,8 @@ In the first example, the -x and the second -v are passed through to the test binary unchanged and with no effect on the go command itself. In the second example, the argument math is passed through to the test binary, instead of being interpreted as the package list. -` +`, +} var HelpTestfunc = &base.Command{ UsageLine: "testfunc", @@ -532,7 +516,7 @@ var testVetFlags = []string{ func runTest(cmd *base.Command, args []string) { modload.LoadTests = true - pkgArgs, testArgs = testFlags(args) + pkgArgs, testArgs = testFlags(cmd.Usage, args) work.FindExecCmd() // initialize cached result diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 73f8c69d9e..ebcf49a4e9 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -87,7 +87,7 @@ func init() { // to allow both // go test fmt -custom-flag-for-fmt-test // go test -x math -func testFlags(args []string) (packageNames, passToTest []string) { +func testFlags(usage func(), args []string) (packageNames, passToTest []string) { args = str.StringList(cmdflag.FindGOFLAGS(testFlagDefn), args) inPkg := false var explicitArgs []string @@ -108,7 +108,7 @@ func testFlags(args []string) (packageNames, passToTest []string) { inPkg = false } - f, value, extraWord := cmdflag.Parse(cmd, testFlagDefn, args, i) + f, value, extraWord := cmdflag.Parse(cmd, usage, testFlagDefn, args, i) if f == nil { // This is a flag we do not know; we must assume // that any args we see after this might be flag diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index b64bf3f8e8..616f774bf6 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -38,7 +38,7 @@ See also: go fmt, go fix. func runVet(cmd *base.Command, args []string) { modload.LoadTests = true - vetFlags, pkgArgs := vetFlags(args) + vetFlags, pkgArgs := vetFlags(cmd.Usage, args) work.BuildInit() work.VetFlags = vetFlags diff --git a/src/cmd/go/internal/vet/vetflag.go b/src/cmd/go/internal/vet/vetflag.go index cfa4352cb9..22bce16cf3 100644 --- a/src/cmd/go/internal/vet/vetflag.go +++ b/src/cmd/go/internal/vet/vetflag.go @@ -40,7 +40,7 @@ var vetTool = os.Getenv("GOVETTOOL") // vetFlags processes the command line, splitting it at the first non-flag // into the list of flags and list of packages. -func vetFlags(args []string) (passToVet, packageNames []string) { +func vetFlags(usage func(), args []string) (passToVet, packageNames []string) { // Query the vet command for its flags. tool := vetTool if tool != "" { @@ -108,7 +108,7 @@ func vetFlags(args []string) (passToVet, packageNames []string) { return args[:i], args[i:] } - f, value, extraWord := cmdflag.Parse("vet", vetFlagDefn, args, i) + f, value, extraWord := cmdflag.Parse("vet", usage, vetFlagDefn, args, i) if f == nil { fmt.Fprintf(os.Stderr, "vet: flag %q not defined\n", args[i]) fmt.Fprintf(os.Stderr, "Run \"go help vet\" for more information\n") diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index d6934ce5e9..6a188262cc 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -235,17 +235,6 @@ func init() { } func mainUsage() { - // special case "go test -h" - if len(os.Args) > 1 && os.Args[1] == "test" { - test.Usage() - } - // Since vet shares code with test in cmdflag, it doesn't show its - // command usage properly. For now, special case it too. - // TODO(mvdan): fix the cmdflag package instead; see - // golang.org/issue/26999 - if len(os.Args) > 1 && os.Args[1] == "vet" { - vet.CmdVet.Usage() - } help.PrintUsage(os.Stderr, base.Go) os.Exit(2) } diff --git a/src/cmd/go/testdata/script/help.txt b/src/cmd/go/testdata/script/help.txt index 656e680100..3d0650880e 100644 --- a/src/cmd/go/testdata/script/help.txt +++ b/src/cmd/go/testdata/script/help.txt @@ -35,7 +35,13 @@ stderr 'Run ''go help mod'' for usage.' stderr 'usage: go vet' stderr 'Run ''go help vet'' for details' +# Earlier versions of Go printed a large document here, instead of these two +# lines. +! go test -h +stderr 'usage: go test' +stderr 'Run ''go help test'' for details' + # go help get shows usage for get go help get stdout 'usage: go get' -stdout 'get when using GOPATH' \ No newline at end of file +stdout 'get when using GOPATH' From 7cd2a51c8c1b84191e518ac39b0890ffd56d852b Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Tue, 6 Nov 2018 22:08:30 +0100 Subject: [PATCH 046/111] cmd/compile: update TestNexting golden file This change updates the expected output of the gdb debugging session in the TestNexting internal/ssa test, aligning it with the changes introduced in CL 147360. Fixes the longtest builder. Change-Id: I5b5c22e1cf5e205967ff8359dc6c1485c815428e Reviewed-on: https://go-review.googlesource.com/c/147957 Run-TryBot: Alberto Donizetti Reviewed-by: Keith Randall Reviewed-by: David Chase TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/testdata/hist.gdb-opt.nexts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cmd/compile/internal/ssa/testdata/hist.gdb-opt.nexts b/src/cmd/compile/internal/ssa/testdata/hist.gdb-opt.nexts index 8664ea77c4..b2f3216707 100644 --- a/src/cmd/compile/internal/ssa/testdata/hist.gdb-opt.nexts +++ b/src/cmd/compile/internal/ssa/testdata/hist.gdb-opt.nexts @@ -123,6 +123,7 @@ t = 0 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) a = 3 @@ -131,6 +132,7 @@ t = 3 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) a = 0 @@ -144,6 +146,7 @@ t = 9 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) a = 1 @@ -152,6 +155,7 @@ t = 17 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) a = 0 From 6fc479166a42f859ec8cfbb3e583f941160e069c Mon Sep 17 00:00:00 2001 From: David Chase Date: Tue, 6 Nov 2018 16:42:13 -0500 Subject: [PATCH 047/111] cmd/compile: update TestNexting golden file for Delve This change updates the expected output of the delve debugging session in the TestNexting internal/ssa test, aligning it with the changes introduced in CL 147360 and earlier. Change-Id: I1cc788d02433624a36f4690f24201569d765e5d3 Reviewed-on: https://go-review.googlesource.com/c/147998 Run-TryBot: David Chase TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/testdata/hist.dlv-opt.nexts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cmd/compile/internal/ssa/testdata/hist.dlv-opt.nexts b/src/cmd/compile/internal/ssa/testdata/hist.dlv-opt.nexts index 7eb1d3a35b..89d0b1b637 100644 --- a/src/cmd/compile/internal/ssa/testdata/hist.dlv-opt.nexts +++ b/src/cmd/compile/internal/ssa/testdata/hist.dlv-opt.nexts @@ -58,11 +58,13 @@ 87: if a == 0 { //gdb-opt=(a,n,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) @@ -70,11 +72,13 @@ 87: if a == 0 { //gdb-opt=(a,n,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 86: for i, a := range hist { 87: if a == 0 { //gdb-opt=(a,n,t) +92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) 91: n += a 90: t += i * a 92: fmt.Fprintf(os.Stderr, "%d\t%d\t%d\t%d\t%d\n", i, a, n, i*a, t) //gdb-dbg=(n,i,t) From 1100df58238a1b1c55af148a880b48caf4be6504 Mon Sep 17 00:00:00 2001 From: Diogo Pinela Date: Tue, 30 Oct 2018 22:58:24 +0000 Subject: [PATCH 048/111] go/internal/gcimporter: ensure tests pass even if GOROOT is read-only This mainly entails writing compiler output files to a temporary directory, as well as the corrupted files in TestVersionHandling. Updates #28387 Change-Id: I6b3619a91fff27011c7d73daa4febd14a6c5c348 Reviewed-on: https://go-review.googlesource.com/c/146119 Run-TryBot: Robert Griesemer TryBot-Result: Gobot Gobot Reviewed-by: Robert Griesemer --- src/go/internal/gcimporter/gcimporter_test.go | 113 +++++++++++------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/src/go/internal/gcimporter/gcimporter_test.go b/src/go/internal/gcimporter/gcimporter_test.go index d496f2e57d..222b36c883 100644 --- a/src/go/internal/gcimporter/gcimporter_test.go +++ b/src/go/internal/gcimporter/gcimporter_test.go @@ -34,16 +34,23 @@ func skipSpecialPlatforms(t *testing.T) { } } -func compile(t *testing.T, dirname, filename string) string { - cmd := exec.Command(testenv.GoToolPath(t), "tool", "compile", filename) +// compile runs the compiler on filename, with dirname as the working directory, +// and writes the output file to outdirname. +func compile(t *testing.T, dirname, filename, outdirname string) string { + // filename must end with ".go" + if !strings.HasSuffix(filename, ".go") { + t.Fatalf("filename doesn't end in .go: %s", filename) + } + basename := filepath.Base(filename) + outname := filepath.Join(outdirname, basename[:len(basename)-2]+"o") + cmd := exec.Command(testenv.GoToolPath(t), "tool", "compile", "-o", outname, filename) cmd.Dir = dirname out, err := cmd.CombinedOutput() if err != nil { t.Logf("%s", out) t.Fatalf("go tool compile %s failed: %s", filename, err) } - // filename should end with ".go" - return filepath.Join(dirname, filename[:len(filename)-2]+"o") + return outname } func testPath(t *testing.T, path, srcDir string) *types.Package { @@ -88,17 +95,30 @@ func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) { return } +func mktmpdir(t *testing.T) string { + tmpdir, err := ioutil.TempDir("", "gcimporter_test") + if err != nil { + t.Fatal("mktmpdir:", err) + } + if err := os.Mkdir(filepath.Join(tmpdir, "testdata"), 0700); err != nil { + os.RemoveAll(tmpdir) + t.Fatal("mktmpdir:", err) + } + return tmpdir +} + func TestImportTestdata(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) } - if outFn := compile(t, "testdata", "exports.go"); outFn != "" { - defer os.Remove(outFn) - } + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) - if pkg := testPath(t, "./testdata/exports", "."); pkg != nil { + compile(t, "testdata", "exports.go", filepath.Join(tmpdir, "testdata")) + + if pkg := testPath(t, "./testdata/exports", tmpdir); pkg != nil { // The package's Imports list must include all packages // explicitly imported by exports.go, plus all packages // referenced indirectly via exported objects in exports.go. @@ -131,6 +151,13 @@ func TestVersionHandling(t *testing.T) { t.Fatal(err) } + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + corruptdir := filepath.Join(tmpdir, "testdata", "versions") + if err := os.Mkdir(corruptdir, 0700); err != nil { + t.Fatal(err) + } + for _, f := range list { name := f.Name() if !strings.HasSuffix(name, ".a") { @@ -178,12 +205,11 @@ func TestVersionHandling(t *testing.T) { } // 4) write the file pkgpath += "_corrupted" - filename := filepath.Join(dir, pkgpath) + ".a" + filename := filepath.Join(corruptdir, pkgpath) + ".a" ioutil.WriteFile(filename, data, 0666) - defer os.Remove(filename) // test that importing the corrupted file results in an error - _, err = Import(make(map[string]*types.Package), pkgpath, dir, nil) + _, err = Import(make(map[string]*types.Package), pkgpath, corruptdir, nil) if err == nil { t.Errorf("import corrupted %q succeeded", pkgpath) } else if msg := err.Error(); !strings.Contains(msg, "version skew") { @@ -315,7 +341,7 @@ func TestIssue5815(t *testing.T) { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) } - pkg := importPkg(t, "strings") + pkg := importPkg(t, "strings", ".") scope := pkg.Scope() for _, name := range scope.Names() { @@ -373,15 +399,22 @@ func TestIssue13566(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "a.go"); f != "" { - defer os.Remove(f) - } - if f := compile(t, "testdata", "b.go"); f != "" { - defer os.Remove(f) + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + testoutdir := filepath.Join(tmpdir, "testdata") + + // b.go needs to be compiled from the output directory so that the compiler can + // find the compiled package a. We pass the full path to compile() so that we + // don't have to copy the file to that directory. + bpath, err := filepath.Abs(filepath.Join("testdata", "b.go")) + if err != nil { + t.Fatal(err) } + compile(t, "testdata", "a.go", testoutdir) + compile(t, testoutdir, bpath, testoutdir) // import must succeed (test for issue at hand) - pkg := importPkg(t, "./testdata/b") + pkg := importPkg(t, "./testdata/b", tmpdir) // make sure all indirectly imported packages have names for _, imp := range pkg.Imports() { @@ -451,9 +484,10 @@ func TestIssue15517(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "p.go"); f != "" { - defer os.Remove(f) - } + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + + compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata")) // Multiple imports of p must succeed without redeclaration errors. // We use an import path that's not cleaned up so that the eventual @@ -469,7 +503,7 @@ func TestIssue15517(t *testing.T) { // The same issue occurs with vendoring.) imports := make(map[string]*types.Package) for i := 0; i < 3; i++ { - if _, err := Import(imports, "./././testdata/p", ".", nil); err != nil { + if _, err := Import(imports, "./././testdata/p", tmpdir, nil); err != nil { t.Fatal(err) } } @@ -489,11 +523,7 @@ func TestIssue15920(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "issue15920.go"); f != "" { - defer os.Remove(f) - } - - importPkg(t, "./testdata/issue15920") + compileAndImportPkg(t, "issue15920") } func TestIssue20046(t *testing.T) { @@ -510,12 +540,8 @@ func TestIssue20046(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "issue20046.go"); f != "" { - defer os.Remove(f) - } - // "./issue20046".V.M must exist - pkg := importPkg(t, "./testdata/issue20046") + pkg := compileAndImportPkg(t, "issue20046") obj := lookupObj(t, pkg.Scope(), "V") if m, index, indirect := types.LookupFieldOrMethod(obj.Type(), false, nil, "M"); m == nil { t.Fatalf("V.M not found (index = %v, indirect = %v)", index, indirect) @@ -535,11 +561,7 @@ func TestIssue25301(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "issue25301.go"); f != "" { - defer os.Remove(f) - } - - importPkg(t, "./testdata/issue25301") + compileAndImportPkg(t, "issue25301") } func TestIssue25596(t *testing.T) { @@ -556,21 +578,24 @@ func TestIssue25596(t *testing.T) { t.Skip("avoid dealing with relative paths/drive letters on windows") } - if f := compile(t, "testdata", "issue25596.go"); f != "" { - defer os.Remove(f) - } - - importPkg(t, "./testdata/issue25596") + compileAndImportPkg(t, "issue25596") } -func importPkg(t *testing.T, path string) *types.Package { - pkg, err := Import(make(map[string]*types.Package), path, ".", nil) +func importPkg(t *testing.T, path, srcDir string) *types.Package { + pkg, err := Import(make(map[string]*types.Package), path, srcDir, nil) if err != nil { t.Fatal(err) } return pkg } +func compileAndImportPkg(t *testing.T, name string) *types.Package { + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata")) + return importPkg(t, "./testdata/"+name, tmpdir) +} + func lookupObj(t *testing.T, scope *types.Scope, name string) types.Object { if obj := scope.Lookup(name); obj != nil { return obj From c0a40e4fe5d4649672d0d430ca26551841fc4852 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Tue, 6 Nov 2018 17:48:08 -0500 Subject: [PATCH 049/111] cmd/vendor: update github.com/google/pprof Sync @ fde099a (Oct 26, 2018) Also update misc/nacl/testzip.proto to include new testdata. Change-Id: If41590be9f395a591056e89a417b589c4ba71b1a Reviewed-on: https://go-review.googlesource.com/c/147979 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- misc/nacl/testzip.proto | 3 + .../github.com/google/pprof/driver/driver.go | 43 +++-- .../pprof/internal/binutils/binutils.go | 109 ++++++++++-- .../pprof/internal/binutils/binutils_test.go | 29 +++ .../internal/binutils/testdata/malformed_elf | 1 + .../binutils/testdata/malformed_macho | 1 + .../google/pprof/internal/driver/cli.go | 7 +- .../google/pprof/internal/driver/commands.go | 38 ++-- .../google/pprof/internal/driver/driver.go | 76 ++++---- .../pprof/internal/driver/driver_test.go | 117 +++++++++++-- .../google/pprof/internal/driver/fetch.go | 60 ++----- .../pprof/internal/driver/fetch_test.go | 165 +++++++++++++++--- .../google/pprof/internal/driver/flags.go | 78 +++++++++ .../pprof/internal/driver/flamegraph.go | 18 +- .../pprof/internal/driver/flamegraph_test.go | 46 ----- .../pprof/internal/driver/interactive_test.go | 19 +- .../google/pprof/internal/driver/options.go | 64 +------ .../pprof.cpu.flat.addresses.noinlines.text | 7 + ...prof.cpu.flat.filefunctions.noinlines.text | 5 + .../pprof.cpu.flat.functions.noinlines.text | 5 + .../driver/testdata/pprof.cpu.lines.topproto | 3 + .../driver/testdata/pprof.longNameFuncs.dot | 9 + .../driver/testdata/pprof.longNameFuncs.text | 5 + .../google/pprof/internal/driver/webhtml.go | 84 ++++++--- .../google/pprof/internal/driver/webui.go | 26 +-- .../google/pprof/internal/elfexec/elfexec.go | 28 +-- .../pprof/internal/elfexec/elfexec_test.go | 11 +- .../google/pprof/internal/graph/dotgraph.go | 1 + .../google/pprof/internal/graph/graph.go | 29 ++- .../google/pprof/internal/graph/graph_test.go | 157 +++++++++++++++++ .../google/pprof/internal/plugin/plugin.go | 15 +- .../google/pprof/internal/report/report.go | 24 ++- .../pprof/internal/report/report_test.go | 21 ++- .../pprof/internal/symbolizer/symbolizer.go | 31 +--- .../internal/symbolizer/symbolizer_test.go | 4 +- .../google/pprof/internal/symbolz/symbolz.go | 35 +++- .../pprof/internal/symbolz/symbolz_test.go | 28 +++ .../pprof/internal/transport/transport.go | 131 ++++++++++++++ .../pprof/profile/legacy_java_profile.go | 2 +- .../google/pprof/profile/profile.go | 6 + .../google/pprof/profile/profile_test.go | 53 ++++++ .../profile/testdata/java.contention.string | 2 +- src/cmd/vendor/vendor.json | 88 +++++----- 43 files changed, 1233 insertions(+), 451 deletions(-) create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_elf create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_macho create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/flags.go delete mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph_test.go create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.addresses.noinlines.text create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.filefunctions.noinlines.text create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.functions.noinlines.text create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.lines.topproto create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.dot create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.text create mode 100644 src/cmd/vendor/github.com/google/pprof/internal/transport/transport.go diff --git a/misc/nacl/testzip.proto b/misc/nacl/testzip.proto index 7f524cac48..c2afa1020a 100644 --- a/misc/nacl/testzip.proto +++ b/misc/nacl/testzip.proto @@ -50,6 +50,9 @@ go src=.. google pprof internal + binutils + testdata + + driver testdata + diff --git a/src/cmd/vendor/github.com/google/pprof/driver/driver.go b/src/cmd/vendor/github.com/google/pprof/driver/driver.go index 3735d6ace9..b1c745bacd 100644 --- a/src/cmd/vendor/github.com/google/pprof/driver/driver.go +++ b/src/cmd/vendor/github.com/google/pprof/driver/driver.go @@ -17,6 +17,7 @@ package driver import ( "io" + "net/http" "regexp" "time" @@ -48,13 +49,14 @@ func (o *Options) internalOptions() *plugin.Options { } } return &plugin.Options{ - Writer: o.Writer, - Flagset: o.Flagset, - Fetch: o.Fetch, - Sym: sym, - Obj: obj, - UI: o.UI, - HTTPServer: httpServer, + Writer: o.Writer, + Flagset: o.Flagset, + Fetch: o.Fetch, + Sym: sym, + Obj: obj, + UI: o.UI, + HTTPServer: httpServer, + HTTPTransport: o.HTTPTransport, } } @@ -64,13 +66,14 @@ type HTTPServerArgs plugin.HTTPServerArgs // Options groups all the optional plugins into pprof. type Options struct { - Writer Writer - Flagset FlagSet - Fetch Fetcher - Sym Symbolizer - Obj ObjTool - UI UI - HTTPServer func(*HTTPServerArgs) error + Writer Writer + Flagset FlagSet + Fetch Fetcher + Sym Symbolizer + Obj ObjTool + UI UI + HTTPServer func(*HTTPServerArgs) error + HTTPTransport http.RoundTripper } // Writer provides a mechanism to write data under a certain name, @@ -100,12 +103,16 @@ type FlagSet interface { // single flag StringList(name string, def string, usage string) *[]*string - // ExtraUsage returns any additional text that should be - // printed after the standard usage message. - // The typical use of ExtraUsage is to show any custom flags - // defined by the specific pprof plugins being used. + // ExtraUsage returns any additional text that should be printed after the + // standard usage message. The extra usage message returned includes all text + // added with AddExtraUsage(). + // The typical use of ExtraUsage is to show any custom flags defined by the + // specific pprof plugins being used. ExtraUsage() string + // AddExtraUsage appends additional text to the end of the extra usage message. + AddExtraUsage(eu string) + // Parse initializes the flags with their values for this run // and returns the non-flag command line arguments. // If an unknown flag is encountered or there are no arguments, diff --git a/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils.go b/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils.go index 12b6a5c4b2..309561112c 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils.go @@ -18,11 +18,14 @@ package binutils import ( "debug/elf" "debug/macho" + "encoding/binary" "fmt" + "io" "os" "os/exec" "path/filepath" "regexp" + "runtime" "strings" "sync" @@ -173,12 +176,8 @@ func (bu *Binutils) Open(name string, start, limit, offset uint64) (plugin.ObjFi b := bu.get() // Make sure file is a supported executable. - // The pprof driver uses Open to sniff the difference - // between an executable and a profile. - // For now, only ELF is supported. - // Could read the first few bytes of the file and - // use a table of prefixes if we need to support other - // systems at some point. + // This uses magic numbers, mainly to provide better error messages but + // it should also help speed. if _, err := os.Stat(name); err != nil { // For testing, do not require file name to exist. @@ -188,21 +187,54 @@ func (bu *Binutils) Open(name string, start, limit, offset uint64) (plugin.ObjFi return nil, err } - if f, err := b.openELF(name, start, limit, offset); err == nil { + // Read the first 4 bytes of the file. + + f, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("error opening %s: %v", name, err) + } + defer f.Close() + + var header [4]byte + if _, err = io.ReadFull(f, header[:]); err != nil { + return nil, fmt.Errorf("error reading magic number from %s: %v", name, err) + } + + elfMagic := string(header[:]) + + // Match against supported file types. + if elfMagic == elf.ELFMAG { + f, err := b.openELF(name, start, limit, offset) + if err != nil { + return nil, fmt.Errorf("error reading ELF file %s: %v", name, err) + } return f, nil } - if f, err := b.openMachO(name, start, limit, offset); err == nil { + + // Mach-O magic numbers can be big or little endian. + machoMagicLittle := binary.LittleEndian.Uint32(header[:]) + machoMagicBig := binary.BigEndian.Uint32(header[:]) + + if machoMagicLittle == macho.Magic32 || machoMagicLittle == macho.Magic64 || + machoMagicBig == macho.Magic32 || machoMagicBig == macho.Magic64 { + f, err := b.openMachO(name, start, limit, offset) + if err != nil { + return nil, fmt.Errorf("error reading Mach-O file %s: %v", name, err) + } return f, nil } - return nil, fmt.Errorf("unrecognized binary: %s", name) + if machoMagicLittle == macho.MagicFat || machoMagicBig == macho.MagicFat { + f, err := b.openFatMachO(name, start, limit, offset) + if err != nil { + return nil, fmt.Errorf("error reading fat Mach-O file %s: %v", name, err) + } + return f, nil + } + + return nil, fmt.Errorf("unrecognized binary format: %s", name) } -func (b *binrep) openMachO(name string, start, limit, offset uint64) (plugin.ObjFile, error) { - of, err := macho.Open(name) - if err != nil { - return nil, fmt.Errorf("error parsing %s: %v", name, err) - } - defer of.Close() +func (b *binrep) openMachOCommon(name string, of *macho.File, start, limit, offset uint64) (plugin.ObjFile, error) { // Subtract the load address of the __TEXT section. Usually 0 for shared // libraries or 0x100000000 for executables. You can check this value by @@ -225,6 +257,53 @@ func (b *binrep) openMachO(name string, start, limit, offset uint64) (plugin.Obj return &fileAddr2Line{file: file{b: b, name: name, base: base}}, nil } +func (b *binrep) openFatMachO(name string, start, limit, offset uint64) (plugin.ObjFile, error) { + of, err := macho.OpenFat(name) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %v", name, err) + } + defer of.Close() + + if len(of.Arches) == 0 { + return nil, fmt.Errorf("empty fat Mach-O file: %s", name) + } + + var arch macho.Cpu + // Use the host architecture. + // TODO: This is not ideal because the host architecture may not be the one + // that was profiled. E.g. an amd64 host can profile a 386 program. + switch runtime.GOARCH { + case "386": + arch = macho.Cpu386 + case "amd64", "amd64p32": + arch = macho.CpuAmd64 + case "arm", "armbe", "arm64", "arm64be": + arch = macho.CpuArm + case "ppc": + arch = macho.CpuPpc + case "ppc64", "ppc64le": + arch = macho.CpuPpc64 + default: + return nil, fmt.Errorf("unsupported host architecture for %s: %s", name, runtime.GOARCH) + } + for i := range of.Arches { + if of.Arches[i].Cpu == arch { + return b.openMachOCommon(name, of.Arches[i].File, start, limit, offset) + } + } + return nil, fmt.Errorf("architecture not found in %s: %s", name, runtime.GOARCH) +} + +func (b *binrep) openMachO(name string, start, limit, offset uint64) (plugin.ObjFile, error) { + of, err := macho.Open(name) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %v", name, err) + } + defer of.Close() + + return b.openMachOCommon(name, of, start, limit, offset) +} + func (b *binrep) openELF(name string, start, limit, offset uint64) (plugin.ObjFile, error) { ef, err := elf.Open(name) if err != nil { diff --git a/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils_test.go b/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils_test.go index d844ed7e4e..17d4225a87 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/binutils/binutils_test.go @@ -22,6 +22,7 @@ import ( "reflect" "regexp" "runtime" + "strings" "testing" "github.com/google/pprof/internal/plugin" @@ -361,3 +362,31 @@ func TestLLVMSymbolizer(t *testing.T) { } } } + +func TestOpenMalformedELF(t *testing.T) { + // Test that opening a malformed ELF file will report an error containing + // the word "ELF". + bu := &Binutils{} + _, err := bu.Open(filepath.Join("testdata", "malformed_elf"), 0, 0, 0) + if err == nil { + t.Fatalf("Open: unexpected success") + } + + if !strings.Contains(err.Error(), "ELF") { + t.Errorf("Open: got %v, want error containing 'ELF'", err) + } +} + +func TestOpenMalformedMachO(t *testing.T) { + // Test that opening a malformed Mach-O file will report an error containing + // the word "Mach-O". + bu := &Binutils{} + _, err := bu.Open(filepath.Join("testdata", "malformed_macho"), 0, 0, 0) + if err == nil { + t.Fatalf("Open: unexpected success") + } + + if !strings.Contains(err.Error(), "Mach-O") { + t.Errorf("Open: got %v, want error containing 'Mach-O'", err) + } +} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_elf b/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_elf new file mode 100644 index 0000000000..f0b503b0b6 --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_elf @@ -0,0 +1 @@ +ELFÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_macho b/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_macho new file mode 100644 index 0000000000..b01ddf69a9 --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/binutils/testdata/malformed_macho @@ -0,0 +1 @@ +Ïúíþÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go index a5153e1511..dfedf9d849 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go @@ -45,8 +45,8 @@ type source struct { func parseFlags(o *plugin.Options) (*source, []string, error) { flag := o.Flagset // Comparisons. - flagBase := flag.StringList("base", "", "Source for base profile for profile subtraction") - flagDiffBase := flag.StringList("diff_base", "", "Source for diff base profile for comparison") + flagDiffBase := flag.StringList("diff_base", "", "Source of base profile for comparison") + flagBase := flag.StringList("base", "", "Source of base profile for profile subtraction") // Source options. flagSymbolize := flag.String("symbolize", "", "Options for profile symbolization") flagBuildID := flag.String("buildid", "", "Override build id for first mapping") @@ -312,7 +312,8 @@ var usageMsgSrc = "\n\n" + " -buildid Override build id for main binary\n" + " -add_comment Free-form annotation to add to the profile\n" + " Displayed on some reports or with pprof -comments\n" + - " -base source Source of profile to use as baseline\n" + + " -diff_base source Source of base profile for comparison\n" + + " -base source Source of base profile for profile subtraction\n" + " profile.pb.gz Profile in compressed protobuf format\n" + " legacy_profile Profile in legacy pprof format\n" + " http://host/profile URL for profile handler to retrieve\n" + diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/commands.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/commands.go index 91d32d1e71..ab073d878d 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/commands.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/commands.go @@ -228,17 +228,17 @@ var pprofVariables = variables{ // Output granularity "functions": &variable{boolKind, "t", "granularity", helpText( "Aggregate at the function level.", - "Takes into account the filename/lineno where the function was defined.")}, + "Ignores the filename where the function was defined.")}, + "filefunctions": &variable{boolKind, "t", "granularity", helpText( + "Aggregate at the function level.", + "Takes into account the filename where the function was defined.")}, "files": &variable{boolKind, "f", "granularity", "Aggregate at the file level."}, "lines": &variable{boolKind, "f", "granularity", "Aggregate at the source code line level."}, "addresses": &variable{boolKind, "f", "granularity", helpText( - "Aggregate at the function level.", + "Aggregate at the address level.", "Includes functions' addresses in the output.")}, - "noinlines": &variable{boolKind, "f", "granularity", helpText( - "Aggregate at the function level.", - "Attributes inlined functions to their first out-of-line caller.")}, - "addressnoinlines": &variable{boolKind, "f", "granularity", helpText( - "Aggregate at the function level, including functions' addresses in the output.", + "noinlines": &variable{boolKind, "f", "", helpText( + "Ignore inlines.", "Attributes inlined functions to their first out-of-line caller.")}, } @@ -337,21 +337,27 @@ func listHelp(c string, redirect bool) string { // browsers returns a list of commands to attempt for web visualization. func browsers() []string { - cmds := []string{"chrome", "google-chrome", "firefox"} + var cmds []string + if userBrowser := os.Getenv("BROWSER"); userBrowser != "" { + cmds = append(cmds, userBrowser) + } switch runtime.GOOS { case "darwin": - return append(cmds, "/usr/bin/open") + cmds = append(cmds, "/usr/bin/open") case "windows": - return append(cmds, "cmd /c start") + cmds = append(cmds, "cmd /c start") default: - userBrowser := os.Getenv("BROWSER") - if userBrowser != "" { - cmds = append([]string{userBrowser, "sensible-browser"}, cmds...) - } else { - cmds = append([]string{"sensible-browser"}, cmds...) + // Commands opening browsers are prioritized over xdg-open, so browser() + // command can be used on linux to open the .svg file generated by the -web + // command (the .svg file includes embedded javascript so is best viewed in + // a browser). + cmds = append(cmds, []string{"chrome", "google-chrome", "chromium", "firefox", "sensible-browser"}...) + if os.Getenv("DISPLAY") != "" { + // xdg-open is only for use in a desktop environment. + cmds = append(cmds, "xdg-open") } - return append(cmds, "xdg-open") } + return cmds } var kcachegrind = []string{"kcachegrind"} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go index 2dabc3017b..45f1846749 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go @@ -152,20 +152,33 @@ func generateReport(p *profile.Profile, cmd []string, vars variables, o *plugin. } func applyCommandOverrides(cmd string, outputFormat int, v variables) variables { - trim, tagfilter, filter := v["trim"].boolValue(), true, true + // Some report types override the trim flag to false below. This is to make + // sure the default heuristics of excluding insignificant nodes and edges + // from the call graph do not apply. One example where it is important is + // annotated source or disassembly listing. Those reports run on a specific + // function (or functions), but the trimming is applied before the function + // data is selected. So, with trimming enabled, the report could end up + // showing no data if the specified function is "uninteresting" as far as the + // trimming is concerned. + trim := v["trim"].boolValue() switch cmd { - case "callgrind", "kcachegrind": - trim = false - v.set("addresses", "t") case "disasm", "weblist": trim = false - v.set("addressnoinlines", "t") + v.set("addresses", "t") + // Force the 'noinlines' mode so that source locations for a given address + // collapse and there is only one for the given address. Without this + // cumulative metrics would be double-counted when annotating the assembly. + // This is because the merge is done by address and in case of an inlined + // stack each of the inlined entries is a separate callgraph node. + v.set("noinlines", "t") case "peek": - trim, tagfilter, filter = false, false, false + trim = false case "list": - v.set("nodecount", "0") + trim = false v.set("lines", "t") + // Do not force 'noinlines' to be false so that specifying + // "-list foo -noinlines" is supported and works as expected. case "text", "top", "topproto": if v["nodecount"].intValue() == -1 { v.set("nodecount", "0") @@ -176,9 +189,11 @@ func applyCommandOverrides(cmd string, outputFormat int, v variables) variables } } - if outputFormat == report.Proto || outputFormat == report.Raw { - trim, tagfilter, filter = false, false, false + switch outputFormat { + case report.Proto, report.Raw, report.Callgrind: + trim = false v.set("addresses", "t") + v.set("noinlines", "f") } if !trim { @@ -186,43 +201,32 @@ func applyCommandOverrides(cmd string, outputFormat int, v variables) variables v.set("nodefraction", "0") v.set("edgefraction", "0") } - if !tagfilter { - v.set("tagfocus", "") - v.set("tagignore", "") - } - if !filter { - v.set("focus", "") - v.set("ignore", "") - v.set("hide", "") - v.set("show", "") - v.set("show_from", "") - } return v } func aggregate(prof *profile.Profile, v variables) error { - var inlines, function, filename, linenumber, address bool + var function, filename, linenumber, address bool + inlines := !v["noinlines"].boolValue() switch { case v["addresses"].boolValue(): - return nil - case v["lines"].boolValue(): - inlines = true - function = true - filename = true - linenumber = true - case v["files"].boolValue(): - inlines = true - filename = true - case v["functions"].boolValue(): - inlines = true - function = true - case v["noinlines"].boolValue(): - function = true - case v["addressnoinlines"].boolValue(): + if inlines { + return nil + } function = true filename = true linenumber = true address = true + case v["lines"].boolValue(): + function = true + filename = true + linenumber = true + case v["files"].boolValue(): + filename = true + case v["functions"].boolValue(): + function = true + case v["filefunctions"].boolValue(): + function = true + filename = true default: return fmt.Errorf("unexpected granularity") } diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go index ff6afe9cff..90f89dc7bc 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go @@ -53,6 +53,9 @@ func TestParse(t *testing.T) { flags, source string }{ {"text,functions,flat", "cpu"}, + {"text,functions,noinlines,flat", "cpu"}, + {"text,filefunctions,noinlines,flat", "cpu"}, + {"text,addresses,noinlines,flat", "cpu"}, {"tree,addresses,flat,nodecount=4", "cpusmall"}, {"text,functions,flat,nodecount=5,call_tree", "unknown"}, {"text,alloc_objects,flat", "heap_alloc"}, @@ -63,6 +66,7 @@ func TestParse(t *testing.T) { {"text,lines,cum,show=[12]00", "cpu"}, {"text,lines,cum,hide=line[X3]0,focus=[12]00", "cpu"}, {"topproto,lines,cum,hide=mangled[X3]0", "cpu"}, + {"topproto,lines", "cpu"}, {"tree,lines,cum,focus=[24]00", "heap"}, {"tree,relative_percentages,cum,focus=[24]00", "heap"}, {"tree,lines,cum,show_from=line2", "cpu"}, @@ -92,6 +96,8 @@ func TestParse(t *testing.T) { {"peek=line.*01", "cpu"}, {"weblist=line[13],addresses,flat", "cpu"}, {"tags,tagfocus=400kb:", "heap_request"}, + {"dot", "longNameFuncs"}, + {"text", "longNameFuncs"}, } baseVars := pprofVariables @@ -108,9 +114,6 @@ func TestParse(t *testing.T) { flags := strings.Split(tc.flags, ",") - // Skip the output format in the first flag, to output to a proto - addFlags(&f, flags[1:]) - // Encode profile into a protobuf and decode it again. protoTempFile, err := ioutil.TempFile("", "profile_proto") if err != nil { @@ -123,11 +126,13 @@ func TestParse(t *testing.T) { if flags[0] == "topproto" { f.bools["proto"] = false f.bools["topproto"] = true + f.bools["addresses"] = true } // First pprof invocation to save the profile into a profile.proto. - o1 := setDefaults(nil) - o1.Flagset = f + // Pass in flag set hen setting defaults, because otherwise default + // transport will try to add flags to the default flag set. + o1 := setDefaults(&plugin.Options{Flagset: f}) o1.Fetch = testFetcher{} o1.Sym = testSymbolizer{} o1.UI = testUI @@ -144,28 +149,28 @@ func TestParse(t *testing.T) { } defer os.Remove(outputTempFile.Name()) defer outputTempFile.Close() + + f = baseFlags() f.strings["output"] = outputTempFile.Name() f.args = []string{protoTempFile.Name()} - var solution string + delete(f.bools, "proto") + addFlags(&f, flags) + solution := solutionFilename(tc.source, &f) // Apply the flags for the second pprof run, and identify name of // the file containing expected results if flags[0] == "topproto" { + addFlags(&f, flags) solution = solutionFilename(tc.source, &f) delete(f.bools, "topproto") f.bools["text"] = true - } else { - delete(f.bools, "proto") - addFlags(&f, flags[:1]) - solution = solutionFilename(tc.source, &f) } - // The add_comment flag is not idempotent so only apply it on the first run. - delete(f.strings, "add_comment") // Second pprof invocation to read the profile from profile.proto // and generate a report. - o2 := setDefaults(nil) - o2.Flagset = f + // Pass in flag set hen setting defaults, because otherwise default + // transport will try to add flags to the default flag set. + o2 := setDefaults(&plugin.Options{Flagset: f}) o2.Sym = testSymbolizeDemangler{} o2.Obj = new(mockObjTool) o2.UI = testUI @@ -250,7 +255,8 @@ func testSourceURL(port int) string { func solutionFilename(source string, f *testFlags) string { name := []string{"pprof", strings.TrimPrefix(source, testSourceURL(8000))} name = addString(name, f, []string{"flat", "cum"}) - name = addString(name, f, []string{"functions", "files", "lines", "addresses"}) + name = addString(name, f, []string{"functions", "filefunctions", "files", "lines", "addresses"}) + name = addString(name, f, []string{"noinlines"}) name = addString(name, f, []string{"inuse_space", "inuse_objects", "alloc_space", "alloc_objects"}) name = addString(name, f, []string{"relative_percentages"}) name = addString(name, f, []string{"seconds"}) @@ -293,6 +299,8 @@ type testFlags struct { func (testFlags) ExtraUsage() string { return "" } +func (testFlags) AddExtraUsage(eu string) {} + func (f testFlags) Bool(s string, d bool, c string) *bool { if b, ok := f.bools[s]; ok { return &b @@ -436,6 +444,8 @@ func (testFetcher) Fetch(s string, d, t time.Duration) (*profile.Profile, string p = contentionProfile() case "symbolz": p = symzProfile() + case "longNameFuncs": + p = longNameFuncsProfile() default: return nil, "", fmt.Errorf("unexpected source: %s", s) } @@ -519,6 +529,83 @@ func fakeDemangler(name string) string { } } +// Returns a profile with function names which should be shortened in +// graph and flame views. +func longNameFuncsProfile() *profile.Profile { + var longNameFuncsM = []*profile.Mapping{ + { + ID: 1, + Start: 0x1000, + Limit: 0x4000, + File: "/path/to/testbinary", + HasFunctions: true, + HasFilenames: true, + HasLineNumbers: true, + HasInlineFrames: true, + }, + } + + var longNameFuncsF = []*profile.Function{ + {ID: 1, Name: "path/to/package1.object.function1", SystemName: "path/to/package1.object.function1", Filename: "path/to/package1.go"}, + {ID: 2, Name: "(anonymous namespace)::Bar::Foo", SystemName: "(anonymous namespace)::Bar::Foo", Filename: "a/long/path/to/package2.cc"}, + {ID: 3, Name: "java.bar.foo.FooBar.run(java.lang.Runnable)", SystemName: "java.bar.foo.FooBar.run(java.lang.Runnable)", Filename: "FooBar.java"}, + } + + var longNameFuncsL = []*profile.Location{ + { + ID: 1000, + Mapping: longNameFuncsM[0], + Address: 0x1000, + Line: []profile.Line{ + {Function: longNameFuncsF[0], Line: 1}, + }, + }, + { + ID: 2000, + Mapping: longNameFuncsM[0], + Address: 0x2000, + Line: []profile.Line{ + {Function: longNameFuncsF[1], Line: 4}, + }, + }, + { + ID: 3000, + Mapping: longNameFuncsM[0], + Address: 0x3000, + Line: []profile.Line{ + {Function: longNameFuncsF[2], Line: 9}, + }, + }, + } + + return &profile.Profile{ + PeriodType: &profile.ValueType{Type: "cpu", Unit: "milliseconds"}, + Period: 1, + DurationNanos: 10e9, + SampleType: []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "milliseconds"}, + }, + Sample: []*profile.Sample{ + { + Location: []*profile.Location{longNameFuncsL[0], longNameFuncsL[1], longNameFuncsL[2]}, + Value: []int64{1000, 1000}, + }, + { + Location: []*profile.Location{longNameFuncsL[0], longNameFuncsL[1]}, + Value: []int64{100, 100}, + }, + { + Location: []*profile.Location{longNameFuncsL[2]}, + Value: []int64{10, 10}, + }, + }, + Location: longNameFuncsL, + Function: longNameFuncsF, + Mapping: longNameFuncsM, + } +} + func cpuProfile() *profile.Profile { var cpuM = []*profile.Mapping{ { diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go index 7a7a1a20f2..b8a69e87fc 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go @@ -16,7 +16,6 @@ package driver import ( "bytes" - "crypto/tls" "fmt" "io" "io/ioutil" @@ -57,7 +56,7 @@ func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) { }) } - p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI) + p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport) if err != nil { return nil, err } @@ -123,7 +122,7 @@ func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) { return p, nil } -func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) { +func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) { wg := sync.WaitGroup{} wg.Add(2) var psrc, pbase *profile.Profile @@ -133,11 +132,11 @@ func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, o var countsrc, countbase int go func() { defer wg.Done() - psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui) + psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr) }() go func() { defer wg.Done() - pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui) + pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr) }() wg.Wait() save := savesrc || savebase @@ -167,7 +166,7 @@ func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, o // chunkedGrab fetches the profiles described in source and merges them into // a single profile. It fetches a chunk of profiles concurrently, with a maximum // chunk size to limit its memory usage. -func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) { +func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) { const chunkSize = 64 var p *profile.Profile @@ -180,7 +179,7 @@ func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTo if end > len(sources) { end = len(sources) } - chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui) + chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr) switch { case chunkErr != nil: return nil, nil, false, 0, chunkErr @@ -204,13 +203,13 @@ func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTo } // concurrentGrab fetches multiple profiles concurrently -func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) { +func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) { wg := sync.WaitGroup{} wg.Add(len(sources)) for i := range sources { go func(s *profileSource) { defer wg.Done() - s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui) + s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr) }(&sources[i]) } wg.Wait() @@ -310,7 +309,7 @@ const testSourceAddress = "pproftest.local" // grabProfile fetches a profile. Returns the profile, sources for the // profile mappings, a bool indicating if the profile was fetched // remotely, and an error. -func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) { +func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) { var src string duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second if fetcher != nil { @@ -321,7 +320,7 @@ func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.Ob } if err != nil || p == nil { // Fetch the profile over HTTP or from a file. - p, src, err = fetch(source, duration, timeout, ui) + p, src, err = fetch(source, duration, timeout, ui, tr) if err != nil { return } @@ -461,7 +460,7 @@ mapping: // fetch fetches a profile from source, within the timeout specified, // producing messages through the ui. It returns the profile and the // url of the actual source of the profile for remote profiles. -func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *profile.Profile, src string, err error) { +func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) { var f io.ReadCloser if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" { @@ -469,7 +468,7 @@ func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *pro if duration > 0 { ui.Print(fmt.Sprintf("Please wait... (%v)", duration)) } - f, err = fetchURL(sourceURL, timeout) + f, err = fetchURL(sourceURL, timeout, tr) src = sourceURL } else if isPerfFile(source) { f, err = convertPerfData(source, ui) @@ -484,8 +483,12 @@ func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *pro } // fetchURL fetches a profile from a URL using HTTP. -func fetchURL(source string, timeout time.Duration) (io.ReadCloser, error) { - resp, err := httpGet(source, timeout) +func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) { + client := &http.Client{ + Transport: tr, + Timeout: timeout + 5*time.Second, + } + resp, err := client.Get(source) if err != nil { return nil, fmt.Errorf("http fetch: %v", err) } @@ -582,30 +585,3 @@ func adjustURL(source string, duration, timeout time.Duration) (string, time.Dur u.RawQuery = values.Encode() return u.String(), timeout } - -// httpGet is a wrapper around http.Get; it is defined as a variable -// so it can be redefined during for testing. -var httpGet = func(source string, timeout time.Duration) (*http.Response, error) { - url, err := url.Parse(source) - if err != nil { - return nil, err - } - - var tlsConfig *tls.Config - if url.Scheme == "https+insecure" { - tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - } - url.Scheme = "https" - source = url.String() - } - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: tlsConfig, - ResponseHeaderTimeout: timeout + 5*time.Second, - }, - } - return client.Get(source) -} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go index e67b2e9f87..b9e9dfe8f4 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go @@ -24,8 +24,8 @@ import ( "fmt" "io/ioutil" "math/big" + "net" "net/http" - "net/url" "os" "path/filepath" "reflect" @@ -39,6 +39,7 @@ import ( "github.com/google/pprof/internal/plugin" "github.com/google/pprof/internal/proftest" "github.com/google/pprof/internal/symbolizer" + "github.com/google/pprof/internal/transport" "github.com/google/pprof/profile" ) @@ -173,12 +174,6 @@ func (testFile) Close() error { func TestFetch(t *testing.T) { const path = "testdata/" - - // Intercept http.Get calls from HTTPFetcher. - savedHTTPGet := httpGet - defer func() { httpGet = savedHTTPGet }() - httpGet = stubHTTPGet - type testcase struct { source, execName string } @@ -188,7 +183,7 @@ func TestFetch(t *testing.T) { {path + "go.nomappings.crash", "/bin/gotest.exe"}, {"http://localhost/profile?file=cppbench.cpu", ""}, } { - p, _, _, err := grabProfile(&source{ExecName: tc.execName}, tc.source, nil, testObj{}, &proftest.TestUI{T: t}) + p, _, _, err := grabProfile(&source{ExecName: tc.execName}, tc.source, nil, testObj{}, &proftest.TestUI{T: t}, &httpTransport{}) if err != nil { t.Fatalf("%s: %s", tc.source, err) } @@ -449,8 +444,9 @@ func TestFetchWithBase(t *testing.T) { f.args = tc.sources o := setDefaults(&plugin.Options{ - UI: &proftest.TestUI{T: t, AllowRx: "Local symbolization failed|Some binary filenames not available"}, - Flagset: f, + UI: &proftest.TestUI{T: t, AllowRx: "Local symbolization failed|Some binary filenames not available"}, + Flagset: f, + HTTPTransport: transport.New(nil), }) src, _, err := parseFlags(o) @@ -503,19 +499,14 @@ func mappingSources(key, source string, start uint64) plugin.MappingSources { } } -// stubHTTPGet intercepts a call to http.Get and rewrites it to use -// "file://" to get the profile directly from a file. -func stubHTTPGet(source string, _ time.Duration) (*http.Response, error) { - url, err := url.Parse(source) - if err != nil { - return nil, err - } +type httpTransport struct{} - values := url.Query() +func (tr *httpTransport) RoundTrip(req *http.Request) (*http.Response, error) { + values := req.URL.Query() file := values.Get("file") if file == "" { - return nil, fmt.Errorf("want .../file?profile, got %s", source) + return nil, fmt.Errorf("want .../file?profile, got %s", req.URL.String()) } t := &http.Transport{} @@ -532,7 +523,7 @@ func closedError() string { return "use of closed" } -func TestHttpsInsecure(t *testing.T) { +func TestHTTPSInsecure(t *testing.T) { if runtime.GOOS == "nacl" || runtime.GOOS == "js" { t.Skip("test assumes tcp available") } @@ -553,7 +544,8 @@ func TestHttpsInsecure(t *testing.T) { pprofVariables = baseVars.makeCopy() defer func() { pprofVariables = baseVars }() - tlsConfig := &tls.Config{Certificates: []tls.Certificate{selfSignedCert(t)}} + tlsCert, _, _ := selfSignedCert(t, "") + tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}} l, err := tls.Listen("tcp", "localhost:0", tlsConfig) if err != nil { @@ -586,8 +578,9 @@ func TestHttpsInsecure(t *testing.T) { Symbolize: "remote", } o := &plugin.Options{ - Obj: &binutils.Binutils{}, - UI: &proftest.TestUI{T: t, AllowRx: "Saved profile in"}, + Obj: &binutils.Binutils{}, + UI: &proftest.TestUI{T: t, AllowRx: "Saved profile in"}, + HTTPTransport: transport.New(nil), } o.Sym = &symbolizer.Symbolizer{Obj: o.Obj, UI: o.UI} p, err := fetchProfiles(s, o) @@ -600,7 +593,122 @@ func TestHttpsInsecure(t *testing.T) { if len(p.Function) == 0 { t.Fatalf("fetchProfiles(%s) got non-symbolized profile: len(p.Function)==0", address) } - if err := checkProfileHasFunction(p, "TestHttpsInsecure"); err != nil { + if err := checkProfileHasFunction(p, "TestHTTPSInsecure"); err != nil { + t.Fatalf("fetchProfiles(%s) %v", address, err) + } +} + +func TestHTTPSWithServerCertFetch(t *testing.T) { + if runtime.GOOS == "nacl" || runtime.GOOS == "js" { + t.Skip("test assumes tcp available") + } + saveHome := os.Getenv(homeEnv()) + tempdir, err := ioutil.TempDir("", "home") + if err != nil { + t.Fatal("creating temp dir: ", err) + } + defer os.RemoveAll(tempdir) + + // pprof writes to $HOME/pprof by default which is not necessarily + // writeable (e.g. on a Debian buildd) so set $HOME to something we + // know we can write to for the duration of the test. + os.Setenv(homeEnv(), tempdir) + defer os.Setenv(homeEnv(), saveHome) + + baseVars := pprofVariables + pprofVariables = baseVars.makeCopy() + defer func() { pprofVariables = baseVars }() + + cert, certBytes, keyBytes := selfSignedCert(t, "localhost") + cas := x509.NewCertPool() + cas.AppendCertsFromPEM(certBytes) + + tlsConfig := &tls.Config{ + RootCAs: cas, + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: cas, + } + + l, err := tls.Listen("tcp", "localhost:0", tlsConfig) + if err != nil { + t.Fatalf("net.Listen: got error %v, want no error", err) + } + + donec := make(chan error, 1) + go func(donec chan<- error) { + donec <- http.Serve(l, nil) + }(donec) + defer func() { + if got, want := <-donec, closedError(); !strings.Contains(got.Error(), want) { + t.Fatalf("Serve got error %v, want %q", got, want) + } + }() + defer l.Close() + + outputTempFile, err := ioutil.TempFile("", "profile_output") + if err != nil { + t.Fatalf("Failed to create tempfile: %v", err) + } + defer os.Remove(outputTempFile.Name()) + defer outputTempFile.Close() + + // Get port from the address, so request to the server can be made using + // the host name specified in certificates. + _, portStr, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + t.Fatalf("cannot get port from URL: %v", err) + } + address := "https://" + "localhost:" + portStr + "/debug/pprof/goroutine" + s := &source{ + Sources: []string{address}, + Seconds: 10, + Timeout: 10, + Symbolize: "remote", + } + + certTempFile, err := ioutil.TempFile("", "cert_output") + if err != nil { + t.Errorf("cannot create cert tempfile: %v", err) + } + defer os.Remove(certTempFile.Name()) + defer certTempFile.Close() + certTempFile.Write(certBytes) + + keyTempFile, err := ioutil.TempFile("", "key_output") + if err != nil { + t.Errorf("cannot create key tempfile: %v", err) + } + defer os.Remove(keyTempFile.Name()) + defer keyTempFile.Close() + keyTempFile.Write(keyBytes) + + f := &testFlags{ + strings: map[string]string{ + "tls_cert": certTempFile.Name(), + "tls_key": keyTempFile.Name(), + "tls_ca": certTempFile.Name(), + }, + } + o := &plugin.Options{ + Obj: &binutils.Binutils{}, + UI: &proftest.TestUI{T: t, AllowRx: "Saved profile in"}, + Flagset: f, + HTTPTransport: transport.New(f), + } + + o.Sym = &symbolizer.Symbolizer{Obj: o.Obj, UI: o.UI, Transport: o.HTTPTransport} + p, err := fetchProfiles(s, o) + if err != nil { + t.Fatal(err) + } + if len(p.SampleType) == 0 { + t.Fatalf("fetchProfiles(%s) got empty profile: len(p.SampleType)==0", address) + } + if len(p.Function) == 0 { + t.Fatalf("fetchProfiles(%s) got non-symbolized profile: len(p.Function)==0", address) + } + if err := checkProfileHasFunction(p, "TestHTTPSWithServerCertFetch"); err != nil { t.Fatalf("fetchProfiles(%s) %v", address, err) } } @@ -614,7 +722,10 @@ func checkProfileHasFunction(p *profile.Profile, fname string) error { return fmt.Errorf("got %s, want function %q", p.String(), fname) } -func selfSignedCert(t *testing.T) tls.Certificate { +// selfSignedCert generates a self-signed certificate, and returns the +// generated certificate, and byte arrays containing the certificate and +// key associated with the certificate. +func selfSignedCert(t *testing.T, host string) (tls.Certificate, []byte, []byte) { privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatalf("failed to generate private key: %v", err) @@ -629,6 +740,8 @@ func selfSignedCert(t *testing.T) tls.Certificate { SerialNumber: big.NewInt(1), NotBefore: time.Now(), NotAfter: time.Now().Add(10 * time.Minute), + IsCA: true, + DNSNames: []string{host}, } b, err = x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, privKey.Public(), privKey) @@ -641,5 +754,5 @@ func selfSignedCert(t *testing.T) tls.Certificate { if err != nil { t.Fatalf("failed to create TLS key pair: %v", err) } - return cert + return cert, bc, bk } diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/flags.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/flags.go new file mode 100644 index 0000000000..c48fb5cd2e --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/flags.go @@ -0,0 +1,78 @@ +package driver + +import ( + "flag" + "strings" +) + +// GoFlags implements the plugin.FlagSet interface. +type GoFlags struct { + UsageMsgs []string +} + +// Bool implements the plugin.FlagSet interface. +func (*GoFlags) Bool(o string, d bool, c string) *bool { + return flag.Bool(o, d, c) +} + +// Int implements the plugin.FlagSet interface. +func (*GoFlags) Int(o string, d int, c string) *int { + return flag.Int(o, d, c) +} + +// Float64 implements the plugin.FlagSet interface. +func (*GoFlags) Float64(o string, d float64, c string) *float64 { + return flag.Float64(o, d, c) +} + +// String implements the plugin.FlagSet interface. +func (*GoFlags) String(o, d, c string) *string { + return flag.String(o, d, c) +} + +// BoolVar implements the plugin.FlagSet interface. +func (*GoFlags) BoolVar(b *bool, o string, d bool, c string) { + flag.BoolVar(b, o, d, c) +} + +// IntVar implements the plugin.FlagSet interface. +func (*GoFlags) IntVar(i *int, o string, d int, c string) { + flag.IntVar(i, o, d, c) +} + +// Float64Var implements the plugin.FlagSet interface. +// the value of the flag. +func (*GoFlags) Float64Var(f *float64, o string, d float64, c string) { + flag.Float64Var(f, o, d, c) +} + +// StringVar implements the plugin.FlagSet interface. +func (*GoFlags) StringVar(s *string, o, d, c string) { + flag.StringVar(s, o, d, c) +} + +// StringList implements the plugin.FlagSet interface. +func (*GoFlags) StringList(o, d, c string) *[]*string { + return &[]*string{flag.String(o, d, c)} +} + +// ExtraUsage implements the plugin.FlagSet interface. +func (f *GoFlags) ExtraUsage() string { + return strings.Join(f.UsageMsgs, "\n") +} + +// AddExtraUsage implements the plugin.FlagSet interface. +func (f *GoFlags) AddExtraUsage(eu string) { + f.UsageMsgs = append(f.UsageMsgs, eu) +} + +// Parse implements the plugin.FlagSet interface. +func (*GoFlags) Parse(usage func()) []string { + flag.Usage = usage + flag.Parse() + args := flag.Args() + if len(args) == 0 { + usage() + } + return args +} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph.go index c9b9a5398f..13613cff86 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph.go @@ -55,7 +55,7 @@ func (ui *webInterface) flamegraph(w http.ResponseWriter, req *http.Request) { v := n.CumValue() fullName := n.Info.PrintableName() node := &treeNode{ - Name: getNodeShortName(fullName), + Name: graph.ShortenFunctionName(fullName), FullName: fullName, Cum: v, CumFormat: config.FormatValue(v), @@ -101,19 +101,3 @@ func (ui *webInterface) flamegraph(w http.ResponseWriter, req *http.Request) { Nodes: nodeArr, }) } - -// getNodeShortName builds a short node name from fullName. -func getNodeShortName(name string) string { - chunks := strings.SplitN(name, "(", 2) - head := chunks[0] - pathSep := strings.LastIndexByte(head, '/') - if pathSep == -1 || pathSep+1 >= len(head) { - return name - } - // Check if name is a stdlib package, i.e. doesn't have "." before "/" - if dot := strings.IndexByte(head, '.'); dot == -1 || dot > pathSep { - return name - } - // Trim package path prefix from node name - return name[pathSep+1:] -} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph_test.go deleted file mode 100644 index c1a887c830..0000000000 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/flamegraph_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package driver - -import "testing" - -func TestGetNodeShortName(t *testing.T) { - type testCase struct { - name string - want string - } - testcases := []testCase{ - { - "root", - "root", - }, - { - "syscall.Syscall", - "syscall.Syscall", - }, - { - "net/http.(*conn).serve", - "net/http.(*conn).serve", - }, - { - "github.com/blah/foo.Foo", - "foo.Foo", - }, - { - "github.com/blah/foo_bar.(*FooBar).Foo", - "foo_bar.(*FooBar).Foo", - }, - { - "encoding/json.(*structEncoder).(encoding/json.encode)-fm", - "encoding/json.(*structEncoder).(encoding/json.encode)-fm", - }, - { - "github.com/blah/blah/vendor/gopkg.in/redis.v3.(*baseClient).(github.com/blah/blah/vendor/gopkg.in/redis.v3.process)-fm", - "redis.v3.(*baseClient).(github.com/blah/blah/vendor/gopkg.in/redis.v3.process)-fm", - }, - } - for _, tc := range testcases { - name := getNodeShortName(tc.name) - if got, want := name, tc.want; got != want { - t.Errorf("for %s, got %q, want %q", tc.name, got, want) - } - } -} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go index 8d775e16bd..758adf9bdc 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go @@ -23,6 +23,7 @@ import ( "github.com/google/pprof/internal/plugin" "github.com/google/pprof/internal/proftest" "github.com/google/pprof/internal/report" + "github.com/google/pprof/internal/transport" "github.com/google/pprof/profile" ) @@ -41,7 +42,10 @@ func TestShell(t *testing.T) { // Random interleave of independent scripts pprofVariables = testVariables(savedVariables) - o := setDefaults(nil) + + // pass in HTTPTransport when setting defaults, because otherwise default + // transport will try to add flags to the default flag set. + o := setDefaults(&plugin.Options{HTTPTransport: transport.New(nil)}) o.UI = newUI(t, interleave(script, 0)) if err := interactive(p, o); err != nil { t.Error("first attempt:", err) @@ -259,12 +263,13 @@ func TestInteractiveCommands(t *testing.T) { { "weblist find -test", map[string]string{ - "functions": "false", - "addressnoinlines": "true", - "nodecount": "0", - "cum": "false", - "flat": "true", - "ignore": "test", + "functions": "false", + "addresses": "true", + "noinlines": "true", + "nodecount": "0", + "cum": "false", + "flat": "true", + "ignore": "test", }, }, { diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/options.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/options.go index 34167d4bf5..6e8f9fca25 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/options.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/options.go @@ -16,7 +16,6 @@ package driver import ( "bufio" - "flag" "fmt" "io" "os" @@ -25,6 +24,7 @@ import ( "github.com/google/pprof/internal/binutils" "github.com/google/pprof/internal/plugin" "github.com/google/pprof/internal/symbolizer" + "github.com/google/pprof/internal/transport" ) // setDefaults returns a new plugin.Options with zero fields sets to @@ -38,7 +38,7 @@ func setDefaults(o *plugin.Options) *plugin.Options { d.Writer = oswriter{} } if d.Flagset == nil { - d.Flagset = goFlags{} + d.Flagset = &GoFlags{} } if d.Obj == nil { d.Obj = &binutils.Binutils{} @@ -46,67 +46,15 @@ func setDefaults(o *plugin.Options) *plugin.Options { if d.UI == nil { d.UI = &stdUI{r: bufio.NewReader(os.Stdin)} } + if d.HTTPTransport == nil { + d.HTTPTransport = transport.New(d.Flagset) + } if d.Sym == nil { - d.Sym = &symbolizer.Symbolizer{Obj: d.Obj, UI: d.UI} + d.Sym = &symbolizer.Symbolizer{Obj: d.Obj, UI: d.UI, Transport: d.HTTPTransport} } return d } -// goFlags returns a flagset implementation based on the standard flag -// package from the Go distribution. It implements the plugin.FlagSet -// interface. -type goFlags struct{} - -func (goFlags) Bool(o string, d bool, c string) *bool { - return flag.Bool(o, d, c) -} - -func (goFlags) Int(o string, d int, c string) *int { - return flag.Int(o, d, c) -} - -func (goFlags) Float64(o string, d float64, c string) *float64 { - return flag.Float64(o, d, c) -} - -func (goFlags) String(o, d, c string) *string { - return flag.String(o, d, c) -} - -func (goFlags) BoolVar(b *bool, o string, d bool, c string) { - flag.BoolVar(b, o, d, c) -} - -func (goFlags) IntVar(i *int, o string, d int, c string) { - flag.IntVar(i, o, d, c) -} - -func (goFlags) Float64Var(f *float64, o string, d float64, c string) { - flag.Float64Var(f, o, d, c) -} - -func (goFlags) StringVar(s *string, o, d, c string) { - flag.StringVar(s, o, d, c) -} - -func (goFlags) StringList(o, d, c string) *[]*string { - return &[]*string{flag.String(o, d, c)} -} - -func (goFlags) ExtraUsage() string { - return "" -} - -func (goFlags) Parse(usage func()) []string { - flag.Usage = usage - flag.Parse() - args := flag.Args() - if len(args) == 0 { - usage() - } - return args -} - type stdUI struct { r *bufio.Reader } diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.addresses.noinlines.text b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.addresses.noinlines.text new file mode 100644 index 0000000000..d53c44dad9 --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.addresses.noinlines.text @@ -0,0 +1,7 @@ +Showing nodes accounting for 1.12s, 100% of 1.12s total +Dropped 1 node (cum <= 0.06s) + flat flat% sum% cum cum% + 1.10s 98.21% 98.21% 1.10s 98.21% 0000000000001000 line1000 testdata/file1000.src:1 + 0.01s 0.89% 99.11% 1.01s 90.18% 0000000000002000 line2000 testdata/file2000.src:4 + 0.01s 0.89% 100% 1.01s 90.18% 0000000000003000 line3000 testdata/file3000.src:6 + 0 0% 100% 0.10s 8.93% 0000000000003001 line3000 testdata/file3000.src:9 diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.filefunctions.noinlines.text b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.filefunctions.noinlines.text new file mode 100644 index 0000000000..88fb760759 --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.filefunctions.noinlines.text @@ -0,0 +1,5 @@ +Showing nodes accounting for 1.12s, 100% of 1.12s total + flat flat% sum% cum cum% + 1.10s 98.21% 98.21% 1.10s 98.21% line1000 testdata/file1000.src + 0.01s 0.89% 99.11% 1.01s 90.18% line2000 testdata/file2000.src + 0.01s 0.89% 100% 1.12s 100% line3000 testdata/file3000.src diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.functions.noinlines.text b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.functions.noinlines.text new file mode 100644 index 0000000000..493b4912de --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.flat.functions.noinlines.text @@ -0,0 +1,5 @@ +Showing nodes accounting for 1.12s, 100% of 1.12s total + flat flat% sum% cum cum% + 1.10s 98.21% 98.21% 1.10s 98.21% line1000 + 0.01s 0.89% 99.11% 1.01s 90.18% line2000 + 0.01s 0.89% 100% 1.12s 100% line3000 diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.lines.topproto b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.lines.topproto new file mode 100644 index 0000000000..33bf6814a4 --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.cpu.lines.topproto @@ -0,0 +1,3 @@ +Showing nodes accounting for 1s, 100% of 1s total + flat flat% sum% cum cum% + 1s 100% 100% 1s 100% mangled1000 testdata/file1000.src:1 diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.dot b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.dot new file mode 100644 index 0000000000..474a5108ba --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.dot @@ -0,0 +1,9 @@ +digraph "testbinary" { +node [style=filled fillcolor="#f8f8f8"] +subgraph cluster_L { "File: testbinary" [shape=box fontsize=16 label="File: testbinary\lType: cpu\lDuration: 10s, Total samples = 1.11s (11.10%)\lShowing nodes accounting for 1.11s, 100% of 1.11s total\l" tooltip="testbinary"] } +N1 [label="package1\nobject\nfunction1\n1.10s (99.10%)" id="node1" fontsize=24 shape=box tooltip="path/to/package1.object.function1 (1.10s)" color="#b20000" fillcolor="#edd5d5"] +N2 [label="FooBar\nrun\n0.01s (0.9%)\nof 1.01s (90.99%)" id="node2" fontsize=10 shape=box tooltip="java.bar.foo.FooBar.run(java.lang.Runnable) (1.01s)" color="#b20400" fillcolor="#edd6d5"] +N3 [label="Bar\nFoo\n0 of 1.10s (99.10%)" id="node3" fontsize=8 shape=box tooltip="(anonymous namespace)::Bar::Foo (1.10s)" color="#b20000" fillcolor="#edd5d5"] +N3 -> N1 [label=" 1.10s" weight=100 penwidth=5 color="#b20000" tooltip="(anonymous namespace)::Bar::Foo -> path/to/package1.object.function1 (1.10s)" labeltooltip="(anonymous namespace)::Bar::Foo -> path/to/package1.object.function1 (1.10s)"] +N2 -> N3 [label=" 1s" weight=91 penwidth=5 color="#b20500" tooltip="java.bar.foo.FooBar.run(java.lang.Runnable) -> (anonymous namespace)::Bar::Foo (1s)" labeltooltip="java.bar.foo.FooBar.run(java.lang.Runnable) -> (anonymous namespace)::Bar::Foo (1s)"] +} diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.text b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.text new file mode 100644 index 0000000000..39cb24ed6a --- /dev/null +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/testdata/pprof.longNameFuncs.text @@ -0,0 +1,5 @@ +Showing nodes accounting for 1.11s, 100% of 1.11s total + flat flat% sum% cum cum% + 1.10s 99.10% 99.10% 1.10s 99.10% path/to/package1.object.function1 + 0.01s 0.9% 100% 1.01s 90.99% java.bar.foo.FooBar.run(java.lang.Runnable) + 0 0% 100% 1.10s 99.10% (anonymous namespace)::Bar::Foo diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/webhtml.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/webhtml.go index c3f9c384f8..74104899ca 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/webhtml.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/webhtml.go @@ -249,6 +249,21 @@ table tr td { + {{$sampleLen := len .SampleTypes}} + {{if gt $sampleLen 1}} +

+ {{end}} +